Lines Matching refs:DAG
1 //===-- RISCVISelLowering.cpp - RISC-V DAG Lowering Implementation -------===//
10 // selection DAG.
2031 SelectionDAG &DAG) const {
2388 ISD::CondCode &CC, SelectionDAG &DAG) {
2406 LHS = DAG.getNode(ISD::SHL, DL, LHS.getValueType(), LHS,
2407 DAG.getConstant(ShAmt, DL, LHS.getValueType()));
2419 RHS = DAG.getConstant(0, DL, RHS.getValueType());
2428 LHS = DAG.getConstant(0, DL, RHS.getValueType());
2717 static MVT getContainerForFixedLengthVector(SelectionDAG &DAG, MVT VT,
2719 return getContainerForFixedLengthVector(DAG.getTargetLoweringInfo(), VT,
2728 static SDValue convertToScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
2735 SDValue Zero = DAG.getVectorIdxConstant(0, DL);
2736 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero);
2740 static SDValue convertFromScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
2747 SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
2748 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero);
2763 SelectionDAG &DAG) {
2765 return DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
2769 SelectionDAG &DAG, const RISCVSubtarget &Subtarget) {
2776 return DAG.getRegister(RISCV::X0, Subtarget.getXLenVT());
2778 return DAG.getConstant(NumElts, DL, Subtarget.getXLenVT());
2782 getDefaultScalableVLOps(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG,
2785 SDValue VL = DAG.getRegister(RISCV::X0, Subtarget.getXLenVT());
2786 SDValue Mask = getAllOnesMask(VecVT, VL, DL, DAG);
2792 SelectionDAG &DAG, const RISCVSubtarget &Subtarget) {
2794 SDValue VL = getVLOp(NumElts, ContainerVT, DL, DAG, Subtarget);
2795 SDValue Mask = getAllOnesMask(ContainerVT, VL, DL, DAG);
2804 getDefaultVLOps(MVT VecVT, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG,
2807 return getDefaultVLOps(VecVT.getVectorNumElements(), ContainerVT, DL, DAG,
2810 return getDefaultScalableVLOps(ContainerVT, DL, DAG, Subtarget);
2814 SelectionDAG &DAG) const {
2816 return DAG.getElementCount(DL, Subtarget.getXLenVT(),
2905 static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG,
2922 Src = DAG.getNode(ISD::FP_EXTEND, SDLoc(Op), MVT::f32, Src);
2935 SDValue FpToInt = DAG.getNode(
2937 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT()));
2940 FpToInt = DAG.getZeroExtendInReg(FpToInt, DL, MVT::i32);
2942 SDValue ZeroInt = DAG.getConstant(0, DL, DstVT);
2943 return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt,
2966 DstContainerVT = getContainerForFixedLengthVector(DAG, DstVT, Subtarget);
2967 SrcContainerVT = getContainerForFixedLengthVector(DAG, SrcVT, Subtarget);
2971 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
2976 auto [Mask, VL] = getDefaultVLOps(DstVT, DstContainerVT, DL, DAG, Subtarget);
2978 SDValue IsNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
2979 {Src, Src, DAG.getCondCode(ISD::SETNE),
2980 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
2987 Src = DAG.getNode(RISCVISD::FP_EXTEND_VL, DL, InterVT, Src, Mask, VL);
2992 SDValue Res = DAG.getNode(RVVOpc, DL, DstContainerVT, Src, Mask, VL);
2994 SDValue SplatZero = DAG.getNode(
2995 RISCVISD::VMV_V_X_VL, DL, DstContainerVT, DAG.getUNDEF(DstContainerVT),
2996 DAG.getConstant(0, DL, Subtarget.getXLenVT()), VL);
2997 Res = DAG.getNode(RISCVISD::VMERGE_VL, DL, DstContainerVT, IsNan, SplatZero,
2998 Res, DAG.getUNDEF(DstContainerVT), VL);
3001 Res = convertFromScalableVector(DstVT, Res, DAG, Subtarget);
3040 lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
3051 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3052 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
3059 Mask = convertToScalableVector(getMaskTypeFor(ContainerVT), Mask, DAG,
3063 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3067 Src = DAG.getFreeze(Src);
3070 SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL);
3075 const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(ContainerVT);
3081 DAG.getConstantFP(MaxVal, DL, ContainerVT.getVectorElementType());
3082 SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT,
3083 DAG.getUNDEF(ContainerVT), MaxValNode, VL);
3088 DAG.getNode(RISCVISD::SETCC_VL, DL, SetccVT,
3089 {Abs, MaxValSplat, DAG.getCondCode(ISD::SETOLT),
3111 Truncated = DAG.getNode(RISCVISD::VFCVT_RM_X_F_VL, DL, IntVT, Src, Mask,
3112 DAG.getTargetConstant(FRM, DL, XLenVT), VL);
3116 Truncated = DAG.getNode(RISCVISD::VFCVT_RTZ_X_F_VL, DL, IntVT, Src,
3121 Truncated = DAG.getNode(RISCVISD::VFCVT_X_F_VL, DL, IntVT, Src, Mask, VL);
3125 Truncated = DAG.getNode(RISCVISD::VFROUND_NOEXCEPT_VL, DL, ContainerVT, Src,
3132 Truncated = DAG.getNode(RISCVISD::SINT_TO_FP_VL, DL, ContainerVT, Truncated,
3136 Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated,
3142 return convertFromScalableVector(VT, Truncated, DAG, Subtarget);
3149 lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
3158 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3159 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
3162 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3165 Src = DAG.getFreeze(Src);
3169 SDValue Unorder = DAG.getNode(RISCVISD::STRICT_FSETCC_VL, DL,
3170 DAG.getVTList(MaskVT, MVT::Other),
3171 {Chain, Src, Src, DAG.getCondCode(ISD::SETUNE),
3172 DAG.getUNDEF(MaskVT), Mask, VL});
3174 Src = DAG.getNode(RISCVISD::STRICT_FADD_VL, DL,
3175 DAG.getVTList(ContainerVT, MVT::Other),
3180 SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL);
3185 const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(ContainerVT);
3191 DAG.getConstantFP(MaxVal, DL, ContainerVT.getVectorElementType());
3192 SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT,
3193 DAG.getUNDEF(ContainerVT), MaxValNode, VL);
3196 Mask = DAG.getNode(
3198 {Abs, MaxValSplat, DAG.getCondCode(ISD::SETOLT), Mask, Mask, VL});
3214 Truncated = DAG.getNode(
3215 RISCVISD::STRICT_VFCVT_RM_X_F_VL, DL, DAG.getVTList(IntVT, MVT::Other),
3216 {Chain, Src, Mask, DAG.getTargetConstant(FRM, DL, XLenVT), VL});
3221 DAG.getNode(RISCVISD::STRICT_VFCVT_RTZ_X_F_VL, DL,
3222 DAG.getVTList(IntVT, MVT::Other), Chain, Src, Mask, VL);
3225 Truncated = DAG.getNode(RISCVISD::STRICT_VFROUND_NOEXCEPT_VL, DL,
3226 DAG.getVTList(ContainerVT, MVT::Other), Chain, Src,
3234 Truncated = DAG.getNode(RISCVISD::STRICT_SINT_TO_FP_VL, DL,
3235 DAG.getVTList(ContainerVT, MVT::Other), Chain,
3241 Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated,
3245 Truncated = convertFromScalableVector(VT, Truncated, DAG, Subtarget);
3246 return DAG.getMergeValues({Truncated, Chain}, DL);
3250 lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
3254 return lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
3256 if (DAG.shouldOptForSize())
3265 const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT);
3270 SDValue MaxValNode = DAG.getConstantFP(MaxVal, DL, VT);
3273 return DAG.getNode(RISCVISD::FROUND, DL, VT, Src, MaxValNode,
3274 DAG.getTargetConstant(FRM, DL, Subtarget.getXLenVT()));
3278 static SDValue lowerVectorXRINT(SDValue Op, SelectionDAG &DAG,
3288 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3289 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
3292 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3294 DAG.getNode(RISCVISD::VFCVT_X_F_VL, DL, ContainerVT, Src, Mask, VL);
3299 return convertFromScalableVector(VT, Truncated, DAG, Subtarget);
3303 getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget,
3309 SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT());
3311 return DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, VT, Ops);
3315 getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL,
3321 SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT());
3323 return DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, VT, Ops);
3472 SelectionDAG &DAG,
3489 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3490 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
3493 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3495 SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, Vec,
3496 Idx, DAG.getUNDEF(ContainerVT), Mask, VL);
3501 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
3513 static SDValue lowerBuildVectorViaDominantValues(SDValue Op, SelectionDAG &DAG,
3518 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3521 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3564 if (!DAG.shouldOptForSize() && NumScalarLoads < NumElts &&
3569 SDValue Vec = DAG.getSplatBuildVector(VT, DL, DominantValue);
3581 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
3585 LastOp = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, LastOp);
3586 Vec = DAG.getNode(OpCode, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Vec,
3588 Vec = convertFromScalableVector(VT, Vec, DAG, Subtarget);
3598 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Vec, V,
3599 DAG.getVectorIdxConstant(OpIdx.index(), DL));
3606 return DAG.getConstant(V == V1, DL, XLenVT);
3608 Vec = DAG.getNode(ISD::VSELECT, DL, VT,
3609 DAG.getBuildVector(SelMaskTy, DL, Ops),
3610 DAG.getSplatBuildVector(VT, DL, V), Vec);
3620 static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG,
3625 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3628 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3635 SDValue VMClr = DAG.getNode(RISCVISD::VMCLR_VL, DL, ContainerVT, VL);
3636 return convertFromScalableVector(VT, VMClr, DAG, Subtarget);
3640 SDValue VMSet = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
3641 return convertFromScalableVector(VT, VMSet, DAG, Subtarget);
3657 if (DAG.shouldOptForSize() && NumElts > NumViaIntegerBits)
3682 SDValue Elt = DAG.getConstant(Bits, DL, XLenVT);
3690 SDValue Vec = DAG.getBuildVector(IntegerViaVecVT, DL, Elts);
3697 Vec = DAG.getBitcast(MVT::v8i1, Vec);
3698 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Vec,
3699 DAG.getConstant(0, DL, XLenVT));
3704 Vec = DAG.getBitcast(VT, Vec);
3714 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Splat);
3716 DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Splat, VL);
3717 return convertFromScalableVector(VT, Splat, DAG, Subtarget);
3752 getContainerForFixedLengthVector(DAG, VIDVT, Subtarget);
3753 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VIDContainerVT, Mask, VL);
3758 VID = convertFromScalableVector(VIDVT, VID, DAG, Subtarget);
3761 SDValue SplatStep = DAG.getConstant(SplatStepVal, DL, VIDVT);
3762 VID = DAG.getNode(StepOpcode, DL, VIDVT, VID, SplatStep);
3766 DAG.getConstant(Log2_64(StepDenominator), DL, VIDVT);
3767 VID = DAG.getNode(ISD::SRL, DL, VIDVT, VID, SplatStep);
3770 SDValue SplatAddend = DAG.getConstant(Addend, DL, VIDVT);
3771 VID = DAG.getNode(Negate ? ISD::SUB : ISD::ADD, DL, VIDVT, SplatAddend,
3776 VID = DAG.getNode(ISD::SINT_TO_FP, DL, VT, VID);
3812 SDValue Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ViaVecVT,
3813 DAG.getUNDEF(ViaVecVT),
3814 DAG.getConstant(SplatValue, DL, XLenVT),
3815 DAG.getVectorIdxConstant(0, DL));
3817 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL,
3819 DAG.getConstant(0, DL, XLenVT));
3820 return DAG.getBitcast(VT, Vec);
3877 DAG.getConstant(ViaVecVT.getVectorNumElements(), DL, XLenVT);
3879 getContainerForFixedLengthVector(DAG, ViaVecVT, Subtarget);
3881 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ViaContainerVT,
3882 DAG.getUNDEF(ViaContainerVT),
3883 DAG.getConstant(SplatValue, DL, XLenVT), ViaVL);
3884 Splat = convertFromScalableVector(ViaVecVT, Splat, DAG, Subtarget);
3886 Splat = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL,
3888 DAG.getConstant(0, DL, XLenVT));
3889 return DAG.getBitcast(VT, Splat);
3901 unsigned SignBits = DAG.ComputeNumSignBits(Op);
3903 SDValue Source = DAG.getBuildVector(VT.changeVectorElementType(MVT::i8),
3906 Source, DAG, Subtarget);
3907 SDValue Res = DAG.getNode(RISCVISD::VSEXT_VL, DL, ContainerVT, Source, Mask, VL);
3908 return convertFromScalableVector(VT, Res, DAG, Subtarget);
3912 if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget))
3941 static SDValue lowerBuildVectorViaPacking(SDValue Op, SelectionDAG &DAG,
3964 SDValue Mask = DAG.getConstant(
3975 DAG.getMachineNode(getPACKOpcode(ElemSizeInBits * 2, Subtarget),
3979 A = DAG.getNode(ISD::AND, SDLoc(A), XLenVT, A, Mask);
3980 B = DAG.getNode(ISD::AND, SDLoc(B), XLenVT, B, Mask);
3981 SDValue ShtAmt = DAG.getConstant(ElemSizeInBits, ElemDL, XLenVT);
3984 return DAG.getNode(ISD::OR, ElemDL, XLenVT, A,
3985 DAG.getNode(ISD::SHL, ElemDL, XLenVT, B, ShtAmt), Flags);
3995 return DAG.getNode(ISD::BITCAST, DL, VT,
3996 DAG.getBuildVector(WideVecVT, DL, NewOperands));
4000 static SDValue lowerBUILD_VECTORvXf16(SDValue Op, SelectionDAG &DAG) {
4005 NewOps[I] = DAG.getBitcast(MVT::i16, Op.getOperand(I));
4006 SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Op), IVT, NewOps);
4007 return DAG.getBitcast(VT, Res);
4010 static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
4018 return lowerBUILD_VECTORvXf16(Op, DAG);
4022 return lowerBuildVectorOfConstants(Op, DAG, Subtarget);
4024 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
4027 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4036 SDValue VecZero = DAG.getConstant(0, DL, WideVecVT);
4042 Splat = DAG.getNode(ISD::AND, DL, Splat.getValueType(), Splat,
4043 DAG.getConstant(1, DL, Splat.getValueType()));
4044 WideVec = DAG.getSplatBuildVector(WideVecVT, DL, Splat);
4047 WideVec = DAG.getBuildVector(WideVecVT, DL, Ops);
4048 SDValue VecOne = DAG.getConstant(1, DL, WideVecVT);
4049 WideVec = DAG.getNode(ISD::AND, DL, WideVecVT, WideVec, VecOne);
4052 return DAG.getSetCC(DL, VT, WideVec, VecZero, ISD::SETNE);
4056 if (auto Gather = matchSplatAsGather(Splat, VT, DL, DAG, Subtarget))
4061 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Splat);
4063 DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Splat, VL);
4064 return convertFromScalableVector(VT, Splat, DAG, Subtarget);
4067 if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget))
4076 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
4078 MVT M1VT = getContainerForFixedLengthVector(DAG, OneRegVT, Subtarget);
4083 // insert_subvector here to avoid DAG combining it back to a large
4087 SDValue Vec = DAG.getUNDEF(ContainerVT);
4091 DAG.getNode(ISD::BUILD_VECTOR, DL, OneRegVT, OneVRegOfOps);
4092 SubBV = convertToScalableVector(M1VT, SubBV, DAG, Subtarget);
4094 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT, Vec, SubBV,
4095 DAG.getVectorIdxConstant(InsertIdx, DL));
4097 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
4104 if (SDValue Res = lowerBuildVectorViaPacking(Op, DAG, Subtarget))
4123 SDValue UndefElem = DAG.getUNDEF(Op->getOperand(0)->getValueType(0));
4136 MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
4141 SDValue SubVecA = DAG.getBuildVector(VT, DL, SubVecAOps);
4142 SDValue SubVecB = DAG.getBuildVector(VT, DL, SubVecBOps);
4144 SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
4145 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, SubVecA, SubVecB);
4211 Vec = DAG.getSplatVector(VT, DL, V);
4212 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4218 const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT());
4219 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
4226 V = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), V);
4227 Vec = DAG.getNode(OpCode, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Vec,
4231 const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT());
4232 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
4235 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
4240 SelectionDAG &DAG) {
4242 Passthru = DAG.getUNDEF(VT);
4249 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
4261 NewVL = DAG.getRegister(RISCV::X0, MVT::i32);
4263 NewVL = DAG.getNode(ISD::ADD, DL, VL.getValueType(), VL, VL);
4268 auto InterVec = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, InterVT,
4269 DAG.getUNDEF(InterVT), Lo, NewVL);
4270 return DAG.getNode(ISD::BITCAST, DL, VT, InterVec);
4279 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
4284 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
4287 return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VT, Passthru, Lo,
4296 SelectionDAG &DAG) {
4299 std::tie(Lo, Hi) = DAG.SplitScalar(Scalar, DL, MVT::i32, MVT::i32);
4300 return splatPartsI64WithVL(DL, VT, Passthru, Lo, Hi, VL, DAG);
4307 MVT VT, const SDLoc &DL, SelectionDAG &DAG,
4311 Passthru = DAG.getUNDEF(VT);
4313 return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, VT, Passthru, Scalar, VL);
4325 Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar);
4326 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Scalar, VL);
4333 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, Passthru,
4334 DAG.getConstant(0, DL, XLenVT), VL);
4337 return splatSplitI64WithVL(DL, VT, Passthru, Scalar, VL, DAG);
4345 const SDLoc &DL, SelectionDAG &DAG,
4350 SDValue Passthru = DAG.getUNDEF(VT);
4362 DAG, ExtractedContainerVT, Subtarget);
4364 ExtractedVal, DAG, Subtarget);
4367 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Passthru,
4368 ExtractedVal, DAG.getVectorIdxConstant(0, DL));
4369 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, ExtractedVal,
4370 DAG.getVectorIdxConstant(0, DL));
4376 return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, VT,
4377 DAG.getUNDEF(VT), Scalar, VL);
4382 return lowerScalarSplat(DAG.getUNDEF(VT), Scalar,
4383 DAG.getConstant(1, DL, XLenVT),
4384 VT, DL, DAG, Subtarget);
4392 Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar);
4393 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT,
4394 DAG.getUNDEF(VT), Scalar, VL);
4562 SelectionDAG &DAG) {
4568 ContainerVT = getContainerForFixedLengthVector(DAG, ContainerVT, Subtarget);
4574 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
4577 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4584 Src = DAG.getBitcast(WideSrcContainerVT, Src);
4592 SDValue SplatShift = DAG.getNode(
4593 RISCVISD::VMV_V_X_VL, DL, IntContainerVT, DAG.getUNDEF(ContainerVT),
4594 DAG.getConstant(Shift, DL, Subtarget.getXLenVT()), VL);
4596 DAG.getNode(RISCVISD::VNSRL_VL, DL, IntContainerVT, Src, SplatShift,
4597 DAG.getUNDEF(IntContainerVT), TrueMask, VL);
4599 Res = DAG.getBitcast(ContainerVT, Res);
4602 Res = convertFromScalableVector(VT, Res, DAG, Subtarget);
4622 SelectionDAG &DAG) {
4671 MVT ContainerVT = getContainerForFixedLengthVector(DAG, SrcVT, Subtarget);
4672 auto [TrueMask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
4674 getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
4675 convertToScalableVector(ContainerVT, Src, DAG, Subtarget),
4676 DAG.getConstant(NewMask[0], DL, XLenVT), TrueMask, VL);
4677 return DAG.getNode(
4679 convertFromScalableVector(SrcVT, Slidedown, DAG, Subtarget),
4680 DAG.getConstant(0, DL, XLenVT));
4699 SelectionDAG &DAG) {
4711 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
4712 auto TrueMask = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).first;
4721 InPlace = convertToScalableVector(ContainerVT, InPlace, DAG, Subtarget);
4722 ToInsert = convertToScalableVector(ContainerVT, ToInsert, DAG, Subtarget);
4723 SDValue VL = DAG.getConstant(NumSubElts + Index, DL, XLenVT);
4729 Res = DAG.getNode(RISCVISD::VMV_V_V_VL, DL, ContainerVT, InPlace, ToInsert,
4732 Res = getVSlideup(DAG, Subtarget, DL, ContainerVT, InPlace, ToInsert,
4733 DAG.getConstant(Index, DL, XLenVT), TrueMask, VL, Policy);
4734 return convertFromScalableVector(VT, Res, DAG, Subtarget);
4743 SelectionDAG &DAG) {
4776 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
4777 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4782 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), Splat);
4783 auto Vec = DAG.getNode(OpCode, DL, ContainerVT,
4784 DAG.getUNDEF(ContainerVT),
4785 convertToScalableVector(ContainerVT, V2, DAG, Subtarget),
4787 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
4794 const SDLoc &DL, SelectionDAG &DAG,
4800 VecContainerVT = getContainerForFixedLengthVector(DAG, VecVT, Subtarget);
4801 EvenV = convertToScalableVector(VecContainerVT, EvenV, DAG, Subtarget);
4802 OddV = convertToScalableVector(VecContainerVT, OddV, DAG, Subtarget);
4816 WideContainerVT = getContainerForFixedLengthVector(DAG, WideVT, Subtarget);
4820 EvenV = DAG.getBitcast(VecContainerVT, EvenV);
4821 OddV = DAG.getBitcast(VecContainerVT, OddV);
4823 auto [Mask, VL] = getDefaultVLOps(VecVT, VecContainerVT, DL, DAG, Subtarget);
4824 SDValue Passthru = DAG.getUNDEF(WideContainerVT);
4832 DAG.getNode(RISCVISD::VZEXT_VL, DL, WideContainerVT, EvenV, Mask, VL);
4836 DAG.getConstant(VecVT.getScalarSizeInBits(), DL, VecContainerVT);
4837 Interleaved = DAG.getNode(RISCVISD::VWSLL_VL, DL, WideContainerVT, OddV,
4840 Interleaved = DAG.getNode(RISCVISD::VWADDU_W_VL, DL, WideContainerVT,
4844 DAG.getNode(RISCVISD::VZEXT_VL, DL, WideContainerVT, OddV, Mask, VL);
4847 DAG.getConstant(VecVT.getScalarSizeInBits(), DL, WideContainerVT);
4848 Interleaved = DAG.getNode(RISCVISD::SHL_VL, DL, WideContainerVT,
4856 Interleaved = DAG.getNode(RISCVISD::VWADDU_VL, DL, WideContainerVT, EvenV,
4860 SDValue AllOnesVec = DAG.getSplatVector(
4861 VecContainerVT, DL, DAG.getAllOnesConstant(DL, Subtarget.getXLenVT()));
4862 SDValue OddsMul = DAG.getNode(RISCVISD::VWMULU_VL, DL, WideContainerVT,
4870 Interleaved = DAG.getNode(RISCVISD::ADD_VL, DL, WideContainerVT,
4878 Interleaved = DAG.getBitcast(ResultContainerVT, Interleaved);
4886 convertFromScalableVector(ResultVT, Interleaved, DAG, Subtarget);
4894 SelectionDAG &DAG,
4910 *DAG.getContext(), EVT::getIntegerVT(*DAG.getContext(), ViaEltSize), 1);
4912 EVT::getVectorVT(*DAG.getContext(), MVT::i1, ViaVT.getScalarSizeInBits());
4925 V = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ViaBitVT, DAG.getUNDEF(ViaBitVT),
4926 V, DAG.getVectorIdxConstant(0, DL));
4929 DAG.getNode(ISD::BITREVERSE, DL, ViaVT, DAG.getBitcast(ViaVT, V));
4934 Res = DAG.getNode(ISD::SRL, DL, ViaVT, Res,
4935 DAG.getConstant(ViaEltSize - NumElts, DL, ViaVT));
4937 Res = DAG.getBitcast(ViaBitVT, Res);
4940 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
4941 DAG.getVectorIdxConstant(0, DL));
4946 SelectionDAG &DAG,
4969 SelectionDAG &DAG,
4976 if (!isLegalBitRotate(SVN, DAG, Subtarget, RotateVT, RotateAmt))
4979 SDValue Op = DAG.getBitcast(RotateVT, SVN->getOperand(0));
4985 Rotate = DAG.getNode(ISD::BSWAP, DL, RotateVT, Op);
4987 Rotate = DAG.getNode(ISD::ROTL, DL, RotateVT, Op,
4988 DAG.getConstant(RotateAmt, DL, RotateVT));
4990 return DAG.getBitcast(VT, Rotate);
4997 SelectionDAG &DAG,
5016 if (isLegalBitRotate(SVN, DAG, Subtarget, RotateVT, RotateAmt))
5049 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
5051 MVT M1VT = getContainerForFixedLengthVector(DAG, OneRegVT, Subtarget);
5054 SDValue Vec = DAG.getUNDEF(ContainerVT);
5057 // to avoid DAG combining it back to a large shuffle_vector again.
5058 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
5059 V2 = convertToScalableVector(ContainerVT, V2, DAG, Subtarget);
5066 SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, M1VT, SrcVec,
5067 DAG.getVectorIdxConstant(ExtractIdx, DL));
5068 SubVec = convertFromScalableVector(OneRegVT, SubVec, DAG, Subtarget);
5069 SubVec = DAG.getVectorShuffle(OneRegVT, DL, SubVec, SubVec, SrcSubMask);
5070 SubVec = convertToScalableVector(M1VT, SubVec, DAG, Subtarget);
5072 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT, Vec, SubVec,
5073 DAG.getVectorIdxConstant(InsertIdx, DL));
5075 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
5078 static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
5091 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
5093 if (SDValue V = lowerBitreverseShuffle(SVN, DAG, Subtarget))
5098 V1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, V1);
5099 V2 = V2.isUndef() ? DAG.getUNDEF(WidenVT)
5100 : DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, V2);
5101 SDValue Shuffled = DAG.getVectorShuffle(WidenVT, DL, V1, V2, SVN->getMask());
5102 return DAG.getSetCC(DL, VT, Shuffled, DAG.getConstant(0, DL, WidenVT),
5106 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
5108 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5132 SDValue NewAddr = DAG.getMemBasePlusOffset(
5137 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
5139 DAG.getTargetConstant(Intrinsic::riscv_vlse, DL, XLenVT);
5142 DAG.getUNDEF(ContainerVT),
5144 DAG.getRegister(RISCV::X0, XLenVT),
5146 SDValue NewLoad = DAG.getMemIntrinsicNode(
5148 DAG.getMachineFunction().getMachineMemOperand(
5150 DAG.makeEquivalentMemoryOrdering(Ld, NewLoad);
5151 return convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
5166 V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr,
5171 V = DAG.getExtLoad(ISD::EXTLOAD, DL, XLenVT, Ld->getChain(), NewAddr,
5175 DAG.makeEquivalentMemoryOrdering(Ld, V);
5180 DAG.getNode(Opc, DL, SplatVT, DAG.getUNDEF(ContainerVT), V, VL);
5181 Splat = DAG.getBitcast(ContainerVT, Splat);
5182 return convertFromScalableVector(VT, Splat, DAG, Subtarget);
5185 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
5187 SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT,
5188 V1, DAG.getConstant(Lane, DL, XLenVT),
5189 DAG.getUNDEF(ContainerVT), TrueMask, VL);
5190 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
5196 if (SDValue V = lowerShuffleViaVRegSplitting(SVN, DAG, Subtarget))
5202 lowerVECTOR_SHUFFLEAsVSlide1(DL, VT, V1, V2, Mask, Subtarget, DAG))
5206 lowerVECTOR_SHUFFLEAsVSlidedown(DL, VT, V1, V2, Mask, Subtarget, DAG))
5212 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
5223 LoV = convertToScalableVector(ContainerVT, LoV, DAG, Subtarget);
5227 HiV = convertToScalableVector(ContainerVT, HiV, DAG, Subtarget);
5234 SDValue Res = DAG.getUNDEF(ContainerVT);
5238 Res = getVSlidedown(DAG, Subtarget, DL, ContainerVT, Res, HiV,
5239 DAG.getConstant(Rotation, DL, XLenVT), TrueMask, VL);
5242 Res = getVSlideup(DAG, Subtarget, DL, ContainerVT, Res, LoV,
5243 DAG.getConstant(InvRotate, DL, XLenVT), TrueMask, VL,
5246 return convertFromScalableVector(VT, Res, DAG, Subtarget);
5253 Subtarget, DAG);
5257 lowerVECTOR_SHUFFLEAsVSlideup(DL, VT, V1, V2, Mask, Subtarget, DAG))
5271 EvenV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, EvenV,
5272 DAG.getVectorIdxConstant(EvenSrc % Size, DL));
5276 OddV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, OddV,
5277 DAG.getVectorIdxConstant(OddSrc % Size, DL));
5279 return getWideningInterleave(EvenV, OddV, DL, DAG, Subtarget);
5289 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
5325 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
5330 ? DAG.getConstant(MaskIndex, DL, XLenVT)
5331 : DAG.getUNDEF(XLenVT));
5333 SDValue LHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesLHS);
5334 LHSIndices = convertToScalableVector(IndexContainerVT, LHSIndices, DAG,
5336 SDValue Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V1, LHSIndices,
5337 DAG.getUNDEF(ContainerVT), TrueMask, VL);
5338 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
5355 bool SwapOps = DAG.isSplatValue(V2) && !DAG.isSplatValue(V1);
5362 V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), ShuffleMaskLHS);
5363 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), ShuffleMaskRHS);
5368 MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
5373 SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
5376 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V1, V2);
5377 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V2, V1);
5404 SelectionDAG &DAG) const {
5415 Mask = convertToScalableVector(getMaskTypeFor(ContainerVT), Mask, DAG,
5431 assert(DAG.getTargetLoweringInfo().isTypeLegal(FloatVT) &&
5437 SDValue Neg = DAG.getNegative(Src, DL, VT);
5438 Src = DAG.getNode(ISD::AND, DL, VT, Src, Neg);
5440 SDValue Neg = DAG.getNode(ISD::VP_SUB, DL, VT, DAG.getConstant(0, DL, VT),
5442 Src = DAG.getNode(ISD::VP_AND, DL, VT, Src, Neg, Mask, VL);
5449 FloatVal = DAG.getNode(ISD::VP_UINT_TO_FP, DL, FloatVT, Src, Mask, VL);
5451 FloatVal = DAG.getNode(ISD::UINT_TO_FP, DL, FloatVT, Src);
5456 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
5459 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5461 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT());
5464 FloatVal = DAG.getNode(RISCVISD::VFCVT_RM_F_XU_VL, DL, ContainerFloatVT,
5467 FloatVal = convertFromScalableVector(FloatVT, FloatVal, DAG, Subtarget);
5471 SDValue Bitcast = DAG.getBitcast(IntVT, FloatVal);
5477 Exp = DAG.getNode(ISD::VP_SRL, DL, IntVT, Bitcast,
5478 DAG.getConstant(ShiftAmt, DL, IntVT), Mask, VL);
5479 Exp = DAG.getVPZExtOrTrunc(DL, VT, Exp, Mask, VL);
5481 Exp = DAG.getNode(ISD::SRL, DL, IntVT, Bitcast,
5482 DAG.getConstant(ShiftAmt, DL, IntVT));
5484 Exp = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Exp);
5486 Exp = DAG.getNode(ISD::TRUNCATE, DL, VT, Exp);
5493 return DAG.getNode(ISD::SUB, DL, VT, Exp,
5494 DAG.getConstant(ExponentBias, DL, VT));
5496 return DAG.getNode(ISD::VP_SUB, DL, VT, Exp,
5497 DAG.getConstant(ExponentBias, DL, VT), Mask, VL);
5504 Res = DAG.getNode(ISD::VP_SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Exp,
5507 Res = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Exp);
5512 Res = DAG.getNode(ISD::UMIN, DL, VT, Res, DAG.getConstant(EltSize, DL, VT));
5514 Res = DAG.getNode(ISD::VP_UMIN, DL, VT, Res,
5515 DAG.getConstant(EltSize, DL, VT), Mask, VL);
5520 SelectionDAG &DAG) const {
5530 Source = convertToScalableVector(ContainerVT, Source, DAG, Subtarget);
5531 Mask = convertToScalableVector(getMaskTypeFor(ContainerVT), Mask, DAG,
5538 SDValue AllZero = DAG.getConstant(0, DL, SrcVT);
5540 Source = DAG.getNode(RISCVISD::SETCC_VL, DL, SrcVT,
5541 {Source, AllZero, DAG.getCondCode(ISD::SETNE),
5542 DAG.getUNDEF(SrcVT), Mask, EVL});
5545 SDValue Res = DAG.getNode(RISCVISD::VFIRST_VL, DL, XLenVT, Source, Mask, EVL);
5552 DAG.getSetCC(DL, XLenVT, Res, DAG.getConstant(0, DL, XLenVT), ISD::SETLT);
5553 Res = DAG.getSelect(DL, XLenVT, SetCC, EVL, Res);
5554 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Res);
5562 SelectionDAG &DAG) const {
5566 if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
5580 SDValue L = DAG.getLoad(NewVT, DL, Load->getChain(), Load->getBasePtr(),
5583 return DAG.getMergeValues({DAG.getBitcast(VT, L), L.getValue(1)}, DL);
5591 SelectionDAG &DAG) const {
5596 if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
5611 StoredVal = DAG.getBitcast(NewVT, StoredVal);
5612 return DAG.getStore(Store->getChain(), DL, StoredVal, Store->getBasePtr(),
5617 static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG,
5640 if (DAG.shouldOptForSize())
5657 static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
5673 return DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
5681 return DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
5686 static SDValue lowerSADDSAT_SSUBSAT(SDValue Op, SelectionDAG &DAG) {
5693 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op.getOperand(0));
5694 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op.getOperand(1));
5696 DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, DL, MVT::i64, LHS, RHS);
5700 SDValue SatMin = DAG.getConstant(MinVal, DL, MVT::i64);
5701 SDValue SatMax = DAG.getConstant(MaxVal, DL, MVT::i64);
5702 Result = DAG.getNode(ISD::SMIN, DL, MVT::i64, Result, SatMax);
5703 Result = DAG.getNode(ISD::SMAX, DL, MVT::i64, Result, SatMin);
5704 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Result);
5707 static SDValue lowerUADDSAT_USUBSAT(SDValue Op, SelectionDAG &DAG) {
5715 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op.getOperand(0));
5716 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op.getOperand(1));
5717 SDValue WideOp = DAG.getNode(Op.getOpcode(), DL, MVT::i64, LHS, RHS);
5718 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, WideOp);
5722 static SDValue lowerSADDO_SSUBO(SDValue Op, SelectionDAG &DAG) {
5730 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op.getOperand(0));
5731 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op.getOperand(1));
5733 DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, DL, MVT::i64, LHS, RHS);
5734 SDValue Res = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, WideOp);
5735 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, WideOp,
5736 DAG.getValueType(MVT::i32));
5737 SDValue Ovf = DAG.getSetCC(DL, Op.getValue(1).getValueType(), WideOp, SExt,
5739 return DAG.getMergeValues({Res, Ovf}, DL);
5743 static SDValue lowerSMULO(SDValue Op, SelectionDAG &DAG) {
5747 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op.getOperand(0));
5748 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op.getOperand(1));
5749 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
5750 SDValue Res = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mul);
5751 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Mul,
5752 DAG.getValueType(MVT::i32));
5753 SDValue Ovf = DAG.getSetCC(DL, Op.getValue(1).getValueType(), Mul, SExt,
5755 return DAG.getMergeValues({Res, Ovf}, DL);
5759 SelectionDAG &DAG) const {
5788 SDValue TDCMaskV = DAG.getConstant(TDCMask, DL, XLenVT);
5796 auto [Mask, VL] = getDefaultScalableVLOps(VT0, DL, DAG, Subtarget);
5801 SDValue FPCLASS = DAG.getNode(RISCVISD::FCLASS_VL, DL, DstVT, Op0, Mask,
5804 return DAG.getSetCC(DL, VT, FPCLASS,
5805 DAG.getConstant(TDCMask, DL, DstVT),
5807 SDValue AND = DAG.getNode(ISD::AND, DL, DstVT, FPCLASS,
5808 DAG.getConstant(TDCMask, DL, DstVT));
5809 return DAG.getSetCC(DL, VT, AND, DAG.getConstant(0, DL, DstVT),
5816 auto [Mask, VL] = getDefaultVLOps(VT0, ContainerVT0, DL, DAG, Subtarget);
5821 Mask = convertToScalableVector(MaskContainerVT, Mask, DAG, Subtarget);
5824 Op0 = convertToScalableVector(ContainerVT0, Op0, DAG, Subtarget);
5826 SDValue FPCLASS = DAG.getNode(RISCVISD::FCLASS_VL, DL, ContainerDstVT, Op0,
5829 TDCMaskV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerDstVT,
5830 DAG.getUNDEF(ContainerDstVT), TDCMaskV, VL);
5833 DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT,
5834 {FPCLASS, TDCMaskV, DAG.getCondCode(ISD::SETEQ),
5835 DAG.getUNDEF(ContainerVT), Mask, VL});
5836 return convertFromScalableVector(VT, VMSEQ, DAG, Subtarget);
5838 SDValue AND = DAG.getNode(RISCVISD::AND_VL, DL, ContainerDstVT, FPCLASS,
5839 TDCMaskV, DAG.getUNDEF(ContainerDstVT), Mask, VL);
5841 SDValue SplatZero = DAG.getConstant(0, DL, XLenVT);
5842 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerDstVT,
5843 DAG.getUNDEF(ContainerDstVT), SplatZero, VL);
5845 SDValue VMSNE = DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT,
5846 {AND, SplatZero, DAG.getCondCode(ISD::SETNE),
5847 DAG.getUNDEF(ContainerVT), Mask, VL});
5848 return convertFromScalableVector(VT, VMSNE, DAG, Subtarget);
5851 SDValue FCLASS = DAG.getNode(RISCVISD::FCLASS, DL, XLenVT, Op.getOperand(0));
5852 SDValue AND = DAG.getNode(ISD::AND, DL, XLenVT, FCLASS, TDCMaskV);
5853 SDValue Res = DAG.getSetCC(DL, XLenVT, AND, DAG.getConstant(0, DL, XLenVT),
5855 return DAG.getNode(ISD::TRUNCATE, DL, VT, Res);
5860 static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG,
5877 if (!Op->getFlags().hasNoNaNs() && !DAG.isKnownNeverNaN(X)) {
5878 SDValue XIsNonNan = DAG.getSetCC(DL, XLenVT, X, X, ISD::SETOEQ);
5879 NewY = DAG.getSelect(DL, VT, XIsNonNan, Y, X);
5883 if (!Op->getFlags().hasNoNaNs() && !DAG.isKnownNeverNaN(Y)) {
5884 SDValue YIsNonNan = DAG.getSetCC(DL, XLenVT, Y, Y, ISD::SETOEQ);
5885 NewX = DAG.getSelect(DL, VT, YIsNonNan, X, Y);
5890 return DAG.getNode(Opc, DL, VT, NewX, NewY);
5894 bool XIsNeverNan = Op->getFlags().hasNoNaNs() || DAG.isKnownNeverNaN(X);
5895 bool YIsNeverNan = Op->getFlags().hasNoNaNs() || DAG.isKnownNeverNaN(Y);
5899 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
5900 X = convertToScalableVector(ContainerVT, X, DAG, Subtarget);
5901 Y = convertToScalableVector(ContainerVT, Y, DAG, Subtarget);
5908 Mask = convertToScalableVector(getMaskTypeFor(ContainerVT), Mask, DAG,
5912 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5917 SDValue XIsNonNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
5918 {X, X, DAG.getCondCode(ISD::SETOEQ),
5919 DAG.getUNDEF(ContainerVT), Mask, VL});
5920 NewY = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, XIsNonNan, Y, X,
5921 DAG.getUNDEF(ContainerVT), VL);
5926 SDValue YIsNonNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
5927 {Y, Y, DAG.getCondCode(ISD::SETOEQ),
5928 DAG.getUNDEF(ContainerVT), Mask, VL});
5929 NewX = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, YIsNonNan, X, Y,
5930 DAG.getUNDEF(ContainerVT), VL);
5937 SDValue Res = DAG.getNode(Opc, DL, ContainerVT, NewX, NewY,
5938 DAG.getUNDEF(ContainerVT), Mask, VL);
5940 Res = convertFromScalableVector(VT, Res, DAG, Subtarget);
6137 static SDValue SplitVectorOp(SDValue Op, SelectionDAG &DAG) {
6138 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(Op.getValueType());
6151 DAG.SplitVector(Op.getOperand(j), DL);
6155 DAG.getNode(Op.getOpcode(), DL, LoVT, LoOperands, Op->getFlags());
6157 DAG.getNode(Op.getOpcode(), DL, HiVT, HiOperands, Op->getFlags());
6159 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Op.getValueType(), LoRes, HiRes);
6162 static SDValue SplitVPOp(SDValue Op, SelectionDAG &DAG) {
6164 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(Op.getValueType());
6173 DAG.SplitEVL(Op.getOperand(j), Op.getValueType(), DL);
6182 DAG.SplitVector(Op.getOperand(j), DL);
6186 DAG.getNode(Op.getOpcode(), DL, LoVT, LoOperands, Op->getFlags());
6188 DAG.getNode(Op.getOpcode(), DL, HiVT, HiOperands, Op->getFlags());
6190 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Op.getValueType(), LoRes, HiRes);
6193 static SDValue SplitVectorReductionOp(SDValue Op, SelectionDAG &DAG) {
6196 auto [Lo, Hi] = DAG.SplitVector(Op.getOperand(1), DL);
6197 auto [MaskLo, MaskHi] = DAG.SplitVector(Op.getOperand(2), DL);
6199 DAG.SplitEVL(Op.getOperand(3), Op.getOperand(1).getValueType(), DL);
6202 DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
6204 return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
6208 static SDValue SplitStrictFPVectorOp(SDValue Op, SelectionDAG &DAG) {
6212 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(Op->getValueType(0));
6214 SDVTList LoVTs = DAG.getVTList(LoVT, Op->getValueType(1));
6215 SDVTList HiVTs = DAG.getVTList(HiVT, Op->getValueType(1));
6229 DAG.SplitVector(Op.getOperand(j), DL);
6233 DAG.getNode(Op.getOpcode(), DL, LoVTs, LoOperands, Op->getFlags());
6236 DAG.getNode(Op.getOpcode(), DL, HiVTs, HiOperands, Op->getFlags());
6238 SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, Op->getValueType(0),
6240 return DAG.getMergeValues({V, HiRes.getValue(1)}, DL);
6244 SelectionDAG &DAG) const {
6249 return LowerATOMIC_FENCE(Op, DAG, Subtarget);
6251 return lowerGlobalAddress(Op, DAG);
6253 return lowerBlockAddress(Op, DAG);
6255 return lowerConstantPool(Op, DAG);
6257 return lowerJumpTable(Op, DAG);
6259 return lowerGlobalTLSAddress(Op, DAG);
6261 return lowerConstant(Op, DAG, Subtarget);
6263 return lowerSELECT(Op, DAG);
6265 return lowerBRCOND(Op, DAG);
6267 return lowerVASTART(Op, DAG);
6269 return lowerFRAMEADDR(Op, DAG);
6271 return lowerRETURNADDR(Op, DAG);
6274 return lowerSADDO_SSUBO(Op, DAG);
6276 return lowerSMULO(Op, DAG);
6278 return lowerShiftLeftParts(Op, DAG);
6280 return lowerShiftRightParts(Op, DAG, true);
6282 return lowerShiftRightParts(Op, DAG, false);
6287 return lowerToScalableOp(Op, DAG);
6304 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Op0);
6305 SDValue FPConv = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::f16, NewOp0);
6310 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Op0);
6311 SDValue FPConv = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::bf16, NewOp0);
6316 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
6318 DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0);
6324 std::tie(Lo, Hi) = DAG.SplitScalar(Op0, DL, MVT::i32, MVT::i32);
6326 DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
6350 EVT BVT = EVT::getVectorVT(*DAG.getContext(), Op0VT, 1);
6353 return DAG.getBitcast(VT, DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, BVT,
6354 DAG.getUNDEF(BVT), Op0,
6355 DAG.getVectorIdxConstant(0, DL)));
6363 EVT BVT = EVT::getVectorVT(*DAG.getContext(), VT, 1);
6366 SDValue BVec = DAG.getBitcast(BVT, Op0);
6367 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec,
6368 DAG.getVectorIdxConstant(0, DL));
6373 return LowerINTRINSIC_WO_CHAIN(Op, DAG);
6375 return LowerINTRINSIC_W_CHAIN(Op, DAG);
6377 return LowerINTRINSIC_VOID(Op, DAG);
6379 return LowerIS_FPCLASS(Op, DAG);
6384 return lowerToScalableOp(Op, DAG);
6390 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, Op.getOperand(0));
6391 return DAG.getNode(RISCVISD::BREV8, DL, VT, BSwap);
6397 return lowerVectorTruncLike(Op, DAG);
6402 return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ 1);
6403 return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VZEXT_VL);
6407 return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ -1);
6408 return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VSEXT_VL);
6410 return lowerSPLAT_VECTOR_PARTS(Op, DAG);
6412 return lowerINSERT_VECTOR_ELT(Op, DAG);
6414 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
6421 SDValue V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, WideVT, Scalar);
6422 return DAG.getNode(ISD::TRUNCATE, DL, VT, V);
6427 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
6428 Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), Scalar);
6429 SDValue V = DAG.getNode(RISCVISD::VMV_S_X_VL, DL, ContainerVT,
6430 DAG.getUNDEF(ContainerVT), Scalar, VL);
6432 V = convertFromScalableVector(VT, V, DAG, Subtarget);
6439 SDValue Res = DAG.getNode(RISCVISD::READ_VLENB, DL, XLenVT);
6452 Res = DAG.getNode(ISD::SRL, DL, XLenVT, Res,
6453 DAG.getConstant(3 - Log2, DL, VT));
6455 Res = DAG.getNode(ISD::SHL, DL, XLenVT, Res,
6456 DAG.getConstant(Log2 - 3, DL, XLenVT));
6460 Res = DAG.getNode(ISD::MUL, DL, XLenVT, Res,
6461 DAG.getConstant(Val / 8, DL, XLenVT));
6463 SDValue VScale = DAG.getNode(ISD::SRL, DL, XLenVT, Res,
6464 DAG.getConstant(3, DL, XLenVT));
6465 Res = DAG.getNode(ISD::MUL, DL, XLenVT, VScale,
6466 DAG.getConstant(Val, DL, XLenVT));
6468 return DAG.getNode(ISD::TRUNCATE, DL, VT, Res);
6476 SDValue Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op.getOperand(0));
6478 DAG.getNode(ISD::FPOWI, DL, MVT::f32, Op0, Op.getOperand(1));
6479 return DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, Powi,
6480 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
6489 return SplitVectorOp(Op, DAG);
6490 return lowerFMAXIMUM_FMINIMUM(Op, DAG, Subtarget);
6497 return DAG.getNode(RISCVISD::FP_EXTEND_BF16, DL, MVT::f32, Op0);
6500 DAG.getNode(RISCVISD::FP_EXTEND_BF16, DL, MVT::f32, Op0);
6501 return DAG.getNode(ISD::FP_EXTEND, DL, MVT::f64, FloatVal);
6506 return lowerVectorFPExtendOrRoundLike(Op, DAG);
6514 return DAG.getNode(RISCVISD::FP_ROUND_BF16, DL, MVT::bf16, Op0);
6518 DAG.getNode(ISD::FP_ROUND, DL, MVT::f32, Op0,
6519 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
6520 return DAG.getNode(RISCVISD::FP_ROUND_BF16, DL, MVT::bf16, FloatVal);
6525 return lowerVectorFPExtendOrRoundLike(Op, DAG);
6529 return lowerStrictFPExtendOrRoundLike(Op, DAG);
6537 return SplitVectorOp(Op, DAG);
6542 SDValue NC = DAG.getNode(Op.getOpcode(), DL, NVT, Op->ops());
6544 return DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(), NC,
6545 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
6556 return SplitVectorOp(Op, DAG);
6561 SDValue WidenVec = DAG.getNode(ISD::FP_EXTEND, DL, NVT, Op1);
6563 return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), WidenVec);
6598 SDValue Ext = DAG.getNode(ExtOpcode, DL, IVecVT, Src);
6600 return DAG.getNode(Op.getOpcode(), DL, Op->getVTList(),
6602 return DAG.getNode(Op.getOpcode(), DL, VT, Ext);
6611 DAG.getStrictFPExtendOrRound(Src, Op.getOperand(0), DL, InterimFVT);
6612 return DAG.getNode(Op.getOpcode(), DL, Op->getVTList(), Chain, FExt);
6614 SDValue FExt = DAG.getFPExtendOrRound(Src, DL, InterimFVT);
6615 return DAG.getNode(Op.getOpcode(), DL, VT, FExt);
6625 SDValue Int2FP = DAG.getNode(Op.getOpcode(), DL,
6626 DAG.getVTList(InterimFVT, MVT::Other),
6629 return DAG.getStrictFPExtendOrRound(Int2FP, Chain, DL, VT).first;
6631 SDValue Int2FP = DAG.getNode(Op.getOpcode(), DL, InterimFVT, Src);
6632 return DAG.getFPExtendOrRound(Int2FP, DL, VT);
6641 DAG.getNode(Op.getOpcode(), DL, DAG.getVTList(IVecVT, MVT::Other),
6643 SDValue Res = DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int);
6644 return DAG.getMergeValues({Res, FP2Int.getValue(1)}, DL);
6646 SDValue FP2Int = DAG.getNode(Op.getOpcode(), DL, IVecVT, Src);
6647 return DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int);
6691 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
6693 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
6695 Src = DAG.getNode(RVVOpc, DL, DAG.getVTList(ContainerVT, MVT::Other),
6697 SDValue SubVec = convertFromScalableVector(VT, Src, DAG, Subtarget);
6698 return DAG.getMergeValues({SubVec, Src.getValue(1)}, DL);
6700 Src = DAG.getNode(RVVOpc, DL, ContainerVT, Src, Mask, VL);
6701 return convertFromScalableVector(VT, Src, DAG, Subtarget);
6705 return lowerFP_TO_INT_SAT(Op, DAG, Subtarget);
6715 makeLibCall(DAG, LC, MVT::f32, Op.getOperand(0), CallOptions, DL).first;
6717 return DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Res);
6718 return DAG.getBitcast(MVT::i32, Res);
6724 Op = DAG.getNode(
6726 DAG.getShiftAmountConstant(16, Op.getOperand(0).getValueType(), DL));
6728 ? DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Op)
6729 : DAG.getBitcast(MVT::f32, Op);
6732 return DAG.getNode(ISD::FP_EXTEND, DL, VT, Res);
6744 makeLibCall(DAG, LC, MVT::f32, Op.getOperand(0), CallOptions, DL).first;
6746 return DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Res);
6747 return DAG.getBitcast(MVT::i32, Res);
6756 ? DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32,
6758 : DAG.getBitcast(MVT::f32, Op.getOperand(0));
6760 makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Arg, CallOptions, DL)
6771 return lowerFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
6774 return lowerVectorXRINT(Op, DAG, Subtarget);
6780 return lowerVECREDUCE(Op, DAG);
6785 return lowerVectorMaskVecReduction(Op, DAG, /*IsVP*/ false);
6786 return lowerVECREDUCE(Op, DAG);
6793 return lowerFPVECREDUCE(Op, DAG);
6808 return SplitVectorReductionOp(Op, DAG);
6809 return lowerVPREDUCE(Op, DAG);
6814 return lowerVectorMaskVecReduction(Op, DAG, /*IsVP*/ true);
6815 return lowerVPREDUCE(Op, DAG);
6818 return lowerVPCttzElements(Op, DAG);
6822 DAG.getUNDEF(ContainerVT), DAG, Subtarget);
6825 return lowerINSERT_SUBVECTOR(Op, DAG);
6827 return lowerEXTRACT_SUBVECTOR(Op, DAG);
6829 return lowerVECTOR_DEINTERLEAVE(Op, DAG);
6831 return lowerVECTOR_INTERLEAVE(Op, DAG);
6833 return lowerSTEP_VECTOR(Op, DAG);
6835 return lowerVECTOR_REVERSE(Op, DAG);
6837 return lowerVECTOR_SPLICE(Op, DAG);
6839 return lowerBUILD_VECTOR(Op, DAG, Subtarget);
6849 return SplitVectorOp(Op, DAG);
6852 DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op.getOperand(0));
6853 SDValue NewSplat = DAG.getNode(
6857 return DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(), NewSplat,
6858 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
6861 return lowerVectorMaskSplat(Op, DAG);
6864 return lowerVECTOR_SHUFFLE(Op, DAG, Subtarget);
6872 ContainerVT = ::getContainerForFixedLengthVector(DAG, VT, Subtarget);
6887 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT,
6889 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT,
6891 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
6896 SDValue Vec = DAG.getUNDEF(VT);
6903 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Vec, SubVec,
6904 DAG.getVectorIdxConstant(OpIdx.index() * NumOpElts, DL));
6909 if (auto V = expandUnalignedRVVLoad(Op, DAG))
6912 return lowerFixedLengthVectorLoadToRVV(Op, DAG);
6915 if (auto V = expandUnalignedRVVStore(Op, DAG))
6918 return lowerFixedLengthVectorStoreToRVV(Op, DAG);
6922 return lowerMaskedLoad(Op, DAG);
6925 return lowerMaskedStore(Op, DAG);
6938 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT);
6941 DAG.getNode(ISD::SETCC, DL, CCVT, Tmp1, Tmp2, CC, Op->getFlags());
6942 return DAG.getSelect(DL, VT, Cond, True, False);
6966 return DAG.getConstant(0, DL, VT);
6969 SDValue SetCC = DAG.getSetCC(
6970 DL, VT, LHS, DAG.getConstant(Imm + 1, DL, OpVT), CCVal);
6971 return DAG.getLogicalNOT(DL, SetCC, VT);
6978 return DAG.getSetCC(DL, VT, RHS, LHS, CCVal);
6984 return SplitVectorOp(Op, DAG);
6986 return lowerFixedLengthVectorSetccToRVV(Op, DAG);
7002 return lowerToScalableOp(Op, DAG);
7007 return lowerToScalableOp(Op, DAG);
7025 return SplitVectorOp(Op, DAG);
7035 return lowerToScalableOp(Op, DAG);
7039 return lowerUADDSAT_USUBSAT(Op, DAG);
7040 return lowerToScalableOp(Op, DAG);
7044 return lowerSADDSAT_SSUBSAT(Op, DAG);
7045 return lowerToScalableOp(Op, DAG);
7050 SDValue LHS = DAG.getFreeze(Op->getOperand(0));
7051 SDValue RHS = DAG.getFreeze(Op->getOperand(1));
7058 SDValue Max = DAG.getNode(MaxOpc, dl, VT, LHS, RHS);
7059 SDValue Min = DAG.getNode(MinOpc, dl, VT, LHS, RHS);
7060 return DAG.getNode(ISD::SUB, dl, VT, Max, Min);
7064 return lowerABS(Op, DAG);
7070 return lowerToScalableOp(Op, DAG);
7072 return lowerCTLZ_CTTZ_ZERO_UNDEF(Op, DAG);
7074 return lowerFixedLengthVectorSelectToRVV(Op, DAG);
7079 return SplitVectorOp(Op, DAG);
7080 return lowerFixedLengthVectorFCOPYSIGNToRVV(Op, DAG);
7090 return SplitStrictFPVectorOp(Op, DAG);
7091 return lowerToScalableOp(Op, DAG);
7094 return lowerVectorStrictFSetcc(Op, DAG);
7102 return lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
7105 return lowerMaskedGather(Op, DAG);
7108 return lowerMaskedScatter(Op, DAG);
7110 return lowerGET_ROUNDING(Op, DAG);
7112 return lowerSET_ROUNDING(Op, DAG);
7114 return lowerEH_DWARF_CFA(Op, DAG);
7130 return lowerVPOp(Op, DAG);
7134 return lowerLogicVPOp(Op, DAG);
7149 return SplitVPOp(Op, DAG);
7154 return lowerVPOp(Op, DAG);
7156 return LowerIS_FPCLASS(Op, DAG);
7160 return lowerVPExtMaskOp(Op, DAG);
7161 return lowerVPOp(Op, DAG);
7163 return lowerVectorTruncLike(Op, DAG);
7166 return lowerVectorFPExtendOrRoundLike(Op, DAG);
7174 return SplitVPOp(Op, DAG);
7179 auto NC = DAG.getNode(Op.getOpcode(), DL, NVT, Op->ops());
7181 return DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(), NC,
7182 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
7193 return SplitVPOp(Op, DAG);
7198 SDValue WidenVec = DAG.getNode(ISD::FP_EXTEND, DL, NVT, Op1);
7200 return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
7203 return lowerVPFPIntConvOp(Op, DAG);
7208 return SplitVPOp(Op, DAG);
7210 return lowerVPSetCCMaskOp(Op, DAG);
7218 return lowerVPOp(Op, DAG);
7222 return lowerVPOp(Op, DAG);
7223 return lowerCTLZ_CTTZ_ZERO_UNDEF(Op, DAG);
7227 return lowerVPOp(Op, DAG);
7228 return lowerCTLZ_CTTZ_ZERO_UNDEF(Op, DAG);
7230 return lowerVPOp(Op, DAG);
7232 return lowerVPStridedLoad(Op, DAG);
7234 return lowerVPStridedStore(Op, DAG);
7245 return SplitVPOp(Op, DAG);
7246 return lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
7252 return SplitVPOp(Op, DAG);
7253 return lowerFMAXIMUM_FMINIMUM(Op, DAG, Subtarget);
7255 return lowerVPSpliceExperimental(Op, DAG);
7257 return lowerVPReverseExperimental(Op, DAG);
7259 return lowerVPSplatExperimental(Op, DAG);
7264 SDValue Flags = DAG.getConstant(0, DL, Subtarget.getXLenVT());
7265 return emitFlushICache(DAG, Op.getOperand(0), Op.getOperand(1),
7271 SDValue RISCVTargetLowering::emitFlushICache(SelectionDAG &DAG, SDValue InChain,
7276 makeLibCall(DAG, RTLIB::RISCV_FLUSH_ICACHE, MVT::isVoid,
7284 SelectionDAG &DAG, unsigned Flags) {
7285 return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags);
7289 SelectionDAG &DAG, unsigned Flags) {
7290 return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, N->getOffset(),
7295 SelectionDAG &DAG, unsigned Flags) {
7296 return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlign(),
7301 SelectionDAG &DAG, unsigned Flags) {
7302 return DAG.getTargetJumpTable(N->getIndex(), Ty, Flags);
7306 SDValue RISCVTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
7309 EVT Ty = getPointerTy(DAG.getDataLayout());
7316 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
7321 return DAG.getNode(RISCVISD::LLA, DL, Ty, Addr);
7327 SDValue(DAG.getMachineNode(RISCV::PseudoLGA, DL, Ty, Addr), 0);
7328 MachineFunction &MF = DAG.getMachineFunction();
7334 DAG.setNodeMemRefs(cast<MachineSDNode>(Load.getNode()), {MemOp});
7344 SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI);
7345 SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO);
7346 SDValue MNHi = DAG.getNode(RISCVISD::HI, DL, Ty, AddrHi);
7347 return DAG.getNode(RISCVISD::ADD_LO, DL, Ty, MNHi, AddrLo);
7350 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
7357 SDValue(DAG.getMachineNode(RISCV::PseudoLGA, DL, Ty, Addr), 0);
7358 MachineFunction &MF = DAG.getMachineFunction();
7364 DAG.setNodeMemRefs(cast<MachineSDNode>(Load.getNode()), {MemOp});
7371 return DAG.getNode(RISCVISD::LLA, DL, Ty, Addr);
7377 SelectionDAG &DAG) const {
7381 return getAddr(N, DAG, GV->isDSOLocal(), GV->hasExternalWeakLinkage());
7385 SelectionDAG &DAG) const {
7388 return getAddr(N, DAG);
7392 SelectionDAG &DAG) const {
7395 return getAddr(N, DAG);
7399 SelectionDAG &DAG) const {
7402 return getAddr(N, DAG);
7406 SelectionDAG &DAG,
7409 EVT Ty = getPointerTy(DAG.getDataLayout());
7418 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
7420 SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_IE, DL, Ty, Addr), 0);
7421 MachineFunction &MF = DAG.getMachineFunction();
7427 DAG.setNodeMemRefs(cast<MachineSDNode>(Load.getNode()), {MemOp});
7430 SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
7431 return DAG.getNode(ISD::ADD, DL, Ty, Load, TPReg);
7439 DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_HI);
7441 DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_ADD);
7443 DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_LO);
7445 SDValue MNHi = DAG.getNode(RISCVISD::HI, DL, Ty, AddrHi);
7446 SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
7448 DAG.getNode(RISCVISD::ADD_TPREL, DL, Ty, MNHi, TPReg, AddrAdd);
7449 return DAG.getNode(RISCVISD::ADD_LO, DL, Ty, MNAdd, AddrLo);
7453 SelectionDAG &DAG) const {
7455 EVT Ty = getPointerTy(DAG.getDataLayout());
7456 IntegerType *CallTy = Type::getIntNTy(*DAG.getContext(), Ty.getSizeInBits());
7462 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
7464 SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_GD, DL, Ty, Addr), 0);
7474 TargetLowering::CallLoweringInfo CLI(DAG);
7476 .setChain(DAG.getEntryNode())
7478 DAG.getExternalSymbol("__tls_get_addr", Ty),
7485 SelectionDAG &DAG) const {
7487 EVT Ty = getPointerTy(DAG.getDataLayout());
7497 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
7498 return SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLSDESC, DL, Ty, Addr), 0);
7502 SelectionDAG &DAG) const {
7506 if (DAG.getTarget().useEmulatedTLS())
7507 return LowerToTLSEmulatedModel(N, DAG);
7511 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
7518 Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/false);
7521 Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/true);
7525 Addr = DAG.getTarget().useTLSDESC() ? getTLSDescAddr(N, DAG)
7526 : getDynamicTLSAddr(N, DAG);
7559 static SDValue combineSelectToBinOp(SDNode *N, SelectionDAG &DAG,
7570 SDValue Neg = DAG.getNegative(CondV, DL, VT);
7571 return DAG.getNode(ISD::OR, DL, VT, Neg, DAG.getFreeze(FalseV));
7575 SDValue Neg = DAG.getNode(ISD::ADD, DL, VT, CondV,
7576 DAG.getAllOnesConstant(DL, VT));
7577 return DAG.getNode(ISD::OR, DL, VT, Neg, DAG.getFreeze(TrueV));
7582 SDValue Neg = DAG.getNode(ISD::ADD, DL, VT, CondV,
7583 DAG.getAllOnesConstant(DL, VT));
7584 return DAG.getNode(ISD::AND, DL, VT, Neg, DAG.getFreeze(FalseV));
7588 SDValue Neg = DAG.getNegative(CondV, DL, VT);
7589 return DAG.getNode(ISD::AND, DL, VT, Neg, DAG.getFreeze(TrueV));
7598 SDValue Neg = DAG.getNegative(CondV, DL, VT);
7599 return DAG.getNode(ISD::XOR, DL, VT, Neg, FalseV);
7614 return DAG.getNode(*MatchResult ? ISD::OR : ISD::AND, DL, VT, TrueV,
7615 DAG.getFreeze(FalseV));
7620 return DAG.getNode(*MatchResult ? ISD::AND : ISD::OR, DL, VT,
7621 DAG.getFreeze(TrueV), FalseV);
7635 foldBinOpIntoSelectIfProfitable(SDNode *BO, SelectionDAG &DAG,
7674 DAG.FoldConstantArithmetic(BO->getOpcode(), DL, VT, NewConstOps);
7686 SDValue NewNonConstOp = DAG.getNode(BO->getOpcode(), DL, VT, NewNonConstOps);
7690 return DAG.getSelect(DL, VT, Sel.getOperand(0), NewT, NewF);
7693 SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
7704 SDValue CondSplat = DAG.getSplat(SplatCondVT, DL, CondV);
7705 return DAG.getNode(ISD::VSELECT, DL, VT, CondSplat, TrueV, FalseV);
7716 return DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV, CondV);
7719 return DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV, CondV);
7724 return DAG.getNode(
7726 DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV, CondV));
7730 return DAG.getNode(
7732 DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV, CondV));
7735 if (SDValue V = combineSelectToBinOp(Op.getNode(), DAG, Subtarget))
7748 SDValue LHSVal = DAG.getConstant(
7751 DAG.getConstant(IsCZERO_NEZ ? TrueVal : FalseVal, DL, VT);
7753 DAG.getNode(IsCZERO_NEZ ? RISCVISD::CZERO_NEZ : RISCVISD::CZERO_EQZ,
7755 return DAG.getNode(ISD::ADD, DL, VT, CMOV, RHSVal);
7761 return DAG.getNode(
7763 DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV, CondV),
7764 DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV, CondV));
7767 if (SDValue V = combineSelectToBinOp(Op.getNode(), DAG, Subtarget))
7772 if (isBinOp(UseOpc) && DAG.isSafeToSpeculativelyExecute(UseOpc)) {
7775 DAG, Subtarget)) {
7776 DAG.ReplaceAllUsesWith(BinOp, &NewSel);
7780 return lowerSELECT(NewSel, DAG);
7792 return DAG.getNode(ISD::SINT_TO_FP, DL, VT, CondV);
7794 SDValue XOR = DAG.getNode(ISD::XOR, DL, XLenVT, CondV,
7795 DAG.getConstant(1, DL, XLenVT));
7796 return DAG.getNode(ISD::SINT_TO_FP, DL, VT, XOR);
7806 SDValue Zero = DAG.getConstant(0, DL, XLenVT);
7807 SDValue SetNE = DAG.getCondCode(ISD::SETNE);
7811 return DAG.getNode(RISCVISD::SELECT_CC, DL, VT, Ops);
7835 return DAG.getNode(ISD::ADD, DL, VT, CondV, FalseV);
7837 return DAG.getNode(ISD::SUB, DL, VT, FalseV, CondV);
7840 translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
7844 LHS = DAG.getConstant(0, DL, VT);
7855 RHS = DAG.getConstant(0, DL, VT);
7858 SDValue TargetCC = DAG.getCondCode(CCVal);
7864 TargetCC = DAG.getCondCode(ISD::getSetCCInverse(CCVal, LHS.getValueType()));
7868 return DAG.getNode(RISCVISD::SELECT_CC, DL, VT, Ops);
7871 SDValue RISCVTargetLowering::lowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
7882 translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
7884 SDValue TargetCC = DAG.getCondCode(CCVal);
7885 return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
7889 return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
7890 CondV, DAG.getConstant(0, DL, XLenVT),
7891 DAG.getCondCode(ISD::SETNE), Op.getOperand(2));
7894 SDValue RISCVTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
7895 MachineFunction &MF = DAG.getMachineFunction();
7899 SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
7905 return DAG.getStore(Op.getOperand(0), DL, FI, Op.getOperand(1),
7910 SelectionDAG &DAG) const {
7912 MachineFunction &MF = DAG.getMachineFunction();
7920 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, FrameReg, VT);
7924 SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr,
7925 DAG.getIntPtrConstant(Offset, DL));
7927 DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo());
7933 SelectionDAG &DAG) const {
7935 MachineFunction &MF = DAG.getMachineFunction();
7941 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
7949 SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
7950 SDValue Offset = DAG.getConstant(Off, DL, VT);
7951 return DAG.getLoad(VT, DL, DAG.getEntryNode(),
7952 DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
7959 return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, XLenVT);
7963 SelectionDAG &DAG) const {
7977 SDValue Zero = DAG.getConstant(0, DL, VT);
7978 SDValue One = DAG.getConstant(1, DL, VT);
7979 SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
7980 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
7981 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
7982 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
7984 SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);
7985 SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One);
7987 DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, XLenMinus1Shamt);
7988 SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);
7989 SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
7990 SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusXLen);
7992 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
7994 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero);
7995 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
7998 return DAG.getMergeValues(Parts, DL);
8001 SDValue RISCVTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
8027 SDValue Zero = DAG.getConstant(0, DL, VT);
8028 SDValue One = DAG.getConstant(1, DL, VT);
8029 SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
8030 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
8031 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
8032 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
8034 SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);
8035 SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One);
8037 DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, XLenMinus1Shamt);
8038 SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi);
8039 SDValue HiTrue = DAG.getNode(ShiftRightOp, DL, VT, Hi, Shamt);
8040 SDValue LoFalse = DAG.getNode(ShiftRightOp, DL, VT, Hi, ShamtMinusXLen);
8042 IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, XLenMinus1) : Zero;
8044 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
8046 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse);
8047 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
8050 return DAG.getMergeValues(Parts, DL);
8056 SelectionDAG &DAG) const {
8062 SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second;
8063 return DAG.getNode(RISCVISD::VMSET_VL, DL, VT, VL);
8066 SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second;
8067 return DAG.getNode(RISCVISD::VMCLR_VL, DL, VT, VL);
8070 SplatVal = DAG.getNode(ISD::AND, DL, SplatVal.getValueType(), SplatVal,
8071 DAG.getConstant(1, DL, SplatVal.getValueType()));
8072 SDValue LHS = DAG.getSplatVector(InterVT, DL, SplatVal);
8073 SDValue Zero = DAG.getConstant(0, DL, InterVT);
8074 return DAG.getSetCC(DL, VT, LHS, Zero, ISD::SETNE);
8082 SelectionDAG &DAG) const {
8096 auto VL = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).second;
8099 splatPartsI64WithVL(DL, ContainerVT, SDValue(), Lo, Hi, VL, DAG);
8102 Res = convertFromScalableVector(VecVT, Res, DAG, Subtarget);
8111 SDValue RISCVTargetLowering::lowerVectorMaskExt(SDValue Op, SelectionDAG &DAG,
8121 SDValue SplatZero = DAG.getConstant(0, DL, VecVT);
8122 SDValue SplatTrueVal = DAG.getConstant(ExtTrueVal, DL, VecVT);
8123 return DAG.getNode(ISD::VSELECT, DL, VecVT, Src, SplatTrueVal, SplatZero);
8130 SDValue CC = convertToScalableVector(I1ContainerVT, Src, DAG, Subtarget);
8132 SDValue VL = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).second;
8135 SDValue SplatZero = DAG.getConstant(0, DL, XLenVT);
8136 SDValue SplatTrueVal = DAG.getConstant(ExtTrueVal, DL, XLenVT);
8138 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
8139 DAG.getUNDEF(ContainerVT), SplatZero, VL);
8140 SplatTrueVal = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
8141 DAG.getUNDEF(ContainerVT), SplatTrueVal, VL);
8143 DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, CC, SplatTrueVal,
8144 SplatZero, DAG.getUNDEF(ContainerVT), VL);
8146 return convertFromScalableVector(VecVT, Select, DAG, Subtarget);
8150 SDValue Op, SelectionDAG &DAG, unsigned ExtendOpc) const {
8166 convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget);
8169 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
8171 SDValue Ext = DAG.getNode(ExtendOpc, DL, ContainerExtVT, Op1, Mask, VL);
8173 return convertFromScalableVector(ExtVT, Ext, DAG, Subtarget);
8180 SelectionDAG &DAG) const {
8199 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
8203 Mask = convertToScalableVector(MaskContainerVT, Mask, DAG, Subtarget);
8209 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
8212 SDValue SplatOne = DAG.getConstant(1, DL, Subtarget.getXLenVT());
8213 SDValue SplatZero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
8215 SplatOne = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
8216 DAG.getUNDEF(ContainerVT), SplatOne, VL);
8217 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
8218 DAG.getUNDEF(ContainerVT), SplatZero, VL);
8221 SDValue Trunc = DAG.getNode(RISCVISD::AND_VL, DL, ContainerVT, Src, SplatOne,
8222 DAG.getUNDEF(ContainerVT), Mask, VL);
8223 Trunc = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskContainerVT,
8224 {Trunc, SplatZero, DAG.getCondCode(ISD::SETNE),
8225 DAG.getUNDEF(MaskContainerVT), Mask, VL});
8227 Trunc = convertFromScalableVector(MaskVT, Trunc, DAG, Subtarget);
8232 SelectionDAG &DAG) const {
8242 return lowerVectorMaskTruncLike(Op, DAG);
8265 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
8268 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
8275 getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
8278 LLVMContext &Context = *DAG.getContext();
8283 Result = DAG.getNode(RISCVISD::TRUNCATE_VECTOR_VL, DL, ResultVT, Result,
8288 Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
8295 SelectionDAG &DAG) const {
8306 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
8309 auto [Mask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
8323 Src = DAG.getNode(InterConvOpc, DL, DAG.getVTList(InterVT, MVT::Other),
8331 SDValue Res = DAG.getNode(ConvOpc, DL, DAG.getVTList(ContainerVT, MVT::Other),
8336 SDValue SubVec = convertFromScalableVector(VT, Res, DAG, Subtarget);
8337 Res = DAG.getMergeValues({SubVec, Res.getValue(1)}, DL);
8344 SelectionDAG &DAG) const {
8381 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
8384 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
8390 getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
8395 Src = DAG.getNode(ConvOpc, DL, ContainerVT, Src, Mask, VL);
8397 Src = convertFromScalableVector(VT, Src, DAG, Subtarget);
8406 DAG.getNode(InterConvOpc, DL, InterVT, Src, Mask, VL);
8408 DAG.getNode(ConvOpc, DL, ContainerVT, IntermediateConv, Mask, VL);
8410 return convertFromScalableVector(VT, Result, DAG, Subtarget);
8420 getSmallestVTForIndex(MVT VecVT, unsigned MaxIdx, SDLoc DL, SelectionDAG &DAG,
8447 SelectionDAG &DAG) const {
8458 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec);
8459 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideVT, Vec, Val, Idx);
8460 return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Vec);
8467 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
8479 DL, DAG, Subtarget)) {
8481 AlignedIdx = DAG.getVectorIdxConstant(0, DL);
8496 AlignedIdx = DAG.getVectorIdxConstant(ExtractIdx, DL);
8497 Idx = DAG.getVectorIdxConstant(RemIdx, DL);
8502 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ContainerVT, Vec,
8517 Val = DAG.getConstant(CVal->getSExtValue(), DL, MVT::i32);
8521 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
8530 Val = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Val);
8531 Vec = DAG.getNode(Opc, DL, ContainerVT, Vec, Val, VL);
8534 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, OrigContainerVT, OrigVec,
8538 return convertFromScalableVector(VecVT, Vec, DAG, Subtarget);
8540 ValInVec = lowerScalarInsert(Val, VL, ContainerVT, DL, DAG, Subtarget);
8547 std::tie(ValLo, ValHi) = DAG.SplitScalar(Val, DL, MVT::i32, MVT::i32);
8551 getDefaultScalableVLOps(I32ContainerVT, DL, DAG, Subtarget).first;
8553 SDValue InsertI64VL = DAG.getConstant(2, DL, XLenVT);
8558 ValInVec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32ContainerVT,
8563 ValInVec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32ContainerVT,
8566 ValInVec = DAG.getBitcast(ContainerVT, ValInVec);
8570 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, OrigContainerVT, OrigVec,
8574 return convertFromScalableVector(VecVT, ValInVec, DAG, Subtarget);
8579 ValInVec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32ContainerVT,
8580 DAG.getUNDEF(I32ContainerVT),
8581 DAG.getUNDEF(I32ContainerVT), ValLo,
8583 ValInVec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32ContainerVT,
8584 DAG.getUNDEF(I32ContainerVT), ValInVec, ValHi,
8587 ValInVec = DAG.getBitcast(ContainerVT, ValInVec);
8592 DAG.getNode(ISD::ADD, DL, XLenVT, Idx, DAG.getConstant(1, DL, XLenVT));
8599 SDValue Slideup = getVSlideup(DAG, Subtarget, DL, ContainerVT, Vec, ValInVec,
8603 Slideup = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, OrigContainerVT, OrigVec,
8607 return convertFromScalableVector(VecVT, Slideup, DAG, Subtarget);
8615 SelectionDAG &DAG) const {
8629 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
8631 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
8633 DAG.getNode(RISCVISD::VFIRST_VL, DL, XLenVT, Vec, Mask, VL);
8634 SDValue Res = DAG.getSetCC(DL, XLenVT, Vfirst,
8635 DAG.getConstant(0, DL, XLenVT), ISD::SETEQ);
8636 return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Res);
8653 ExtractElementIdx = DAG.getConstant(0, DL, XLenVT);
8659 ExtractElementIdx = DAG.getNode(
8661 DAG.getConstant(Log2_64(WideEltVT.getSizeInBits()), DL, XLenVT));
8663 ExtractBitIdx = DAG.getNode(
8665 DAG.getConstant(WideEltVT.getSizeInBits() - 1, DL, XLenVT));
8668 Vec = DAG.getNode(ISD::BITCAST, DL, WideVT, Vec);
8669 SDValue ExtractElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, XLenVT,
8673 DAG.getNode(ISD::SRL, DL, XLenVT, ExtractElt, ExtractBitIdx);
8674 SDValue Res = DAG.getNode(ISD::AND, DL, XLenVT, ShiftRight,
8675 DAG.getConstant(1, DL, XLenVT));
8676 return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Res);
8681 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec);
8682 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec, Idx);
8689 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
8707 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, M1VT, Vec,
8708 DAG.getVectorIdxConstant(ExtractIdx, DL));
8709 Idx = DAG.getVectorIdxConstant(RemIdx, DL);
8722 getSmallestVTForIndex(ContainerVT, *MaxIdx, DL, DAG, Subtarget)) {
8724 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ContainerVT, Vec,
8725 DAG.getConstant(0, DL, XLenVT));
8748 auto [Mask, VL] = getDefaultVLOps(1, ContainerVT, DL, DAG, Subtarget);
8749 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT,
8750 DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
8755 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec,
8756 DAG.getVectorIdxConstant(0, DL));
8759 SDValue Elt0 = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
8760 return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Elt0);
8765 static SDValue lowerVectorIntrinsicScalars(SDValue Op, SelectionDAG &DAG,
8806 ScalarOp = DAG.getNode(ExtOpc, DL, XLenVT, ScalarOp);
8807 return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
8825 if (DAG.ComputeNumSignBits(ScalarOp) > 32) {
8826 ScalarOp = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, ScalarOp);
8827 return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
8841 SDValue Vec = DAG.getBitcast(I32VT, Operands[2]);
8844 DAG.SplitScalar(ScalarOp, DL, MVT::i32, MVT::i32);
8857 I32VL = DAG.getConstant(2 * AVLInt, DL, XLenVT);
8861 SDValue LMUL = DAG.getConstant(Lmul, DL, XLenVT);
8863 SDValue SEW = DAG.getConstant(Sew, DL, XLenVT);
8864 SDValue SETVLMAX = DAG.getTargetConstant(
8866 I32VL = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, XLenVT, SETVLMAX, SEW,
8876 SDValue LMUL = DAG.getConstant(Lmul, DL, XLenVT);
8878 SDValue SEW = DAG.getConstant(Sew, DL, XLenVT);
8880 DAG.getTargetConstant(Intrinsic::riscv_vsetvli, DL, MVT::i32);
8883 SDValue VL = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, XLenVT, SETVL, AVL,
8886 DAG.getNode(ISD::SHL, DL, XLenVT, VL, DAG.getConstant(1, DL, XLenVT));
8889 SDValue I32Mask = getAllOnesMask(I32VT, I32VL, DL, DAG);
8895 Passthru = DAG.getUNDEF(I32VT);
8897 Passthru = DAG.getBitcast(I32VT, Operands[1]);
8901 Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Passthru, Vec,
8903 Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Passthru, Vec,
8906 Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Passthru, Vec,
8908 Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Passthru, Vec,
8913 Vec = DAG.getBitcast(VT, Vec);
8927 return DAG.getNode(RISCVISD::VMERGE_VL, DL, VT, Mask, Vec, MaskedOff,
8928 DAG.getUNDEF(VT), AVL);
8931 return DAG.getNode(RISCVISD::VMERGE_VL, DL, VT, Mask, Vec, MaskedOff,
8939 ScalarOp = splatSplitI64WithVL(DL, VT, SDValue(), ScalarOp, VL, DAG);
8940 return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
8953 static SDValue lowerGetVectorLength(SDNode *N, SelectionDAG &DAG,
8977 SDValue LMul = DAG.getTargetConstant(VLMUL, DL, XLenVT);
8978 SDValue Sew = DAG.getTargetConstant(VSEW, DL, XLenVT);
8980 SDValue AVL = DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, N->getOperand(1));
8982 SDValue ID = DAG.getTargetConstant(Intrinsic::riscv_vsetvli, DL, XLenVT);
8984 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, XLenVT, ID, AVL, Sew, LMul);
8985 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), Res);
8988 static SDValue lowerCttzElts(SDNode *N, SelectionDAG &DAG,
8994 ContainerVT = getContainerForFixedLengthVector(DAG, OpVT, Subtarget);
8995 Op0 = convertToScalableVector(ContainerVT, Op0, DAG, Subtarget);
8999 auto [Mask, VL] = getDefaultVLOps(OpVT, ContainerVT, DL, DAG, Subtarget);
9000 SDValue Res = DAG.getNode(RISCVISD::VFIRST_VL, DL, XLenVT, Op0, Mask, VL);
9006 DAG.getSetCC(DL, XLenVT, Res, DAG.getConstant(0, DL, XLenVT), ISD::SETLT);
9007 VL = DAG.getElementCount(DL, XLenVT, OpVT.getVectorElementCount());
9008 return DAG.getSelect(DL, XLenVT, Setcc, VL, Res);
9013 SelectionDAG &DAG) {
9015 DAG.getMachineFunction().getSubtarget<RISCVSubtarget>();
9043 ScalarOp = DAG.getNode(ExtOpc, DL, XLenVT, ScalarOp);
9051 SelectionDAG &DAG) {
9052 promoteVCIXScalar(OrigOp, Operands, DAG);
9054 DAG.getMachineFunction().getSubtarget<RISCVSubtarget>();
9061 V = DAG.getBitcast(InterimIVT, V);
9065 DAG, V.getSimpleValueType(), Subtarget);
9066 V = convertToScalableVector(OpContainerVT, V, DAG, Subtarget);
9080 SelectionDAG &DAG) const {
9089 EVT PtrVT = getPointerTy(DAG.getDataLayout());
9090 return DAG.getRegister(RISCV::X4, PtrVT);
9114 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1));
9115 SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp);
9116 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res);
9119 return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1));
9128 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1));
9130 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(2));
9132 DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1, Op.getOperand(3));
9133 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res);
9136 return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2),
9143 return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1));
9148 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1));
9149 SDValue Res = DAG.getNode(
9151 DAG.getTargetConstant(Op.getConstantOperandVal(2), DL, MVT::i64));
9152 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res);
9154 return DAG.getNode(RISCVISD::MOPR, DL, XLenVT, Op.getOperand(1),
9161 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1));
9163 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(2));
9164 SDValue Res = DAG.getNode(
9166 DAG.getTargetConstant(Op.getConstantOperandVal(3), DL, MVT::i64));
9167 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res);
9169 return DAG.getNode(RISCVISD::MOPRR, DL, XLenVT, Op.getOperand(1),
9175 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1));
9177 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(2));
9178 SDValue Res = DAG.getNode(RISCVISD::CLMUL, DL, MVT::i64, NewOp0, NewOp1);
9179 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res);
9181 return DAG.getNode(RISCVISD::CLMUL, DL, XLenVT, Op.getOperand(1),
9189 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1));
9191 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(2));
9192 NewOp0 = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0,
9193 DAG.getConstant(32, DL, MVT::i64));
9194 NewOp1 = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp1,
9195 DAG.getConstant(32, DL, MVT::i64));
9196 SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1);
9197 Res = DAG.getNode(ISD::SRL, DL, MVT::i64, Res,
9198 DAG.getConstant(32, DL, MVT::i64));
9199 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res);
9202 return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2));
9205 return lowerGetVectorLength(Op.getNode(), DAG, Subtarget);
9207 return lowerCttzElts(Op.getNode(), DAG, Subtarget);
9209 SDValue Res = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Op.getOperand(1));
9210 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Res);
9213 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Op.getValueType(),
9214 Op.getOperand(1), DAG.getVectorIdxConstant(0, DL));
9217 Op.getOperand(3), Op.getSimpleValueType(), DL, DAG,
9220 return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, Op.getValueType(),
9226 Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Scalar);
9227 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, Op.getValueType(),
9252 SDValue SplattedVal = splatSplitI64WithVL(DL, VT, SDValue(), Scalar, VL, DAG);
9256 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT),
9257 DAG.getConstant(0, DL, MVT::i32), VL);
9260 SDValue Mask = getAllOnesMask(VT, VL, DL, DAG);
9261 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
9263 DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT,
9264 {VID, SplattedIdx, DAG.getCondCode(ISD::SETEQ),
9265 DAG.getUNDEF(MaskVT), Mask, VL});
9266 return DAG.getNode(RISCVISD::VMERGE_VL, DL, VT, SelectCond, SplattedVal,
9267 Vec, DAG.getUNDEF(VT), VL);
9270 return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, Op.getSimpleValueType(),
9331 processVCIXOperands(Op, Operands, DAG);
9340 SDValue NewNode = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, RetVT, Operands);
9343 NewNode = convertFromScalableVector(VT, NewNode, DAG, Subtarget);
9345 NewNode = DAG.getBitcast(VT, NewNode);
9354 return lowerVectorIntrinsicScalars(Op, DAG, Subtarget);
9357 static inline SDValue getVCIXISDNodeWCHAIN(SDValue &Op, SelectionDAG &DAG,
9364 DAG.getMachineFunction().getSubtarget<RISCVSubtarget>();
9375 RetVT = getContainerForFixedLengthVector(DAG.getTargetLoweringInfo(), RetVT,
9378 processVCIXOperands(Op, Operands, DAG);
9380 SDVTList VTs = DAG.getVTList({RetVT, MVT::Other});
9381 SDValue NewNode = DAG.getNode(Type, DL, VTs, Operands);
9385 NewNode = convertFromScalableVector(FloatVT, NewNode, DAG, Subtarget);
9387 NewNode = DAG.getBitcast(VT, NewNode);
9389 NewNode = DAG.getMergeValues({NewNode, Chain}, DL);
9394 static inline SDValue getVCIXISDNodeVOID(SDValue &Op, SelectionDAG &DAG,
9398 processVCIXOperands(Op, Operands, DAG);
9400 return DAG.getNode(Type, SDLoc(Op), Op.getValueType(), Operands);
9404 SelectionDAG &DAG) const {
9427 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
9428 PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
9433 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
9443 DAG.getExtLoad(ISD::EXTLOAD, DL, XLenVT, Load->getChain(), Ptr,
9446 Result = lowerScalarSplat(SDValue(), ScalarLoad, VL, ContainerVT, DL, DAG,
9449 SDValue ScalarLoad = DAG.getLoad(ScalarVT, DL, Load->getChain(), Ptr,
9452 Result = DAG.getSplat(ContainerVT, DL, ScalarLoad);
9454 SDValue IntID = DAG.getTargetConstant(
9460 Ops.push_back(DAG.getUNDEF(ContainerVT));
9470 DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT);
9474 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
9476 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
9481 Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
9482 return DAG.getMergeValues({Result, Chain}, DL);
9503 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG,
9505 SDValue IntID = DAG.getTargetConstant(VlsegInts[NF - 2], DL, XLenVT);
9509 SDVTList VTs = DAG.getVTList(ContainerVTs);
9511 Ops.insert(Ops.end(), NF, DAG.getUNDEF(ContainerVT));
9515 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
9520 DAG, Subtarget));
9522 return DAG.getMergeValues(Results, DL);
9525 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_X_SE);
9527 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_I_SE);
9529 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_XV_SE);
9531 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_IV_SE);
9533 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_VV_SE);
9535 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_FV_SE);
9537 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_XVV_SE);
9539 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_IVV_SE);
9541 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_VVV_SE);
9543 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_FVV_SE);
9545 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_XVW_SE);
9547 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_IVW_SE);
9549 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_VVW_SE);
9551 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_FVW_SE);
9554 return lowerVectorIntrinsicScalars(Op, DAG, Subtarget);
9558 SelectionDAG &DAG) const {
9577 Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
9582 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
9585 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
9587 SDValue IntID = DAG.getTargetConstant(
9600 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, Store->getVTList(),
9624 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG,
9626 SDValue IntID = DAG.getTargetConstant(VssegInts[NF - 2], DL, XLenVT);
9633 ContainerVT, FixedIntrinsic->getOperand(2 + i), DAG, Subtarget));
9636 return DAG.getMemIntrinsicNode(
9637 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops,
9641 return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_XV_SE);
9643 return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_IV_SE);
9645 return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_VV_SE);
9647 return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_FV_SE);
9649 return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_XVV_SE);
9651 return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_IVV_SE);
9653 return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_VVV_SE);
9655 return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_FVV_SE);
9657 return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_XVW_SE);
9659 return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_IVW_SE);
9661 return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_VVW_SE);
9663 return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_FVW_SE);
9666 return lowerVectorIntrinsicScalars(Op, DAG, Subtarget);
9712 SelectionDAG &DAG,
9730 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
9739 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
9744 SDValue Zero = DAG.getConstant(0, DL, XLenVT);
9752 SDValue TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
9753 Vec = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Vec, TrueMask, VL);
9754 Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
9762 Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
9769 SDValue One = DAG.getConstant(1, DL, XLenVT);
9770 Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
9771 Vec = DAG.getNode(ISD::AND, DL, XLenVT, Vec, One);
9778 SDValue SetCC = DAG.getSetCC(DL, XLenVT, Vec, Zero, CC);
9779 SetCC = DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), SetCC);
9790 return DAG.getNode(BaseOpc, DL, Op.getValueType(), SetCC, Op.getOperand(0));
9804 SDValue VL, const SDLoc &DL, SelectionDAG &DAG,
9817 auto InnerVL = NonZeroAVL ? VL : DAG.getConstant(1, DL, XLenVT);
9819 DAG, Subtarget);
9822 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, M1VT, DAG.getUNDEF(M1VT),
9823 InitialValue, DAG.getVectorIdxConstant(0, DL));
9824 SDValue PassThru = NonZeroAVL ? DAG.getUNDEF(M1VT) : InitialValue;
9825 SDValue Policy = DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT);
9827 SDValue Reduction = DAG.getNode(RVVOpcode, DL, M1VT, Ops);
9828 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Reduction,
9829 DAG.getVectorIdxConstant(0, DL));
9833 SelectionDAG &DAG) const {
9842 while (getTypeAction(*DAG.getContext(), VecEVT) ==
9844 auto [Lo, Hi] = DAG.SplitVector(Vec, DL);
9846 Vec = DAG.getNode(BaseOpc, DL, VecEVT, Lo, Hi);
9861 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
9864 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
9866 SDValue StartV = DAG.getNeutralElement(BaseOpc, DL, VecEltVT, SDNodeFlags());
9874 StartV = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Vec,
9875 DAG.getVectorIdxConstant(0, DL));
9878 Mask, VL, DL, DAG, Subtarget);
9885 getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT,
9896 DAG.getConstantFP(Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, EltVT);
9907 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op.getOperand(0),
9908 DAG.getVectorIdxConstant(0, DL));
9919 SelectionDAG &DAG) const {
9926 getRVVFPReductionOpAndOperands(Op, DAG, VecEltVT, Subtarget);
9932 VectorVal = convertToScalableVector(ContainerVT, VectorVal, DAG, Subtarget);
9936 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
9938 VL, DL, DAG, Subtarget);
9948 DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
9949 {VectorVal, VectorVal, DAG.getCondCode(ISD::SETNE),
9950 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
9952 SDValue CPop = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, IsNan, Mask, VL);
9953 SDValue NoNaNs = DAG.getSetCC(DL, XLenVT, CPop,
9954 DAG.getConstant(0, DL, XLenVT), ISD::SETEQ);
9955 return DAG.getSelect(
9957 DAG.getConstantFP(APFloat::getNaN(DAG.EVTToAPFloatSemantics(ResVT)), DL,
9962 SelectionDAG &DAG) const {
9980 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
9987 Vec, Mask, VL, DL, DAG, Subtarget);
9995 SDValue IsNaN = DAG.getNode(
9997 {Vec, Vec, DAG.getCondCode(ISD::SETNE), DAG.getUNDEF(PredVT), Mask, VL});
9998 SDValue VCPop = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, IsNaN, Mask, VL);
10000 SDValue StartIsNaN = DAG.getSetCC(DL, XLenVT, Start, Start, ISD::SETUO);
10001 VCPop = DAG.getNode(ISD::OR, DL, XLenVT, VCPop, StartIsNaN);
10002 SDValue NoNaNs = DAG.getSetCC(DL, XLenVT, VCPop,
10003 DAG.getConstant(0, DL, XLenVT), ISD::SETEQ);
10005 return DAG.getSelect(
10007 DAG.getConstantFP(APFloat::getNaN(DAG.EVTToAPFloatSemantics(ResVT)), DL,
10012 SelectionDAG &DAG) const {
10042 Vec = DAG.getBitcast(VecVT, Vec);
10043 SubVec = DAG.getBitcast(SubVecVT, SubVec);
10051 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
10052 SubVec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtSubVecVT, SubVec);
10053 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ExtVecVT, Vec, SubVec,
10055 SDValue SplatZero = DAG.getConstant(0, DL, ExtVecVT);
10056 return DAG.getSetCC(DL, VecVT, Vec, SplatZero, ISD::SETNE);
10072 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
10076 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT,
10077 DAG.getUNDEF(ContainerVT), SubVec,
10078 DAG.getVectorIdxConstant(0, DL));
10079 SubVec = convertFromScalableVector(VecVT, SubVec, DAG, Subtarget);
10080 return DAG.getBitcast(Op.getValueType(), SubVec);
10083 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT,
10084 DAG.getUNDEF(ContainerVT), SubVec,
10085 DAG.getVectorIdxConstant(0, DL));
10087 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
10091 SDValue VL = getVLOp(EndIndex, ContainerVT, DL, DAG, Subtarget);
10102 DAG.getNode(RISCVISD::VMV_V_V_VL, DL, ContainerVT, Vec, SubVec, VL);
10104 SDValue SlideupAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
10105 SubVec = getVSlideup(DAG, Subtarget, DL, ContainerVT, Vec, SubVec,
10110 SubVec = convertFromScalableVector(VecVT, SubVec, DAG, Subtarget);
10111 return DAG.getBitcast(Op.getValueType(), SubVec);
10117 Vec = convertToScalableVector(ContainerVecVT, Vec, DAG, Subtarget);
10123 SubVec = convertToScalableVector(ContainerSubVecVT, SubVec, DAG, Subtarget);
10178 DAG.getTargetInsertSubreg(SubRegIdx, DL, ContainerVecVT, Vec, SubVec);
10180 Insert = convertFromScalableVector(VecVT, Insert, DAG, Subtarget);
10200 AlignedExtract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InterSubVT, Vec,
10201 DAG.getVectorIdxConstant(AlignedIdx, DL));
10204 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InterSubVT,
10205 DAG.getUNDEF(InterSubVT), SubVec,
10206 DAG.getVectorIdxConstant(0, DL));
10208 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVecVT, DL, DAG, Subtarget);
10211 VL = DAG.getElementCount(DL, XLenVT, SubVecVT.getVectorElementCount());
10222 SubVec = DAG.getNode(RISCVISD::VMV_V_V_VL, DL, InterSubVT, AlignedExtract,
10225 SDValue SlideupAmt = DAG.getElementCount(DL, XLenVT, RemIdx);
10228 VL = DAG.getNode(ISD::ADD, DL, XLenVT, SlideupAmt, VL);
10230 SubVec = getVSlideup(DAG, Subtarget, DL, InterSubVT, AlignedExtract, SubVec,
10237 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVecVT, Vec, SubVec,
10238 DAG.getVectorIdxConstant(AlignedIdx, DL));
10241 SubVec = convertFromScalableVector(VecVT, SubVec, DAG, Subtarget);
10245 return DAG.getBitcast(Op.getSimpleValueType(), SubVec);
10249 SelectionDAG &DAG) const {
10277 Vec = DAG.getBitcast(VecVT, Vec);
10288 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
10289 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtSubVecVT, Vec,
10291 SDValue SplatZero = DAG.getConstant(0, DL, ExtSubVecVT);
10292 return DAG.getSetCC(DL, SubVecVT, Vec, SplatZero, ISD::SETNE);
10312 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
10318 getSmallestVTForIndex(ContainerVT, LastIdx, DL, DAG, Subtarget)) {
10320 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ContainerVT, Vec,
10321 DAG.getVectorIdxConstant(0, DL));
10325 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
10328 SDValue VL = getVLOp(SubVecVT.getVectorNumElements(), ContainerVT, DL, DAG,
10330 SDValue SlidedownAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
10332 getVSlidedown(DAG, Subtarget, DL, ContainerVT,
10333 DAG.getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
10335 Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown,
10336 DAG.getVectorIdxConstant(0, DL));
10337 return DAG.getBitcast(Op.getValueType(), Slidedown);
10342 Vec = convertToScalableVector(VecVT, Vec, DAG, Subtarget);
10376 Vec = DAG.getTargetExtractSubreg(SubRegIdx, DL, ContainerSubVecVT, Vec);
10377 return convertFromScalableVector(SubVecVT, Vec, DAG, Subtarget);
10396 Vec = DAG.getTargetExtractSubreg(SubRegIdx, DL, InterSubVT, Vec);
10401 SDValue SlidedownAmt = DAG.getElementCount(DL, XLenVT, RemIdx);
10402 auto [Mask, VL] = getDefaultScalableVLOps(InterSubVT, DL, DAG, Subtarget);
10404 VL = getVLOp(SubVecVT.getVectorNumElements(), InterSubVT, DL, DAG,
10407 getVSlidedown(DAG, Subtarget, DL, InterSubVT, DAG.getUNDEF(InterSubVT),
10412 Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown,
10413 DAG.getVectorIdxConstant(0, DL));
10417 return DAG.getBitcast(Op.getSimpleValueType(), Slidedown);
10423 SelectionDAG &DAG) {
10430 WideOps.push_back(DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Op));
10435 SDVTList VTs = DAG.getVTList(SmallVector<EVT, 4>(
10437 SDValue WideN = DAG.getNode(N.getOpcode(), DL, VTs, WideOps);
10441 DAG.getSetCC(DL, N->getSimpleValueType(I), WideN.getValue(I),
10442 DAG.getConstant(0, DL, WideVT), ISD::SETNE));
10446 return DAG.getMergeValues(TruncVals, DL);
10451 SelectionDAG &DAG) const {
10460 return widenVectorOpsToi8(Op, DL, DAG);
10465 auto [Op0Lo, Op0Hi] = DAG.SplitVectorOperand(Op.getNode(), 0);
10466 auto [Op1Lo, Op1Hi] = DAG.SplitVectorOperand(Op.getNode(), 1);
10469 SDValue ResLo = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
10470 DAG.getVTList(SplitVT, SplitVT), Op0Lo, Op0Hi);
10471 SDValue ResHi = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
10472 DAG.getVTList(SplitVT, SplitVT), Op1Lo, Op1Hi);
10474 SDValue Even = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT,
10476 SDValue Odd = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, ResLo.getValue(1),
10478 return DAG.getMergeValues({Even, Odd}, DL);
10485 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT,
10489 auto [Mask, VL] = getDefaultScalableVLOps(ConcatVT, DL, DAG, Subtarget);
10490 SDValue Passthru = DAG.getUNDEF(ConcatVT);
10496 getDeinterleaveViaVNSRL(DL, VecVT, Concat, true, Subtarget, DAG);
10498 getDeinterleaveViaVNSRL(DL, VecVT, Concat, false, Subtarget, DAG);
10499 return DAG.getMergeValues({Even, Odd}, DL);
10506 DAG.getStepVector(DL, IdxVT, APInt(IdxVT.getScalarSizeInBits(), 2));
10509 DAG.getNode(ISD::ADD, DL, IdxVT, EvenIdx, DAG.getConstant(1, DL, IdxVT));
10512 SDValue EvenWide = DAG.getNode(RISCVISD::VRGATHER_VV_VL, DL, ConcatVT,
10514 SDValue OddWide = DAG.getNode(RISCVISD::VRGATHER_VV_VL, DL, ConcatVT,
10518 SDValue Even = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VecVT, EvenWide,
10519 DAG.getVectorIdxConstant(0, DL));
10520 SDValue Odd = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VecVT, OddWide,
10521 DAG.getVectorIdxConstant(0, DL));
10523 return DAG.getMergeValues({Even, Odd}, DL);
10527 SelectionDAG &DAG) const {
10536 return widenVectorOpsToi8(Op, DL, DAG);
10539 SDValue VL = DAG.getRegister(RISCV::X0, XLenVT);
10543 auto [Op0Lo, Op0Hi] = DAG.SplitVectorOperand(Op.getNode(), 0);
10544 auto [Op1Lo, Op1Hi] = DAG.SplitVectorOperand(Op.getNode(), 1);
10547 SDValue ResLo = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL,
10548 DAG.getVTList(SplitVT, SplitVT), Op0Lo, Op1Lo);
10549 SDValue ResHi = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL,
10550 DAG.getVTList(SplitVT, SplitVT), Op0Hi, Op1Hi);
10552 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT,
10554 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT,
10556 return DAG.getMergeValues({Lo, Hi}, DL);
10565 DAG, Subtarget);
10571 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT,
10577 SDValue StepVec = DAG.getStepVector(DL, IdxVT);
10580 SDValue Ones = DAG.getSplatVector(IdxVT, DL, DAG.getConstant(1, DL, XLenVT));
10583 SDValue OddMask = DAG.getNode(ISD::AND, DL, IdxVT, StepVec, Ones);
10584 OddMask = DAG.getSetCC(
10586 DAG.getSplatVector(IdxVT, DL, DAG.getConstant(0, DL, XLenVT)),
10589 SDValue VLMax = DAG.getSplatVector(IdxVT, DL, computeVLMax(VecVT, DL, DAG));
10593 SDValue Idx = DAG.getNode(ISD::SRL, DL, IdxVT, StepVec, Ones);
10596 DAG.getNode(RISCVISD::ADD_VL, DL, IdxVT, Idx, VLMax, Idx, OddMask, VL);
10600 SDValue TrueMask = getAllOnesMask(IdxVT, VL, DL, DAG);
10601 Interleaved = DAG.getNode(RISCVISD::VRGATHEREI16_VV_VL, DL, ConcatVT,
10602 Concat, Idx, DAG.getUNDEF(ConcatVT), TrueMask, VL);
10606 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VecVT, Interleaved,
10607 DAG.getVectorIdxConstant(0, DL));
10608 SDValue Hi = DAG.getNode(
10610 DAG.getVectorIdxConstant(VecVT.getVectorMinNumElements(), DL));
10612 return DAG.getMergeValues({Lo, Hi}, DL);
10618 SelectionDAG &DAG) const {
10623 auto [Mask, VL] = getDefaultScalableVLOps(VT, DL, DAG, Subtarget);
10624 SDValue StepVec = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
10629 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT),
10630 DAG.getConstant(Log2_64(StepValImm), DL, XLenVT), VL);
10631 StepVec = DAG.getNode(ISD::SHL, DL, VT, StepVec, StepVal);
10634 SDValue(), DAG.getConstant(StepValImm, DL, VT.getVectorElementType()),
10635 VL, VT, DL, DAG, Subtarget);
10636 StepVec = DAG.getNode(ISD::MUL, DL, VT, StepVec, StepVal);
10648 SelectionDAG &DAG) const {
10653 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, Op.getOperand(0));
10654 SDValue Op2 = DAG.getNode(ISD::VECTOR_REVERSE, DL, WidenVT, Op1);
10655 return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Op2);
10676 auto [Lo, Hi] = DAG.SplitVectorOperand(Op.getNode(), 0);
10677 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VecVT);
10678 Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, LoVT, Lo);
10679 Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, HiVT, Hi);
10683 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, DAG.getUNDEF(VecVT), Hi,
10684 DAG.getVectorIdxConstant(0, DL));
10685 return DAG.getNode(
10687 DAG.getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL));
10696 auto [Mask, VL] = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
10699 SDValue VLMinus1 = DAG.getNode(ISD::SUB, DL, XLenVT,
10700 computeVLMax(VecVT, DL, DAG),
10701 DAG.getConstant(1, DL, XLenVT));
10708 SplatVL = DAG.getSplatVector(IntVT, DL, VLMinus1);
10710 SplatVL = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntVT, DAG.getUNDEF(IntVT),
10711 VLMinus1, DAG.getRegister(RISCV::X0, XLenVT));
10713 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, IntVT, Mask, VL);
10714 SDValue Indices = DAG.getNode(RISCVISD::SUB_VL, DL, IntVT, SplatVL, VID,
10715 DAG.getUNDEF(IntVT), Mask, VL);
10717 return DAG.getNode(GatherOpc, DL, VecVT, Op.getOperand(0), Indices,
10718 DAG.getUNDEF(VecVT), Mask, VL);
10722 SelectionDAG &DAG) const {
10729 SDValue VLMax = computeVLMax(VecVT, DL, DAG);
10736 DownOffset = DAG.getConstant(ImmValue, DL, XLenVT);
10737 UpOffset = DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, DownOffset);
10741 UpOffset = DAG.getConstant(-ImmValue, DL, XLenVT);
10742 DownOffset = DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, UpOffset);
10745 SDValue TrueMask = getAllOnesMask(VecVT, VLMax, DL, DAG);
10748 getVSlidedown(DAG, Subtarget, DL, VecVT, DAG.getUNDEF(VecVT), V1,
10750 return getVSlideup(DAG, Subtarget, DL, VecVT, SlideDown, V2, UpOffset,
10751 TrueMask, DAG.getRegister(RISCV::X0, XLenVT),
10757 SelectionDAG &DAG) const {
10761 assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
10778 DAG.getLoad(ContainerVT, DL, Load->getChain(), Load->getBasePtr(),
10781 SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
10782 return DAG.getMergeValues({Result, NewLoad.getValue(1)}, DL);
10785 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG, Subtarget);
10788 SDValue IntID = DAG.getTargetConstant(
10792 Ops.push_back(DAG.getUNDEF(ContainerVT));
10795 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
10797 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
10800 SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
10801 return DAG.getMergeValues({Result, NewLoad.getValue(1)}, DL);
10806 SelectionDAG &DAG) const {
10810 assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
10823 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getConstant(0, DL, VT),
10824 StoreVal, DAG.getVectorIdxConstant(0, DL));
10830 convertToScalableVector(ContainerVT, StoreVal, DAG, Subtarget);
10840 return DAG.getStore(Store->getChain(), DL, NewValue, Store->getBasePtr(),
10845 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG,
10849 SDValue IntID = DAG.getTargetConstant(
10851 return DAG.getMemIntrinsicNode(
10852 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other),
10858 SelectionDAG &DAG) const {
10871 PassThru = DAG.getUNDEF(VT);
10886 PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
10889 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
10894 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
10898 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
10900 Ops.push_back(DAG.getUNDEF(ContainerVT));
10908 Ops.push_back(DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT));
10910 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
10913 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO);
10917 Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
10919 return DAG.getMergeValues({Result, Chain}, DL);
10923 SelectionDAG &DAG) const {
10955 Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
10958 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
10963 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
10966 Val = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
10967 DAG.getConstant(Intrinsic::riscv_vcompress, DL, XLenVT),
10968 DAG.getUNDEF(ContainerVT), Val, Mask, VL);
10970 DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Mask,
10971 getAllOnesMask(Mask.getSimpleValueType(), VL, DL, DAG), VL);
10976 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
10983 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
10984 DAG.getVTList(MVT::Other), Ops, MemVT, MMO);
10989 SelectionDAG &DAG) const {
10996 convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget);
10998 convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget);
11002 DAG, Subtarget);
11006 DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT,
11007 {Op1, Op2, Op.getOperand(2), DAG.getUNDEF(MaskVT), Mask, VL});
11009 return convertFromScalableVector(VT, Cmp, DAG, Subtarget);
11013 SelectionDAG &DAG) const {
11031 SDValue OLECCVal = DAG.getCondCode(ISD::SETOLE);
11032 SDValue Tmp1 = DAG.getNode(ISD::STRICT_FSETCCS, DL, VTList, Chain, Op1,
11034 SDValue Tmp2 = DAG.getNode(ISD::STRICT_FSETCCS, DL, VTList, Chain, Op2,
11036 SDValue OutChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
11040 Tmp1 = DAG.getNode(ISD::AND, DL, VT, Tmp1, Tmp2);
11041 return DAG.getMergeValues({Tmp1, OutChain}, DL);
11046 SDValue OEQCCVal = DAG.getCondCode(ISD::SETOEQ);
11047 SDValue OEQ = DAG.getNode(ISD::STRICT_FSETCCS, DL, VTList, Chain, Op1,
11049 SDValue Res = DAG.getNOT(DL, OEQ, VT);
11050 return DAG.getMergeValues({Res, OEQ.getValue(1)}, DL);
11057 Op1 = convertToScalableVector(ContainerInVT, Op1, DAG, Subtarget);
11058 Op2 = convertToScalableVector(ContainerInVT, Op2, DAG, Subtarget);
11062 auto [Mask, VL] = getDefaultVLOps(InVT, ContainerInVT, DL, DAG, Subtarget);
11070 SDValue True = getAllOnesMask(ContainerInVT, VL, DL, DAG);
11071 SDValue OrderMask1 = DAG.getNode(
11072 RISCVISD::STRICT_FSETCC_VL, DL, DAG.getVTList(MaskVT, MVT::Other),
11073 {Chain, Op1, Op1, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
11075 SDValue OrderMask2 = DAG.getNode(
11076 RISCVISD::STRICT_FSETCC_VL, DL, DAG.getVTList(MaskVT, MVT::Other),
11077 {Chain, Op2, Op2, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
11080 DAG.getNode(RISCVISD::VMAND_VL, DL, MaskVT, OrderMask1, OrderMask2, VL);
11083 Res = DAG.getNode(RISCVISD::STRICT_FSETCCS_VL, DL,
11084 DAG.getVTList(MaskVT, MVT::Other),
11089 Res = DAG.getNode(RVVOpc, DL, DAG.getVTList(MaskVT, MVT::Other),
11090 {Chain, Op1, Op2, CC, DAG.getUNDEF(MaskVT), Mask, VL});
11094 SDValue SubVec = convertFromScalableVector(VT, Res, DAG, Subtarget);
11095 return DAG.getMergeValues({SubVec, Res.getValue(1)}, DL);
11101 SDValue RISCVTargetLowering::lowerABS(SDValue Op, SelectionDAG &DAG) const {
11112 X = convertToScalableVector(ContainerVT, X, DAG, Subtarget);
11119 Mask = convertToScalableVector(getMaskTypeFor(ContainerVT), Mask, DAG,
11123 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
11125 SDValue SplatZero = DAG.getNode(
11126 RISCVISD::VMV_V_X_VL, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
11127 DAG.getConstant(0, DL, Subtarget.getXLenVT()), VL);
11128 SDValue NegX = DAG.getNode(RISCVISD::SUB_VL, DL, ContainerVT, SplatZero, X,
11129 DAG.getUNDEF(ContainerVT), Mask, VL);
11130 SDValue Max = DAG.getNode(RISCVISD::SMAX_VL, DL, ContainerVT, X, NegX,
11131 DAG.getUNDEF(ContainerVT), Mask, VL);
11134 Max = convertFromScalableVector(VT, Max, DAG, Subtarget);
11139 SDValue Op, SelectionDAG &DAG) const {
11148 Mag = convertToScalableVector(ContainerVT, Mag, DAG, Subtarget);
11149 Sign = convertToScalableVector(ContainerVT, Sign, DAG, Subtarget);
11151 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
11153 SDValue CopySign = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Mag,
11154 Sign, DAG.getUNDEF(ContainerVT), Mask, VL);
11156 return convertFromScalableVector(VT, CopySign, DAG, Subtarget);
11160 SDValue Op, SelectionDAG &DAG) const {
11168 convertToScalableVector(I1ContainerVT, Op.getOperand(0), DAG, Subtarget);
11170 convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget);
11172 convertToScalableVector(ContainerVT, Op.getOperand(2), DAG, Subtarget);
11175 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
11177 SDValue Select = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, CC, Op1,
11178 Op2, DAG.getUNDEF(ContainerVT), VL);
11180 return convertFromScalableVector(VT, Select, DAG, Subtarget);
11184 SelectionDAG &DAG) const {
11206 Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget));
11210 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
11212 Ops.push_back(DAG.getUNDEF(ContainerVT));
11221 DAG.getNode(NewOpc, DL, DAG.getVTList(ContainerVT, MVT::Other), Ops,
11223 SDValue SubVec = convertFromScalableVector(VT, ScalableRes, DAG, Subtarget);
11224 return DAG.getMergeValues({SubVec, ScalableRes.getValue(1)}, DL);
11228 DAG.getNode(NewOpc, DL, ContainerVT, Ops, Op->getFlags());
11229 return convertFromScalableVector(VT, ScalableRes, DAG, Subtarget);
11237 SDValue RISCVTargetLowering::lowerVPOp(SDValue Op, SelectionDAG &DAG) const {
11258 Ops.push_back(DAG.getUNDEF(ContainerVT));
11267 Ops.push_back(DAG.getUNDEF(ContainerVT));
11281 Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget));
11285 return DAG.getNode(RISCVISDOpc, DL, VT, Ops, Op->getFlags());
11287 SDValue VPOp = DAG.getNode(RISCVISDOpc, DL, ContainerVT, Ops, Op->getFlags());
11289 return convertFromScalableVector(VT, VPOp, DAG, Subtarget);
11293 SelectionDAG &DAG) const {
11305 Src = convertToScalableVector(SrcVT, Src, DAG, Subtarget);
11309 SDValue Zero = DAG.getConstant(0, DL, XLenVT);
11310 SDValue ZeroSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
11311 DAG.getUNDEF(ContainerVT), Zero, VL);
11313 SDValue SplatValue = DAG.getConstant(
11315 SDValue Splat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
11316 DAG.getUNDEF(ContainerVT), SplatValue, VL);
11318 SDValue Result = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, Src, Splat,
11319 ZeroSplat, DAG.getUNDEF(ContainerVT), VL);
11322 return convertFromScalableVector(VT, Result, DAG, Subtarget);
11326 SelectionDAG &DAG) const {
11339 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
11340 Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget);
11344 SDValue AllOneMask = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
11351 Result = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op1, Op2, VL);
11356 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op1, Op2, VL);
11358 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Temp, AllOneMask, VL);
11366 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op1, AllOneMask, VL);
11367 Result = DAG.getNode(RISCVISD::VMAND_VL, DL, ContainerVT, Temp, Op2, VL);
11375 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op2, AllOneMask, VL);
11376 Result = DAG.getNode(RISCVISD::VMAND_VL, DL, ContainerVT, Op1, Temp, VL);
11384 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op1, AllOneMask, VL);
11385 Result = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Temp, Op2, VL);
11393 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op2, AllOneMask, VL);
11394 Result = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Temp, Op1, VL);
11401 return convertFromScalableVector(VT, Result, DAG, Subtarget);
11406 SelectionDAG &DAG) const {
11419 Src = convertToScalableVector(SrcVT, Src, DAG, Subtarget);
11421 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
11440 SDValue Zero = DAG.getConstant(0, DL, XLenVT);
11441 SDValue ZeroSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntVT,
11442 DAG.getUNDEF(IntVT), Zero, VL);
11443 SDValue One = DAG.getConstant(
11445 SDValue OneSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntVT,
11446 DAG.getUNDEF(IntVT), One, VL);
11447 Src = DAG.getNode(RISCVISD::VMERGE_VL, DL, IntVT, Src, OneSplat,
11448 ZeroSplat, DAG.getUNDEF(IntVT), VL);
11453 Src = DAG.getNode(RISCVISDExtOpc, DL, IntVT, Src, Mask, VL);
11456 Result = DAG.getNode(RISCVISDOpc, DL, DstVT, Src, Mask, VL);
11467 DAG.getNode(RISCVISD::FP_EXTEND_VL, DL, InterimFVT, Src, Mask, VL);
11470 Result = DAG.getNode(RISCVISDOpc, DL, DstVT, Src, Mask, VL);
11485 Result = DAG.getNode(RISCVISDOpc, DL, InterimFVT, Src, Mask, VL);
11489 Result = DAG.getNode(RISCVISD::FP_ROUND_VL, DL, DstVT, Src, Mask, VL);
11503 Result = DAG.getNode(RISCVISDOpc, DL, InterimIVT, Src, Mask, VL);
11508 SDValue SplatZero = DAG.getConstant(0, DL, XLenVT);
11509 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, InterimIVT,
11510 DAG.getUNDEF(InterimIVT), SplatZero, VL);
11511 Result = DAG.getNode(RISCVISD::SETCC_VL, DL, DstVT,
11512 {Result, SplatZero, DAG.getCondCode(ISD::SETNE),
11513 DAG.getUNDEF(DstVT), Mask, VL});
11518 Result = DAG.getNode(RISCVISDOpc, DL, InterimIVT, Src, Mask, VL);
11525 Result = DAG.getNode(RISCVISD::TRUNCATE_VECTOR_VL, DL, InterimIVT,
11535 return convertFromScalableVector(VT, Result, DAG, Subtarget);
11540 SelectionDAG &DAG) const {
11555 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
11556 Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget);
11558 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
11562 EVL1 = DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, EVL1);
11569 SDValue SplatOneOp1 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
11570 DAG.getUNDEF(ContainerVT),
11571 DAG.getConstant(1, DL, XLenVT), EVL1);
11572 SDValue SplatZeroOp1 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
11573 DAG.getUNDEF(ContainerVT),
11574 DAG.getConstant(0, DL, XLenVT), EVL1);
11575 Op1 = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, Op1, SplatOneOp1,
11576 SplatZeroOp1, DAG.getUNDEF(ContainerVT), EVL1);
11578 SDValue SplatOneOp2 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
11579 DAG.getUNDEF(ContainerVT),
11580 DAG.getConstant(1, DL, XLenVT), EVL2);
11581 SDValue SplatZeroOp2 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
11582 DAG.getUNDEF(ContainerVT),
11583 DAG.getConstant(0, DL, XLenVT), EVL2);
11584 Op2 = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, Op2, SplatOneOp2,
11585 SplatZeroOp2, DAG.getUNDEF(ContainerVT), EVL2);
11593 DownOffset = DAG.getConstant(ImmValue, DL, XLenVT);
11594 UpOffset = DAG.getNode(ISD::SUB, DL, XLenVT, EVL1, DownOffset);
11598 UpOffset = DAG.getConstant(-ImmValue, DL, XLenVT);
11599 DownOffset = DAG.getNode(ISD::SUB, DL, XLenVT, EVL1, UpOffset);
11603 getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
11605 SDValue Result = getVSlideup(DAG, Subtarget, DL, ContainerVT, SlideDown, Op2,
11610 Result = DAG.getNode(
11612 {Result, DAG.getConstant(0, DL, ContainerVT),
11613 DAG.getCondCode(ISD::SETNE), DAG.getUNDEF(getMaskTypeFor(ContainerVT)),
11619 return convertFromScalableVector(VT, Result, DAG, Subtarget);
11623 SelectionDAG &DAG) const {
11634 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
11638 lowerScalarSplat(SDValue(), Val, VL, ContainerVT, DL, DAG, Subtarget);
11642 return convertFromScalableVector(VT, Result, DAG, Subtarget);
11647 SelectionDAG &DAG) const {
11659 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
11661 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
11672 SDValue SplatOne = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IndicesVT,
11673 DAG.getUNDEF(IndicesVT),
11674 DAG.getConstant(1, DL, XLenVT), EVL);
11675 SDValue SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IndicesVT,
11676 DAG.getUNDEF(IndicesVT),
11677 DAG.getConstant(0, DL, XLenVT), EVL);
11678 Op1 = DAG.getNode(RISCVISD::VMERGE_VL, DL, IndicesVT, Op1, SplatOne,
11679 SplatZero, DAG.getUNDEF(IndicesVT), EVL);
11701 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(GatherVT);
11702 auto [Lo, Hi] = DAG.SplitVector(Op1, DL);
11704 SDValue LoRev = DAG.getNode(ISD::VECTOR_REVERSE, DL, LoVT, Lo);
11705 SDValue HiRev = DAG.getNode(ISD::VECTOR_REVERSE, DL, HiVT, Hi);
11712 DAG.getNode(ISD::CONCAT_VECTORS, DL, GatherVT, HiRev, LoRev);
11718 DAG.getVScale(DL, XLenVT, APInt(XLenVT.getSizeInBits(), MinElts));
11719 SDValue Diff = DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, EVL);
11721 Result = getVSlidedown(DAG, Subtarget, DL, GatherVT,
11722 DAG.getUNDEF(GatherVT), Result, Diff, Mask, EVL);
11727 DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT,
11728 {Result, DAG.getConstant(0, DL, GatherVT),
11729 DAG.getCondCode(ISD::SETNE),
11730 DAG.getUNDEF(getMaskTypeFor(ContainerVT)), Mask, EVL});
11735 return convertFromScalableVector(VT, Result, DAG, Subtarget);
11743 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, IndicesVT, Mask, EVL);
11745 DAG.getNode(ISD::SUB, DL, XLenVT, EVL, DAG.getConstant(1, DL, XLenVT));
11746 SDValue VecLenSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IndicesVT,
11747 DAG.getUNDEF(IndicesVT), VecLen, EVL);
11748 SDValue VRSUB = DAG.getNode(RISCVISD::SUB_VL, DL, IndicesVT, VecLenSplat, VID,
11749 DAG.getUNDEF(IndicesVT), Mask, EVL);
11750 SDValue Result = DAG.getNode(GatherOpc, DL, GatherVT, Op1, VRSUB,
11751 DAG.getUNDEF(GatherVT), Mask, EVL);
11755 Result = DAG.getNode(
11757 {Result, DAG.getConstant(0, DL, GatherVT), DAG.getCondCode(ISD::SETNE),
11758 DAG.getUNDEF(getMaskTypeFor(ContainerVT)), Mask, EVL});
11763 return convertFromScalableVector(VT, Result, DAG, Subtarget);
11767 SelectionDAG &DAG) const {
11770 return lowerVPOp(Op, DAG);
11781 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
11782 Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget);
11786 SDValue Val = DAG.getNode(getRISCVVLOp(Op), DL, ContainerVT, Op1, Op2, VL);
11789 return convertFromScalableVector(VT, Val, DAG, Subtarget);
11793 SelectionDAG &DAG) const {
11801 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
11808 SDValue IntID = DAG.getTargetConstant(IsUnmasked ? Intrinsic::riscv_vlse
11812 DAG.getUNDEF(ContainerVT), VPNode->getBasePtr(),
11817 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
11823 SDValue Policy = DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT);
11828 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
11833 Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
11835 return DAG.getMergeValues({Result, Chain}, DL);
11839 SelectionDAG &DAG) const {
11849 StoreVal = convertToScalableVector(ContainerVT, StoreVal, DAG, Subtarget);
11856 SDValue IntID = DAG.getTargetConstant(IsUnmasked ? Intrinsic::riscv_vsse
11864 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
11870 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, VPNode->getVTList(),
11882 SelectionDAG &DAG) const {
11898 PassThru = DAG.getUNDEF(VT);
11931 Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget);
11935 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
11936 PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
11941 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
11945 Index = DAG.getNode(ISD::TRUNCATE, DL, IndexVT, Index);
11950 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
11952 Ops.push_back(DAG.getUNDEF(ContainerVT));
11961 Ops.push_back(DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT));
11963 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
11965 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO);
11969 Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
11971 return DAG.getMergeValues({Result, Chain}, DL);
11981 SelectionDAG &DAG) const {
12029 Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget);
12030 Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
12034 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
12039 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
12043 Index = DAG.getNode(ISD::TRUNCATE, DL, IndexVT, Index);
12048 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
12056 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
12057 DAG.getVTList(MVT::Other), Ops, MemVT, MMO);
12061 SelectionDAG &DAG) const {
12065 SDValue SysRegNo = DAG.getTargetConstant(
12067 SDVTList VTs = DAG.getVTList(XLenVT, MVT::Other);
12068 SDValue RM = DAG.getNode(RISCVISD::READ_CSR, DL, VTs, Chain, SysRegNo);
12082 DAG.getNode(ISD::SHL, DL, XLenVT, RM, DAG.getConstant(2, DL, XLenVT));
12083 SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT,
12084 DAG.getConstant(Table, DL, XLenVT), Shift);
12085 SDValue Masked = DAG.getNode(ISD::AND, DL, XLenVT, Shifted,
12086 DAG.getConstant(7, DL, XLenVT));
12088 return DAG.getMergeValues({Masked, Chain}, DL);
12092 SelectionDAG &DAG) const {
12097 SDValue SysRegNo = DAG.getTargetConstant(
12111 RMValue = DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, RMValue);
12113 SDValue Shift = DAG.getNode(ISD::SHL, DL, XLenVT, RMValue,
12114 DAG.getConstant(2, DL, XLenVT));
12115 SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT,
12116 DAG.getConstant(Table, DL, XLenVT), Shift);
12117 RMValue = DAG.getNode(ISD::AND, DL, XLenVT, Shifted,
12118 DAG.getConstant(0x7, DL, XLenVT));
12119 return DAG.getNode(RISCVISD::WRITE_CSR, DL, MVT::Other, Chain, SysRegNo,
12124 SelectionDAG &DAG) const {
12125 MachineFunction &MF = DAG.getMachineFunction();
12128 EVT PtrVT = getPointerTy(DAG.getDataLayout());
12131 return DAG.getFrameIndex(FI, PtrVT);
12164 static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG,
12168 SDValue NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0));
12169 SDValue NewOp1 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(1));
12170 SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
12172 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes);
12177 static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG) {
12179 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
12180 SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12181 SDValue NewWOp = DAG.getNode(N->getOpcode(), DL, MVT::i64, NewOp0, NewOp1);
12182 SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
12183 DAG.getValueType(MVT::i32));
12184 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes);
12189 SelectionDAG &DAG) const {
12204 if (getTypeAction(*DAG.getContext(), Op0.getValueType()) !=
12213 Op0 = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other},
12219 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other);
12220 SDValue Res = DAG.getNode(
12222 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, MVT::i64));
12223 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12232 Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op0);
12236 DAG.getNode(Opc, DL, MVT::i64, Op0,
12237 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, MVT::i64));
12238 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12256 makeLibCall(DAG, LC, N->getValueType(0), Op0, CallOptions, DL, Chain);
12265 if (getTypeAction(*DAG.getContext(), Op0.getValueType()) !=
12272 Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op0);
12275 DAG.getNode(RISCVISD::FCVT_W_RV64, DL, MVT::i64, Op0,
12276 DAG.getTargetConstant(RISCVFPRndMode::RMM, DL, MVT::i64));
12277 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12288 SDValue Result = makeLibCall(DAG, LC, MVT::i64, Op0, CallOptions, DL).first;
12289 Result = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Result);
12301 LoCounter = DAG.getTargetConstant(
12303 HiCounter = DAG.getTargetConstant(
12306 LoCounter = DAG.getTargetConstant(
12308 HiCounter = DAG.getTargetConstant(
12311 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
12312 SDValue RCW = DAG.getNode(RISCVISD::READ_COUNTER_WIDE, DL, VTs,
12316 DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, RCW, RCW.getValue(1)));
12329 SDValue Res = DAG.getExtLoad(ISD::SEXTLOAD, dl, MVT::i64, Ld->getChain(),
12332 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Res));
12346 bool LHSIsU = DAG.MaskedValueIsZero(LHS, HighMask);
12347 bool RHSIsU = DAG.MaskedValueIsZero(RHS, HighMask);
12354 S = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, S);
12355 U = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, U);
12356 SDValue Lo = DAG.getNode(ISD::MUL, DL, XLenVT, S, U);
12357 SDValue Hi = DAG.getNode(RISCVISD::MULHSU, DL, XLenVT, S, U);
12358 return DAG.getNode(ISD::BUILD_PAIR, DL, N->getValueType(0), Lo, Hi);
12361 bool LHSIsS = DAG.ComputeNumSignBits(LHS) > XLen;
12362 bool RHSIsS = DAG.ComputeNumSignBits(RHS) > XLen;
12379 Results.push_back(customLegalizeToWOpWithSExt(N, DAG));
12391 Results.push_back(customLegalizeToWOp(N, DAG));
12401 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
12403 DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(1));
12404 SDValue NewWOp = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0, NewOp1);
12405 SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
12406 DAG.getValueType(MVT::i32));
12407 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
12421 Results.push_back(customLegalizeToWOp(N, DAG));
12431 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
12435 SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp0);
12436 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12448 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
12461 Results.push_back(customLegalizeToWOp(N, DAG, ExtOpc));
12473 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
12474 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(1));
12475 SDValue Res = DAG.getNode(ISD::ADD, DL, MVT::i64, LHS, RHS);
12476 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Res,
12477 DAG.getValueType(MVT::i32));
12479 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
12488 SDValue ResultLowerThanLHS = DAG.getSetCC(DL, OType, Res, LHS, ISD::SETLT);
12489 SDValue ConditionRHS = DAG.getSetCC(DL, OType, RHS, Zero, ISD::SETLT);
12492 DAG.getNode(ISD::XOR, DL, OType, ConditionRHS, ResultLowerThanLHS);
12493 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12503 SDValue LHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
12504 SDValue RHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12506 DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, DL, MVT::i64, LHS, RHS);
12507 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Res,
12508 DAG.getValueType(MVT::i32));
12517 Overflow = DAG.getSetCC(DL, N->getValueType(1), Res,
12518 DAG.getConstant(0, DL, MVT::i64), ISD::SETEQ);
12521 Overflow = DAG.getSetCC(DL, N->getValueType(1), N->getOperand(0),
12522 DAG.getConstant(0, DL, MVT::i32), ISD::SETNE);
12527 LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
12528 Overflow = DAG.getSetCC(DL, N->getValueType(1), Res, LHS,
12532 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12545 DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
12547 DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(1));
12548 SDValue Res = DAG.getNode(N->getOpcode(), DL, MVT::i64, LHS, RHS);
12549 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12555 Results.push_back(expandAddSubSat(N, DAG));
12562 Results.push_back(expandAddSubSat(N, DAG));
12573 SDValue Src = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64,
12575 SDValue Abs = DAG.getNode(RISCVISD::ABSW, DL, MVT::i64, Src);
12576 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Abs));
12581 SDValue Src = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
12584 Src = DAG.getFreeze(Src);
12587 SDValue SignFill = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Src,
12588 DAG.getValueType(MVT::i32));
12589 SignFill = DAG.getNode(ISD::SRA, DL, MVT::i64, SignFill,
12590 DAG.getConstant(31, DL, MVT::i64));
12592 SDValue NewRes = DAG.getNode(ISD::XOR, DL, MVT::i64, Src, SignFill);
12593 NewRes = DAG.getNode(ISD::SUB, DL, MVT::i64, NewRes, SignFill);
12597 NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewRes,
12598 DAG.getValueType(MVT::i32));
12599 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
12610 SDValue FPConv = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Op0);
12611 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FPConv));
12614 SDValue FPConv = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Op0);
12615 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FPConv));
12619 DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Op0);
12620 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv));
12623 SDValue NewReg = DAG.getNode(RISCVISD::SplitF64, DL,
12624 DAG.getVTList(MVT::i32, MVT::i32), Op0);
12625 SDValue RetReg = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64,
12634 EVT BVT = EVT::getVectorVT(*DAG.getContext(), VT, 1);
12636 SDValue BVec = DAG.getBitcast(BVT, Op0);
12637 Results.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec,
12638 DAG.getVectorIdxConstant(0, DL)));
12652 SDValue NewOp = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, N->getOperand(0));
12653 SDValue NewRes = DAG.getNode(N->getOpcode(), DL, XLenVT, NewOp);
12656 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NewRes));
12685 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
12691 auto [Mask, VL] = getDefaultVLOps(1, ContainerVT, DL, DAG, Subtarget);
12696 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT,
12697 DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
12701 SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
12705 SDValue ThirtyTwoV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
12706 DAG.getUNDEF(ContainerVT),
12707 DAG.getConstant(32, DL, XLenVT), VL);
12709 DAG.getNode(RISCVISD::SRL_VL, DL, ContainerVT, Vec, ThirtyTwoV,
12710 DAG.getUNDEF(ContainerVT), Mask, VL);
12712 SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
12714 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi));
12724 SDValue Res = lowerGetVectorLength(N, DAG, Subtarget);
12725 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12729 SDValue Res = lowerCttzElts(N, DAG, Subtarget);
12731 DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), Res));
12757 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12758 SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp);
12759 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12767 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12769 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
12771 DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1, N->getOperand(3));
12772 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12779 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12780 SDValue Res = DAG.getNode(
12782 DAG.getTargetConstant(N->getConstantOperandVal(2), DL, MVT::i64));
12783 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12790 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12792 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
12793 SDValue Res = DAG.getNode(
12795 DAG.getTargetConstant(N->getConstantOperandVal(3), DL, MVT::i64));
12796 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12804 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12806 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
12807 SDValue Res = DAG.getNode(RISCVISD::CLMUL, DL, MVT::i64, NewOp0, NewOp1);
12808 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12826 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12828 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
12829 NewOp0 = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0,
12830 DAG.getConstant(32, DL, MVT::i64));
12831 NewOp1 = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp1,
12832 DAG.getConstant(32, DL, MVT::i64));
12835 SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1);
12836 Res = DAG.getNode(ISD::SRL, DL, MVT::i64, Res,
12837 DAG.getConstant(32, DL, MVT::i64));
12838 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12846 SDValue Extract = DAG.getNode(RISCVISD::VMV_X_S, DL,
12848 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Extract));
12860 SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
12864 auto [Mask, VL] = getDefaultVLOps(1, VecVT, DL, DAG, Subtarget);
12867 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VecVT, DAG.getUNDEF(VecVT),
12868 DAG.getConstant(32, DL, XLenVT), VL);
12869 SDValue LShr32 = DAG.getNode(RISCVISD::SRL_VL, DL, VecVT, Vec, ThirtyTwoV,
12870 DAG.getUNDEF(VecVT), Mask, VL);
12871 SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
12874 DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi));
12888 if (SDValue V = lowerVECREDUCE(SDValue(N, 0), DAG))
12899 if (SDValue V = lowerVPREDUCE(SDValue(N, 0), DAG))
12903 SDVTList VTs = DAG.getVTList(Subtarget.getXLenVT(), MVT::Other);
12904 SDValue Res = DAG.getNode(ISD::GET_ROUNDING, DL, VTs, N->getOperand(0));
12947 combineBinOpOfExtractToReduceTree(SDNode *N, SelectionDAG &DAG,
12953 if (DAG.NewNodesMustHaveLegalTypes)
13008 EVT ReduceVT = EVT::getVectorVT(*DAG.getContext(), VT, 2);
13009 SDValue Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ReduceVT, SrcVec,
13010 DAG.getVectorIdxConstant(0, DL));
13011 return DAG.getNode(ReduceOpc, DL, VT, Vec, N->getFlags());
13030 EVT ReduceVT = EVT::getVectorVT(*DAG.getContext(), VT, RHSIdx + 1);
13031 SDValue Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ReduceVT, SrcVec,
13032 DAG.getVectorIdxConstant(0, DL));
13035 return DAG.getNode(ReduceOpc, DL, VT, Vec, Flags);
13043 static SDValue combineBinOpToReduce(SDNode *N, SelectionDAG &DAG,
13130 ScalarV.getSimpleValueType(), DL, DAG, Subtarget);
13135 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ScalarVT, DAG.getUNDEF(ScalarVT),
13136 NewScalarV, DAG.getVectorIdxConstant(0, DL));
13142 DAG.getNode(Reduce.getOpcode(), DL, Reduce.getValueType(), Ops);
13143 return DAG.getNode(Extract.getOpcode(), DL, Extract.getValueType(), NewReduce,
13149 static SDValue transformAddShlImm(SDNode *N, SelectionDAG &DAG,
13187 SDValue SHADD = DAG.getNode(RISCVISD::SHL_ADD, DL, VT, NL,
13188 DAG.getConstant(Diff, DL, VT), NS);
13189 return DAG.getNode(ISD::SHL, DL, VT, SHADD, DAG.getConstant(Bits, DL, VT));
13205 SelectionDAG &DAG, bool AllOnes,
13254 FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT, OtherOp, NonConstantVal);
13260 return DAG.getNode(RISCVISD::SELECT_CC, SDLoc(N), VT,
13264 return DAG.getNode(ISD::SELECT, SDLoc(N), VT,
13269 static SDValue combineSelectAndUseCommutative(SDNode *N, SelectionDAG &DAG,
13274 if (SDValue Result = combineSelectAndUse(N, N0, N1, DAG, AllOnes, Subtarget))
13276 if (SDValue Result = combineSelectAndUse(N, N1, N0, DAG, AllOnes, Subtarget))
13299 static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG,
13341 SDValue New0 = DAG.getNode(ISD::ADD, DL, VT, N0->getOperand(0),
13342 DAG.getConstant(CA, DL, VT));
13344 DAG.getNode(ISD::MUL, DL, VT, New0, DAG.getConstant(C0, DL, VT));
13345 return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getConstant(CB, DL, VT));
13359 static SDValue combineBinOpOfZExt(SDNode *N, SelectionDAG &DAG) {
13362 if (!VT.isVector() || !DAG.getTargetLoweringInfo().isTypeLegal(VT))
13375 if (!DAG.getTargetLoweringInfo().isTypeLegal(SrcVT) ||
13380 LLVMContext &C = *DAG.getContext();
13384 Src0 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src0), NarrowVT, Src0);
13385 Src1 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src1), NarrowVT, Src1);
13395 return DAG.getNode(
13397 DAG.getNode(N->getOpcode(), SDLoc(N), NarrowVT, Src0, Src1));
13401 static SDValue combineAddOfBooleanXor(SDNode *N, SelectionDAG &DAG) {
13417 if (!DAG.MaskedValueIsZero(N0.getOperand(0), Mask))
13421 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
13428 SelectionDAG &DAG = DCI.DAG;
13429 if (SDValue V = combineAddOfBooleanXor(N, DAG))
13431 if (SDValue V = transformAddImmMulImm(N, DAG, Subtarget))
13434 if (SDValue V = transformAddShlImm(N, DAG, Subtarget))
13436 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
13438 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
13440 if (SDValue V = combineBinOpOfZExt(N, DAG))
13445 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget);
13449 static SDValue combineSubOfBoolean(SDNode *N, SelectionDAG &DAG) {
13476 DAG.getSetCC(SDLoc(N1), VT, N1.getOperand(0), N1.getOperand(1), CCVal);
13485 SDValue NewRHS = DAG.getConstant(ImmValMinus1, DL, VT);
13486 return DAG.getNode(ISD::ADD, DL, VT, NewLHS, NewRHS);
13491 static SDValue combineSubShiftToOrcB(SDNode *N, SelectionDAG &DAG,
13512 if (!DAG.MaskedValueIsZero(N1, Mask))
13515 return DAG.getNode(RISCVISD::ORC_B, SDLoc(N), VT, N1);
13518 static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG,
13520 if (SDValue V = combineSubOfBoolean(N, DAG))
13533 return DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0),
13534 DAG.getConstant(ShAmt, DL, VT));
13538 if (SDValue V = combineBinOpOfZExt(N, DAG))
13540 if (SDValue V = combineSubShiftToOrcB(N, DAG, Subtarget))
13545 return combineSelectAndUse(N, N1, N0, DAG, /*AllOnes*/ false, Subtarget);
13551 static SDValue combineDeMorganOfBoolean(SDNode *N, SelectionDAG &DAG) {
13585 if (!DAG.MaskedValueIsZero(N00, Mask) || !DAG.MaskedValueIsZero(N10, Mask))
13591 SDValue Logic = DAG.getNode(Opc, DL, VT, N00, N10);
13592 return DAG.getNode(ISD::XOR, DL, VT, Logic, DAG.getConstant(1, DL, VT));
13601 static SDValue combineTruncSelectToSMaxUSat(SDNode *N, SelectionDAG &DAG) {
13606 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13658 DAG.getNode(ISD::SMAX, DL, SrcVT, True, DAG.getConstant(0, DL, SrcVT));
13660 DAG.getNode(ISD::SMIN, DL, SrcVT, Max,
13661 DAG.getConstant((1ULL << ScalarBits) - 1, DL, SrcVT));
13662 return DAG.getNode(ISD::TRUNCATE, DL, VT, Min);
13665 static SDValue performTRUNCATECombine(SDNode *N, SelectionDAG &DAG,
13679 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N0.getOperand(0));
13680 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N0.getOperand(1));
13681 SDValue Srl = DAG.getNode(ISD::SRL, DL, MVT::i64, Op0, Op1);
13682 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Srl);
13685 return combineTruncSelectToSMaxUSat(N, DAG);
13694 SelectionDAG &DAG = DCI.DAG;
13707 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N0.getOperand(0));
13708 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N0.getOperand(1));
13709 SDValue Srl = DAG.getNode(ISD::SRL, DL, MVT::i64, Op0, Op1);
13710 SDValue And = DAG.getNode(ISD::AND, DL, MVT::i64, Srl,
13711 DAG.getConstant(1, DL, MVT::i64));
13712 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, And);
13715 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
13717 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
13721 if (SDValue V = combineDeMorganOfBoolean(N, DAG))
13726 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ true, Subtarget);
13732 SelectionDAG &DAG) {
13757 SDValue NewN0 = DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV.getOperand(0),
13759 SDValue NewN1 = DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV.getOperand(0),
13761 SDValue NewOr = DAG.getNode(ISD::OR, DL, VT, NewN0, NewN1);
13762 return DAG.getNode(ISD::XOR, DL, VT, NewOr, TrueV.getOperand(1));
13767 SelectionDAG &DAG = DCI.DAG;
13769 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
13771 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
13775 if (SDValue V = combineDeMorganOfBoolean(N, DAG))
13782 if (SDValue V = combineOrOfCZERO(N, N0, N1, DAG))
13784 if (SDValue V = combineOrOfCZERO(N, N1, N0, DAG))
13789 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget);
13792 static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG,
13805 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N0.getOperand(0));
13806 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N0.getOperand(1));
13807 SDValue Shl = DAG.getNode(ISD::SHL, DL, MVT::i64, Op0, Op1);
13808 SDValue And = DAG.getNOT(DL, Shl, MVT::i64);
13809 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, And);
13814 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13819 return DAG.getNode(RISCVISD::ROLW, DL, MVT::i64,
13820 DAG.getConstant(~1, DL, MVT::i64), N0.getOperand(1));
13832 return DAG.getSetCC(DL, VT, N0.getOperand(1),
13833 DAG.getConstant(Imm + 1, DL, VT), CC);
13849 SDValue Setcc = DAG.getSetCC(SDLoc(N00), N0.getOperand(0).getValueType(),
13851 return DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N->getValueType(0), Setcc);
13854 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
13856 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
13861 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget);
13865 static SDValue expandMul(SDNode *N, SelectionDAG &DAG,
13872 if (DAG.getMachineFunction().getFunction().hasMinSize())
13908 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, X,
13909 DAG.getConstant(Log2_64(MulAmt2), DL, VT));
13910 return DAG.getNode(RISCVISD::SHL_ADD, DL, VT, Shl,
13911 DAG.getConstant(Log2_64(Divisor - 1), DL, VT),
13917 DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
13918 DAG.getConstant(Log2_64(Divisor - 1), DL, VT), X);
13919 return DAG.getNode(ISD::SHL, DL, VT, Mul359,
13920 DAG.getConstant(Log2_64(MulAmt2), DL, VT));
13927 DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
13928 DAG.getConstant(Log2_64(Divisor - 1), DL, VT), X);
13929 return DAG.getNode(RISCVISD::SHL_ADD, DL, VT, Mul359,
13930 DAG.getConstant(Log2_64(MulAmt2 - 1), DL, VT),
13944 DAG.getNode(ISD::SHL, DL, VT, X, DAG.getConstant(ShiftAmt, DL, VT));
13945 return DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
13946 DAG.getConstant(ScaleShift, DL, VT), Shift1);
13963 DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
13964 DAG.getConstant(Log2_64(Divisor - 1), DL, VT), X);
13965 return DAG.getNode(RISCVISD::SHL_ADD, DL, VT, Mul359,
13966 DAG.getConstant(TZ, DL, VT), X);
13977 DAG.getNode(ISD::SHL, DL, VT, X, DAG.getConstant(ShiftAmt, DL, VT));
13978 return DAG.getNode(ISD::ADD, DL, VT, Shift1,
13979 DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
13980 DAG.getConstant(ScaleShift, DL, VT), X));
13989 DAG.getNode(ISD::SHL, DL, VT, X,
13990 DAG.getConstant(Log2_64(MulAmt + Offset), DL, VT));
13992 DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
13993 DAG.getConstant(Log2_64(Offset - 1), DL, VT), X);
13994 return DAG.getNode(ISD::SUB, DL, VT, Shift1, Mul359);
14004 SDValue Shift1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
14005 DAG.getConstant(Log2_64(ShiftAmt1), DL, VT));
14007 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
14008 DAG.getConstant(Log2_64(MulAmtLowBit), DL, VT));
14009 return DAG.getNode(ISD::SUB, DL, VT, Shift1, Shift2);
14018 static SDValue combineVectorMulToSraBitcast(SDNode *N, SelectionDAG &DAG) {
14020 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
14044 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(),
14045 EVT::getIntegerVT(*DAG.getContext(), HalfSize),
14048 SDValue Cast = DAG.getNode(ISD::BITCAST, DL, HalfVT, Srl.getOperand(0));
14049 SDValue Sra = DAG.getNode(ISD::SRA, DL, HalfVT, Cast,
14050 DAG.getConstant(HalfSize - 1, DL, HalfVT));
14051 return DAG.getNode(ISD::BITCAST, DL, VT, Sra);
14054 static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG,
14059 return expandMul(N, DAG, DCI, Subtarget);
14085 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N1, MulOper);
14086 return DAG.getNode(AddSubOpc, DL, VT, N1, MulVal);
14090 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N0, MulOper);
14091 return DAG.getNode(AddSubOpc, DL, VT, N0, MulVal);
14094 if (SDValue V = combineBinOpOfZExt(N, DAG))
14097 if (SDValue V = combineVectorMulToSraBitcast(N, DAG))
14105 static bool narrowIndex(SDValue &N, ISD::MemIndexType IndexType, SelectionDAG &DAG) {
14123 KnownBits Known = DAG.computeKnownBits(N);
14125 LLVMContext &C = *DAG.getContext();
14128 N = DAG.getNode(ISD::TRUNCATE, DL,
14161 EVT NewEltVT = EVT::getIntegerVT(*DAG.getContext(), NewElen);
14164 SDValue NewExt = DAG.getNode(N0->getOpcode(), DL, NewVT, N0->ops());
14165 SDValue NewShAmtVec = DAG.getConstant(ShAmtV, DL, NewVT);
14166 N = DAG.getNode(ISD::SHL, DL, NewVT, NewExt, NewShAmtVec);
14174 static SDValue performSETCCCombine(SDNode *N, SelectionDAG &DAG,
14203 if (DAG.MaskedValueIsZero(N0.getOperand(0), SignMask))
14212 return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT);
14214 SDValue SExtOp = DAG.getNode(ISD::SIGN_EXTEND_INREG, N, OpVT,
14215 N0.getOperand(0), DAG.getValueType(MVT::i32));
14216 return DAG.getSetCC(dl, VT, SExtOp, DAG.getConstant(C1.trunc(32).sext(64),
14221 performSIGN_EXTEND_INREGCombine(SDNode *N, SelectionDAG &DAG,
14229 return DAG.getNode(RISCVISD::FMV_X_SIGNEXTH, SDLoc(N), VT,
14320 SDValue getOrCreateExtendedOp(SDNode *Root, SelectionDAG &DAG,
14337 auto [Mask, VL] = getMaskAndVL(Root, DAG, Subtarget);
14344 return DAG.getNode(ExtOpc, DL, NarrowVT, Source, Mask, VL);
14346 return DAG.getSplat(NarrowVT, DL, Source.getOperand(0));
14348 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, NarrowVT,
14349 DAG.getUNDEF(NarrowVT), Source.getOperand(1), VL);
14355 return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, NarrowVT,
14356 DAG.getUNDEF(NarrowVT), Source, VL);
14489 void fillUpExtensionSupportForSplat(SDNode *Root, SelectionDAG &DAG,
14518 if (DAG.SignBitIsZero(Op))
14531 if (DAG.ComputeMaxSignificantBits(Op) <= NarrowSize)
14534 if (DAG.MaskedValueIsZero(Op,
14543 void fillUpExtensionSupport(SDNode *Root, SelectionDAG &DAG,
14581 fillUpExtensionSupportForSplat(Root, DAG, Subtarget);
14645 NodeExtensionHelper(SDNode *Root, unsigned OperandIdx, SelectionDAG &DAG,
14651 assert(DAG.getTargetLoweringInfo().isTypeLegal(Root->getValueType(0)));
14680 fillUpExtensionSupport(Root, DAG, Subtarget);
14687 getMaskAndVL(const SDNode *Root, SelectionDAG &DAG,
14698 return getDefaultScalableVLOps(VT, DL, DAG, Subtarget);
14768 SDValue materialize(SelectionDAG &DAG,
14772 NodeExtensionHelper::getMaskAndVL(Root, DAG, Subtarget);
14782 Merge = DAG.getUNDEF(Root->getValueType(0));
14785 return DAG.getNode(TargetOpcode, SDLoc(Root), Root->getValueType(0),
14786 LHS.getOrCreateExtendedOp(Root, DAG, Subtarget, LHSExt),
14787 RHS.getOrCreateExtendedOp(Root, DAG, Subtarget, RHSExt),
14805 uint8_t AllowExtMask, SelectionDAG &DAG,
14830 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
14833 Root, LHS, RHS, ExtKind::ZExt | ExtKind::SExt | ExtKind::FPExt, DAG,
14843 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
14871 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
14873 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::SExt, DAG,
14883 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
14885 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::ZExt, DAG,
14895 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
14897 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::FPExt, DAG,
14907 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
14986 SelectionDAG &DAG = DCI.DAG;
15004 NodeExtensionHelper LHS(Root, 0, DAG, Subtarget);
15005 NodeExtensionHelper RHS(Root, 1, DAG, Subtarget);
15033 FoldingStrategy(Root, LHS, RHS, DAG, Subtarget);
15062 SDValue NewValue = Res.materialize(DAG, Subtarget);
15072 DAG.ReplaceAllUsesOfValueWith(OldNewValues.first, OldNewValues.second);
15081 static SDValue combineVWADDSUBWSelect(SDNode *N, SelectionDAG &DAG) {
15118 return DAG.getNode(Opc, SDLoc(N), N->getValueType(0),
15133 return combineVWADDSUBWSelect(N, DCI.DAG);
15139 static SDValue tryMemPairCombine(SelectionDAG &DAG, LSBaseSDNode *LSNode1,
15149 MachineFunction &MF = DAG.getMachineFunction();
15168 SDValue Res = DAG.getMemIntrinsicNode(
15169 Opcode, SDLoc(LSNode1), DAG.getVTList({XLenVT, XLenVT, MVT::Other}),
15171 DAG.getConstant(Imm, SDLoc(LSNode1), XLenVT)},
15175 DAG.getMergeValues({Res.getValue(0), Res.getValue(2)}, SDLoc(LSNode1));
15177 DAG.getMergeValues({Res.getValue(1), Res.getValue(2)}, SDLoc(LSNode2));
15179 DAG.ReplaceAllUsesWith(LSNode2, Node2.getNode());
15184 SDValue Res = DAG.getMemIntrinsicNode(
15185 Opcode, SDLoc(LSNode1), DAG.getVTList(MVT::Other),
15187 BasePtr, DAG.getConstant(Imm, SDLoc(LSNode1), XLenVT)},
15190 DAG.ReplaceAllUsesWith(LSNode2, Res.getNode());
15199 SelectionDAG &DAG = DCI.DAG;
15200 MachineFunction &MF = DAG.getMachineFunction();
15269 tryMemPairCombine(DAG, LSNode1, LSNode2, Base1, Offset1))
15287 SelectionDAG &DAG = DCI.DAG;
15288 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
15330 SrcContainerVT = getContainerForFixedLengthVector(DAG, SrcVT, Subtarget);
15331 XVal = convertToScalableVector(SrcContainerVT, XVal, DAG, Subtarget);
15333 getContainerForFixedLengthVector(DAG, ContainerVT, Subtarget);
15337 getDefaultVLOps(SrcVT, SrcContainerVT, DL, DAG, Subtarget);
15345 FpToInt = DAG.getNode(Opc, DL, ContainerVT, XVal, Mask, VL);
15349 FpToInt = DAG.getNode(Opc, DL, ContainerVT, XVal, Mask, VL);
15353 FpToInt = DAG.getNode(Opc, DL, ContainerVT, XVal, Mask,
15354 DAG.getTargetConstant(FRM, DL, XLenVT), VL);
15359 FpToInt = convertFromScalableVector(VT, FpToInt, DAG, Subtarget);
15375 SDValue FpToInt = DAG.getNode(Opc, DL, XLenVT, Src.getOperand(0),
15376 DAG.getTargetConstant(FRM, DL, XLenVT));
15377 return DAG.getNode(ISD::TRUNCATE, DL, VT, FpToInt);
15390 SelectionDAG &DAG = DCI.DAG;
15391 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
15434 SDValue FpToInt = DAG.getNode(Opc, DL, XLenVT, Src,
15435 DAG.getTargetConstant(FRM, DL, XLenVT));
15440 FpToInt = DAG.getZeroExtendInReg(FpToInt, DL, MVT::i32);
15444 SDValue ZeroInt = DAG.getConstant(0, DL, DstVT);
15445 return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO);
15450 static SDValue performBITREVERSECombine(SDNode *N, SelectionDAG &DAG,
15464 return DAG.getNode(RISCVISD::BREV8, DL, VT, Src.getOperand(0));
15508 static SDValue combineVFMADD_VLWithVFNEG_VL(SDNode *N, SelectionDAG &DAG) {
15539 return DAG.getNode(NewOpcode, SDLoc(N), N->getVTList(),
15541 return DAG.getNode(NewOpcode, SDLoc(N), N->getValueType(0), A, B, C, Mask,
15545 static SDValue performVFMADD_VLCombine(SDNode *N, SelectionDAG &DAG,
15547 if (SDValue V = combineVFMADD_VLWithVFNEG_VL(N, DAG))
15600 return DAG.getNode(NewOpc, SDLoc(N), N->getValueType(0), Op0, Op1,
15604 static SDValue performSRACombine(SDNode *N, SelectionDAG &DAG,
15630 SDValue Shl = DAG.getNode(ISD::SHL, ShlDL, MVT::i64,
15632 DAG.getConstant(LShAmt + 32, ShlDL, MVT::i64));
15634 return DAG.getNode(ISD::SRA, DL, MVT::i64, Shl,
15635 DAG.getConstant(ShAmt + 32, DL, MVT::i64));
15695 DAG.getConstant(AddC->getAPIntValue().lshr(32), DL, MVT::i64);
15697 In = DAG.getNode(ISD::ADD, DL, MVT::i64, In, ShiftedAddC);
15699 In = DAG.getNode(ISD::SUB, DL, MVT::i64, ShiftedAddC, In);
15702 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, In,
15703 DAG.getValueType(MVT::i32));
15707 return DAG.getNode(
15709 DAG.getConstant(32 - ShAmt, DL, MVT::i64));
15716 static SDValue tryDemorganOfBooleanCondition(SDValue Cond, SelectionDAG &DAG) {
15745 if (!DAG.MaskedValueIsZero(Xor0, Mask))
15756 Setcc = DAG.getSetCC(SDLoc(Setcc), VT, Setcc.getOperand(0),
15760 Setcc = DAG.getSetCC(SDLoc(Setcc), VT, Setcc.getOperand(1),
15761 DAG.getConstant(1, SDLoc(Setcc), VT), CCVal);
15764 Setcc = DAG.getSetCC(SDLoc(Setcc), VT,
15765 DAG.getConstant(0, SDLoc(Setcc), VT),
15771 return DAG.getNode(Opc, SDLoc(Cond), VT, Setcc, Xor.getOperand(0));
15776 SelectionDAG &DAG, const RISCVSubtarget &Subtarget) {
15805 translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
15807 CC = DAG.getCondCode(CCVal);
15828 CC = DAG.getCondCode(CCVal);
15834 DAG.getNode(ISD::SHL, DL, LHS.getValueType(), LHS0.getOperand(0),
15835 DAG.getConstant(ShAmt, DL, LHS.getValueType()));
15844 if (isOneConstant(RHS) && DAG.MaskedValueIsZero(LHS, Mask)) {
15846 CC = DAG.getCondCode(CCVal);
15847 RHS = DAG.getConstant(0, DL, LHS.getValueType());
15852 if (SDValue NewCond = tryDemorganOfBooleanCondition(LHS, DAG)) {
15854 CC = DAG.getCondCode(CCVal);
15868 static SDValue tryFoldSelectIntoOp(SDNode *N, SelectionDAG &DAG,
15904 DAG.getNeutralElement(Opc, DL, OtherOpVT, N->getFlags());
15906 IdentityOperand = DAG.getConstant(0, DL, OtherOpVT);
15912 DAG.getSelect(DL, OtherOpVT, N->getOperand(0), OtherOp, IdentityOperand);
15913 return DAG.getNode(TrueVal.getOpcode(), DL, VT, FalseVal, NewSel);
15918 static SDValue foldSelectOfCTTZOrCTLZ(SDNode *N, SelectionDAG &DAG) {
15961 CountZeroes = DAG.getNode(ISD::CTTZ, SDLoc(CountZeroes),
15964 CountZeroes = DAG.getNode(ISD::CTLZ, SDLoc(CountZeroes),
15970 DAG.getConstant(BitWidth - 1, SDLoc(N), CountZeroes.getValueType());
15972 auto AndNode = DAG.getNode(ISD::AND, SDLoc(N), CountZeroes.getValueType(),
15974 return DAG.getZExtOrTrunc(AndNode, SDLoc(N), N->getValueType(0));
15977 static SDValue useInversedSetcc(SDNode *N, SelectionDAG &DAG,
16000 return DAG.getSelect(DL, VT,
16001 DAG.getSetCC(DL, CondVT, LHS, RHS, ISD::SETNE),
16008 static SDValue performSELECTCombine(SDNode *N, SelectionDAG &DAG,
16010 if (SDValue Folded = foldSelectOfCTTZOrCTLZ(N, DAG))
16013 if (SDValue V = useInversedSetcc(N, DAG, Subtarget))
16021 if (SDValue V = tryFoldSelectIntoOp(N, DAG, TrueVal, FalseVal, /*Swapped*/false))
16023 return tryFoldSelectIntoOp(N, DAG, FalseVal, TrueVal, /*Swapped*/true);
16031 static SDValue performBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG,
16059 if (!DAG.isSafeToSpeculativelyExecute(Opcode))
16085 return DAG.getNode(Opcode, DL, VT, DAG.getBuildVector(VT, DL, LHSOps),
16086 DAG.getBuildVector(VT, DL, RHSOps));
16089 static SDValue performINSERT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG,
16123 SDValue LHS = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT,
16125 SDValue RHS = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT,
16127 return DAG.getNode(InVecOpcode, DL, VT, LHS, RHS);
16144 SDValue NewIdx = DAG.getVectorIdxConstant(Elt % ConcatNumElts, DL);
16148 ConcatOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ConcatVT,
16154 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps);
16161 static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG,
16200 auto GetPtrDiff = [&DAG](LoadSDNode *Ld1,
16204 BaseIndexOffset BIO1 = BaseIndexOffset::match(Ld1, DAG);
16205 BaseIndexOffset BIO2 = BaseIndexOffset::match(Ld2, DAG);
16206 if (BIO1.equalBaseIndex(BIO2, DAG))
16232 // matchers above and below into a DAG combine?
16251 : DAG.getConstant(std::get<int64_t>(StrideVariant), DL,
16254 Stride = DAG.getNegative(Stride, DL, Stride.getValueType());
16257 DAG.getSplat(WideVecVT.changeVectorElementType(MVT::i1), DL,
16258 DAG.getConstant(1, DL, MVT::i1));
16271 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
16275 SDValue StridedLoad = DAG.getStridedLoadVP(
16278 DAG.getConstant(N->getNumOperands(), DL, Subtarget.getXLenVT()), MMO);
16281 DAG.makeEquivalentMemoryOrdering(cast<LoadSDNode>(Ld), StridedLoad);
16283 return DAG.getBitcast(VT.getSimpleVT(), StridedLoad);
16286 static SDValue combineToVWMACC(SDNode *N, SelectionDAG &DAG,
16325 auto [AddMask, AddVL] = [](SDNode *N, SelectionDAG &DAG,
16329 return getDefaultScalableVLOps(N->getSimpleValueType(0), DL, DAG,
16333 }(N, DAG, Subtarget);
16355 return DAG.getNode(Opc, DL, VT, Ops);
16364 SelectionDAG &DAG = DCI.DAG;
16366 DAG.getMachineFunction().getSubtarget<RISCVSubtarget>().getXLenVT();
16380 Index = DAG.getNode(ISD::SIGN_EXTEND, DL,
16473 static SDValue combineTruncOfSraSext(SDNode *N, SelectionDAG &DAG) {
16517 DAG.getNode(ISD::SMIN, SDLoc(N1), N->getValueType(0), N10,
16518 DAG.getConstant(MaxShAmt, SDLoc(N1), N->getValueType(0)));
16519 return DAG.getNode(ISD::SRA, SDLoc(N), N->getValueType(0), N00, SMin);
16527 static SDValue combineTruncToVnclip(SDNode *N, SelectionDAG &DAG,
16595 return DAG.getNode(RISCVISD::SMAX_VL, DL, V.getValueType(), SMinOp,
16596 V.getOperand(1), DAG.getUNDEF(V.getValueType()),
16647 Val = DAG.getNode(
16649 {Val, DAG.getConstant(0, DL, ValVT), DAG.getUNDEF(VT), Mask,
16650 DAG.getTargetConstant(RISCVVXRndMode::RNU, DL, Subtarget.getXLenVT()),
16659 SelectionDAG &DAG = DCI.DAG;
16688 SDValue Lo = DAG.getUNDEF(MVT::i32);
16689 SDValue Hi = DAG.getUNDEF(MVT::i32);
16698 SDValue Lo = DAG.getConstant(V.trunc(32), DL, MVT::i32);
16699 SDValue Hi = DAG.getConstant(V.lshr(32).trunc(32), DL, MVT::i32);
16711 DAG.getNode(RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32),
16717 SDValue NewHi = DAG.getNode(ISD::XOR, DL, MVT::i32, Hi,
16718 DAG.getConstant(SignBit, DL, MVT::i32));
16722 SDValue NewHi = DAG.getNode(ISD::AND, DL, MVT::i32, Hi,
16723 DAG.getConstant(~SignBit, DL, MVT::i32));
16778 SDValue NewFMV = DAG.getNode(N->getOpcode(), DL, VT, Op0.getOperand(0));
16782 return DAG.getNode(ISD::XOR, DL, VT, NewFMV,
16783 DAG.getConstant(SignBit, DL, VT));
16786 return DAG.getNode(ISD::AND, DL, VT, NewFMV,
16787 DAG.getConstant(~SignBit, DL, VT));
16797 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT,
16798 DAG.getNode(ISD::ABS, DL, Src.getValueType(), Src));
16805 if (SDValue V = combineToVWMACC(N, DAG, Subtarget))
16812 return performSUBCombine(N, DAG, Subtarget);
16822 return performXORCombine(N, DAG, Subtarget);
16826 return performMULCombine(N, DAG, DCI, Subtarget);
16831 if (SDValue V = combineBinOpOfZExt(N, DAG))
16841 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
16843 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
16848 return performSETCCCombine(N, DAG, Subtarget);
16850 return performSIGN_EXTEND_INREGCombine(N, DAG, Subtarget);
16859 return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), MVT::i64,
16863 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other);
16864 SDValue Res = DAG.getNode(ISD::STRICT_FP_TO_UINT, SDLoc(N), VTs,
16867 DAG.ReplaceAllUsesOfValueWith(Src.getValue(1), Res.getValue(1));
16874 if (SDValue V = combineTruncOfSraSext(N, DAG))
16876 return combineTruncToVnclip(N, DAG, Subtarget);
16878 return performTRUNCATECombine(N, DAG, Subtarget);
16880 return performSELECTCombine(N, DAG, Subtarget);
16900 if (DAG.MaskedValueIsZero(NewCond, Mask))
16901 return DAG.getNode(InvOpc, SDLoc(N), N->getValueType(0), Val, NewCond);
16910 return DAG.getNode(CCVal == ISD::SETNE ? Opc : InvOpc, SDLoc(N),
16945 DAG.getNode(ISD::SRA, DL, VT, LHS,
16946 DAG.getConstant(Subtarget.getXLen() - 1, DL, VT));
16948 DAG.getNode(ISD::AND, DL, VT, SRA,
16949 DAG.getConstant(TrueSImm - FalseSImm, DL, VT));
16950 return DAG.getNode(ISD::ADD, DL, VT, AND, FalseV);
16957 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget))
16958 return DAG.getNode(RISCVISD::SELECT_CC, DL, N->getValueType(0),
16964 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, CCVal);
16965 SDValue Neg = DAG.getNegative(C, DL, VT);
16966 return DAG.getNode(ISD::OR, DL, VT, Neg, FalseV);
16971 DAG.getSetCC(DL, VT, LHS, RHS, ISD::getSetCCInverse(CCVal, VT));
16972 SDValue Neg = DAG.getNegative(C, DL, VT);
16973 return DAG.getNode(ISD::OR, DL, VT, Neg, TrueV);
16979 DAG.getSetCC(DL, VT, LHS, RHS, ISD::getSetCCInverse(CCVal, VT));
16980 SDValue Neg = DAG.getNegative(C, DL, VT);
16981 return DAG.getNode(ISD::AND, DL, VT, Neg, FalseV);
16985 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, CCVal);
16986 SDValue Neg = DAG.getNegative(C, DL, VT);
16987 return DAG.getNode(ISD::AND, DL, VT, Neg, TrueV);
16997 LHS = DAG.getFreeze(LHS);
16998 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, ISD::CondCode::SETEQ);
16999 return DAG.getNode(ISD::ADD, DL, VT, LHS, C);
17011 SDValue NewSel = DAG.getNode(RISCVISD::SELECT_CC, DL, VT, LHS, RHS, CC,
17013 return DAG.getNode(ISD::XOR, DL, VT, NewSel, TrueV.getOperand(1));
17024 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget))
17025 return DAG.getNode(RISCVISD::BR_CC, DL, N->getValueType(0),
17031 return performBITREVERSECombine(N, DAG, Subtarget);
17058 SDValue NewFPExtRound = DAG.getFPExtendOrRound(In2.getOperand(0), DL, VT);
17059 return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N->getOperand(0),
17060 DAG.getNode(ISD::FNEG, DL, VT, NewFPExtRound));
17073 return DAG.getMaskedGather(
17079 if (narrowIndex(Index, IndexType, DAG))
17080 return DAG.getMaskedGather(
17100 const EVT PtrVT = getPointerTy(DAG.getDataLayout());
17103 SDValue BasePtr = DAG.getNode(ISD::ADD, DL, PtrVT, MGN->getBasePtr(),
17104 DAG.getConstant(Addend, DL, PtrVT));
17106 SDValue EVL = DAG.getElementCount(DL, Subtarget.getXLenVT(),
17109 DAG.getStridedLoadVP(VT, DL, MGN->getChain(), BasePtr,
17110 DAG.getConstant(StepNumerator, DL, XLenVT),
17112 SDValue VPSelect = DAG.getNode(ISD::VP_SELECT, DL, VT, MGN->getMask(),
17114 return DAG.getMergeValues({VPSelect, SDValue(StridedLoad.getNode(), 1)},
17122 SDValue Load = DAG.getMaskedLoad(VT, DL, MGN->getChain(),
17123 MGN->getBasePtr(), DAG.getUNDEF(XLenVT),
17124 MGN->getMask(), DAG.getUNDEF(VT),
17128 DAG.getVectorShuffle(VT, DL, Load, DAG.getUNDEF(VT), ShuffleMask);
17129 return DAG.getMergeValues({Shuffle, Load.getValue(1)}, DL);
17139 .getHalfNumVectorElementsVT(*DAG.getContext());
17140 Index = DAG.getBuildVector(IndexVT, DL, NewIndices);
17146 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), WideScalarVT,
17148 SDValue Passthru = DAG.getBitcast(WideVT, MGN->getPassThru());
17149 EVT MaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
17151 SDValue Mask = DAG.getSplat(MaskVT, DL, DAG.getConstant(1, DL, MVT::i1));
17154 DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other), WideVT, DL,
17158 SDValue Result = DAG.getBitcast(VT, Gather.getValue(0));
17159 return DAG.getMergeValues({Result, Gather.getValue(1)}, DL);
17173 return DAG.getMaskedScatter(
17179 if (narrowIndex(Index, IndexType, DAG))
17180 return DAG.getMaskedScatter(
17190 SDValue Shuffle = DAG.getVectorShuffle(VT, DL, MSN->getValue(),
17191 DAG.getUNDEF(VT), ShuffleMask);
17192 return DAG.getMaskedStore(MSN->getChain(), DL, Shuffle, MSN->getBasePtr(),
17193 DAG.getUNDEF(XLenVT), MSN->getMask(),
17209 return DAG.getGatherVP(N->getVTList(), VPGN->getMemoryVT(), DL,
17215 if (narrowIndex(Index, IndexType, DAG))
17216 return DAG.getGatherVP(N->getVTList(), VPGN->getMemoryVT(), DL,
17234 return DAG.getScatterVP(N->getVTList(), VPSN->getMemoryVT(), DL,
17240 if (narrowIndex(Index, IndexType, DAG))
17241 return DAG.getScatterVP(N->getVTList(), VPSN->getMemoryVT(), DL,
17260 ShAmt = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT),
17262 return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt,
17268 if (SDValue V = performSRACombine(N, DAG, Subtarget))
17282 ShAmt = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT),
17284 DAG.getRegister(RISCV::X0, Subtarget.getXLenVT()));
17285 return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt);
17292 return combineToVWMACC(N, DAG, Subtarget);
17309 return performVFMADD_VLCombine(N, DAG, Subtarget);
17366 allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
17368 SDValue NewV = DAG.getConstant(NewC, DL, NewVT);
17369 return DAG.getStore(Chain, DL, NewV, Store->getBasePtr(),
17385 if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
17387 allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
17389 SDValue NewL = DAG.getLoad(NewVT, DL, L->getChain(), L->getBasePtr(),
17392 return DAG.getStore(Chain, DL, NewL, Store->getBasePtr(),
17412 return DAG.getStoreVP(
17414 DAG.getConstant(1, DL, MaskVT),
17415 DAG.getConstant(1, DL, Subtarget.getXLenVT()), MemVT,
17429 DAG, Subtarget))
17434 if (SDValue V = performBUILD_VECTORCombine(N, DAG, Subtarget, *this))
17438 if (SDValue V = performCONCAT_VECTORSCombine(N, DAG, Subtarget, *this))
17442 if (SDValue V = performINSERT_VECTOR_ELTCombine(N, DAG, Subtarget, *this))
17453 return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, VT, Passthru, Scalar, VL);
17476 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, Passthru, Scalar, VL);
17513 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, M1VT, Passthru,
17514 DAG.getVectorIdxConstant(0, DL));
17516 DAG.getNode(N->getOpcode(), DL, M1VT, M1Passthru, Scalar, VL);
17517 Result = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Passthru, Result,
17518 DAG.getVectorIdxConstant(0, DL));
17528 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Scalar, VL);
17537 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, M1VT, Vec,
17538 DAG.getVectorIdxConstant(0, DL));
17539 return DAG.getNode(RISCVISD::VMV_X_S, DL, N->getSimpleValueType(0), Vec);
17565 return DAG.getMaskedLoad(VT, DL, Load->getChain(), Base,
17566 DAG.getUNDEF(XLenVT), Mask, PassThru,
17583 return DAG.getMaskedStore(Store->getChain(), DL, Value, Base,
17584 DAG.getUNDEF(XLenVT), Mask,
17604 return DAG.getConstant(-1, DL, VT);
17605 return DAG.getConstant(0, DL, VT);
17619 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getUNDEF(SrcVT));
17622 N0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i1, Ops);
17623 N0 = DAG.getBitcast(MVT::i8, N0);
17624 return DAG.getNode(ISD::TRUNCATE, DL, VT, N0);
17740 SDValue NewC = TLO.DAG.getConstant(NewMask, DL, Op.getValueType());
17741 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
17813 const SelectionDAG &DAG,
17828 Known = DAG.computeKnownBits(Op.getOperand(4), Depth + 1);
17832 KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(3), Depth + 1);
17840 Known = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
17847 Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
17848 Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
17857 Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
17858 Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
17867 Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
17868 Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
17875 KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
17882 KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
17892 Known = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
17952 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
17959 DAG.ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth + 1);
17962 DAG.ComputeNumSignBits(Op.getOperand(4), DemandedElts, Depth + 1);
17969 return DAG.ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
17974 DAG.ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
18038 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
18051 Op, DemandedElts, DAG, PoisonOnly, ConsiderFlags, Depth);
19316 static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val,
19324 Val = convertFromScalableVector(VA.getValVT(), Val, DAG, Subtarget);
19329 Val = DAG.getNode(RISCVISD::FMV_H_X, DL, VA.getValVT(), Val);
19332 Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Val);
19333 Val = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Val);
19335 Val = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Val);
19338 Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
19347 static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain,
19351 MachineFunction &MF = DAG.getMachineFunction();
19358 Val = DAG.getCopyFromReg(Chain, DL, VReg, LocVT);
19377 return convertLocVTToValVT(DAG, Val, VA, DL, TLI.getSubtarget());
19380 static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val,
19390 Val = convertToScalableVector(LocVT, Val, DAG, Subtarget);
19395 Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, LocVT, Val);
19398 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Val);
19399 Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Val);
19401 Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Val);
19404 Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val);
19413 static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
19415 MachineFunction &MF = DAG.getMachineFunction();
19419 EVT PtrVT = MVT::getIntegerVT(DAG.getDataLayout().getPointerSizeInBits(0));
19428 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
19441 Val = DAG.getExtLoad(
19443 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), ValVT);
19447 static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain,
19453 MachineFunction &MF = DAG.getMachineFunction();
19461 SDValue Lo = DAG.getCopyFromReg(Chain, DL, LoVReg, MVT::i32);
19467 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
19468 Hi = DAG.getLoad(MVT::i32, DL, Chain, FIN,
19474 Hi = DAG.getCopyFromReg(Chain, DL, HiVReg, MVT::i32);
19476 return DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
19663 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
19665 MachineFunction &MF = DAG.getMachineFunction();
19698 EVT PtrVT = getPointerTy(DAG.getDataLayout());
19706 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
19722 ArgValue = unpackF64OnRV32DSoftABI(DAG, Chain, VA, ArgLocs[++i], DL);
19724 ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL, Ins[InsIdx], *this);
19726 ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL);
19734 InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
19742 SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL);
19744 Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset);
19745 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue, Offset);
19746 InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
19791 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
19798 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, XLenVT);
19799 SDValue Store = DAG.getStore(
19804 DAG.getMemBasePlusOffset(FIN, TypeSize::getFixed(XLenInBytes), DL);
19818 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains);
19886 static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG) {
19887 return DAG.getDataLayout().getPrefTypeAlign(
19888 VT.getTypeForEVT(*DAG.getContext()));
19895 SelectionDAG &DAG = CLI.DAG;
19905 EVT PtrVT = getPointerTy(DAG.getDataLayout());
19908 MachineFunction &MF = DAG.getMachineFunction();
19912 CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
19949 SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
19950 SDValue SizeNode = DAG.getConstant(Size, DL, XLenVT);
19952 Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Alignment,
19960 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL);
19976 SDValue SplitF64 = DAG.getNode(
19977 RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32), ArgValue);
19990 StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
19992 DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
19993 DAG.getIntPtrConstant(HiVA.getLocMemOffset(), DL));
19996 DAG.getStore(Chain, DL, Hi, Address, MachinePointerInfo()));
20010 std::max(getPrefTypeAlign(Outs[OutIdx].ArgVT, DAG),
20011 getPrefTypeAlign(ArgValue.getValueType(), DAG));
20028 SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL);
20031 Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset);
20033 StackAlign = std::max(StackAlign, getPrefTypeAlign(PartVT, DAG));
20038 SDValue SpillSlot = DAG.CreateStackTemporary(StoredSize, StackAlign);
20041 DAG.getStore(Chain, DL, ArgValue, SpillSlot,
20047 DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot, PartOffset);
20049 DAG.getStore(Chain, DL, PartValue, Address,
20054 ArgValue = convertValVTToLocVT(DAG, ArgValue, VA, DL, Subtarget);
20071 StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
20073 DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
20074 DAG.getIntPtrConstant(VA.getLocMemOffset(), DL));
20078 DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo()));
20084 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
20090 Chain = DAG.getCopyToReg(Chain, DL, Reg.first, Reg.second, Glue);
20109 Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, RISCVII::MO_CALL);
20111 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, RISCVII::MO_CALL);
20122 Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
20129 Ops.push_back(DAG.getRegisterMask(Mask));
20140 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
20144 SDValue Ret = DAG.getNode(RISCVISD::TAIL, DL, NodeTys, Ops);
20147 DAG.addNoMergeSiteInfo(Ret.getNode(), CLI.NoMerge);
20151 Chain = DAG.getNode(RISCVISD::CALL, DL, NodeTys, Ops);
20154 DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
20158 Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, Glue, DL);
20163 CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
20171 DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), Glue);
20178 SDValue RetValue2 = DAG.getCopyFromReg(Chain, DL, RVLocs[++i].getLocReg(),
20182 RetValue = DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, RetValue,
20186 RetValue = convertLocVTToValVT(DAG, RetValue, VA, DL, Subtarget);
20219 const SDLoc &DL, SelectionDAG &DAG) const {
20220 MachineFunction &MF = DAG.getMachineFunction();
20227 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
20228 *DAG.getContext());
20230 analyzeOutputArgs(DAG.getMachineFunction(), CCInfo, Outs, /*IsRet=*/true,
20249 SDValue SplitF64 = DAG.getNode(RISCVISD::SplitF64, DL,
20250 DAG.getVTList(MVT::i32, MVT::i32), Val);
20262 Chain = DAG.getCopyToReg(Chain, DL, RegLo, Lo, Glue);
20264 RetOps.push_back(DAG.getRegister(RegLo, MVT::i32));
20265 Chain = DAG.getCopyToReg(Chain, DL, RegHi, Hi, Glue);
20267 RetOps.push_back(DAG.getRegister(RegHi, MVT::i32));
20270 Val = convertValVTToLocVT(DAG, Val, VA, DL, Subtarget);
20271 Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Glue);
20280 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
20297 const Function &Func = DAG.getMachineFunction().getFunction();
20303 MachineFunction &MF = DAG.getMachineFunction();
20313 return DAG.getNode(RetOpc, DL, MVT::Other, RetOps);
20885 SelectionDAG &DAG) const {
20895 DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
20902 DAG.getTargetConstant(0, SDLoc(Op), Subtarget.getXLenVT()));
20910 DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
20914 TargetLowering::LowerAsmOperandForConstraint(Op, "s", Ops, DAG);
20920 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
21191 SelectionDAG &DAG) const {
21227 SelectionDAG &DAG) const {
21239 if (!getIndexedAddressParts(Ptr.getNode(), Base, Offset, AM, DAG))
21250 SelectionDAG &DAG) const {
21282 if (!getIndexedAddressParts(Op, Base, Offset, AM, DAG))
21479 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
21487 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Val);
21488 Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Val);
21489 Val = DAG.getNode(ISD::OR, DL, MVT::i32, Val,
21490 DAG.getConstant(0xFFFF0000, DL, MVT::i32));
21491 Val = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Val);
21497 LLVMContext &Context = *DAG.getContext();
21516 Val = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SameEltTypeVT,
21517 DAG.getUNDEF(SameEltTypeVT), Val,
21518 DAG.getVectorIdxConstant(0, DL));
21520 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
21523 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
21524 Val, DAG.getVectorIdxConstant(0, DL));
21534 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
21542 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Val);
21543 Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Val);
21544 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
21549 LLVMContext &Context = *DAG.getContext();
21569 Val = DAG.getNode(ISD::BITCAST, DL, SameEltTypeVT, Val);
21571 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
21572 DAG.getVectorIdxConstant(0, DL));
22012 SelectionDAG &DAG,
22014 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
22028 return TargetLowering::buildSDIVPow2WithCMov(N, Divisor, DAG, Created);
22220 SelectionDAG &DAG) const {
22224 SDValue JTInfo = DAG.getJumpTableDebugInfo(JTI, Value, dl);
22225 return DAG.getNode(RISCVISD::SW_GUARDED_BRIND, dl, MVT::Other, JTInfo,
22228 return TargetLowering::expandIndirectJTBranch(dl, Value, Addr, JTI, DAG);