Lines Matching defs:ContainerVT
215 MVT ContainerVT = getContainerForFixedLengthVector(VT);
216 unsigned RCID = getRegClassIDForVecVT(ContainerVT);
2768 static SDValue getVLOp(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL,
2774 RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget);
2791 getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL,
2793 assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
2794 SDValue VL = getVLOp(NumElts, ContainerVT, DL, DAG, Subtarget);
2795 SDValue Mask = getAllOnesMask(ContainerVT, VL, DL, DAG);
2800 // VecVT is a vector type, either fixed-length or scalable, and ContainerVT is
2802 // VecVT is scalable, then ContainerVT should be the same as VecVT.
2804 getDefaultVLOps(MVT VecVT, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG,
2807 return getDefaultVLOps(VecVT.getVectorNumElements(), ContainerVT, DL, DAG,
2809 assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
2810 return getDefaultScalableVLOps(ContainerVT, DL, DAG, Subtarget);
3049 MVT ContainerVT = VT;
3051 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3052 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
3059 Mask = convertToScalableVector(getMaskTypeFor(ContainerVT), Mask, DAG,
3063 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3070 SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL);
3075 const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(ContainerVT);
3081 DAG.getConstantFP(MaxVal, DL, ContainerVT.getVectorElementType());
3082 SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT,
3083 DAG.getUNDEF(ContainerVT), MaxValNode, VL);
3086 MVT SetccVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
3093 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
3125 Truncated = DAG.getNode(RISCVISD::VFROUND_NOEXCEPT_VL, DL, ContainerVT, Src,
3132 Truncated = DAG.getNode(RISCVISD::SINT_TO_FP_VL, DL, ContainerVT, Truncated,
3136 Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated,
3156 MVT ContainerVT = VT;
3158 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3159 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
3162 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3175 DAG.getVTList(ContainerVT, MVT::Other),
3180 SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL);
3185 const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(ContainerVT);
3191 DAG.getConstantFP(MaxVal, DL, ContainerVT.getVectorElementType());
3192 SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT,
3193 DAG.getUNDEF(ContainerVT), MaxValNode, VL);
3201 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
3226 DAG.getVTList(ContainerVT, MVT::Other), Chain, Src,
3235 DAG.getVTList(ContainerVT, MVT::Other), Chain,
3241 Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated,
3285 MVT ContainerVT = VT;
3288 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3289 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
3292 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3294 DAG.getNode(RISCVISD::VFCVT_X_F_VL, DL, ContainerVT, Src, Mask, VL);
3487 MVT ContainerVT = VT;
3489 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3490 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
3493 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3495 SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, Vec,
3496 Idx, DAG.getUNDEF(ContainerVT), Mask, VL);
3518 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3521 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3581 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
3586 Vec = DAG.getNode(OpCode, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Vec,
3625 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3628 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3635 SDValue VMClr = DAG.getNode(RISCVISD::VMCLR_VL, DL, ContainerVT, VL);
3640 SDValue VMSet = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
3716 DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Splat, VL);
3905 Source = convertToScalableVector(ContainerVT.changeVectorElementType(MVT::i8),
3907 SDValue Res = DAG.getNode(RISCVISD::VSEXT_VL, DL, ContainerVT, Source, Mask, VL);
4024 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
4027 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4063 DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Splat, VL);
4076 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
4087 SDValue Vec = DAG.getUNDEF(ContainerVT);
4094 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT, Vec, SubBV,
4120 ContainerVT.bitsLE(getLMUL1VT(ContainerVT))) {
4159 switch (RISCVTargetLowering::getLMUL(ContainerVT)) {
4212 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4219 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
4227 Vec = DAG.getNode(OpCode, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Vec,
4232 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
4404 static bool isDeinterleaveShuffle(MVT VT, MVT ContainerVT, SDValue V1,
4564 MVT ContainerVT = VT;
4566 if (ContainerVT.isFixedLengthVector()) {
4568 ContainerVT = getContainerForFixedLengthVector(DAG, ContainerVT, Subtarget);
4572 MVT::getVectorVT(ContainerVT.getVectorElementType(),
4573 ContainerVT.getVectorElementCount() * 2);
4577 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4581 unsigned EltBits = ContainerVT.getScalarSizeInBits();
4583 MVT::getIntegerVT(EltBits * 2), ContainerVT.getVectorElementCount());
4587 MVT IntContainerVT = ContainerVT.changeVectorElementTypeToInteger();
4593 RISCVISD::VMV_V_X_VL, DL, IntContainerVT, DAG.getUNDEF(ContainerVT),
4599 Res = DAG.getBitcast(ContainerVT, Res);
4671 MVT ContainerVT = getContainerForFixedLengthVector(DAG, SrcVT, Subtarget);
4672 auto [TrueMask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
4674 getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
4675 convertToScalableVector(ContainerVT, Src, DAG, Subtarget),
4711 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
4712 auto TrueMask = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).first;
4721 InPlace = convertToScalableVector(ContainerVT, InPlace, DAG, Subtarget);
4722 ToInsert = convertToScalableVector(ContainerVT, ToInsert, DAG, Subtarget);
4729 Res = DAG.getNode(RISCVISD::VMV_V_V_VL, DL, ContainerVT, InPlace, ToInsert,
4732 Res = getVSlideup(DAG, Subtarget, DL, ContainerVT, InPlace, ToInsert,
4776 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
4777 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4783 auto Vec = DAG.getNode(OpCode, DL, ContainerVT,
4784 DAG.getUNDEF(ContainerVT),
4785 convertToScalableVector(ContainerVT, V2, DAG, Subtarget),
5049 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
5054 SDValue Vec = DAG.getUNDEF(ContainerVT);
5058 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
5059 V2 = convertToScalableVector(ContainerVT, V2, DAG, Subtarget);
5072 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT, Vec, SubVec,
5106 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
5108 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5137 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
5142 DAG.getUNDEF(ContainerVT),
5154 MVT SplatVT = ContainerVT;
5159 SplatVT = ContainerVT.changeVectorElementType(SVT);
5180 DAG.getNode(Opc, DL, SplatVT, DAG.getUNDEF(ContainerVT), V, VL);
5181 Splat = DAG.getBitcast(ContainerVT, Splat);
5185 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
5187 SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT,
5189 DAG.getUNDEF(ContainerVT), TrueMask, VL);
5223 LoV = convertToScalableVector(ContainerVT, LoV, DAG, Subtarget);
5227 HiV = convertToScalableVector(ContainerVT, HiV, DAG, Subtarget);
5234 SDValue Res = DAG.getUNDEF(ContainerVT);
5238 Res = getVSlidedown(DAG, Subtarget, DL, ContainerVT, Res, HiV,
5242 Res = getVSlideup(DAG, Subtarget, DL, ContainerVT, Res, LoV,
5251 if (isDeinterleaveShuffle(VT, ContainerVT, V1, V2, Mask, Subtarget)) {
5323 ContainerVT.changeVectorElementType(IndexVT.getScalarType());
5325 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
5336 SDValue Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V1, LHSIndices,
5337 DAG.getUNDEF(ContainerVT), TrueMask, VL);
5409 MVT ContainerVT = VT;
5415 Mask = convertToScalableVector(getMaskTypeFor(ContainerVT), Mask, DAG,
5455 ContainerVT = getContainerForFixedLengthVector(VT);
5456 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
5459 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5463 MVT::getVectorVT(FloatEltVT, ContainerVT.getVectorElementCount());
5529 MVT ContainerVT = getContainerForFixedLengthVector(SrcVT);
5530 Source = convertToScalableVector(ContainerVT, Source, DAG, Subtarget);
5531 Mask = convertToScalableVector(getMaskTypeFor(ContainerVT), Mask, DAG,
5533 SrcVT = ContainerVT;
5814 MVT ContainerVT = getContainerForFixedLengthVector(VT);
5833 DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT,
5835 DAG.getUNDEF(ContainerVT), Mask, VL});
5845 SDValue VMSNE = DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT,
5847 DAG.getUNDEF(ContainerVT), Mask, VL});
5897 MVT ContainerVT = VT;
5899 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
5900 X = convertToScalableVector(ContainerVT, X, DAG, Subtarget);
5901 Y = convertToScalableVector(ContainerVT, Y, DAG, Subtarget);
5908 Mask = convertToScalableVector(getMaskTypeFor(ContainerVT), Mask, DAG,
5912 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5919 DAG.getUNDEF(ContainerVT), Mask, VL});
5920 NewY = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, XIsNonNan, Y, X,
5921 DAG.getUNDEF(ContainerVT), VL);
5928 DAG.getUNDEF(ContainerVT), Mask, VL});
5929 NewX = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, YIsNonNan, X, Y,
5930 DAG.getUNDEF(ContainerVT), VL);
5937 SDValue Res = DAG.getNode(Opc, DL, ContainerVT, NewX, NewY,
5938 DAG.getUNDEF(ContainerVT), Mask, VL);
6424 MVT ContainerVT = VT;
6426 ContainerVT = getContainerForFixedLengthVector(VT);
6427 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
6429 SDValue V = DAG.getNode(RISCVISD::VMV_S_X_VL, DL, ContainerVT,
6430 DAG.getUNDEF(ContainerVT), Scalar, VL);
6686 MVT ContainerVT = getContainerForFixedLengthVector(VT);
6688 assert(ContainerVT.getVectorElementCount() == SrcContainerVT.getVectorElementCount() &&
6691 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
6695 Src = DAG.getNode(RVVOpc, DL, DAG.getVTList(ContainerVT, MVT::Other),
6700 Src = DAG.getNode(RVVOpc, DL, ContainerVT, Src, Mask, VL);
6820 MVT ContainerVT = getContainerForFixedLengthVector(Op.getSimpleValueType());
6822 DAG.getUNDEF(ContainerVT), DAG, Subtarget);
6870 MVT ContainerVT = VT;
6872 ContainerVT = ::getContainerForFixedLengthVector(DAG, VT, Subtarget);
6883 ContainerVT.bitsGE(getLMUL1VT(ContainerVT))) {
8092 MVT ContainerVT = VecVT;
8094 ContainerVT = getContainerForFixedLengthVector(VecVT);
8096 auto VL = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).second;
8099 splatPartsI64WithVL(DL, ContainerVT, SDValue(), Lo, Hi, VL, DAG);
8126 MVT ContainerVT = getContainerForFixedLengthVector(VecVT);
8128 MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
8132 SDValue VL = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).second;
8138 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
8139 DAG.getUNDEF(ContainerVT), SplatZero, VL);
8140 SplatTrueVal = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
8141 DAG.getUNDEF(ContainerVT), SplatTrueVal, VL);
8143 DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, CC, SplatTrueVal,
8144 SplatZero, DAG.getUNDEF(ContainerVT), VL);
8162 MVT ContainerVT = MVT::getVectorVT(VT.getVectorElementType(),
8166 convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget);
8169 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
8195 MVT ContainerVT = VecVT;
8198 ContainerVT = getContainerForFixedLengthVector(VecVT);
8199 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
8209 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
8215 SplatOne = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
8216 DAG.getUNDEF(ContainerVT), SplatOne, VL);
8217 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
8218 DAG.getUNDEF(ContainerVT), SplatZero, VL);
8220 MVT MaskContainerVT = ContainerVT.changeVectorElementType(MVT::i1);
8221 SDValue Trunc = DAG.getNode(RISCVISD::AND_VL, DL, ContainerVT, Src, SplatOne,
8222 DAG.getUNDEF(ContainerVT), Mask, VL);
8257 MVT ContainerVT = SrcVT;
8264 ContainerVT = getContainerForFixedLengthVector(SrcVT);
8265 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
8267 MVT MaskVT = getMaskTypeFor(ContainerVT);
8275 getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
8279 const ElementCount Count = ContainerVT.getVectorElementCount();
8301 MVT ContainerVT = VT;
8304 ContainerVT =
8309 auto [Mask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
8322 MVT InterVT = ContainerVT.changeVectorElementType(MVT::f32);
8331 SDValue Res = DAG.getNode(ConvOpc, DL, DAG.getVTList(ContainerVT, MVT::Other),
8371 MVT ContainerVT = VT;
8379 ContainerVT =
8383 MVT MaskVT = getMaskTypeFor(ContainerVT);
8390 getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
8395 Src = DAG.getNode(ConvOpc, DL, ContainerVT, Src, Mask, VL);
8404 MVT InterVT = ContainerVT.changeVectorElementType(MVT::f32);
8408 DAG.getNode(ConvOpc, DL, ContainerVT, IntermediateConv, Mask, VL);
8463 MVT ContainerVT = VecVT;
8466 ContainerVT = getContainerForFixedLengthVector(VecVT);
8467 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
8472 MVT OrigContainerVT = ContainerVT;
8478 if (auto ShrunkVT = getSmallestVTForIndex(ContainerVT, OrigIdx,
8480 ContainerVT = *ShrunkVT;
8487 const MVT M1VT = getLMUL1VT(ContainerVT);
8489 VLEN && ContainerVT.bitsGT(M1VT)) {
8498 ContainerVT = M1VT;
8502 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ContainerVT, Vec,
8521 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
8531 Vec = DAG.getNode(Opc, DL, ContainerVT, Vec, Val, VL);
8540 ValInVec = lowerScalarInsert(Val, VL, ContainerVT, DL, DAG, Subtarget);
8549 MVT::getVectorVT(MVT::i32, ContainerVT.getVectorElementCount() * 2);
8566 ValInVec = DAG.getBitcast(ContainerVT, ValInVec);
8587 ValInVec = DAG.getBitcast(ContainerVT, ValInVec);
8599 SDValue Slideup = getVSlideup(DAG, Subtarget, DL, ContainerVT, Vec, ValInVec,
8626 MVT ContainerVT = VecVT;
8628 ContainerVT = getContainerForFixedLengthVector(VecVT);
8629 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
8631 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
8686 MVT ContainerVT = VecVT;
8688 ContainerVT = getContainerForFixedLengthVector(VecVT);
8689 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
8699 MVT M1VT = getLMUL1VT(ContainerVT);
8710 ContainerVT = M1VT;
8722 getSmallestVTForIndex(ContainerVT, *MaxIdx, DL, DAG, Subtarget)) {
8723 ContainerVT = *SmallerVT;
8724 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ContainerVT, Vec,
8741 MVT LMUL2VT = getLMUL1VT(ContainerVT).getDoubleNumVectorElementsVT();
8742 if (ContainerVT.bitsGT(LMUL2VT) && VecVT.isFixedLengthVector())
8748 auto [Mask, VL] = getDefaultVLOps(1, ContainerVT, DL, DAG, Subtarget);
8749 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT,
8750 DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
8992 MVT ContainerVT = OpVT;
8994 ContainerVT = getContainerForFixedLengthVector(DAG, OpVT, Subtarget);
8995 Op0 = convertToScalableVector(ContainerVT, Op0, DAG, Subtarget);
8999 auto [Mask, VL] = getDefaultVLOps(OpVT, ContainerVT, DL, DAG, Subtarget);
9419 MVT ContainerVT = VT;
9421 ContainerVT = getContainerForFixedLengthVector(VT);
9425 MVT MaskVT = getMaskTypeFor(ContainerVT);
9428 PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
9433 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
9440 MVT ScalarVT = ContainerVT.getVectorElementType();
9441 if (IsUnmasked && isNullConstant(Stride) && ContainerVT.isInteger()) {
9446 Result = lowerScalarSplat(SDValue(), ScalarLoad, VL, ContainerVT, DL, DAG,
9452 Result = DAG.getSplat(ContainerVT, DL, ScalarLoad);
9460 Ops.push_back(DAG.getUNDEF(ContainerVT));
9474 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
9501 MVT ContainerVT = getContainerForFixedLengthVector(VT);
9503 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG,
9507 SmallVector<EVT, 9> ContainerVTs(NF, ContainerVT);
9511 Ops.insert(Ops.end(), NF, DAG.getUNDEF(ContainerVT));
9574 MVT ContainerVT = VT;
9576 ContainerVT = getContainerForFixedLengthVector(VT);
9577 Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
9580 MVT MaskVT = getMaskTypeFor(ContainerVT);
9585 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
9622 MVT ContainerVT = getContainerForFixedLengthVector(VT);
9624 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG,
9633 ContainerVT, FixedIntrinsic->getOperand(2 + i), DAG, Subtarget));
9727 MVT ContainerVT = VecVT;
9729 ContainerVT = getContainerForFixedLengthVector(VecVT);
9730 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
9739 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
9752 SDValue TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
9753 Vec = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Vec, TrueMask, VL);
9858 MVT ContainerVT = VecVT;
9860 ContainerVT = getContainerForFixedLengthVector(VecVT);
9861 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
9864 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
9929 MVT ContainerVT = VecVT;
9931 ContainerVT = getContainerForFixedLengthVector(VecVT);
9932 VectorVal = convertToScalableVector(ContainerVT, VectorVal, DAG, Subtarget);
9936 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
9979 auto ContainerVT = getContainerForFixedLengthVector(VecVT);
9980 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
10069 MVT ContainerVT = VecVT;
10071 ContainerVT = getContainerForFixedLengthVector(VecVT);
10072 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
10076 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT,
10077 DAG.getUNDEF(ContainerVT), SubVec,
10083 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT,
10084 DAG.getUNDEF(ContainerVT), SubVec,
10087 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
10091 SDValue VL = getVLOp(EndIndex, ContainerVT, DL, DAG, Subtarget);
10102 DAG.getNode(RISCVISD::VMV_V_V_VL, DL, ContainerVT, Vec, SubVec, VL);
10105 SubVec = getVSlideup(DAG, Subtarget, DL, ContainerVT, Vec, SubVec,
10309 MVT ContainerVT = VecVT;
10311 ContainerVT = getContainerForFixedLengthVector(VecVT);
10312 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
10318 getSmallestVTForIndex(ContainerVT, LastIdx, DL, DAG, Subtarget)) {
10319 ContainerVT = *ShrunkVT;
10320 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ContainerVT, Vec,
10325 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
10328 SDValue VL = getVLOp(SubVecVT.getVectorNumElements(), ContainerVT, DL, DAG,
10332 getVSlidedown(DAG, Subtarget, DL, ContainerVT,
10333 DAG.getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
10768 MVT ContainerVT = getContainerForFixedLengthVector(VT);
10773 RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget);
10775 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
10778 DAG.getLoad(ContainerVT, DL, Load->getChain(), Load->getBasePtr(),
10785 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG, Subtarget);
10792 Ops.push_back(DAG.getUNDEF(ContainerVT));
10795 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
10827 MVT ContainerVT = getContainerForFixedLengthVector(VT);
10830 convertToScalableVector(ContainerVT, StoreVal, DAG, Subtarget);
10836 RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget);
10838 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
10845 SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG,
10883 MVT ContainerVT = VT;
10885 ContainerVT = getContainerForFixedLengthVector(VT);
10886 PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
10888 MVT MaskVT = getMaskTypeFor(ContainerVT);
10894 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
10900 Ops.push_back(DAG.getUNDEF(ContainerVT));
10910 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
10951 MVT ContainerVT = VT;
10953 ContainerVT = getContainerForFixedLengthVector(VT);
10955 Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
10957 MVT MaskVT = getMaskTypeFor(ContainerVT);
10963 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
10966 Val = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
10968 DAG.getUNDEF(ContainerVT), Val, Mask, VL);
10991 MVT ContainerVT = getContainerForFixedLengthVector(InVT);
10996 convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget);
10998 convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget);
11001 auto [Mask, VL] = getDefaultVLOps(VT.getVectorNumElements(), ContainerVT, DL,
11003 MVT MaskVT = getMaskTypeFor(ContainerVT);
11109 MVT ContainerVT = VT;
11111 ContainerVT = getContainerForFixedLengthVector(VT);
11112 X = convertToScalableVector(ContainerVT, X, DAG, Subtarget);
11119 Mask = convertToScalableVector(getMaskTypeFor(ContainerVT), Mask, DAG,
11123 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
11126 RISCVISD::VMV_V_X_VL, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
11128 SDValue NegX = DAG.getNode(RISCVISD::SUB_VL, DL, ContainerVT, SplatZero, X,
11129 DAG.getUNDEF(ContainerVT), Mask, VL);
11130 SDValue Max = DAG.getNode(RISCVISD::SMAX_VL, DL, ContainerVT, X, NegX,
11131 DAG.getUNDEF(ContainerVT), Mask, VL);
11147 MVT ContainerVT = getContainerForFixedLengthVector(VT);
11148 Mag = convertToScalableVector(ContainerVT, Mag, DAG, Subtarget);
11149 Sign = convertToScalableVector(ContainerVT, Sign, DAG, Subtarget);
11151 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
11153 SDValue CopySign = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Mag,
11154 Sign, DAG.getUNDEF(ContainerVT), Mask, VL);
11162 MVT ContainerVT = getContainerForFixedLengthVector(VT);
11165 MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
11170 convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget);
11172 convertToScalableVector(ContainerVT, Op.getOperand(2), DAG, Subtarget);
11175 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
11177 SDValue Select = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, CC, Op1,
11178 Op2, DAG.getUNDEF(ContainerVT), VL);
11190 MVT ContainerVT = getContainerForFixedLengthVector(VT);
11206 Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget));
11210 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
11212 Ops.push_back(DAG.getUNDEF(ContainerVT));
11221 DAG.getNode(NewOpc, DL, DAG.getVTList(ContainerVT, MVT::Other), Ops,
11228 DAG.getNode(NewOpc, DL, ContainerVT, Ops, Op->getFlags());
11245 MVT ContainerVT = VT;
11247 ContainerVT = getContainerForFixedLengthVector(VT);
11258 Ops.push_back(DAG.getUNDEF(ContainerVT));
11267 Ops.push_back(DAG.getUNDEF(ContainerVT));
11278 MVT ContainerVT = getContainerForFixedLengthVector(OpVT);
11281 Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget));
11287 SDValue VPOp = DAG.getNode(RISCVISDOpc, DL, ContainerVT, Ops, Op->getFlags());
11301 MVT ContainerVT = VT;
11303 ContainerVT = getContainerForFixedLengthVector(VT);
11304 MVT SrcVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
11310 SDValue ZeroSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
11311 DAG.getUNDEF(ContainerVT), Zero, VL);
11315 SDValue Splat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
11316 DAG.getUNDEF(ContainerVT), SplatValue, VL);
11318 SDValue Result = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, Src, Splat,
11319 ZeroSplat, DAG.getUNDEF(ContainerVT), VL);
11336 MVT ContainerVT = VT;
11338 ContainerVT = getContainerForFixedLengthVector(VT);
11339 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
11340 Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget);
11344 SDValue AllOneMask = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
11351 Result = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op1, Op2, VL);
11356 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op1, Op2, VL);
11358 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Temp, AllOneMask, VL);
11366 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op1, AllOneMask, VL);
11367 Result = DAG.getNode(RISCVISD::VMAND_VL, DL, ContainerVT, Temp, Op2, VL);
11375 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op2, AllOneMask, VL);
11376 Result = DAG.getNode(RISCVISD::VMAND_VL, DL, ContainerVT, Op1, Temp, VL);
11384 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op1, AllOneMask, VL);
11385 Result = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Temp, Op2, VL);
11393 DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Op2, AllOneMask, VL);
11394 Result = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Temp, Op1, VL);
11552 MVT ContainerVT = VT;
11554 ContainerVT = getContainerForFixedLengthVector(VT);
11555 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
11556 Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget);
11557 MVT MaskVT = getMaskTypeFor(ContainerVT);
11566 ContainerVT = ContainerVT.changeVectorElementType(MVT::i8);
11569 SDValue SplatOneOp1 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
11570 DAG.getUNDEF(ContainerVT),
11572 SDValue SplatZeroOp1 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
11573 DAG.getUNDEF(ContainerVT),
11575 Op1 = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, Op1, SplatOneOp1,
11576 SplatZeroOp1, DAG.getUNDEF(ContainerVT), EVL1);
11578 SDValue SplatOneOp2 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
11579 DAG.getUNDEF(ContainerVT),
11581 SDValue SplatZeroOp2 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
11582 DAG.getUNDEF(ContainerVT),
11584 Op2 = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, Op2, SplatOneOp2,
11585 SplatZeroOp2, DAG.getUNDEF(ContainerVT), EVL2);
11603 getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
11605 SDValue Result = getVSlideup(DAG, Subtarget, DL, ContainerVT, SlideDown, Op2,
11611 RISCVISD::SETCC_VL, DL, ContainerVT.changeVectorElementType(MVT::i1),
11612 {Result, DAG.getConstant(0, DL, ContainerVT),
11613 DAG.getCondCode(ISD::SETNE), DAG.getUNDEF(getMaskTypeFor(ContainerVT)),
11630 MVT ContainerVT = VT;
11632 ContainerVT = getContainerForFixedLengthVector(VT);
11633 MVT MaskVT = getMaskTypeFor(ContainerVT);
11638 lowerScalarSplat(SDValue(), Val, VL, ContainerVT, DL, DAG, Subtarget);
11656 MVT ContainerVT = VT;
11658 ContainerVT = getContainerForFixedLengthVector(VT);
11659 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
11660 MVT MaskVT = getMaskTypeFor(ContainerVT);
11664 MVT GatherVT = ContainerVT;
11665 MVT IndicesVT = ContainerVT.changeVectorElementTypeToInteger();
11667 bool IsMaskVector = ContainerVT.getVectorElementType() == MVT::i1;
11669 GatherVT = IndicesVT = ContainerVT.changeVectorElementType(MVT::i8);
11727 DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT,
11730 DAG.getUNDEF(getMaskTypeFor(ContainerVT)), Mask, EVL});
11756 RISCVISD::SETCC_VL, DL, ContainerVT,
11758 DAG.getUNDEF(getMaskTypeFor(ContainerVT)), Mask, EVL});
11777 MVT ContainerVT = VT;
11780 ContainerVT = getContainerForFixedLengthVector(VT);
11781 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
11782 Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget);
11786 SDValue Val = DAG.getNode(getRISCVVLOp(Op), DL, ContainerVT, Op1, Op2, VL);
11797 MVT ContainerVT = VT;
11799 ContainerVT = getContainerForFixedLengthVector(VT);
11801 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
11812 DAG.getUNDEF(ContainerVT), VPNode->getBasePtr(),
11816 MVT MaskVT = ContainerVT.changeVectorElementType(MVT::i1);
11846 MVT ContainerVT = VT;
11848 ContainerVT = getContainerForFixedLengthVector(VT);
11849 StoreVal = convertToScalableVector(ContainerVT, StoreVal, DAG, Subtarget);
11863 MVT MaskVT = ContainerVT.changeVectorElementType(MVT::i1);
11925 MVT ContainerVT = VT;
11927 ContainerVT = getContainerForFixedLengthVector(VT);
11929 ContainerVT.getVectorElementCount());
11934 MVT MaskVT = getMaskTypeFor(ContainerVT);
11936 PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
11941 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
11952 Ops.push_back(DAG.getUNDEF(ContainerVT));
11963 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
12023 MVT ContainerVT = VT;
12025 ContainerVT = getContainerForFixedLengthVector(VT);
12027 ContainerVT.getVectorElementCount());
12030 Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
12033 MVT MaskVT = getMaskTypeFor(ContainerVT);
12039 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
12682 MVT ContainerVT = VecVT;
12684 ContainerVT = getContainerForFixedLengthVector(VecVT);
12685 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
12691 auto [Mask, VL] = getDefaultVLOps(1, ContainerVT, DL, DAG, Subtarget);
12696 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT,
12697 DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
12705 SDValue ThirtyTwoV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
12706 DAG.getUNDEF(ContainerVT),
12709 DAG.getNode(RISCVISD::SRL_VL, DL, ContainerVT, Vec, ThirtyTwoV,
12710 DAG.getUNDEF(ContainerVT), Mask, VL);
15317 MVT ContainerVT = VT.getSimpleVT();
15332 ContainerVT =
15333 getContainerForFixedLengthVector(DAG, ContainerVT, Subtarget);
15345 FpToInt = DAG.getNode(Opc, DL, ContainerVT, XVal, Mask, VL);
15349 FpToInt = DAG.getNode(Opc, DL, ContainerVT, XVal, Mask, VL);
15353 FpToInt = DAG.getNode(Opc, DL, ContainerVT, XVal, Mask,
21632 MVT ContainerVT = VT.getSimpleVT();
21642 ContainerVT = getContainerForFixedLengthVector(VT.getSimpleVT());
21651 auto [LMUL, Fractional] = RISCVVType::decodeVLMUL(getLMUL(ContainerVT));