Lines Matching +full:nand +full:- +full:style
1 //===-- RISCVISelLowering.cpp - RISC-V DAG Lowering Implementation -------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file defines the interfaces that RISC-V uses to lower LLVM code into a
12 //===----------------------------------------------------------------------===//
51 #define DEBUG_TYPE "riscv-lower"
56 DEBUG_TYPE "-ext-max-web-size", cl::Hidden,
62 AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden,
68 DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden,
74 FPImmCost(DEBUG_TYPE "-fpimm-cost", cl::Hidden,
76 "use for creating a floating-point immediate value"),
80 RV64LegalI32("riscv-experimental-rv64-legal-i32", cl::ReallyHidden,
92 errs() << "Hard-float 'f' ABI can't be used for a target that "
94 "target-abi)\n";
98 errs() << "Hard-float 'd' ABI can't be used for a target that "
100 "target-abi)\n";
629 // TODO: On M-mode only targets, the cycle[h]/time[h] CSR may not be present.
731 // We must custom-lower certain vXi64 operations on RV32 due to the vector
751 // Mask VTs are custom-expanded into a series of standard nodes
778 // RVV has native int->float & float->int conversions where the
779 // element type sizes are within one power-of-two of each other. Any
834 // Custom-lower extensions and truncations from/to mask types.
838 // RVV has native int->float & float->int conversions where the
839 // element type sizes are within one power-of-two of each other. Any
858 // Custom-lower insert/extract operations to simplify patterns.
862 // Custom-lower reduction operations to set up the corresponding custom
935 // purposes; they are expanded to their swapped-operand CCs (LT,OLT,LE,OLE),
936 // and we pattern-match those back to the "original", swapping operands once
967 // Sets common operation actions on RVV floating-point vector types.
971 // sizes are within one power-of-two of each other. Therefore conversions
976 // Custom-lower insert/extract operations to simplify patterns.
1043 // Sets common extload/truncstore actions on RVV floating-point vector
1258 // Custom-lower reduction operations to set up the corresponding custom
1398 // Custom-legalize bitcasts from fixed-length vectors to scalar types.
1580 I.getArgOperand(PtrOp)->getType()->getPointerAddressSpace();
1584 MemTy = I.getArgOperand(0)->getType();
1588 if (MemTy->isStructTy())
1589 MemTy = MemTy->getStructElementType(0);
1592 MemTy = MemTy->getScalarType();
1595 Info.align = Align(DL.getTypeSizeInBits(MemTy->getScalarType()) / 8);
1649 return SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 2,
1698 return SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 2,
1715 return SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
1739 return SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
1763 return SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 5,
1773 return SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 2,
1783 return SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
1807 return SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
1831 return SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
1849 // Require a 12-bit signed offset.
1875 // On RV32, 64-bit integers are split into their high and low parts and held
1878 // FIXME: Should we consider i64->i32 free on RV64 to match the EVT version of
1881 if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
1883 unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
1884 unsigned DestBits = DstTy->getPrimitiveSizeInBits();
1889 // We consider i64->i32 free on RV64 since we have good selection of W
1916 // Don't advertise i32->i64 zextload as being free for RV64. It interacts
1919 EVT MemVT = LD->getMemoryVT();
1921 (LD->getExtensionType() == ISD::NON_EXTLOAD ||
1922 LD->getExtensionType() == ISD::ZEXTLOAD))
1934 return Subtarget.is64Bit() && CI->getType()->isIntegerTy(32);
1959 return !Mask->getValue().isSignedIntN(12) && Mask->getValue().isPowerOf2();
1970 (!isa<ConstantSDNode>(Y) || cast<ConstantSDNode>(Y)->isOpaque());
1982 return C && C->getAPIntValue().ule(10);
1999 assert(Ty->isIntegerTy());
2001 unsigned BitSize = Ty->getIntegerBitWidth();
2005 // Fast path, assume 32-bit immediates are cheap.
2037 if (XC && OldShiftOpcode == ISD::SRL && XC->isOne())
2041 if (NewShiftOpcode == ISD::SRL && CC->isOne())
2082 if (!I->getType()->isVectorTy() || !Subtarget.hasVInstructions())
2085 if (canSplatOperand(I->getOpcode(), Operand))
2092 switch (II->getIntrinsicID()) {
2147 if (!I->getType()->isVectorTy() || !Subtarget.hasVInstructions())
2158 for (auto OpIdx : enumerate(I->operands())) {
2164 if (!Op || any_of(Ops, [&](Use *U) { return U->get() == Op; }))
2173 if (cast<VectorType>(Op->getType())->getElementType()->isIntegerTy(1))
2178 for (Use &U : Op->uses()) {
2184 Ops.push_back(&Op->getOperandUse(0));
2194 // TODO - do we have any exceptions?
2222 // (1) `{0-31 value, false}` if FLI is available for Imm's type and FP value.
2223 // (2) `{0-31 value, true}` if Imm is negative and FLI is available for its
2227 // (3) `{-1, _}` if there is no way FLI can be used to materialize Imm.
2231 return std::make_pair(-1, false);
2244 return std::make_pair(-1, false);
2249 return std::make_pair(RISCVLoadFPImm::getLoadFPImm(-Imm), true);
2272 // Cannot create a 64 bit floating-point immediate value for rv32.
2274 // td can handle +0.0 or -0.0 already.
2275 // -0.0 can be created by fmv + fneg.
2385 // in the RISC-V ISA. May adjust compares to favor compare with 0 over compare
2386 // with 1/-1.
2399 ShAmt = LHS.getValueSizeInBits() - 1 - Log2_64(Mask);
2401 ShAmt = LHS.getValueSizeInBits() - llvm::bit_width(Mask);
2413 int64_t C = RHSC->getSExtValue();
2417 // Convert X > -1 to X >= 0.
2418 if (C == -1) {
2540 // nxv16i32@12 -> nxv2i32: sub_vrm4_1_then_sub_vrm2_1_then_sub_vrm1_0
2550 SubRegIdx = TRI->composeSubRegIndices(SubRegIdx,
2553 InsertExtractIdx -= VecVT.getVectorElementCount().getKnownMinValue();
2603 return Op.getOperand(II->VLOperand + 1 + HasChain);
2614 // Therefore -- since the largest is v1024i8/v512i16/etc -- the largest
2615 // fixed-length vector type we support is 1024 bytes.
2799 // Gets the two common "VL" operands: an all-ones mask and the vector length.
2800 // VecVT is a vector type, either fixed-length or scalable, and ContainerVT is
2801 // the vector type that the fixed-length vector is contained in. Otherwise if
2854 // implementation-defined.
2884 /// or may track the vrgather.vv cost. It is implementation-dependent.
2892 /// or may track the vrgather.vv cost. It is implementation-dependent.
2900 /// or may track the vrgather.vv cost. It is implementation-dependent.
2907 // RISC-V FP-to-int conversions saturate to the destination register size, but
2913 EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3056 if (Op->isVPOpcode()) {
3078 MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
3135 // Restore the original sign so that -0.0 is preserved.
3188 MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
3240 // Restore the original sign so that -0.0 is preserved.
3268 MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
3345 // We use an arbitrary rounding mode here. If a floating-point is an exact
3360 // Try to match an arithmetic-sequence BUILD_VECTOR [X,X+S,X+2*S,...,X+(N-1)*S]
3361 // to the (non-zero) step S and start value X. This can be then lowered as the
3365 // sequences in which either the numerator is +/- 1 or the denominator is 1. It
3373 if (!cast<BuildVectorSDNode>(Op)->isConstant())
3386 for (auto [Idx, Elt] : enumerate(Op->op_values())) {
3392 Elts[Idx] = Elt->getAsZExtVal() & maskTrailingOnes<uint64_t>(OpSize);
3395 getExactInteger(cast<ConstantFPSDNode>(Elt)->getValueAPF(), OpSize);
3409 // Calculate the step since the last non-undef element, and ensure
3411 unsigned IdxDiff = Idx - PrevElt->second;
3412 int64_t ValDiff = SignExtend64(*Elt - PrevElt->first, EltSizeInBits);
3414 // A zero-value value difference means that we're somewhere in the middle
3441 // Record this non-undef element for later.
3442 if (!PrevElt || PrevElt->first != *Elt)
3457 int64_t Addend = SignExtend64(*Elt - ExpectedVal, EltSizeInBits);
3505 /// Try and optimize BUILD_VECTORs with "dominant values" - these are values
3510 /// upper-most element is the "dominant" one, allowing us to use a splat to
3530 count_if(Op->op_values(), [](const SDValue &V) { return V.isUndef(); });
3533 // any non-zero floating-point constant. Other kinds of element are either
3536 // vector-insertion instructions is not known.
3539 for (SDValue V : Op->op_values()) {
3547 NumScalarLoads += !CFP->isExactlyValue(+0.0);
3558 assert(DominantValue && "Not expecting an all-undef BUILD_VECTOR");
3559 unsigned NumDefElts = NumElts - NumUndefElts;
3560 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
3578 if (SDValue LastOp = Op->getOperand(Op->getNumOperands() - 1);
3593 for (const auto &OpIdx : enumerate(Op->ops())) {
3605 transform(Op->op_values(), std::back_inserter(Ops), [&](SDValue V1) {
3645 // scalar integer chunks whose bit-width depends on the number of mask
3649 // according to the size of the final vector - use i8 chunks rather than
3672 bool BitValue = !V.isUndef() && V->getAsZExtVal();
3694 // type, bitcast to the equivalent (known-legal) mask type, and extract
3710 if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
3721 // with optional modifications. An all-undef vector is matched by
3724 int64_t StepNumerator = SimpleVID->StepNumerator;
3725 unsigned StepDenominator = SimpleVID->StepDenominator;
3726 int64_t Addend = SimpleVID->Addend;
3741 // Only emit VIDs with suitably-small steps/addends. We use imm5 is a
3756 // these, we'd lose the fixed-length vector info and generate worse
3800 for (const auto &OpIdx : enumerate(Op->op_values())) {
3804 ((SeqV->getAsZExtVal() & EltMask) << (OpIdx.index() * EltBitSize));
3807 // On RV64, sign-extend from 32 to 64 bits where possible in order to
3825 // when re-interpreted as a vector with a larger element type. For example,
3829 // TODO: This optimization could also work on non-constant splats, but it
3830 // would require bit-manipulation instructions to construct the splat value.
3835 BV->getRepeatedSequence(Sequence) &&
3860 ((SeqV->getAsZExtVal() & EltMask) << (EltIdx * EltBitSize));
3864 // On RV64, sign-extend from 32 to 64 bits where possible in order to
3870 // perform an i64 splat on RV32 if it is its own sign-extended value. That
3902 if (EltBitSize - SignBits < 8) {
3904 DL, Op->ops());
3939 /// operations are generally lower latency, and for out-of-order cores
3968 // definition of the element - this tends to reduce register
4032 // A BUILD_VECTOR can be lowered as a SETCC. For each fixed-length mask
4033 // vector type, we have a legal equivalently-sized i8 type, so we can use
4039 if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
4046 SmallVector<SDValue, 8> Ops(Op->op_values());
4055 if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
4085 SmallVector<SDValue> BuildVectorOps(Op->op_begin(), Op->op_end());
4107 // For m1 vectors, if we have non-undef values in both halves of our vector,
4112 // length of the sub-vectors here, so if both fallback to the generic
4117 count_if(Op->op_values(), [](const SDValue &V) { return V.isUndef(); });
4118 unsigned NumDefElts = NumElts - NumUndefElts;
4123 SDValue UndefElem = DAG.getUNDEF(Op->getOperand(0)->getValueType(0));
4127 SDValue Elem = Op->getOperand(i);
4175 for (const SDValue &V : Op->ops()) {
4181 LinearBudget -= PerSlideCost;
4184 LinearBudget -= PerSlideCost;
4187 LinearBudget -= PerSlideCost;
4201 for (SDValue V : Op->ops()) {
4244 int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
4245 int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
4255 // FIXME: Support non-constant VLs by saturating?
4260 cast<RegisterSDNode>(VL)->getReg() == RISCV::X0))
4262 else if (isa<ConstantSDNode>(VL) && isUInt<4>(VL->getAsZExtVal()))
4343 // to be non-zero.
4437 if (Mask[i] != Mask[i - 1] + 2)
4486 /// vector of the rotate or -1 for undef. \p HiSrc indicates the second vector
4487 /// of the rotate or -1 for undef. At least one of \p LoSrc and \p HiSrc will be
4498 // [-1, 12, 13, 14, -1, -1, 1, -1]
4499 // [-1, -1, -1, -1, -1, -1, 1, 2]
4501 // [-1, 4, 5, 6, -1, -1, 9, -1]
4502 // [-1, 4, 5, 6, -1, -1, -1, -1]
4504 LoSrc = -1;
4505 HiSrc = -1;
4512 int StartIdx = i - (M % Size);
4515 return -1;
4520 int CandidateRotation = StartIdx < 0 ? -StartIdx : Size - StartIdx;
4526 return -1;
4543 return -1;
4555 // [a, p, b, q, c, r, d, s] -> [a, b, c, d] (EvenElts == true)
4556 // -> [p, q, r, s] (EvenElts == false)
4579 // Bitcast the source vector from <m x n*2 x ty> -> <m x n x ty*2>
4624 [](SDValue Parent) -> std::pair<SDValue, uint64_t> {
4627 // EXTRACT_SUBVECTOR can be used to extract a fixed-width vector from
4647 if (NewMask[i] == -1)
4655 NewMask[i] = NewMask[i] - NewMask.size() + V2IndexOffset;
4659 // First index must be known and non-zero. It will be used as the slidedown
4666 if (NewMask[i - 1] + 1 != NewMask[i])
4687 // ->
4692 // ->
4738 /// N-1 elements to make room for an inserted scalar at one end.
4751 SDValue Splat = cast<BuildVectorSDNode>(V1)->getSplatValue();
4755 // Return true if the mask could describe a slide of Mask.size() - 1
4758 const unsigned S = (Offset > 0) ? 0 : -Offset;
4759 const unsigned E = Mask.size() - ((Offset > 0) ? Offset : 0);
4768 if (!IsVSlidedown && !isSlideMask(Mask, OpsSwapped ? 0 : NumElts, -1))
4771 const int InsertIdx = Mask[IsVSlidedown ? (NumElts - 1) : 0];
4859 // Then get OddV * by 2^(VecVT.getScalarSizeInBits() - 1)
4897 MVT VT = SVN->getSimpleValueType(0);
4898 SDValue V = SVN->getOperand(0);
4903 if (!ShuffleVectorInst::isReverseMask(SVN->getMask(),
4904 SVN->getMask().size()) ||
4905 !SVN->getOperand(1).isUndef())
4916 if (!Subtarget.getTargetLowering()->isOperationLegalOrCustom(ISD::BITREVERSE,
4918 !Subtarget.getTargetLowering()->isTypeLegal(ViaBitVT))
4935 DAG.getConstant(ViaEltSize - NumElts, DL, ViaVT));
4951 EVT VT = SVN->getValueType(0);
4955 if (!ShuffleVectorInst::isBitRotateMask(SVN->getMask(), EltSizeInBits, 2,
4962 return Subtarget.getTargetLowering()->isTypeLegal(RotateVT);
4973 EVT VT = SVN->getValueType(0);
4979 SDValue Op = DAG.getBitcast(RotateVT, SVN->getOperand(0));
5000 MVT VT = SVN->getSimpleValueType(0);
5001 SDValue V1 = SVN->getOperand(0);
5002 SDValue V2 = SVN->getOperand(1);
5003 ArrayRef<int> Mask = SVN->getMask();
5012 // Avoid picking up bitrotate patterns which we have a linear-in-lmul
5024 OutMasks(VRegsPerSrc, {-1, {}});
5026 // Check if our mask can be done as a 1-to-1 mapping from source
5037 if (OutMasks[DstVecIdx].first == -1)
5045 OutMasks[DstVecIdx].second.resize(ElemsPerVReg, -1);
5062 if (SrcVecIdx == -1)
5101 SDValue Shuffled = DAG.getVectorShuffle(WidenVT, DL, V1, V2, SVN->getMask());
5110 if (SVN->isSplat()) {
5111 const int Lane = SVN->getSplatIndex();
5129 if (ISD::isNormalLoad(V.getNode()) && cast<LoadSDNode>(V)->isSimple()) {
5133 Ld->getBasePtr(), TypeSize::getFixed(Offset), DL);
5140 SDValue Ops[] = {Ld->getChain(),
5149 Ld->getMemOperand(), Offset, SVT.getStoreSize()));
5166 V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr,
5167 Ld->getPointerInfo().getWithOffset(Offset),
5168 Ld->getOriginalAlign(),
5169 Ld->getMemOperand()->getFlags());
5171 V = DAG.getExtLoad(ISD::EXTLOAD, DL, XLenVT, Ld->getChain(), NewAddr,
5172 Ld->getPointerInfo().getWithOffset(Offset), SVT,
5173 Ld->getOriginalAlign(),
5174 Ld->getMemOperand()->getFlags());
5199 ArrayRef<int> Mask = SVN->getMask();
5231 // to slide LoV up by (NumElts - Rotation).
5232 unsigned InvRotate = NumElts - Rotation;
5261 // (vmaccu.vx (vwaddu.vx lohalf(V1), lohalf(V2)), lohalf(V2), (2^eltbits - 1))
5297 // user-supplied maximum fixed-length LMUL.
5301 // Base case for the two operand recursion below - handle the worst case
5316 if (IndexVT.getScalarType().bitsGT(MVT::i16) && isUInt<16>(NumElts - 1) &&
5350 ? MaskIndex : -1);
5351 ShuffleMaskRHS.push_back(IsLHSOrUndefIndex ? -1 : (MaskIndex - NumElts));
5361 // second permute (vrgather, or other) by the post-isel combine.
5371 assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
5412 if (Op->isVPOpcode()) {
5434 // For CTTZ_ZERO_UNDEF, we need to extract the lowest set bit using X & -X.
5448 if (Op->isVPOpcode())
5458 if (!Op->isVPOpcode())
5476 if (Op->isVPOpcode()) {
5500 // leading zeros. We can do this by subtracting from (Bias + (EltSize - 1)).
5501 unsigned Adjust = ExponentBias + (EltSize - 1);
5503 if (Op->isVPOpcode())
5523 SDValue Source = Op->getOperand(0);
5525 SDValue Mask = Op->getOperand(1);
5526 SDValue EVL = Op->getOperand(2);
5546 if (Op->getOpcode() == ISD::VP_CTTZ_ELTS_ZERO_UNDEF)
5547 // In this case, we can interpret poison as -1, so nothing to do further.
5550 // Convert -1 to VL.
5558 // legal equivalently-sized byte-typed vector instead. This method is
5559 // responsible for re-expressing a ISD::LOAD via a correctly-aligned type. If
5560 // the load is already correctly-aligned, it returns SDValue().
5564 assert(Load && Load->getMemoryVT().isVector() && "Expected vector load");
5567 Load->getMemoryVT(),
5568 *Load->getMemOperand()))
5579 "Expecting equally-sized RVV vector types to be legal");
5580 SDValue L = DAG.getLoad(NewVT, DL, Load->getChain(), Load->getBasePtr(),
5581 Load->getPointerInfo(), Load->getOriginalAlign(),
5582 Load->getMemOperand()->getFlags());
5587 // legal equivalently-sized byte-typed vector instead. This method is
5588 // responsible for re-expressing a ISD::STORE via a correctly-aligned type. It
5593 assert(Store && Store->getValue().getValueType().isVector() &&
5597 Store->getMemoryVT(),
5598 *Store->getMemOperand()))
5602 SDValue StoredVal = Store->getValue();
5610 "Expecting equally-sized RVV vector types to be legal");
5612 return DAG.getStore(Store->getChain(), DL, StoredVal, Store->getBasePtr(),
5613 Store->getPointerInfo(), Store->getOriginalAlign(),
5614 Store->getMemOperand()->getFlags());
5621 int64_t Imm = cast<ConstantSDNode>(Op)->getSExtValue();
5666 // The only fence that needs an instruction is a sequentially-consistent
5667 // cross-thread fence.
5672 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
5680 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
5802 VL, Op->getFlags());
5827 Mask, VL, Op->getFlags());
5877 if (!Op->getFlags().hasNoNaNs() && !DAG.isKnownNeverNaN(X)) {
5883 if (!Op->getFlags().hasNoNaNs() && !DAG.isKnownNeverNaN(Y)) {
5894 bool XIsNeverNan = Op->getFlags().hasNoNaNs() || DAG.isKnownNeverNaN(X);
5895 bool YIsNeverNan = Op->getFlags().hasNoNaNs() || DAG.isKnownNeverNaN(Y);
5905 if (Op->isVPOpcode()) {
5934 Op.getOpcode() == ISD::FMAXIMUM || Op->getOpcode() == ISD::VP_FMAXIMUM
5944 /// Get a RISC-V target specified VL op for a given SDNode.
5952 // clang-format off
5955 llvm_unreachable("don't have RISC-V specified VL op for this SDNode");
6085 // clang-format on
6090 /// Return true if a RISC-V target specified op has a merge operand.
6094 "not a RISC-V target specific op");
6095 static_assert(RISCVISD::LAST_VL_VECTOR_OP - RISCVISD::FIRST_VL_VECTOR_OP ==
6097 RISCVISD::LAST_RISCV_STRICTFP_OPCODE -
6116 /// Return true if a RISC-V target specified op has a mask operand.
6120 "not a RISC-V target specific op");
6121 static_assert(RISCVISD::LAST_VL_VECTOR_OP - RISCVISD::FIRST_VL_VECTOR_OP ==
6123 RISCVISD::LAST_RISCV_STRICTFP_OPCODE -
6155 DAG.getNode(Op.getOpcode(), DL, LoVT, LoOperands, Op->getFlags());
6157 DAG.getNode(Op.getOpcode(), DL, HiVT, HiOperands, Op->getFlags());
6186 DAG.getNode(Op.getOpcode(), DL, LoVT, LoOperands, Op->getFlags());
6188 DAG.getNode(Op.getOpcode(), DL, HiVT, HiOperands, Op->getFlags());
6203 {Op.getOperand(0), Lo, MaskLo, EVLLo}, Op->getFlags());
6205 {ResLo, Hi, MaskHi, EVLHi}, Op->getFlags());
6210 assert(Op->isStrictFPOpcode());
6212 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(Op->getValueType(0));
6214 SDVTList LoVTs = DAG.getVTList(LoVT, Op->getValueType(1));
6215 SDVTList HiVTs = DAG.getVTList(HiVT, Op->getValueType(1));
6233 DAG.getNode(Op.getOpcode(), DL, LoVTs, LoOperands, Op->getFlags());
6236 DAG.getNode(Op.getOpcode(), DL, HiVTs, HiOperands, Op->getFlags());
6238 SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, Op->getValueType(0),
6330 // Consider other scalar<->scalar casts as legal if the types are legal.
6346 // When bitcasting from scalar to fixed-length vector, insert the scalar
6347 // into a one-element vector of the result type, and perform a vector
6359 // Custom-legalize bitcasts from fixed-length vector types to scalar types
6360 // thus: bitcast the vector to a one-element vector type whose element type
6394 // Only custom-lower vector truncates
6407 return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ -1);
6453 DAG.getConstant(3 - Log2, DL, VT));
6456 DAG.getConstant(Log2 - 3, DL, XLenVT));
6538 // int -> f32
6542 SDValue NC = DAG.getNode(Op.getOpcode(), DL, NVT, Op->ops());
6543 // f32 -> f16
6557 // f16 -> f32
6562 // f32 -> int
6570 // RVV can only do fp<->int conversions to types half/double the size as
6571 // the source. We custom-lower any conversions that do two hops into
6577 bool IsStrict = Op->isStrictFPOpcode();
6600 return DAG.getNode(Op.getOpcode(), DL, Op->getVTList(),
6612 return DAG.getNode(Op.getOpcode(), DL, Op->getVTList(), Chain, FExt);
6650 // Scalable vectors can exit here. Patterns will handle equally-sized
6655 // For fixed-length vectors we lower to a custom "VL" node.
6877 // ->
6888 Op->ops().take_front(HalfNumOps));
6890 Op->ops().drop_front(HalfNumOps));
6897 for (const auto &OpIdx : enumerate(Op->ops())) {
6941 DAG.getNode(ISD::SETCC, DL, CCVT, Tmp1, Tmp2, CC, Op->getFlags());
6950 ISD::CondCode CCVal = cast<CondCodeSDNode>(Op.getOperand(2))->get();
6956 // If the RHS is a constant in the range [-2049, 0) or (0, 2046], we can
6961 int64_t Imm = cast<ConstantSDNode>(RHS)->getSExtValue();
6963 // If this is an unsigned compare and the constant is -1, incrementing
6965 if (CCVal == ISD::SETUGT && Imm == -1)
6967 // Using getSetCCSwappedOperands will convert SET(U)GT->SET(U)LT.
7049 EVT VT = Op->getValueType(0);
7050 SDValue LHS = DAG.getFreeze(Op->getOperand(0));
7051 SDValue RHS = DAG.getFreeze(Op->getOperand(1));
7052 bool IsSigned = Op->getOpcode() == ISD::ABDS;
7054 // abds(lhs, rhs) -> sub(smax(lhs,rhs), smin(lhs,rhs))
7055 // abdu(lhs, rhs) -> sub(umax(lhs,rhs), umin(lhs,rhs))
7175 // int -> f32
7179 auto NC = DAG.getNode(Op.getOpcode(), DL, NVT, Op->ops());
7180 // f32 -> f16
7194 // f16 -> f32
7199 // f32 -> int
7285 return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags);
7290 return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, N->getOffset(),
7296 return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlign(),
7297 N->getOffset(), Flags);
7302 return DAG.getTargetJumpTable(N->getIndex(), Ty, Flags);
7313 // is incompatible with existing code models. This also applies to non-pic
7318 // Use PC-relative addressing to access the symbol. This generates the
7323 // Use PC-relative addressing to access the GOT for this symbol, then load
7353 // not be within 2GiB of PC, so use GOT-indirect addressing to access the
7379 assert(N->getOffset() == 0 && "unexpected offset in global node");
7380 const GlobalValue *GV = N->getGlobal();
7381 return getAddr(N, DAG, GV->isDSOLocal(), GV->hasExternalWeakLinkage());
7410 const GlobalValue *GV = N->getGlobal();
7414 // Use PC-relative addressing to access the GOT for this TLS symbol, then
7457 const GlobalValue *GV = N->getGlobal();
7459 // Use a PC-relative addressing mode to access the global dynamic GOT address.
7488 const GlobalValue *GV = N->getGlobal();
7490 // Use a PC-relative addressing mode to access the global dynamic GOT address.
7504 assert(N->getOffset() == 0 && "unexpected offset in global node");
7509 TLSModel::Model Model = getTargetMachine().getTLSModel(N->getGlobal());
7538 assert(Val->getOpcode() == ISD::SETCC);
7541 ISD::CondCode CC2 = cast<CondCodeSDNode>(Val.getOperand(2))->get();
7561 SDValue CondV = N->getOperand(0);
7562 SDValue TrueV = N->getOperand(1);
7563 SDValue FalseV = N->getOperand(2);
7564 MVT VT = N->getSimpleValueType(0);
7568 // (select c, -1, y) -> -c | y
7573 // (select c, y, -1) -> (c-1) | y
7580 // (select c, 0, y) -> (c-1) & y
7586 // (select c, y, 0) -> -c & y
7593 // select c, ~x, x --> xor -c, x
7595 const APInt &TrueVal = TrueV->getAsAPIntVal();
7596 const APInt &FalseVal = FalseV->getAsAPIntVal();
7609 ISD::CondCode CC = cast<CondCodeSDNode>(CondV.getOperand(2))->get();
7611 // (select x, x, y) -> x | y
7612 // (select !x, x, y) -> x & y
7617 // (select x, y, x) -> x & y
7618 // (select !x, y, x) -> x | y
7631 // being `0` or `-1`. In such cases we can replace `select` with `and`.
7641 SDValue Sel = BO->getOperand(0);
7644 Sel = BO->getOperand(1);
7652 if (!dyn_cast<ConstantSDNode>(Sel->getOperand(ConstSelOpNo))) {
7656 SDValue ConstSelOp = Sel->getOperand(ConstSelOpNo);
7658 if (!ConstSelOpNode || ConstSelOpNode->isOpaque())
7661 SDValue ConstBinOp = BO->getOperand(SelOpNo ^ 1);
7663 if (!ConstBinOpNode || ConstBinOpNode->isOpaque())
7667 EVT VT = BO->getValueType(0);
7674 DAG.FoldConstantArithmetic(BO->getOpcode(), DL, VT, NewConstOps);
7678 const APInt &NewConstAPInt = NewConstOp->getAsAPIntVal();
7682 SDValue OtherSelOp = Sel->getOperand(OtherSelOpNo);
7686 SDValue NewNonConstOp = DAG.getNode(BO->getOpcode(), DL, VT, NewNonConstOps);
7711 // sequence or RISCVISD::SELECT_CC node (branch-based select).
7714 // (select c, t, 0) -> (czero_eqz t, c)
7717 // (select c, 0, f) -> (czero_nez f, c)
7721 // (select c, (and f, x), f) -> (or (and f, x), (czero_nez f, c))
7727 // (select c, t, (and t, x)) -> (or (czero_eqz t, c), (and t, x))
7738 // (select c, c1, c2) -> (add (czero_nez c2 - c1, c), c1)
7739 // (select c, c1, c2) -> (add (czero_eqz c1 - c2, c), c2)
7741 const APInt &TrueVal = TrueV->getAsAPIntVal();
7742 const APInt &FalseVal = FalseV->getAsAPIntVal();
7749 IsCZERO_NEZ ? FalseVal - TrueVal : TrueVal - FalseVal, DL, VT);
7758 // (select c, t, f) -> (or (czero_eqz t, c), (czero_nez f, c))
7771 unsigned UseOpc = Op->use_begin()->getOpcode();
7773 SDNode *BinOp = *Op->use_begin();
7774 if (SDValue NewSel = foldBinOpIntoSelectIfProfitable(*Op->use_begin(),
7786 // (select cc, 1.0, 0.0) -> (sint_to_fp (zext cc))
7787 // (select cc, 0.0, 1.0) -> (sint_to_fp (zext (xor cc, 1)))
7791 if (FPTV->isExactlyValue(1.0) && FPFV->isExactlyValue(0.0))
7793 if (FPTV->isExactlyValue(0.0) && FPFV->isExactlyValue(1.0)) {
7803 // -> (riscvisd::select_cc condv, zero, setne, truev, falsev)
7818 // -> (riscvisd::select_cc lhs, rhs, cc, truev, falsev)
7821 ISD::CondCode CCVal = cast<CondCodeSDNode>(CondV.getOperand(2))->get();
7832 const APInt &TrueVal = TrueV->getAsAPIntVal();
7833 const APInt &FalseVal = FalseV->getAsAPIntVal();
7834 if (TrueVal - 1 == FalseVal)
7841 // 1 < x ? x : 1 -> 0 < x ? x : 1
7852 // x <s -1 ? x : -1 -> x <s 0 ? x : -1
7862 // -> (select (setcc lhs, rhs, InverseCC), falsev, constant)
7880 ISD::CondCode CCVal = cast<CondCodeSDNode>(CondV.getOperand(2))->get();
7899 SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
7904 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
7922 while (Depth--) {
7923 int Offset = -(XLenInBytes * 2);
7948 int Off = -XLenInBytes;
7957 // live-in.
7970 // if Shamt-XLEN < 0: // Shamt < XLEN
7972 // Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (XLEN-1 - Shamt))
7975 // Hi = Lo << (Shamt-XLEN)
7979 SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
7980 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
8010 // if Shamt-XLEN < 0: // Shamt < XLEN
8011 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - ShAmt))
8014 // Lo = Hi >>s (Shamt-XLEN);
8015 // Hi = Hi >>s (XLEN-1)
8018 // if Shamt-XLEN < 0: // Shamt < XLEN
8019 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - ShAmt))
8022 // Lo = Hi >>u (Shamt-XLEN);
8029 SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
8030 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
8054 // legal equivalently-sized i8 type, so we can use that as a go-between.
8060 // All-zeros or all-ones splats are handled specially.
8077 // Custom-lower a SPLAT_VECTOR_PARTS where XLEN<SEW, as the SEW element type is
8079 // FIXME: We could also catch non-constant sign-extended i32 values and lower
8107 // Custom-lower extensions from mask vectors by using a vselect either with 1
8108 // for zero/any-extension or -1 for sign-extension:
8109 // (vXiN = (s|z)ext vXi1:vmask) -> (vXiN = vselect vmask, (-1 or 1), 0)
8110 // Note that any-extension is lowered identically to zero-extension.
8116 // Only custom-lower extensions from mask types
8152 // Only custom-lower extensions from fixed-length vector types.
8176 // Custom-lower truncations from vectors to mask vectors by using a mask and a
8178 // (vXi1 = trunc vXiN vec) -> (vXi1 = setcc (and vec, 1), 0, ne)
8184 // Only expect to custom-lower truncations to mask types
8237 // Only custom-lower vector truncates
8244 // RVV only has truncates which operate from SEW*2->SEW, so lower arbitrary
8318 // For double rounding, the intermediate rounding should be round-to-odd.
8350 // custom-lower f64->f16 rounds via RVV's round-to-odd float
8370 // Prepare any fixed-length vector operands.
8440 // Custom-legalize INSERT_VECTOR_ELT so that the value is inserted into the
8445 // at VL-1, which is replaced with the desired value.
8464 // If the operand is a fixed-length vector, convert to a scalable one.
8476 const unsigned OrigIdx = IdxC->getZExtValue();
8509 // Even i64-element vectors on RV32 can be lowered without scalar
8510 // legalization if the most-significant 32 bits of the value are not affected
8511 // by the sign-extension of the lower 32 bits.
8512 // TODO: We could also catch sign extensions of a 32-bit value.
8515 if (isInt<32>(CVal->getSExtValue())) {
8517 Val = DAG.getConstant(CVal->getSExtValue(), DL, MVT::i32);
8542 // On RV32, i64-element vectors must be specially handled to place the
8544 // the i32 split lo/hi value. Use an equivalently-sized i32 vector for
8597 Idx->getAsZExtVal() + 1 == VecVT.getVectorNumElements())
8610 // Custom-lower EXTRACT_VECTOR_ELT operations to slide the vector down, then
8665 DAG.getConstant(WideEltVT.getSizeInBits() - 1, DL, XLenVT));
8700 unsigned OrigIdx = IdxC->getZExtValue();
8717 MaxIdx = VecVT.getVectorNumElements() - 1;
8719 MaxIdx = IdxC->getZExtValue();
8754 // Floating-point extracts are handled in TableGen.
8783 if (!II || !II->hasScalarOperand())
8786 unsigned SplatOp = II->ScalarOperand + 1 + HasChain;
8789 SmallVector<SDValue, 8> Operands(Op->op_begin(), Op->op_end());
8807 return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
8816 assert(II->ScalarOperand > 0 && "Unexpected splat operand!");
8817 MVT VT = Op.getOperand(SplatOp - 1).getSimpleValueType();
8823 // If this is a sign-extended 32-bit value, we can truncate it and rely on the
8824 // instruction to sign-extend since SEW>XLEN.
8827 return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
8855 uint64_t AVLInt = AVL->getAsZExtVal();
8918 SDValue Mask = Operands[NumOps - 3];
8921 uint64_t Policy = Operands[NumOps - 1]->getAsZExtVal();
8940 return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
8966 [[maybe_unused]] unsigned VF = N->getConstantOperandVal(2);
8980 SDValue AVL = DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, N->getOperand(1));
8985 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), Res);
8990 SDValue Op0 = N->getOperand(1);
9001 if (isOneConstant(N->getOperand(2)))
9004 // Convert -1 to VL.
9024 if (!II || !II->hasScalarOperand())
9027 unsigned SplatOp = II->ScalarOperand + 1;
9288 !isValidEGW(4, Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9289 !isValidEGW(4, Op->getOperand(2).getSimpleValueType(), Subtarget))
9297 !isValidEGW(8, Op->getOperand(1).getSimpleValueType(), Subtarget))
9305 if (Op->getSimpleValueType(0).getScalarSizeInBits() == 64 &&
9309 !isValidEGW(4, Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9310 !isValidEGW(4, Op->getOperand(2).getSimpleValueType(), Subtarget))
9330 SmallVector<SDValue> Operands{Op->op_values()};
9360 SmallVector<SDValue> Operands{Op->op_values()};
9396 SmallVector<SDValue> Operands{Op->op_values()};
9418 MVT VT = Op->getSimpleValueType(0);
9443 DAG.getExtLoad(ISD::EXTLOAD, DL, XLenVT, Load->getChain(), Ptr,
9444 ScalarVT, Load->getMemOperand());
9449 SDValue ScalarLoad = DAG.getLoad(ScalarVT, DL, Load->getChain(), Ptr,
9450 Load->getMemOperand());
9458 SmallVector<SDValue, 8> Ops{Load->getChain(), IntID};
9477 Load->getMemoryVT(), Load->getMemOperand());
9497 unsigned NF = Op->getNumValues() - 1;
9500 MVT VT = Op->getSimpleValueType(0);
9505 SDValue IntID = DAG.getTargetConstant(VlsegInts[NF - 2], DL, XLenVT);
9510 SmallVector<SDValue, 12> Ops = {Load->getChain(), IntID};
9516 Load->getMemoryVT(), Load->getMemOperand());
9592 SmallVector<SDValue, 8> Ops{Store->getChain(), IntID};
9600 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, Store->getVTList(),
9601 Ops, Store->getMemoryVT(),
9602 Store->getMemOperand());
9618 unsigned NF = Op->getNumOperands() - 4;
9621 MVT VT = Op->getOperand(2).getSimpleValueType();
9626 SDValue IntID = DAG.getTargetConstant(VssegInts[NF - 2], DL, XLenVT);
9627 SDValue Ptr = Op->getOperand(NF + 2);
9630 SmallVector<SDValue, 12> Ops = {FixedIntrinsic->getChain(), IntID};
9633 ContainerVT, FixedIntrinsic->getOperand(2 + i), DAG, Subtarget));
9638 FixedIntrinsic->getMemoryVT(), FixedIntrinsic->getMemOperand());
9788 // AND gives us (0 == 0) -> 1 and OR/XOR give us (0 != 0) -> 0. Therefore we
9796 return (RegisterAVL && RegisterAVL->getReg() == RISCV::X0) ||
9797 (ImmAVL && ImmAVL->getZExtValue() >= 1);
9815 // prove it is non-zero. For the AVL=0 case, we need the scalar to
9888 auto Flags = Op->getFlags();
9896 DAG.getConstantFP(Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, EltVT);
9943 if (Op->getFlags().hasNoNaNs())
9989 Op->getFlags().hasNoNaNs())
10025 // equivalent i8 vectors. Note that when inserting a fixed-length vector
10047 // can't be re-expressed as a larger type. Just choose the slow path and
10060 // If the subvector vector is a fixed-length type and we don't know VLEN
10193 unsigned AlignedIdx = OrigIdx - RemIdx.getKnownMinValue();
10261 // equivalent i8 vectors. Note that when extracting a fixed-length vector
10281 // can't be re-expressed as a larger type. Just choose the slow path and
10296 // With an index of 0 this is a cast-like subvector, which can be performed
10303 // If the subvector vector is a fixed-length type and we don't know VLEN
10316 unsigned LastIdx = OrigIdx + SubVecVT.getVectorNumElements() - 1;
10334 // Now we can use a cast-like subvector extract to get the result.
10388 // If the vector type is an LMUL-group type, extract a subvector equal to the
10427 for (SDValue Op : N->ops()) {
10433 unsigned NumVals = N->getNumValues();
10441 DAG.getSetCC(DL, N->getSimpleValueType(I), WideN.getValue(I),
10456 "vector_interleave on non-scalable vector!");
10532 "vector_interleave on non-scalable vector!");
10615 // Lower step_vector to the vid instruction. Any non-identity step value must
10643 // subtracting the id of each element from (VLMAX-1). This will convert
10645 // (0, 1,..., VLMAX-2, VLMAX-1) -> (VLMAX-1, VLMAX-2,..., 1, 0).
10698 // Calculate VLMAX-1 for the desired SEW.
10703 // Splat VLMAX-1 taking care to handle SEW==64 on RV32.
10731 int64_t ImmValue = cast<ConstantSDNode>(Op.getOperand(2))->getSExtValue();
10741 UpOffset = DAG.getConstant(-ImmValue, DL, XLenVT);
10762 Load->getMemoryVT(),
10763 *Load->getMemOperand()) &&
10764 "Expecting a correctly-aligned load");
10776 MachineMemOperand *MMO = Load->getMemOperand();
10778 DAG.getLoad(ContainerVT, DL, Load->getChain(), Load->getBasePtr(),
10779 MMO->getPointerInfo(), MMO->getBaseAlign(), MMO->getFlags(),
10780 MMO->getAAInfo(), MMO->getRanges());
10790 SmallVector<SDValue, 4> Ops{Load->getChain(), IntID};
10793 Ops.push_back(Load->getBasePtr());
10798 Load->getMemoryVT(), Load->getMemOperand());
10811 Store->getMemoryVT(),
10812 *Store->getMemOperand()) &&
10813 "Expecting a correctly-aligned store");
10815 SDValue StoreVal = Store->getValue();
10839 MachineMemOperand *MMO = Store->getMemOperand();
10840 return DAG.getStore(Store->getChain(), DL, NewValue, Store->getBasePtr(),
10841 MMO->getPointerInfo(), MMO->getBaseAlign(),
10842 MMO->getFlags(), MMO->getAAInfo());
10853 {Store->getChain(), IntID, NewValue, Store->getBasePtr(), VL},
10854 Store->getMemoryVT(), Store->getMemOperand());
10863 EVT MemVT = MemSD->getMemoryVT();
10864 MachineMemOperand *MMO = MemSD->getMemOperand();
10865 SDValue Chain = MemSD->getChain();
10866 SDValue BasePtr = MemSD->getBasePtr();
10870 Mask = VPLoad->getMask();
10872 VL = VPLoad->getVectorLength();
10875 Mask = MLoad->getMask();
10876 PassThru = MLoad->getPassThru();
10927 EVT MemVT = MemSD->getMemoryVT();
10928 MachineMemOperand *MMO = MemSD->getMemOperand();
10929 SDValue Chain = MemSD->getChain();
10930 SDValue BasePtr = MemSD->getBasePtr();
10935 Val = VPStore->getValue();
10936 Mask = VPStore->getMask();
10937 VL = VPStore->getVectorLength();
10940 Val = MStore->getValue();
10941 Mask = MStore->getMask();
10942 IsCompressingStore = MStore->isCompressingStore();
11020 ISD::CondCode CCVal = cast<CondCodeSDNode>(CC)->get();
11029 SDVTList VTList = Op->getVTList();
11116 if (Op->getOpcode() == ISD::VP_ABS) {
11117 Mask = Op->getOperand(1);
11121 VL = Op->getOperand(2);
11194 for (const SDValue &V : Op->op_values()) {
11197 // Pass through non-vector operands.
11219 if (Op->isStrictFPOpcode()) {
11222 Op->getFlags());
11228 DAG.getNode(NewOpc, DL, ContainerVT, Ops, Op->getFlags());
11235 // * Fixed-length vectors are converted to their scalable-vector container
11249 for (const auto &OpIdx : enumerate(Op->ops())) {
11271 // Pass through operands which aren't fixed-length vectors.
11285 return DAG.getNode(RISCVISDOpc, DL, VT, Ops, Op->getFlags());
11287 SDValue VPOp = DAG.getNode(RISCVISDOpc, DL, ContainerVT, Ops, Op->getFlags());
11314 Op.getOpcode() == ISD::VP_ZERO_EXTEND ? 1 : -1, DL, XLenVT);
11332 ISD::CondCode Condition = cast<CondCodeSDNode>(Op.getOperand(2))->get();
11349 // X != Y --> (X^Y)
11353 // X == Y --> ~(X^Y)
11361 // X >s Y --> X == 0 & Y == 1 --> ~X & Y
11362 // X <u Y --> X == 0 & Y == 1 --> ~X & Y
11370 // X <s Y --> X == 1 & Y == 0 --> ~Y & X
11371 // X >u Y --> X == 1 & Y == 0 --> ~Y & X
11379 // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
11380 // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
11388 // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
11389 // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
11404 // Lower Floating-Point/Integer Type-Convert VP SDNodes
11428 if (DstEltSize >= SrcEltSize) { // Single-width and widening conversion.
11436 // Do we need to do any pre-widening before converting?
11444 RISCVISDExtOpc == RISCVISD::VZEXT_VL ? 1 : -1, DL, XLenVT);
11505 // Compare the integer result to 0. The integer should be 0 or 1/-1,
11588 int64_t ImmValue = cast<ConstantSDNode>(Offset)->getSExtValue();
11598 UpOffset = DAG.getConstant(-ImmValue, DL, XLenVT);
11699 // Slide the concatenated result by (VLMax - VL).
11772 // It is safe to drop mask parameter as masked-off elements are undef.
11773 SDValue Op1 = Op->getOperand(0);
11774 SDValue Op2 = Op->getOperand(1);
11775 SDValue VL = Op->getOperand(3);
11805 SDValue Mask = VPNode->getMask();
11811 SmallVector<SDValue, 8> Ops{VPNode->getChain(), IntID,
11812 DAG.getUNDEF(ContainerVT), VPNode->getBasePtr(),
11813 VPNode->getStride()};
11821 Ops.push_back(VPNode->getVectorLength());
11829 VPNode->getMemoryVT(), VPNode->getMemOperand());
11844 SDValue StoreVal = VPNode->getValue();
11853 SDValue Mask = VPNode->getMask();
11859 SmallVector<SDValue, 8> Ops{VPNode->getChain(), IntID, StoreVal,
11860 VPNode->getBasePtr(), VPNode->getStride()};
11868 Ops.push_back(VPNode->getVectorLength());
11870 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, VPNode->getVTList(),
11871 Ops, VPNode->getMemoryVT(),
11872 VPNode->getMemOperand());
11878 // zero-extended or truncated to XLEN and are treated as byte offsets. Any
11887 EVT MemVT = MemSD->getMemoryVT();
11888 MachineMemOperand *MMO = MemSD->getMemOperand();
11889 SDValue Chain = MemSD->getChain();
11890 SDValue BasePtr = MemSD->getBasePtr();
11896 Index = VPGN->getIndex();
11897 Mask = VPGN->getMask();
11899 VL = VPGN->getVectorLength();
11905 Index = MGN->getIndex();
11906 Mask = MGN->getMask();
11907 PassThru = MGN->getPassThru();
11908 LoadExtType = MGN->getExtensionType();
11917 // Targets have to explicitly opt-in for extending vector loads.
11977 // zero-extended or truncated to XLEN and are treated as byte offsets. Any
11984 EVT MemVT = MemSD->getMemoryVT();
11985 MachineMemOperand *MMO = MemSD->getMemOperand();
11986 SDValue Chain = MemSD->getChain();
11987 SDValue BasePtr = MemSD->getBasePtr();
11993 Index = VPSN->getIndex();
11994 Mask = VPSN->getMask();
11995 Val = VPSN->getValue();
11996 VL = VPSN->getVectorLength();
12002 Index = MSN->getIndex();
12003 Mask = MSN->getMask();
12004 Val = MSN->getValue();
12005 IsTruncatingStore = MSN->isTruncatingStore();
12015 // Targets have to explicitly opt-in for extending vector loads and
12064 SDValue Chain = Op->getOperand(0);
12066 RISCVSysReg::lookupSysRegByName("FRM")->Encoding, DL, XLenVT);
12070 // Encoding used for rounding mode in RISC-V differs from that used in
12071 // FLT_ROUNDS. To convert it the RISC-V rounding mode is used as an index in a
12072 // table, which consists of a sequence of 4-bit fields, each representing
12095 SDValue Chain = Op->getOperand(0);
12096 SDValue RMValue = Op->getOperand(1);
12098 RISCVSysReg::lookupSysRegByName("FRM")->Encoding, DL, XLenVT);
12100 // Encoding used for rounding mode in RISC-V differs from that used in
12102 // a table, which consists of a sequence of 4-bit fields, each representing
12103 // corresponding RISC-V mode.
12134 // Returns the opcode of the target-specific SDNode that implements the 32-bit
12159 // Converts the given i8/i16/i32 operation to a target-specific SelectionDAG
12167 RISCVISD::NodeType WOpcode = getRISCVWOpcode(N->getOpcode());
12168 SDValue NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0));
12169 SDValue NewOp1 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(1));
12172 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes);
12175 // Converts the given 32-bit operation to a i64 operation with signed extension
12179 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
12180 SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12181 SDValue NewWOp = DAG.getNode(N->getOpcode(), DL, MVT::i64, NewOp0, NewOp1);
12191 switch (N->getOpcode()) {
12198 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
12200 bool IsStrict = N->isStrictFPOpcode();
12201 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
12202 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
12203 SDValue Op0 = IsStrict ? N->getOperand(1) : N->getOperand(0);
12209 SDValue Chain = N->getOperand(0);
12247 LC = RTLIB::getFPTOSINT(Op0.getValueType(), N->getValueType(0));
12249 LC = RTLIB::getFPTOUINT(Op0.getValueType(), N->getValueType(0));
12252 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
12253 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
12256 makeLibCall(DAG, LC, N->getValueType(0), Op0, CallOptions, DL, Chain);
12263 SDValue Op0 = N->getOperand(0);
12300 if (N->getOpcode() == ISD::READCYCLECOUNTER) {
12302 RISCVSysReg::lookupSysRegByName("CYCLE")->Encoding, DL, XLenVT);
12304 RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding, DL, XLenVT);
12307 RISCVSysReg::lookupSysRegByName("TIME")->Encoding, DL, XLenVT);
12309 RISCVSysReg::lookupSysRegByName("TIMEH")->Encoding, DL, XLenVT);
12313 N->getOperand(0), LoCounter, HiCounter);
12329 SDValue Res = DAG.getExtLoad(ISD::SEXTLOAD, dl, MVT::i64, Ld->getChain(),
12330 Ld->getBasePtr(), Ld->getMemoryVT(),
12331 Ld->getMemOperand());
12337 unsigned Size = N->getSimpleValueType(0).getSizeInBits();
12342 SDValue LHS = N->getOperand(0);
12343 SDValue RHS = N->getOperand(1);
12358 return DAG.getNode(ISD::BUILD_PAIR, DL, N->getValueType(0), Lo, Hi);
12377 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
12384 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
12386 if (N->getOperand(1).getOpcode() != ISD::Constant) {
12388 if (N->getOpcode() == ISD::SHL && Subtarget.hasStdExtZbs() &&
12389 isOneConstant(N->getOperand(0)))
12398 if (N->getOpcode() == ISD::SHL) {
12401 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
12403 DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(1));
12413 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
12418 if (!isa<ConstantSDNode>(N->getOperand(1)) &&
12427 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
12431 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
12433 N->getOpcode() == ISD::CTTZ || N->getOpcode() == ISD::CTTZ_ZERO_UNDEF;
12442 MVT VT = N->getSimpleValueType(0);
12449 if (N->getOperand(1).getOpcode() == ISD::Constant &&
12450 !isIntDivCheap(N->getValueType(0), Attr))
12458 ExtOpc = N->getOpcode() == ISD::SDIV ? ISD::SIGN_EXTEND
12465 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
12470 if (!isa<ConstantSDNode>(N->getOperand(1)))
12473 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
12474 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(1));
12485 // (LHS) if and only if the other operand (RHS) is (non-zero) positive,
12487 EVT OType = N->getValueType(1);
12499 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
12501 bool IsAdd = N->getOpcode() == ISD::UADDO;
12503 SDValue LHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
12504 SDValue RHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12517 Overflow = DAG.getSetCC(DL, N->getValueType(1), Res,
12520 // Special case uaddo X, -1 overflowed if X != 0.
12521 Overflow = DAG.getSetCC(DL, N->getValueType(1), N->getOperand(0),
12527 LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
12528 Overflow = DAG.getSetCC(DL, N->getValueType(1), Res, LHS,
12538 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
12545 DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
12547 DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(1));
12548 SDValue Res = DAG.getNode(N->getOpcode(), DL, MVT::i64, LHS, RHS);
12560 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
12566 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
12574 N->getOperand(0));
12581 SDValue Src = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
12603 EVT VT = N->getValueType(0);
12605 SDValue Op0 = N->getOperand(0);
12630 // Custom-legalize bitcasts from fixed-length vector types to illegal
12632 // one-element vector type whose element type is the same as the result
12645 MVT VT = N->getSimpleValueType(0);
12649 assert(((N->getOpcode() == RISCVISD::BREV8 && Subtarget.hasStdExtZbkb()) ||
12650 (N->getOpcode() == RISCVISD::ORC_B && Subtarget.hasStdExtZbb())) &&
12652 SDValue NewOp = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, N->getOperand(0));
12653 SDValue NewRes = DAG.getNode(N->getOpcode(), DL, XLenVT, NewOp);
12660 // Custom-legalize an EXTRACT_VECTOR_ELT where XLEN<SEW, as the SEW element
12662 // With vmv.x.s, when SEW > XLEN, only the least-significant XLEN bits are
12664 // upper- and lower- halves of the SEW-bit vector element, slid down to the
12666 SDValue Vec = N->getOperand(0);
12667 SDValue Idx = N->getOperand(1);
12677 assert(!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64 &&
12704 // element right by 32 bits and re-extract the lower XLEN bits.
12718 unsigned IntNo = N->getConstantOperandVal(0);
12731 DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), Res));
12742 if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32)
12757 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12767 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12769 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
12771 DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1, N->getOperand(3));
12776 if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32)
12779 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12782 DAG.getTargetConstant(N->getConstantOperandVal(2), DL, MVT::i64));
12787 if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32)
12790 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12792 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
12795 DAG.getTargetConstant(N->getConstantOperandVal(3), DL, MVT::i64));
12800 if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32)
12804 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12806 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
12813 if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32)
12817 // to the full 128-bit clmul result of multiplying two xlen values.
12826 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12828 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
12842 EVT VT = N->getValueType(0);
12847 Subtarget.getXLenVT(), N->getOperand(1));
12856 SDValue Vec = N->getOperand(1);
12863 // element right by 32 bits and re-extract the lower XLEN bits.
12904 SDValue Res = DAG.getNode(ISD::GET_ROUNDING, DL, VTs, N->getOperand(0));
12962 const EVT VT = N->getValueType(0);
12963 const unsigned Opc = N->getOpcode();
12970 (Opc != ISD::FADD || !N->getFlags().hasAllowReassociation()))
12976 SDValue LHS = N->getOperand(0);
12977 SDValue RHS = N->getOperand(1);
12989 uint64_t RHSIdx = cast<ConstantSDNode>(RHS.getOperand(1))->getLimitedValue();
13006 cast<ConstantSDNode>(LHS.getOperand(1))->getLimitedValue();
13011 return DAG.getNode(ReduceOpc, DL, VT, Vec, N->getFlags());
13033 auto Flags = ReduceVec->getFlags();
13034 Flags.intersectWith(N->getFlags());
13080 unsigned Opc = N->getOpcode();
13082 if (IsReduction(N->getOperand(0), Opc))
13084 else if (IsReduction(N->getOperand(1), Opc))
13090 if (Opc == ISD::FADD && !N->getFlags().hasAllowReassociation())
13093 SDValue Extract = N->getOperand(ReduceIdx);
13101 ScalarV.getOperand(0)->isUndef() &&
13116 if (!isNeutralConstant(N->getOpcode(), N->getFlags(), ScalarV.getOperand(1),
13125 SDValue NewStart = N->getOperand(1 - ReduceIdx);
13147 // Optimize (add (shl x, c0), (shl y, c1)) ->
13148 // (SLLI (SH*ADD x, y), c0), if c1-c0 equals to [1|2|3].
13156 EVT VT = N->getValueType(0);
13161 SDValue N0 = N->getOperand(0);
13162 SDValue N1 = N->getOperand(1);
13163 if (N0->getOpcode() != ISD::SHL || N1->getOpcode() != ISD::SHL ||
13164 !N0->hasOneUse() || !N1->hasOneUse())
13168 auto *N0C = dyn_cast<ConstantSDNode>(N0->getOperand(1));
13169 auto *N1C = dyn_cast<ConstantSDNode>(N1->getOperand(1));
13172 int64_t C0 = N0C->getSExtValue();
13173 int64_t C1 = N1C->getSExtValue();
13179 int64_t Diff = std::abs(C0 - C1);
13185 SDValue NS = (C0 < C1) ? N0->getOperand(0) : N1->getOperand(0);
13186 SDValue NL = (C0 > C1) ? N0->getOperand(0) : N1->getOperand(0);
13194 // (and (select cond, -1, c), x)
13195 // -> (select cond, x, (and x, c)) [AllOnes=1]
13197 // -> (select cond, x, (or x, c)) [AllOnes=0]
13199 // -> (select cond, x, (xor x, c)) [AllOnes=0]
13201 // -> (select cond, x, (add x, c)) [AllOnes=0]
13203 // -> (select cond, x, (sub x, c)) [AllOnes=0]
13207 EVT VT = N->getValueType(0);
13217 N->getOpcode() != ISD::AND)
13254 FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT, OtherOp, NonConstantVal);
13272 SDValue N0 = N->getOperand(0);
13273 SDValue N1 = N->getOperand(1);
13281 // Transform (add (mul x, c0), c1) ->
13286 // Or transform (add (mul x, c0), c1) ->
13287 // (add (mul (add x, c1/c0+1), c0), c1%c0-c0),
13288 // if c1/c0+1 and c1%c0-c0 are simm12, while c1 is not. A special corner
13291 // Or transform (add (mul x, c0), c1) ->
13292 // (add (mul (add x, c1/c0-1), c0), c1%c0+c0),
13293 // if c1/c0-1 and c1%c0+c0 are simm12, while c1 is not. A special corner
13294 // case that should be excluded is when c0*(c1/c0-1) is simm12, which will
13296 // Or transform (add (mul x, c0), c1) ->
13302 EVT VT = N->getValueType(0);
13306 SDValue N0 = N->getOperand(0);
13307 if (!N0->hasOneUse() || N0->getOpcode() != ISD::MUL)
13310 auto *N0C = dyn_cast<ConstantSDNode>(N0->getOperand(1));
13311 auto *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1));
13317 if (!N0C->hasOneUse())
13319 int64_t C0 = N0C->getSExtValue();
13320 int64_t C1 = N1C->getSExtValue();
13322 if (C0 == -1 || C0 == 0 || C0 == 1 || isInt<12>(C1))
13324 // Search for proper CA (non-zero) and CB that both are simm12.
13330 isInt<12>(C1 % C0 - C0) && !isInt<12>(C0 * (C1 / C0 + 1))) {
13332 CB = C1 % C0 - C0;
13333 } else if ((C1 / C0 - 1) != 0 && isInt<12>(C1 / C0 - 1) &&
13334 isInt<12>(C1 % C0 + C0) && !isInt<12>(C0 * (C1 / C0 - 1))) {
13335 CA = C1 / C0 - 1;
13341 SDValue New0 = DAG.getNode(ISD::ADD, DL, VT, N0->getOperand(0),
13348 // add (zext, zext) -> zext (add (zext, zext))
13349 // sub (zext, zext) -> sext (sub (zext, zext))
13350 // mul (zext, zext) -> zext (mul (zext, zext))
13351 // sdiv (zext, zext) -> zext (sdiv (zext, zext))
13352 // udiv (zext, zext) -> zext (udiv (zext, zext))
13353 // srem (zext, zext) -> zext (srem (zext, zext))
13354 // urem (zext, zext) -> zext (urem (zext, zext))
13361 EVT VT = N->getValueType(0);
13365 SDValue N0 = N->getOperand(0);
13366 SDValue N1 = N->getOperand(1);
13393 N->getOpcode() == ISD::SUB ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
13397 DAG.getNode(N->getOpcode(), SDLoc(N), NarrowVT, Src0, Src1));
13400 // Try to turn (add (xor bool, 1) -1) into (neg bool).
13402 SDValue N0 = N->getOperand(0);
13403 SDValue N1 = N->getOperand(1);
13404 EVT VT = N->getValueType(0);
13407 // RHS should be -1.
13443 // fold (add (select lhs, rhs, cc, 0, y), x) ->
13450 SDValue N0 = N->getOperand(0);
13451 SDValue N1 = N->getOperand(1);
13452 EVT VT = N->getValueType(0);
13462 APInt ImmValMinus1 = N0C->getAPIntValue() - 1;
13468 // (sub constant, (setcc x, y, eq/neq)) ->
13469 // (add (setcc x, y, neq/eq), constant - 1)
13470 ISD::CondCode CCVal = cast<CondCodeSDNode>(N1.getOperand(2))->get();
13479 // (sub C, (xor (setcc), 1)) -> (add (setcc), C-1).
13480 // Since setcc returns a bool the xor is equivalent to 1-setcc.
13490 // non-zero. Replace with orc.b.
13496 EVT VT = N->getValueType(0);
13501 SDValue N0 = N->getOperand(0);
13502 SDValue N1 = N->getOperand(1);
13508 if (!ShAmtC || ShAmtC->getZExtValue() != 8)
13523 EVT VT = N->getValueType(0);
13524 SDValue N0 = N->getOperand(0);
13525 SDValue N1 = N->getOperand(1);
13526 // fold (sub 0, (setcc x, 0, setlt)) -> (sra x, xlen - 1)
13529 ISD::CondCode CCVal = cast<CondCodeSDNode>(N1.getOperand(2))->get();
13532 unsigned ShAmt = N0.getValueSizeInBits() - 1;
13543 // fold (sub x, (select lhs, rhs, cc, 0, y)) ->
13552 SDValue N0 = N->getOperand(0);
13553 SDValue N1 = N->getOperand(1);
13554 bool IsAnd = N->getOpcode() == ISD::AND;
13566 // (xor X, -1) based on the upper bits of the other operand being 0. If the
13567 // operation is And, allow one of the Xors to use -1.
13572 // N01 and N11 being 1 was already handled. Handle N11==1 and N01==-1.
13578 EVT VT = N->getValueType(0);
13602 EVT VT = N->getValueType(0);
13603 SDValue N0 = N->getOperand(0);
13622 ISD::CondCode CCVal = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
13636 if (!CondRHSC || CondRHSC->getAPIntValue() != (1ULL << ScalarBits))
13648 if (!FalseRHSC || !FalseRHSC->isZero())
13651 ISD::CondCode CCVal2 = cast<CondCodeSDNode>(False.getOperand(2))->get();
13661 DAG.getConstant((1ULL << ScalarBits) - 1, DL, SrcVT));
13667 SDValue N0 = N->getOperand(0);
13668 EVT VT = N->getValueType(0);
13670 // Pre-promote (i1 (truncate (srl X, Y))) on RV64 with Zbs without zero
13696 SDValue N0 = N->getOperand(0);
13697 // Pre-promote (i32 (and (srl X, Y), 1)) on RV64 with Zbs without zero
13703 N->getValueType(0) == MVT::i32 && isOneConstant(N->getOperand(1)) &&
13724 // fold (and (select lhs, rhs, cc, -1, y), x) ->
13733 assert(N->getOpcode() == ISD::OR && "Unexpected opcode");
13754 EVT VT = N->getValueType(0);
13780 SDValue N0 = N->getOperand(0);
13781 SDValue N1 = N->getOperand(1);
13787 // fold (or (select cond, 0, y), x) ->
13794 SDValue N0 = N->getOperand(0);
13795 SDValue N1 = N->getOperand(1);
13797 // Pre-promote (i32 (xor (shl -1, X), ~0)) on RV64 with Zbs so we can use
13798 // (ADDI (BSET X0, X), -1). If we wait until/ type legalization, we'll create
13801 N->getValueType(0) == MVT::i32 && isAllOnesConstant(N1) &&
13812 // fold (xor (sllw 1, x), -1) -> (rolw ~1, x)
13823 // Fold (xor (setcc constant, y, setlt), 1) -> (setcc y, constant + 1, setlt)
13826 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
13830 const APInt &Imm = ConstN00->getAPIntValue();
13837 // Combine (xor (trunc (X cc Y)) 1) -> (trunc (X !cc Y)). This is needed with
13840 if (N->getValueType(0) == MVT::i32 && N0.getOpcode() == ISD::TRUNCATE &&
13847 ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(),
13851 return DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N->getValueType(0), Setcc);
13859 // fold (xor (select cond, 0, y), x) ->
13869 EVT VT = N->getValueType(0);
13884 ConstantSDNode *CNode = dyn_cast<ConstantSDNode>(N->getOperand(1));
13887 uint64_t MulAmt = CNode->getZExtValue();
13893 SDValue X = N->getOperand(0);
13900 // 3/5/9 * 2^N -> shl (shXadd X, X), N
13903 SDValue X = N->getOperand(0);
13911 DAG.getConstant(Log2_64(Divisor - 1), DL, VT),
13918 DAG.getConstant(Log2_64(Divisor - 1), DL, VT), X);
13923 // 3/5/9 * 3/5/9 -> shXadd (shYadd X, X), (shYadd X, X)
13928 DAG.getConstant(Log2_64(Divisor - 1), DL, VT), X);
13930 DAG.getConstant(Log2_64(MulAmt2 - 1), DL, VT),
13938 if (isPowerOf2_64(MulAmt & (MulAmt - 1))) {
13941 unsigned ShiftAmt = Log2_64((MulAmt & (MulAmt - 1)));
13950 // 2^(1,2,3) * 3,5,9 + 1 -> (shXadd (shYadd x, x), x)
13954 // 2^(C1>3) * 3,5,9 +/- 1
13956 uint64_t C = MulAmt - 1;
13964 DAG.getConstant(Log2_64(Divisor - 1), DL, VT), X);
13970 // 2^n + 2/4/8 + 1 -> (add (shl X, C1), (shXadd X, X))
13971 if (MulAmt > 2 && isPowerOf2_64((MulAmt - 1) & (MulAmt - 2))) {
13972 unsigned ScaleShift = llvm::countr_zero(MulAmt - 1);
13974 unsigned ShiftAmt = Log2_64(((MulAmt - 1) & (MulAmt - 2)));
13984 // 2^N - 3/5/9 --> (sub (shl X, C1), (shXadd X, x))
13993 DAG.getConstant(Log2_64(Offset - 1), DL, VT), X);
13999 // 2^N - 2^M -> (sub (shl X, C1), (shl X, C2))
14000 uint64_t MulAmtLowBit = MulAmt & (-MulAmt);
14004 SDValue Shift1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
14007 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
14015 // Combine vXi32 (mul (and (lshr X, 15), 0x10001), 0xffff) ->
14019 EVT VT = N->getValueType(0);
14026 if (N->getOperand(0).getOpcode() != ISD::AND ||
14027 N->getOperand(0).getOperand(0).getOpcode() != ISD::SRL)
14030 SDValue And = N->getOperand(0);
14034 if (!ISD::isConstantSplatVector(N->getOperand(1).getNode(), V1) ||
14041 V3 != (HalfSize - 1))
14050 DAG.getConstant(HalfSize - 1, DL, HalfVT));
14057 EVT VT = N->getValueType(0);
14062 SDValue N0 = N->getOperand(0);
14063 SDValue N1 = N->getOperand(1);
14067 // vmadd: (mul (add x, 1), y) -> (add (mul x, y), y)
14068 // (mul x, add (y, 1)) -> (add x, (mul x, y))
14069 // vnmsub: (mul (sub 1, x), y) -> (sub y, (mul x, y))
14070 // (mul x, (sub 1, y)) -> (sub x, (mul x, y))
14071 auto IsAddSubWith1 = [&](SDValue V) -> bool {
14072 AddSubOpc = V->getOpcode();
14073 if ((AddSubOpc == ISD::ADD || AddSubOpc == ISD::SUB) && V->hasOneUse()) {
14074 SDValue Opnd = V->getOperand(1);
14075 MulOper = V->getOperand(0);
14103 /// According to the property that indexed load/store instructions zero-extend
14109 if (!N->hasOneUse())
14142 if (!N0->hasOneUse())
14164 SDValue NewExt = DAG.getNode(N0->getOpcode(), DL, NewVT, N0->ops());
14176 SDValue N0 = N->getOperand(0);
14177 SDValue N1 = N->getOperand(1);
14178 EVT VT = N->getValueType(0);
14196 ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(2))->get();
14206 const APInt &C1 = N1C->getAPIntValue();
14209 // If the constant is larger than 2^32 - 1 it is impossible for both sides
14223 SDValue Src = N->getOperand(0);
14224 EVT VT = N->getValueType(0);
14226 // Fold (sext_inreg (fmv_x_anyexth X), i16) -> (fmv_x_signexth X)
14228 cast<VTSDNode>(N->getOperand(1))->getVT().bitsGE(MVT::i16))
14243 /// add | add_vl | or disjoint -> vwadd(u) | vwadd(u)_w
14244 /// sub | sub_vl -> vwsub(u) | vwsub(u)_w
14245 /// mul | mul_vl -> vwmul(u) | vwmul_su
14246 /// shl | shl_vl -> vwsll
14247 /// fadd -> vfwadd | vfwadd_w
14248 /// fsub -> vfwsub | vfwsub_w
14249 /// fmul -> vfwmul
14259 /// - VWADD_W is conceptually == add(op0, sext(op1))
14260 /// - VWADDU_W == add(op0, zext(op1))
14261 /// - VWSUB_W == sub(op0, sext(op1))
14262 /// - VWSUBU_W == sub(op0, zext(op1))
14263 /// - VFWADD_W == fadd(op0, fpext(op1))
14264 /// - VFWSUB_W == fsub(op0, fpext(op1))
14275 /// Records if this operand is like being floating-Point extended.
14329 assert(Subtarget.getTargetLowering()->isTypeLegal(Source.getValueType()));
14367 /// element by 2. E.g., if Root's type <2xi16> -> narrow type <2xi8>.
14370 MVT VT = Root->getSimpleValueType(0);
14386 /// Opcode(sext(a), sext(b)) -> newOpcode(a, b)
14409 /// Opcode(zext(a), zext(b)) -> newOpcode(a, b)
14435 /// Opcode(fpext(a), fpext(b)) -> newOpcode(a, b)
14451 /// Get the opcode to materialize \p Opcode(sext(a), zext(b)) ->
14460 /// \p Opcode(a, s|z|fpext(b)) -> newOpcode(a, b).
14609 switch (Root->getOpcode()) {
14613 return Root->getValueType(0).isScalableVector();
14616 return Root->getValueType(0).isScalableVector() &&
14617 Root->getFlags().hasDisjoint();
14627 // Vector Widening Floating-Point Add/Sub/Mul Instructions
14635 return Root->getValueType(0).isScalableVector() &&
14651 assert(DAG.getTargetLoweringInfo().isTypeLegal(Root->getValueType(0)));
14652 OrigOperand = Root->getOperand(OperandIdx);
14654 unsigned Opc = Root->getOpcode();
14657 // VW<ADD|SUB>_W(LHS, RHS) -> <ADD|SUB>(LHS, SEXT(RHS))
14658 // VW<ADD|SUB>U_W(LHS, RHS) -> <ADD|SUB>(LHS, ZEXT(RHS))
14659 // VFW<ADD|SUB>_W(LHS, RHS) -> F<ADD|SUB>(LHS, FPEXT(RHS))
14690 switch (Root->getOpcode()) {
14697 MVT VT = Root->getSimpleValueType(0);
14701 return std::make_pair(Root->getOperand(3), Root->getOperand(4));
14708 switch (N->getOpcode()) {
14773 switch (Root->getOpcode()) {
14775 Merge = Root->getOperand(2);
14782 Merge = DAG.getUNDEF(Root->getValueType(0));
14785 return DAG.getNode(TargetOpcode, SDLoc(Root), Root->getValueType(0),
14808 return CombineResult(NodeExtensionHelper::getZExtOpcode(Root->getOpcode()),
14812 return CombineResult(NodeExtensionHelper::getSExtOpcode(Root->getOpcode()),
14816 return CombineResult(NodeExtensionHelper::getFPExtOpcode(Root->getOpcode()),
14847 NodeExtensionHelper::getWOpcode(Root->getOpcode(), ExtKind::FPExt),
14856 NodeExtensionHelper::getWOpcode(Root->getOpcode(), ExtKind::ZExt), Root,
14860 NodeExtensionHelper::getWOpcode(Root->getOpcode(), ExtKind::SExt), Root,
14912 return CombineResult(NodeExtensionHelper::getSUOpcode(Root->getOpcode()),
14920 switch (Root->getOpcode()) {
14928 // add|sub|fadd|fsub-> vwadd(u)|vwsub(u)|vfwadd|vfwsub
14930 // add|sub|fadd|fsub -> vwadd(u)_w|vwsub(u)_w}|vfwadd_w|vfwsub_w
14938 // mul -> vwmul(u)
14940 // mul -> vwmulsu
14945 // shl -> vwsll
14950 // vwadd_w|vwsub_w -> vwadd|vwsub
14955 // vwaddu_w|vwsubu_w -> vwaddu|vwsubu
14960 // vfwadd_w|vfwsub_w -> vfwadd|vfwsub
14972 /// add | add_vl | or disjoint -> vwadd(u) | vwadd(u)_w
14973 /// sub | sub_vl -> vwsub(u) | vwsub(u)_w
14974 /// mul | mul_vl -> vwmul(u) | vwmul_su
14975 /// shl | shl_vl -> vwsll
14976 /// fadd_vl -> vfwadd | vfwadd_w
14977 /// fsub_vl -> vfwsub | vfwsub_w
14978 /// fmul_vl -> vfwmul
14979 /// vwadd_w(u) -> vwadd(u)
14980 /// vwsub_w(u) -> vwsub(u)
14981 /// vfwadd_w -> vfwadd
14982 /// vfwsub_w -> vfwsub
15009 for (SDNode *TheUse : Op.OrigOperand->uses()) {
15040 if (Res->LHSExt.has_value())
15042 if (Res->RHSExt.has_value())
15056 // nodes may be feeding some of the yet-to-be-replaced nodes. Put differently,
15058 // yet-to-be-visited CombinesToApply roots.
15078 // Fold (vwadd(u).wv y, (vmerge cond, x, 0)) -> vwadd(u).wv y, x, y, cond
15079 // (vwsub(u).wv y, (vmerge cond, x, 0)) -> vwsub(u).wv y, x, y, cond
15082 unsigned Opc = N->getOpcode();
15086 SDValue Y = N->getOperand(0);
15087 SDValue MergeOp = N->getOperand(1);
15093 SDValue X = MergeOp->getOperand(1);
15099 SDValue Passthru = N->getOperand(2);
15104 SDValue Mask = N->getOperand(3);
15109 SDValue Z = MergeOp->getOperand(2);
15118 return DAG.getNode(Opc, SDLoc(N), N->getValueType(0),
15119 {Y, X, Y, MergeOp->getOperand(0), N->getOperand(4)},
15120 N->getFlags());
15126 [[maybe_unused]] unsigned Opc = N->getOpcode();
15154 EVT MemVT = LSNode1->getMemoryVT();
15156 MachineMemOperand *MMO = LSNode1->getMemOperand();
15158 MMO, MMO->getPointerInfo(), MemVT == MVT::i32 ? 8 : 16);
15160 if (LSNode1->getOpcode() == ISD::LOAD) {
15161 auto Ext = cast<LoadSDNode>(LSNode1)->getExtensionType();
15170 {LSNode1->getChain(), BasePtr,
15186 {LSNode1->getChain(), LSNode1->getOperand(1), LSNode2->getOperand(1),
15208 EVT MemVT = LSNode1->getMemoryVT();
15209 unsigned OpNum = LSNode1->getOpcode() == ISD::LOAD ? 1 : 2;
15212 if (!LSNode1->isSimple() || LSNode1->isIndexed())
15216 auto ExtractBaseAndOffset = [](SDValue Ptr) -> std::pair<SDValue, uint64_t> {
15217 if (Ptr->getOpcode() == ISD::ADD)
15218 if (auto *C1 = dyn_cast<ConstantSDNode>(Ptr->getOperand(1)))
15219 return {Ptr->getOperand(0), C1->getZExtValue()};
15223 auto [Base1, Offset1] = ExtractBaseAndOffset(LSNode1->getOperand(OpNum));
15225 SDValue Chain = N->getOperand(0);
15226 for (SDNode::use_iterator UI = Chain->use_begin(), UE = Chain->use_end();
15230 Use.getUser()->getOpcode() == N->getOpcode()) {
15234 if (!LSNode2->isSimple() || LSNode2->isIndexed())
15238 if (LSNode1->getOpcode() == ISD::LOAD)
15239 if (cast<LoadSDNode>(LSNode2)->getExtensionType() !=
15240 cast<LoadSDNode>(LSNode1)->getExtensionType())
15243 if (LSNode1->getMemoryVT() != LSNode2->getMemoryVT())
15246 auto [Base2, Offset2] = ExtractBaseAndOffset(LSNode2->getOperand(OpNum));
15255 // Check for adjacent i32 values and a 2-bit index.
15259 // Check for adjacent i64 values and a 2-bit index.
15278 // (fp_to_int (froundeven X)) -> fcvt X, rne
15279 // (fp_to_int (ftrunc X)) -> fcvt X, rtz
15280 // (fp_to_int (ffloor X)) -> fcvt X, rdn
15281 // (fp_to_int (fceil X)) -> fcvt X, rup
15282 // (fp_to_int (fround X)) -> fcvt X, rmm
15283 // (fp_to_int (frint X)) -> fcvt X
15291 SDValue Src = N->getOperand(0);
15293 // Don't do this for strict-fp Src.
15294 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
15311 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT;
15312 EVT VT = N->getValueType(0);
15328 // Make fixed-length vectors scalable first
15357 // If converted from fixed-length to scalable, convert back
15381 // (fp_to_int_sat (froundeven X)) -> (select X == nan, 0, (fcvt X, rne))
15382 // (fp_to_int_sat (ftrunc X)) -> (select X == nan, 0, (fcvt X, rtz))
15383 // (fp_to_int_sat (ffloor X)) -> (select X == nan, 0, (fcvt X, rdn))
15384 // (fp_to_int_sat (fceil X)) -> (select X == nan, 0, (fcvt X, rup))
15385 // (fp_to_int_sat (fround X)) -> (select X == nan, 0, (fcvt X, rmm))
15386 // (fp_to_int_sat (frint X)) -> (select X == nan, 0, (fcvt X, dyn))
15396 EVT DstVT = N->getValueType(0);
15400 SDValue Src = N->getOperand(0);
15402 // Don't do this for strict-fp Src.
15403 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
15414 EVT SatVT = cast<VTSDNode>(N->getOperand(1))->getVT();
15420 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT_SAT;
15442 // RISC-V FP-to-int conversions saturate to the destination register size, but
15454 SDValue Src = N->getOperand(0);
15458 EVT VT = N->getValueType(0);
15471 // Negating the multiply result changes ADD<->SUB and toggles 'N'.
15473 // clang-format off
15485 // clang-format on
15488 // Negating the accumulator changes ADD<->SUB.
15490 // clang-format off
15502 // clang-format on
15510 // The first operand of strict-fp is chain.
15511 unsigned Offset = N->isTargetStrictFPOpcode();
15512 SDValue A = N->getOperand(0 + Offset);
15513 SDValue B = N->getOperand(1 + Offset);
15514 SDValue C = N->getOperand(2 + Offset);
15515 SDValue Mask = N->getOperand(3 + Offset);
15516 SDValue VL = N->getOperand(4 + Offset);
15537 unsigned NewOpcode = negateFMAOpcode(N->getOpcode(), NegA != NegB, NegC);
15538 if (N->isTargetStrictFPOpcode())
15539 return DAG.getNode(NewOpcode, SDLoc(N), N->getVTList(),
15540 {N->getOperand(0), A, B, C, Mask, VL});
15541 return DAG.getNode(NewOpcode, SDLoc(N), N->getValueType(0), A, B, C, Mask,
15550 if (N->getValueType(0).getVectorElementType() == MVT::f32 &&
15555 if (N->isTargetStrictFPOpcode())
15559 SDValue Op0 = N->getOperand(0);
15560 SDValue Op1 = N->getOperand(1);
15561 SDValue Mask = N->getOperand(3);
15562 SDValue VL = N->getOperand(4);
15571 (Op0 != Op1 || !Op0->hasNUsesOfValue(2, 0)))
15580 switch (N->getOpcode()) {
15600 return DAG.getNode(NewOpc, SDLoc(N), N->getValueType(0), Op0, Op1,
15601 N->getOperand(2), Mask, VL);
15606 assert(N->getOpcode() == ISD::SRA && "Unexpected opcode");
15608 if (N->getValueType(0) != MVT::i64 || !Subtarget.is64Bit())
15611 if (!isa<ConstantSDNode>(N->getOperand(1)))
15613 uint64_t ShAmt = N->getConstantOperandVal(1);
15617 SDValue N0 = N->getOperand(0);
15619 // Combine (sra (sext_inreg (shl X, C1), i32), C2) ->
15624 cast<VTSDNode>(N0.getOperand(1))->getVT() == MVT::i32 &&
15639 // Combine (sra (shl X, 32), 32 - C) -> (shl (sext_inreg X, i32), C)
15643 // (sra (add (shl X, 32), C1), 32 - C) -> (shl (sext_inreg (add X, C1), C)
15644 // (sra (sub C1, (shl X, 32)), 32 - C) -> (shl (sext_inreg (sub C1, X), C)
15657 if (AddC->getAPIntValue().countr_zero() < 32)
15663 for (SDNode *U : N0->uses()) {
15664 if (U->getOpcode() != ISD::SRA ||
15665 !isa<ConstantSDNode>(U->getOperand(1)) ||
15666 U->getConstantOperandVal(1) > 32)
15695 DAG.getConstant(AddC->getAPIntValue().lshr(32), DL, MVT::i64);
15709 DAG.getConstant(32 - ShAmt, DL, MVT::i64));
15753 ISD::CondCode CCVal = cast<CondCodeSDNode>(Setcc.getOperand(2))->get();
15777 ISD::CondCode CCVal = cast<CondCodeSDNode>(CC)->get();
15781 // Fold setlt (sra X, N), 0 -> setlt X, 0 and
15782 // setge (sra X, N), 0 -> setge X, 0
15792 // Fold ((setlt X, Y), 0, ne) -> (X, Y, lt)
15799 CCVal = cast<CondCodeSDNode>(LHS.getOperand(2))->get();
15811 // Fold ((xor X, Y), 0, eq/ne) -> (X, Y, eq/ne)
15818 // Fold ((srl (and X, 1<<C), C), 0, eq/ne) -> ((shl X, XLen-1-C), 0, ge/lt)
15830 ShAmt = LHS.getValueSizeInBits() - 1 - ShAmt;
15841 // (X, 1, setne) -> // (X, 0, seteq) if we can prove X is 0/1.
15864 // (select C, (add Y, X), Y) -> (add Y, (select C, X, 0)).
15865 // (select C, (sub Y, X), Y) -> (sub Y, (select C, X, 0)).
15866 // (select C, (or Y, X), Y) -> (or Y, (select C, X, 0)).
15867 // (select C, (xor Y, X), Y) -> (xor Y, (select C, X, 0)).
15899 EVT VT = N->getValueType(0);
15901 SDValue OtherOp = TrueVal.getOperand(1 - OpToFold);
15904 DAG.getNeutralElement(Opc, DL, OtherOpVT, N->getFlags());
15912 DAG.getSelect(DL, OtherOpVT, N->getOperand(0), OtherOp, IdentityOperand);
15919 SDValue Cond = N->getOperand(0);
15929 if (!isNullConstant(Cond->getOperand(1)))
15932 ISD::CondCode CCVal = cast<CondCodeSDNode>(Cond->getOperand(2))->get();
15934 CountZeroes = N->getOperand(2);
15935 ValOnZero = N->getOperand(1);
15937 CountZeroes = N->getOperand(1);
15938 ValOnZero = N->getOperand(2);
15956 SDValue CountZeroesArgument = CountZeroes->getOperand(0);
15957 if (Cond->getOperand(0) != CountZeroesArgument)
15970 DAG.getConstant(BitWidth - 1, SDLoc(N), CountZeroes.getValueType());
15974 return DAG.getZExtOrTrunc(AndNode, SDLoc(N), N->getValueType(0));
15979 SDValue Cond = N->getOperand(0);
15980 SDValue True = N->getOperand(1);
15981 SDValue False = N->getOperand(2);
15983 EVT VT = N->getValueType(0);
15995 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
16019 SDValue TrueVal = N->getOperand(1);
16020 SDValue FalseVal = N->getOperand(2);
16035 EVT VT = N->getValueType(0);
16042 const unsigned Opcode = N->op_begin()->getNode()->getOpcode();
16050 // truncates through binops is non-trivial.
16051 if (N->op_begin()->getValueType() != VT.getVectorElementType())
16056 for (SDValue Op : N->ops()) {
16092 SDValue InVec = N->getOperand(0);
16093 SDValue InVal = N->getOperand(1);
16094 SDValue EltNo = N->getOperand(2);
16107 const unsigned InVecOpcode = InVec->getOpcode();
16108 if (InVecOpcode == InVal->getOpcode() && TLI.isBinOp(InVecOpcode) &&
16110 SDValue InVecLHS = InVec->getOperand(0);
16111 SDValue InVecRHS = InVec->getOperand(1);
16112 SDValue InValLHS = InVal->getOperand(0);
16113 SDValue InValRHS = InVal->getOperand(1);
16138 unsigned Elt = IndexC->getZExtValue();
16152 ConcatOps.append(InVec->op_begin(), InVec->op_end());
16165 EVT VT = N->getValueType(0);
16175 auto *BaseLd = dyn_cast<LoadSDNode>(N->getOperand(0));
16176 if (!BaseLd || !BaseLd->isSimple() || !ISD::isNormalLoad(BaseLd) ||
16180 EVT BaseLdVT = BaseLd->getValueType(0);
16185 Align Align = BaseLd->getAlign();
16186 for (SDValue Op : N->ops().drop_front()) {
16188 if (!Ld || !Ld->isSimple() || !Op.hasOneUse() ||
16189 Ld->getChain() != BaseLd->getChain() || !ISD::isNormalLoad(Ld) ||
16190 Ld->getValueType(0) != BaseLdVT)
16196 Align = std::min(Align, Ld->getAlign());
16201 LoadSDNode *Ld2) -> std::optional<PtrDiff> {
16207 return {{BIO2.getOffset() - BIO1.getOffset(), false}};
16210 SDValue P1 = Ld1->getBasePtr();
16211 SDValue P2 = Ld2->getBasePtr();
16226 for (auto *It = Lds.begin() + 1; It != Lds.end() - 1; It++)
16234 // Get the widened scalar type, e.g. v4i8 -> i64
16239 // Get the vector type for the strided load, e.g. 4 x v4i8 -> v4i64
16240 MVT WideVecVT = MVT::getVectorVT(WideScalarVT, N->getNumOperands());
16252 Lds[0]->getOffset().getValueType());
16262 ConstStride && ConstStride->getSExtValue() >= 0)
16263 // total size = (elsize * n) + (stride - elsize) * (n-1)
16264 // = elsize + stride * (n-1)
16266 ConstStride->getSExtValue() * (N->getNumOperands() - 1);
16272 BaseLd->getPointerInfo(), BaseLd->getMemOperand()->getFlags(), MemSize,
16276 WideVecVT, DL, BaseLd->getChain(), BaseLd->getBasePtr(), Stride,
16278 DAG.getConstant(N->getNumOperands(), DL, Subtarget.getXLenVT()), MMO);
16280 for (SDValue Ld : N->ops())
16289 assert(N->getOpcode() == RISCVISD::ADD_VL || N->getOpcode() == ISD::ADD);
16291 if (N->getValueType(0).isFixedLengthVector())
16294 SDValue Addend = N->getOperand(0);
16295 SDValue MulOp = N->getOperand(1);
16297 if (N->getOpcode() == RISCVISD::ADD_VL) {
16298 SDValue AddMergeOp = N->getOperand(2);
16327 if (N->getOpcode() == ISD::ADD) {
16329 return getDefaultScalableVLOps(N->getSimpleValueType(0), DL, DAG,
16332 return std::make_pair(N->getOperand(3), N->getOperand(4));
16341 unsigned Opc = RISCVISD::VWMACC_VL + MulOp.getOpcode() - RISCVISD::VWMUL_VL;
16352 EVT VT = N->getValueType(0);
16370 // RISC-V indexed loads only support the "unsigned unscaled" addressing
16404 for (unsigned i = 0; i < Index->getNumOperands(); i++) {
16407 if (Index->getOperand(i)->isUndef())
16409 uint64_t C = Index->getConstantOperandVal(i);
16447 for (unsigned i = 0; i < Index->getNumOperands(); i++) {
16450 if (Index->getOperand(i)->isUndef())
16454 uint64_t C = Index->getConstantOperandVal(i);
16460 uint64_t Last = Index->getConstantOperandVal(i-1);
16467 // trunc (sra sext (X), zext (Y)) -> sra (X, smin (Y, scalarsize(Y) - 1))
16470 // n-levels TRUNCATE_VECTOR_VL to satisfy RVV's SEW*2->SEW truncate
16474 SDValue Mask = N->getOperand(1);
16475 SDValue VL = N->getOperand(2);
16479 cast<RegisterSDNode>(VL)->getReg() == RISCV::X0);
16489 SDValue Op = N->getOperand(0);
16512 N->getValueType(0) != N10.getValueType())
16515 unsigned MaxShAmt = N10.getValueType().getScalarSizeInBits() - 1;
16517 DAG.getNode(ISD::SMIN, SDLoc(N1), N->getValueType(0), N10,
16518 DAG.getConstant(MaxShAmt, SDLoc(N1), N->getValueType(0)));
16519 return DAG.getNode(ISD::SRA, SDLoc(N), N->getValueType(0), N00, SMin);
16522 // Combine (truncate_vector_vl (umin X, C)) -> (vnclipu_vl X) if C is the
16524 // Combine (truncate_vector_vl (smin (smax X, C2), C1)) -> (vnclip_vl X) if C1
16529 assert(N->getOpcode() == RISCVISD::TRUNCATE_VECTOR_VL);
16531 MVT VT = N->getSimpleValueType(0);
16533 SDValue Mask = N->getOperand(1);
16534 SDValue VL = N->getOperand(2);
16561 Op1->getAPIntValue().sextOrTrunc(Op.getScalarValueSizeInBits());
16624 SDValue Src = N->getOperand(0);
16667 SDValue Op = N->getOperand(OpNo);
16672 if (N->getOpcode() != ISD::DELETED_NODE)
16677 switch (N->getOpcode()) {
16681 SDValue Op0 = N->getOperand(0);
16684 if (Op0->getOpcode() == RISCVISD::BuildPairF64)
16687 if (Op0->isUndef()) {
16693 // It's cheaper to materialise two 32-bit integers than to load a double
16697 APInt V = C->getValueAPF().bitcastToAPInt();
16703 // This is a target-specific version of a DAGCombine performed in
16705 // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
16706 // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
16708 !Op0.getNode()->hasOneUse())
16749 SDValue Op0 = N->getOperand(0);
16757 SDValue Op0 = N->getOperand(0);
16758 MVT VT = N->getSimpleValueType(0);
16762 if ((N->getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64 &&
16763 Op0->getOpcode() == RISCVISD::FMV_W_X_RV64) ||
16764 (N->getOpcode() == RISCVISD::FMV_X_ANYEXTH &&
16765 Op0->getOpcode() == RISCVISD::FMV_H_X)) {
16771 // This is a target-specific version of a DAGCombine performed in
16773 // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
16774 // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
16776 !Op0.getNode()->hasOneUse())
16778 SDValue NewFMV = DAG.getNode(N->getOpcode(), DL, VT, Op0.getOperand(0));
16779 unsigned FPBits = N->getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64 ? 32 : 16;
16790 EVT VT = N->getValueType(0);
16791 SDValue N0 = N->getOperand(0);
16792 // abs (sext) -> zext (abs)
16793 // abs (zext) -> zext (handled elsewhere)
16855 if (N->getValueType(0) == MVT::i64 && Subtarget.is64Bit()) {
16856 SDValue Src = N->getOperand(0);
16883 SDValue Val = N->getOperand(0);
16884 SDValue Cond = N->getOperand(1);
16886 unsigned Opc = N->getOpcode();
16888 // czero_eqz x, x -> x
16895 // czero_eqz X, (xor Y, 1) -> czero_nez X, Y if Y is 0 or 1.
16896 // czero_nez X, (xor Y, 1) -> czero_eqz X, Y if Y is 0 or 1.
16901 return DAG.getNode(InvOpc, SDLoc(N), N->getValueType(0), Val, NewCond);
16903 // czero_eqz x, (setcc y, 0, ne) -> czero_eqz x, y
16904 // czero_nez x, (setcc y, 0, ne) -> czero_nez x, y
16905 // czero_eqz x, (setcc y, 0, eq) -> czero_nez x, y
16906 // czero_nez x, (setcc y, 0, eq) -> czero_eqz x, y
16908 ISD::CondCode CCVal = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
16911 N->getValueType(0), Val, Cond.getOperand(0));
16917 SDValue LHS = N->getOperand(0);
16918 SDValue RHS = N->getOperand(1);
16919 SDValue CC = N->getOperand(2);
16920 ISD::CondCode CCVal = cast<CondCodeSDNode>(CC)->get();
16921 SDValue TrueV = N->getOperand(3);
16922 SDValue FalseV = N->getOperand(4);
16924 EVT VT = N->getValueType(0);
16930 // (select (x < 0), y, z) -> x >> (XLEN - 1) & (y - z) + z
16931 // (select (x >= 0), y, z) -> x >> (XLEN - 1) & (z - y) + y
16938 int64_t TrueSImm = cast<ConstantSDNode>(TrueV)->getSExtValue();
16939 int64_t FalseSImm = cast<ConstantSDNode>(FalseV)->getSExtValue();
16943 isInt<12>(TrueSImm - FalseSImm)) {
16946 DAG.getConstant(Subtarget.getXLen() - 1, DL, VT));
16949 DAG.getConstant(TrueSImm - FalseSImm, DL, VT));
16958 return DAG.getNode(RISCVISD::SELECT_CC, DL, N->getValueType(0),
16962 // (select c, -1, y) -> -c | y
16968 // (select c, y, -1) -> -!c | y
16976 // (select c, 0, y) -> -!c & y
16983 // (select c, y, 0) -> -c & y
16989 // (riscvisd::select_cc x, 0, ne, x, 1) -> (add x, (setcc x, 0, eq))
16990 // (riscvisd::select_cc x, 0, eq, 1, x) -> (add x, (setcc x, 0, eq))
17019 SDValue LHS = N->getOperand(1);
17020 SDValue RHS = N->getOperand(2);
17021 SDValue CC = N->getOperand(3);
17025 return DAG.getNode(RISCVISD::BR_CC, DL, N->getValueType(0),
17026 N->getOperand(0), LHS, RHS, CC, N->getOperand(4));
17039 EVT VT = N->getValueType(0);
17046 SDValue In2 = N->getOperand(1);
17059 return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N->getOperand(0),
17064 const EVT VT = N->getValueType(0);
17065 SDValue Index = MGN->getIndex();
17066 SDValue ScaleOp = MGN->getScale();
17067 ISD::MemIndexType IndexType = MGN->getIndexType();
17068 assert(!MGN->isIndexScaled() &&
17074 N->getVTList(), MGN->getMemoryVT(), DL,
17075 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
17076 MGN->getBasePtr(), Index, ScaleOp},
17077 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
17081 N->getVTList(), MGN->getMemoryVT(), DL,
17082 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
17083 MGN->getBasePtr(), Index, ScaleOp},
17084 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
17087 MGN->getExtensionType() == ISD::NON_EXTLOAD && isTypeLegal(VT)) {
17092 SimpleVID && SimpleVID->StepDenominator == 1) {
17093 const int64_t StepNumerator = SimpleVID->StepNumerator;
17094 const int64_t Addend = SimpleVID->Addend;
17101 assert(MGN->getBasePtr()->getValueType(0) == PtrVT);
17103 SDValue BasePtr = DAG.getNode(ISD::ADD, DL, PtrVT, MGN->getBasePtr(),
17109 DAG.getStridedLoadVP(VT, DL, MGN->getChain(), BasePtr,
17111 MGN->getMask(), EVL, MGN->getMemOperand());
17112 SDValue VPSelect = DAG.getNode(ISD::VP_SELECT, DL, VT, MGN->getMask(),
17113 StridedLoad, MGN->getPassThru(), EVL);
17120 if (MGN->getExtensionType() == ISD::NON_EXTLOAD &&
17121 matchIndexAsShuffle(VT, Index, MGN->getMask(), ShuffleMask)) {
17122 SDValue Load = DAG.getMaskedLoad(VT, DL, MGN->getChain(),
17123 MGN->getBasePtr(), DAG.getUNDEF(XLenVT),
17124 MGN->getMask(), DAG.getUNDEF(VT),
17125 MGN->getMemoryVT(), MGN->getMemOperand(),
17132 if (MGN->getExtensionType() == ISD::NON_EXTLOAD &&
17133 matchIndexAsWiderOp(VT, Index, MGN->getMask(),
17134 MGN->getMemOperand()->getBaseAlign(), Subtarget)) {
17136 for (unsigned i = 0; i < Index->getNumOperands(); i += 2)
17148 SDValue Passthru = DAG.getBitcast(WideVT, MGN->getPassThru());
17155 {MGN->getChain(), Passthru, Mask, MGN->getBasePtr(),
17157 MGN->getMemOperand(), IndexType, ISD::NON_EXTLOAD);
17165 SDValue Index = MSN->getIndex();
17166 SDValue ScaleOp = MSN->getScale();
17167 ISD::MemIndexType IndexType = MSN->getIndexType();
17168 assert(!MSN->isIndexScaled() &&
17174 N->getVTList(), MSN->getMemoryVT(), DL,
17175 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
17177 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
17181 N->getVTList(), MSN->getMemoryVT(), DL,
17182 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
17184 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
17186 EVT VT = MSN->getValue()->getValueType(0);
17188 if (!MSN->isTruncatingStore() &&
17189 matchIndexAsShuffle(VT, Index, MSN->getMask(), ShuffleMask)) {
17190 SDValue Shuffle = DAG.getVectorShuffle(VT, DL, MSN->getValue(),
17192 return DAG.getMaskedStore(MSN->getChain(), DL, Shuffle, MSN->getBasePtr(),
17193 DAG.getUNDEF(XLenVT), MSN->getMask(),
17194 MSN->getMemoryVT(), MSN->getMemOperand(),
17201 SDValue Index = VPGN->getIndex();
17202 SDValue ScaleOp = VPGN->getScale();
17203 ISD::MemIndexType IndexType = VPGN->getIndexType();
17204 assert(!VPGN->isIndexScaled() &&
17209 return DAG.getGatherVP(N->getVTList(), VPGN->getMemoryVT(), DL,
17210 {VPGN->getChain(), VPGN->getBasePtr(), Index,
17211 ScaleOp, VPGN->getMask(),
17212 VPGN->getVectorLength()},
17213 VPGN->getMemOperand(), IndexType);
17216 return DAG.getGatherVP(N->getVTList(), VPGN->getMemoryVT(), DL,
17217 {VPGN->getChain(), VPGN->getBasePtr(), Index,
17218 ScaleOp, VPGN->getMask(),
17219 VPGN->getVectorLength()},
17220 VPGN->getMemOperand(), IndexType);
17226 SDValue Index = VPSN->getIndex();
17227 SDValue ScaleOp = VPSN->getScale();
17228 ISD::MemIndexType IndexType = VPSN->getIndexType();
17229 assert(!VPSN->isIndexScaled() &&
17234 return DAG.getScatterVP(N->getVTList(), VPSN->getMemoryVT(), DL,
17235 {VPSN->getChain(), VPSN->getValue(),
17236 VPSN->getBasePtr(), Index, ScaleOp,
17237 VPSN->getMask(), VPSN->getVectorLength()},
17238 VPSN->getMemOperand(), IndexType);
17241 return DAG.getScatterVP(N->getVTList(), VPSN->getMemoryVT(), DL,
17242 {VPSN->getChain(), VPSN->getValue(),
17243 VPSN->getBasePtr(), Index, ScaleOp,
17244 VPSN->getMask(), VPSN->getVectorLength()},
17245 VPSN->getMemOperand(), IndexType);
17254 SDValue ShAmt = N->getOperand(1);
17256 // We don't need the upper 32 bits of a 64-bit element for a shift amount.
17258 SDValue VL = N->getOperand(4);
17259 EVT VT = N->getValueType(0);
17262 return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt,
17263 N->getOperand(2), N->getOperand(3), N->getOperand(4));
17273 if (N->getOpcode() == ISD::SHL) {
17277 SDValue ShAmt = N->getOperand(1);
17279 // We don't need the upper 32 bits of a 64-bit element for a shift amount.
17281 EVT VT = N->getValueType(0);
17285 return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt);
17315 if (N->getValueType(0).getVectorElementType() == MVT::f32 &&
17326 if (N->getOpcode() != ISD::STORE)
17330 SDValue Chain = Store->getChain();
17331 EVT MemVT = Store->getMemoryVT();
17332 SDValue Val = Store->getValue();
17337 Store->isSimple() &&
17343 // any power-of-two size up to XLen bits, provided that they aren't too
17348 // ->
17367 NewVT, *Store->getMemOperand())) {
17369 return DAG.getStore(Chain, DL, NewV, Store->getBasePtr(),
17370 Store->getPointerInfo(), Store->getOriginalAlign(),
17371 Store->getMemOperand()->getFlags());
17380 L && DCI.isBeforeLegalize() && IsScalarizable && L->isSimple() &&
17381 L->hasNUsesOfValue(1, 0) && L->hasNUsesOfValue(1, 1) &&
17382 Store->getChain() == SDValue(L, 1) && ISD::isNormalLoad(L) &&
17383 L->getMemoryVT() == MemVT) {
17386 NewVT, *Store->getMemOperand()) &&
17388 NewVT, *L->getMemOperand())) {
17389 SDValue NewL = DAG.getLoad(NewVT, DL, L->getChain(), L->getBasePtr(),
17390 L->getPointerInfo(), L->getOriginalAlign(),
17391 L->getMemOperand()->getFlags());
17392 return DAG.getStore(Chain, DL, NewL, Store->getBasePtr(),
17393 Store->getPointerInfo(), Store->getOriginalAlign(),
17394 Store->getMemOperand()->getFlags());
17408 if (!Store->isIndexed() && VecVT.isScalableVector() &&
17413 Store->getChain(), DL, Src, Store->getBasePtr(), Store->getOffset(),
17416 Store->getMemOperand(), Store->getAddressingMode(),
17417 Store->isTruncatingStore(), /*IsCompress*/ false);
17424 EVT VT = N->getValueType(0);
17428 if (auto Gather = matchSplatAsGather(N->getOperand(0), VT.getSimpleVT(), N,
17446 const MVT VT = N->getSimpleValueType(0);
17447 SDValue Passthru = N->getOperand(0);
17448 SDValue Scalar = N->getOperand(1);
17449 SDValue VL = N->getOperand(2);
17457 const MVT VT = N->getSimpleValueType(0);
17458 SDValue Passthru = N->getOperand(0);
17459 SDValue Scalar = N->getOperand(1);
17460 SDValue VL = N->getOperand(2);
17474 (!Const || Const->isZero() ||
17475 !Const->getAPIntValue().sextOrTrunc(EltWidth).isSignedIntN(5)))
17481 SDValue Src = N->getOperand(1);
17482 // Try to remove vector->scalar->vector if the scalar->vector is inserting
17484 // TODO: Could use a vslide or vmv.v.v for non-undef.
17485 if (N->getOperand(0).isUndef() &&
17489 EVT VT = N->getValueType(0);
17500 const MVT VT = N->getSimpleValueType(0);
17501 SDValue Passthru = N->getOperand(0);
17502 SDValue Scalar = N->getOperand(1);
17503 SDValue VL = N->getOperand(2);
17506 Scalar.getOperand(0).getValueType() == N->getValueType(0))
17516 DAG.getNode(N->getOpcode(), DL, M1VT, M1Passthru, Scalar, VL);
17526 Const && !Const->isZero() && isInt<5>(Const->getSExtValue()) &&
17533 SDValue Vec = N->getOperand(0);
17534 MVT VecVT = N->getOperand(0).getSimpleValueType();
17539 return DAG.getNode(RISCVISD::VMV_X_S, DL, N->getSimpleValueType(0), Vec);
17546 unsigned IntOpNo = N->getOpcode() == ISD::INTRINSIC_WO_CHAIN ? 0 : 1;
17547 unsigned IntNo = N->getConstantOperandVal(IntOpNo);
17553 MVT VT = N->getSimpleValueType(0);
17555 SDValue PassThru = N->getOperand(2);
17556 SDValue Base = N->getOperand(3);
17557 SDValue Stride = N->getOperand(4);
17558 SDValue Mask = N->getOperand(5);
17564 StrideC && StrideC->getZExtValue() == ElementSize)
17565 return DAG.getMaskedLoad(VT, DL, Load->getChain(), Base,
17567 Load->getMemoryVT(), Load->getMemOperand(),
17573 SDValue Value = N->getOperand(2);
17574 SDValue Base = N->getOperand(3);
17575 SDValue Stride = N->getOperand(4);
17576 SDValue Mask = N->getOperand(5);
17582 StrideC && StrideC->getZExtValue() == ElementSize)
17583 return DAG.getMaskedStore(Store->getChain(), DL, Value, Base,
17585 Value.getValueType(), Store->getMemOperand(),
17593 SDValue VL = N->getOperand(2);
17596 VL = N->getOperand(3);
17599 // If VL is 0, vcpop -> li 0, vfirst -> li -1.
17601 EVT VT = N->getValueType(0);
17604 return DAG.getConstant(-1, DL, VT);
17611 SDValue N0 = N->getOperand(0);
17612 EVT VT = N->getValueType(0);
17655 assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA ||
17656 N->getOpcode() == ISD::SRL) &&
17662 // (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
17663 // (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
17664 SDValue N0 = N->getOperand(0);
17668 auto *C1 = dyn_cast<ConstantSDNode>(N0->getOperand(1));
17669 auto *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1));
17671 const APInt &C1Int = C1->getAPIntValue();
17672 APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
17724 const APInt &Mask = C->getAPIntValue();
17726 // Clear all non-demanded bits initially.
17733 auto IsLegalMask = [ShrunkMask, ExpandedMask](const APInt &Mask) -> bool {
17736 auto UseMask = [Mask, Op, &TLO](const APInt &NewMask) -> bool {
17781 else if (!C->isOpaque() && MinSignedBits <= 32 && !ShrunkMask.isSignedIntN(32))
17890 // FIXME: This is based on the non-ratified Zbp GREV and GORC where a
17990 // TODO: As the result is sign-extended, this is conservatively correct. A
17998 // element type is wider than XLen, the least-significant XLEN bits are
18003 return XLen - EltBits + 1;
18060 SDValue Ptr = Ld->getBasePtr();
18063 auto GetSupportedConstantPool = [](SDValue Ptr) -> ConstantPoolSDNode * {
18065 if (!CNode || CNode->isMachineConstantPoolEntry() ||
18066 CNode->getOffset() != 0)
18075 if (!CNode || CNode->getTargetFlags() != 0)
18078 return CNode->getConstVal();
18089 if (!CNodeLo || CNodeLo->getTargetFlags() != RISCVII::MO_LO ||
18090 !CNodeHi || CNodeHi->getTargetFlags() != RISCVII::MO_HI)
18093 if (CNodeLo->getConstVal() != CNodeHi->getConstVal())
18096 return CNodeLo->getConstVal();
18103 // To read a 64-bit counter CSR on a 32-bit target, we read the two halves.
18115 MachineFunction &MF = *BB->getParent();
18116 const BasicBlock *LLVMBB = BB->getBasicBlock();
18117 MachineFunction::iterator It = ++BB->getIterator();
18126 DoneMBB->splice(DoneMBB->begin(), BB,
18127 std::next(MachineBasicBlock::iterator(MI)), BB->end());
18128 DoneMBB->transferSuccessorsAndUpdatePHIs(BB);
18130 BB->addSuccessor(LoopMBB);
18141 BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), HiReg)
18144 BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), LoReg)
18147 BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), ReadAgainReg)
18151 BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
18156 LoopMBB->addSuccessor(LoopMBB);
18157 LoopMBB->addSuccessor(DoneMBB);
18169 MachineFunction &MF = *BB->getParent();
18178 int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);
18205 MachineFunction &MF = *BB->getParent();
18214 int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);
18260 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
18262 const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
18271 MIB->setFlag(MachineInstr::MIFlag::NoFPExcept);
18282 MIB2->setFlag(MachineInstr::MIFlag::NoFPExcept);
18294 // Without this, custom-inserter would have generated:
18330 const BasicBlock *LLVM_BB = ThisMBB->getBasicBlock();
18331 MachineFunction *F = ThisMBB->getParent();
18332 MachineBasicBlock *FirstMBB = F->CreateMachineBasicBlock(LLVM_BB);
18333 MachineBasicBlock *SecondMBB = F->CreateMachineBasicBlock(LLVM_BB);
18334 MachineBasicBlock *SinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
18335 MachineFunction::iterator It = ++ThisMBB->getIterator();
18336 F->insert(It, FirstMBB);
18337 F->insert(It, SecondMBB);
18338 F->insert(It, SinkMBB);
18341 SinkMBB->splice(SinkMBB->begin(), ThisMBB,
18343 ThisMBB->end());
18344 SinkMBB->transferSuccessorsAndUpdatePHIs(ThisMBB);
18347 ThisMBB->addSuccessor(FirstMBB);
18349 FirstMBB->addSuccessor(SecondMBB);
18350 ThisMBB->addSuccessor(SinkMBB);
18351 FirstMBB->addSuccessor(SinkMBB);
18353 SecondMBB->addSuccessor(SinkMBB);
18378 BuildMI(*SinkMBB, SinkMBB->begin(), DL, TII.get(RISCV::PHI), DestReg)
18396 // control-flow pattern. The incoming instructions know the destination vreg
18412 // - They are not pseudo instructions.
18413 // - They are debug instructions. Otherwise,
18414 // - They do not have side-effects, do not access memory and their inputs do
18415 // not depend on the results of the select pseudo-instructions.
18422 // is checked here and handled by a separate function -
18425 auto Next = next_nodbg(MI.getIterator(), BB->instr_end());
18428 Next != BB->end() && Next->getOpcode() == MI.getOpcode() &&
18429 Next->getOperand(5).getReg() == MI.getOperand(0).getReg() &&
18430 Next->getOperand(5).isKill())
18444 for (auto E = BB->end(), SequenceMBBI = MachineBasicBlock::iterator(MI);
18446 if (SequenceMBBI->isDebugInstr())
18449 if (SequenceMBBI->getOperand(1).getReg() != LHS ||
18450 !SequenceMBBI->getOperand(2).isReg() ||
18451 SequenceMBBI->getOperand(2).getReg() != RHS ||
18452 SequenceMBBI->getOperand(3).getImm() != CC ||
18453 SelectDests.count(SequenceMBBI->getOperand(4).getReg()) ||
18454 SelectDests.count(SequenceMBBI->getOperand(5).getReg()))
18457 SequenceMBBI->collectDebugValues(SelectDebugValues);
18458 SelectDests.insert(SequenceMBBI->getOperand(0).getReg());
18461 if (SequenceMBBI->hasUnmodeledSideEffects() ||
18462 SequenceMBBI->mayLoadOrStore() ||
18463 SequenceMBBI->usesCustomInsertionHook())
18465 if (llvm::any_of(SequenceMBBI->operands(), [&](MachineOperand &MO) {
18472 const BasicBlock *LLVM_BB = BB->getBasicBlock();
18474 MachineFunction::iterator I = ++BB->getIterator();
18477 MachineFunction *F = BB->getParent();
18478 MachineBasicBlock *TailMBB = F->CreateMachineBasicBlock(LLVM_BB);
18479 MachineBasicBlock *IfFalseMBB = F->CreateMachineBasicBlock(LLVM_BB);
18481 F->insert(I, IfFalseMBB);
18482 F->insert(I, TailMBB);
18486 IfFalseMBB->setCallFrameSize(CallFrameSize);
18487 TailMBB->setCallFrameSize(CallFrameSize);
18491 TailMBB->push_back(DebugInstr->removeFromParent());
18495 TailMBB->splice(TailMBB->end(), HeadMBB,
18496 std::next(LastSelectPseudo->getIterator()), HeadMBB->end());
18497 // Update machine-CFG edges by transferring all successors of the current
18499 TailMBB->transferSuccessorsAndUpdatePHIs(HeadMBB);
18501 HeadMBB->addSuccessor(IfFalseMBB);
18502 HeadMBB->addSuccessor(TailMBB);
18517 IfFalseMBB->addSuccessor(TailMBB);
18519 // Create PHIs for all of the select pseudo-instructions.
18521 auto SelectEnd = std::next(LastSelectPseudo->getIterator());
18522 auto InsertionPoint = TailMBB->begin();
18527 BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
18528 TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
18529 .addReg(SelectMBBI->getOperand(4).getReg())
18531 .addReg(SelectMBBI->getOperand(5).getReg())
18533 SelectMBBI->eraseFromParent();
18538 F->getProperties().reset(MachineFunctionProperties::Property::NoPHIs);
18549 RISCV::lookupMaskedIntrinsicByUnmasked(Inverse->Pseudo);
18559 const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
18561 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
18571 BB->getParent()->getSubtarget().getRegisterInfo();
18593 ->MaskedPseudo;
18657 assert(Subtarget.is64Bit() && "Expected 64-bit GPR.");
18666 assert(Subtarget.is64Bit() && "Expected 64-bit GPR.");
18676 const BasicBlock *BB = MBB->getBasicBlock();
18678 MachineFunction::iterator I = ++MBB->getIterator();
18680 MachineFunction *F = MBB->getParent();
18681 MachineBasicBlock *CvtMBB = F->CreateMachineBasicBlock(BB);
18682 MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(BB);
18684 F->insert(I, CvtMBB);
18685 F->insert(I, DoneMBB);
18687 DoneMBB->splice(DoneMBB->end(), MBB, MachineBasicBlock::iterator(MI),
18688 MBB->end());
18689 // Update machine-CFG edges by transferring all successors of the current
18691 DoneMBB->transferSuccessorsAndUpdatePHIs(MBB);
18693 MBB->addSuccessor(CvtMBB);
18694 MBB->addSuccessor(DoneMBB);
18702 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
18712 MIB->setFlag(MachineInstr::MIFlag::NoFPExcept);
18720 CvtMBB->addSuccessor(DoneMBB);
18726 MIB->setFlag(MachineInstr::MIFlag::NoFPExcept);
18732 MIB->setFlag(MachineInstr::MIFlag::NoFPExcept);
18739 BuildMI(*DoneMBB, DoneMBB->begin(), DL, TII.get(RISCV::PHI), DstReg)
18827 // has implicit def. This def is early-clobber as it will be set at
18840 "supported on 64-bit targets");
18868 // role of the frontend as simple and well-defined as possible. The rules can
18878 // * If a struct is less than 2*XLEN, then coerce to either a two-element
18879 // word-sized array or a 2*XLEN scalar (depending on alignment).
18886 // register-size fields in the same situations they would be for fixed
18931 // for save-restore libcall, so we don't use them.
18964 // TODO: To be compatible with GCC's behaviors, we force them to have 4-byte
18993 // Implements the RISC-V calling convention. Returns true upon failure.
19017 // UseGPRForF16_F32 if targeting one of the soft-float ABIs, if passing a
19020 // UseGPRForF64 if targeting soft-float ABIs or an FLEN=32 ABI, if passing a
19064 // If this is a variadic argument, the RISC-V calling convention requires
19065 // that it is assigned an 'even' or 'aligned' register if it has 8-byte
19066 // alignment (RV32) or 16-byte alignment (RV64). An aligned register should
19120 // Fixed-length vectors are located in the corresponding scalable-vector
19171 // FIXME: The proposed vector ABI only mandates v8-v15 for return values,
19183 // Pass fixed-length vectors on the stack.
19198 // If we reach this point and PendingLocs is non-empty, we must be at the
19225 // When a scalar floating-point value is passed on the stack, no
19226 // bit-conversion is needed.
19269 ArgTy = FType->getReturnType();
19271 ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
19294 for (const TargetLowering::ArgListEntry &Arg : CLI->getArgs())
19301 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty : nullptr;
19363 if (OrigArg->getType()->isIntegerTy()) {
19364 unsigned BitWidth = OrigArg->getType()->getIntegerBitWidth();
19369 RVFI->addSExt32Register(VReg);
19567 // Fixed-length vectors are located in the corresponding scalable-vector
19741 unsigned PartOffset = Ins[InsIdx + 1].PartOffset - ArgPartOffset;
19758 MF.getInfo<RISCVMachineFunctionInfo>()->setIsVectorCall();
19769 // zero or large enough to hold a0-a7.
19770 int VarArgsSaveSize = XLenInBytes * (ArgRegs.size() - Idx);
19779 int VaArgOffset = -VarArgsSaveSize;
19783 // ensure that the frame pointer is 2*XLEN-aligned, which in turn ensures
19784 // offsets to even-numbered registered remain 2*XLEN-aligned.
19787 XLenInBytes, VaArgOffset - static_cast<int>(XLenInBytes), true);
19801 MachinePointerInfo::getFixedStack(MF, FI, (I - Idx) * XLenInBytes));
19810 RVFI->setVarArgsFrameIndex(FI);
19811 RVFI->setVarArgsSaveSize(VarArgsSaveSize);
19824 /// isEligibleForTailCallOptimization - Check whether the call is eligible
19836 // Exception-handling functions need a special set of instructions to
19837 // indicate a return to the hardware. Tail-calling another function would
19839 // TODO: The "interrupt" attribute isn't currently defined by RISC-V. This
19869 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
19871 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
19872 if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
19929 else if (CLI.CB && CLI.CB->isMustTailCall())
20027 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
20039 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
20088 // Build a sequence of copy-to-reg nodes, chained and glued together.
20108 const GlobalValue *GV = S->getGlobal();
20111 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, RISCVII::MO_CALL);
20125 // Add a register mask operand representing the call-preserved registers.
20127 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
20136 assert((!CLI.CFIType || CLI.CB->isIndirectCall()) &&
20146 Ret.getNode()->setCFIType(CLI.CFIType->getZExtValue());
20153 Chain.getNode()->setCFIType(CLI.CFIType->getZExtValue());
20293 MF.getInfo<RISCVMachineFunctionInfo>()->setIsVectorCall();
20299 if (!Func.getReturnType()->isVoidTy())
20330 // otherwise we can't perform a tail-call.
20332 if (N->getNumValues() != 1)
20334 if (!N->hasNUsesOfValue(1, 0))
20337 SDNode *Copy = *N->use_begin();
20339 if (Copy->getOpcode() == ISD::BITCAST) {
20343 // TODO: Handle additional opcodes in order to support tail-calling libcalls
20345 if (Copy->getOpcode() != ISD::CopyToReg) {
20351 if (Copy->getOperand(Copy->getNumOperands() - 1).getValueType() == MVT::Glue)
20356 for (SDNode *Node : Copy->uses()) {
20357 if (Node->getOpcode() != RISCVISD::RET_GLUE)
20364 Chain = Copy->getOperand(0);
20369 return CI->isTailCall();
20376 // clang-format off
20631 // clang-format on
20636 /// getConstraintType - Given a constraint letter, return the type of
20667 // First, see if this is a constraint that directly corresponds to a RISC-V
20696 if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy))
20700 if (TRI->isTypeLegalForClass(RISCV::VMV0RegClass, VT.SimpleTy))
20706 // users of these frontends to use the ABI names for registers in LLVM-style
20788 assert(RISCV::F0_F <= FReg && FReg <= RISCV::F31_F && "Unknown fp-reg");
20790 unsigned RegNo = FReg - RISCV::F0_F;
20797 unsigned RegNo = FReg - RISCV::F0_F;
20840 if (TRI->isTypeLegalForClass(RISCV::VMRegClass, VT.SimpleTy))
20842 if (TRI->isTypeLegalForClass(RISCV::VRRegClass, VT.SimpleTy))
20846 if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy)) {
20847 VReg = TRI->getMatchingSuperReg(VReg, RISCV::sub_vrm1_0, RC);
20890 // Validate & create a 12-bit signed immediate operand.
20892 uint64_t CVal = C->getSExtValue();
20905 // Validate & create a 5-bit unsigned immediate operand.
20907 uint64_t CVal = C->getZExtValue();
20958 // atomicrmw {fadd,fsub} must be expanded to use compare-exchange, as floating
20960 // forward-progress guarantee.
20961 if (AI->isFloatingPointOperation() ||
20962 AI->getOperation() == AtomicRMWInst::UIncWrap ||
20963 AI->getOperation() == AtomicRMWInst::UDecWrap)
20970 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
20971 if (AI->getOperation() == AtomicRMWInst::Nand) {
20997 case AtomicRMWInst::Nand:
21020 case AtomicRMWInst::Nand:
21039 // In the case of an atomicrmw xchg with a constant 0/-1 operand, replace
21043 if (AI->getOperation() == AtomicRMWInst::Xchg &&
21044 isa<ConstantInt>(AI->getValOperand())) {
21045 ConstantInt *CVal = cast<ConstantInt>(AI->getValOperand());
21046 if (CVal->isZero())
21049 AI->getAlign(), Ord);
21050 if (CVal->isMinusOne())
21052 AI->getAlign(), Ord);
21057 Builder.getIntN(XLen, static_cast<uint64_t>(AI->getOrdering()));
21058 Type *Tys[] = {AlignedAddr->getType()};
21060 AI->getModule(),
21061 getIntrinsicForMaskedAtomicRMWBinOp(XLen, AI->getOperation()), Tys);
21073 // bits to shift the value into position. Pass XLen-ShiftAmt-ValWidth, which
21075 // sign-extend.
21076 if (AI->getOperation() == AtomicRMWInst::Min ||
21077 AI->getOperation() == AtomicRMWInst::Max) {
21078 const DataLayout &DL = AI->getDataLayout();
21080 DL.getTypeStoreSizeInBits(AI->getValOperand()->getType());
21082 Builder.CreateSub(Builder.getIntN(XLen, XLen - ValWidth), ShiftAmt);
21102 unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits();
21121 Type *Tys[] = {AlignedAddr->getType()};
21123 Intrinsic::getDeclaration(CI->getModule(), CmpXchgIntrID, Tys);
21173 return MCSymbolRefExpr::create(MBB->getSymbol(), Ctx);
21196 if (Op->getOpcode() != ISD::ADD && Op->getOpcode() != ISD::SUB)
21199 Base = Op->getOperand(0);
21200 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Op->getOperand(1))) {
21201 int64_t RHSC = RHS->getSExtValue();
21202 if (Op->getOpcode() == ISD::SUB)
21203 RHSC = -(uint64_t)RHSC;
21217 Offset = Op->getOperand(1);
21231 VT = LD->getMemoryVT();
21232 Ptr = LD->getBasePtr();
21234 VT = ST->getMemoryVT();
21235 Ptr = ST->getBasePtr();
21252 if (Op->getOpcode() != ISD::ADD)
21256 Base = LS->getBasePtr();
21260 if (Base == Op->getOperand(0))
21261 Offset = Op->getOperand(1);
21262 else if (Base == Op->getOperand(1))
21263 Offset = Op->getOperand(0);
21274 VT = LD->getMemoryVT();
21275 Ptr = LD->getBasePtr();
21277 VT = ST->getMemoryVT();
21278 Ptr = ST->getBasePtr();
21284 // Post-indexing updates the base, so it's not a valid transform
21361 const APInt &Imm = ConstNode->getAPIntValue();
21362 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
21363 (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
21368 ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
21369 (Imm - 8).isPowerOf2()))
21375 ConstNode->hasOneUse()) {
21377 if ((ImmS + 1).isPowerOf2() || (ImmS - 1).isPowerOf2() ||
21378 (1 - ImmS).isPowerOf2())
21400 const APInt &C1 = C1Node->getAPIntValue();
21401 const APInt &C2 = C2Node->getAPIntValue();
21444 // We use LMUL1 memory operations here for a non-obvious reason. Our caller
21458 // Prefer i8 for non-zero memset as it allows us to avoid materializing
21590 unsigned Opc = N->getOpcode();
21597 Module *M = IRB.GetInsertBlock()->getParent()->getParent();
21608 return useTpOffset(IRB, -0x10);
21614 return useTpOffset(IRB, -0x18);
21628 !allowsMemoryAccessForAlignment(VTy->getContext(), DL, VT, AddrSpace,
21639 if (FVTy->getNumElements() < 2)
21700 auto *VTy = cast<FixedVectorType>(Shuffles[0]->getType());
21701 if (!isLegalInterleavedAccessType(VTy, Factor, LI->getAlign(),
21702 LI->getPointerAddressSpace(),
21703 LI->getDataLayout()))
21706 auto *XLenTy = Type::getIntNTy(LI->getContext(), Subtarget.getXLen());
21709 Intrinsic::getDeclaration(LI->getModule(), FixedVlsegIntrIds[Factor - 2],
21710 {VTy, LI->getPointerOperandType(), XLenTy});
21712 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
21715 Builder.CreateCall(VlsegNFunc, {LI->getPointerOperand(), VL});
21719 Shuffles[i]->replaceAllUsesWith(SubVec);
21751 auto *ShuffleVTy = cast<FixedVectorType>(SVI->getType());
21753 auto *VTy = FixedVectorType::get(ShuffleVTy->getElementType(),
21754 ShuffleVTy->getNumElements() / Factor);
21755 if (!isLegalInterleavedAccessType(VTy, Factor, SI->getAlign(),
21756 SI->getPointerAddressSpace(),
21757 SI->getDataLayout()))
21760 auto *XLenTy = Type::getIntNTy(SI->getContext(), Subtarget.getXLen());
21763 Intrinsic::getDeclaration(SI->getModule(), FixedVssegIntrIds[Factor - 2],
21764 {VTy, SI->getPointerOperandType(), XLenTy});
21766 auto Mask = SVI->getShuffleMask();
21771 SVI->getOperand(0), SVI->getOperand(1),
21772 createSequentialMask(Mask[i], VTy->getNumElements(), 0));
21778 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
21779 Ops.append({SI->getPointerOperand(), VL});
21788 assert(LI->isSimple());
21792 if (DI->getIntrinsicID() != Intrinsic::vector_deinterleave2)
21797 VectorType *VTy = cast<VectorType>(DI->getOperand(0)->getType());
21798 VectorType *ResVTy = cast<VectorType>(DI->getType()->getContainedType(0));
21800 if (!isLegalInterleavedAccessType(ResVTy, Factor, LI->getAlign(),
21801 LI->getPointerAddressSpace(),
21802 LI->getDataLayout()))
21807 Type *XLenTy = Type::getIntNTy(LI->getContext(), Subtarget.getXLen());
21812 LI->getModule(), FixedVlsegIntrIds[Factor - 2],
21813 {ResVTy, LI->getPointerOperandType(), XLenTy});
21814 VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
21822 VlsegNFunc = Intrinsic::getDeclaration(LI->getModule(), IntrIds[Factor - 2],
21828 Ops.append({LI->getPointerOperand(), VL});
21831 DI->replaceAllUsesWith(Vlseg);
21838 assert(SI->isSimple());
21842 if (II->getIntrinsicID() != Intrinsic::vector_interleave2)
21847 VectorType *VTy = cast<VectorType>(II->getType());
21848 VectorType *InVTy = cast<VectorType>(II->getOperand(0)->getType());
21850 if (!isLegalInterleavedAccessType(InVTy, Factor, SI->getAlign(),
21851 SI->getPointerAddressSpace(),
21852 SI->getDataLayout()))
21857 Type *XLenTy = Type::getIntNTy(SI->getContext(), Subtarget.getXLen());
21861 SI->getModule(), FixedVssegIntrIds[Factor - 2],
21862 {InVTy, SI->getPointerOperandType(), XLenTy});
21863 VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
21871 VssegNFunc = Intrinsic::getDeclaration(SI->getModule(), IntrIds[Factor - 2],
21876 Builder.CreateCall(VssegNFunc, {II->getOperand(0), II->getOperand(1),
21877 SI->getPointerOperand(), VL});
21886 assert(MBBI->isCall() && MBBI->getCFIType() &&
21889 MBBI->getOpcode()));
21891 MachineOperand &Target = MBBI->getOperand(0);
21894 return BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(RISCV::KCFI_CHECK))
21896 .addImm(MBBI->getCFIType())
21912 BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF);
21914 report_fatal_error(Twine("Trying to obtain non-reserved register \"" +
21927 // 2 -> __RISCV_NTLH_INNERMOST_PRIVATE
21928 // 3 -> __RISCV_NTLH_ALL_PRIVATE
21929 // 4 -> __RISCV_NTLH_INNERMOST_SHARED
21930 // 5 -> __RISCV_NTLH_ALL
21933 I.getMetadata("riscv-nontemporal-domain");
21937 cast<ConstantAsMetadata>(RISCVNontemporalInfo->getOperand(0))
21938 ->getValue())
21939 ->getZExtValue();
21942 "RISC-V target doesn't support this non-temporal domain.");
21944 NontemporalLevel -= 2;
21957 MachineMemOperand::Flags NodeFlags = Node.getMemOperand()->getFlags();
21994 if (Inst.getType()->isScalableTy())
21998 if (Inst.getOperand(i)->getType()->isScalableTy() &&
22003 if (AI->getAllocatedType()->isScalableTy())
22015 if (isIntDivCheap(N->getValueType(0), Attr))
22021 EVT VT = N->getValueType(0);
22025 // Ensure 2**k-1 < 2048 so that we can just emit a single addi/addiw.
22026 if (Divisor.sgt(2048) || Divisor.slt(-2048))
22050 MVT FirstArgRegType = It->VT;
22053 if (It == ArgList.end() || It->Flags.isSplit())
22066 if (It->Flags.isSplit() || It->VT != FirstArgRegType)
22081 // Skip non-RVV register type
22086 RegisterVT = TLI->getContainerForFixedLengthVector(RegisterVT);
22102 const DataLayout &DL = MF->getDataLayout();
22103 const Function &F = MF->getFunction();
22109 if (STy && STy->containsHomogeneousScalableVectorTypes()) {
22110 Type *ElemTy = STy->getTypeAtIndex(0U);
22111 EVT VT = TLI->getValueType(DL, ElemTy);
22113 TLI->getRegisterTypeForCallingConv(Context, F.getCallingConv(), VT);
22115 TLI->getNumRegistersForCallingConv(Context, F.getCallingConv(), VT);
22118 {NumRegs * STy->getNumElements(), RegisterVT, false});
22127 TLI->getRegisterTypeForCallingConv(Context, F.getCallingConv(), VT);
22129 TLI->getNumRegistersForCallingConv(Context, F.getCallingConv(), VT);
22131 // Skip non-RVV register type
22136 RegisterVT = TLI->getContainerForFixedLengthVector(RegisterVT);
22142 --NumRegs;
22175 AllocatedPhysRegs.push_back(VRArrays[(StartReg - 8) / LMul + i]);
22197 uint32_t Map = ((1 << TotalRegsNeeded) - 1) << StartReg;