Lines Matching +full:mid +full:- +full:scale
1 //=- LoongArchISelLowering.cpp - LoongArch DAG Lowering Implementation ---===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
12 //===----------------------------------------------------------------------===//
37 #define DEBUG_TYPE "loongarch-isel-lowering"
41 static cl::opt<bool> ZeroDivCheck("loongarch-check-zero-division", cl::Hidden,
102 // Expand bitreverse.i16 with native-width bitrev and shift for now, before in LoongArchTargetLowering()
107 // LA32 does not have REVB.2W and REVB.D due to the 64-bit operands, and in LoongArchTargetLowering()
109 // and i32 could still be byte-swapped relatively cheaply. in LoongArchTargetLowering()
443 if (*I != -1 && *I != ExpectedIndex) in fitsRegularPattern()
467 int SplatIndex = -1; in lowerVECTOR_SHUFFLE_VREPLVEI()
469 if (M != -1) { in lowerVECTOR_SHUFFLE_VREPLVEI()
475 if (SplatIndex == -1) in lowerVECTOR_SHUFFLE_VREPLVEI()
514 int SubMask[4] = {-1, -1, -1, -1}; in lowerVECTOR_SHUFFLE_VSHUF4I()
519 // Convert from vector index to 4-element subvector index in lowerVECTOR_SHUFFLE_VSHUF4I()
521 if (Idx != -1) { in lowerVECTOR_SHUFFLE_VSHUF4I()
522 Idx -= 4 * (j / 4); in lowerVECTOR_SHUFFLE_VSHUF4I()
529 if (SubMask[i] == -1) in lowerVECTOR_SHUFFLE_VSHUF4I()
531 // Check that non-undef values are the same as in the mask. If they in lowerVECTOR_SHUFFLE_VSHUF4I()
533 else if (Idx != -1 && Idx != SubMask[i]) in lowerVECTOR_SHUFFLE_VSHUF4I()
540 for (int i = 3; i >= 0; --i) { in lowerVECTOR_SHUFFLE_VSHUF4I()
543 if (Idx == -1) in lowerVECTOR_SHUFFLE_VSHUF4I()
636 /// VILVH interleaves consecutive elements from the left (highest-indexed) half
679 /// VILVL interleaves consecutive elements from the right (lowest-indexed) half
738 const auto &Mid = Mask.begin() + Mask.size() / 2; in lowerVECTOR_SHUFFLE_VPICKEV() local
742 if (fitsRegularPattern<int>(Begin, 1, Mid, 0, 2)) in lowerVECTOR_SHUFFLE_VPICKEV()
744 else if (fitsRegularPattern<int>(Begin, 1, Mid, Mask.size(), 2)) in lowerVECTOR_SHUFFLE_VPICKEV()
749 if (fitsRegularPattern<int>(Mid, 1, End, 0, 2)) in lowerVECTOR_SHUFFLE_VPICKEV()
751 else if (fitsRegularPattern<int>(Mid, 1, End, Mask.size(), 2)) in lowerVECTOR_SHUFFLE_VPICKEV()
780 const auto &Mid = Mask.begin() + Mask.size() / 2; in lowerVECTOR_SHUFFLE_VPICKOD() local
784 if (fitsRegularPattern<int>(Begin, 1, Mid, 1, 2)) in lowerVECTOR_SHUFFLE_VPICKOD()
786 else if (fitsRegularPattern<int>(Begin, 1, Mid, Mask.size() + 1, 2)) in lowerVECTOR_SHUFFLE_VPICKOD()
791 if (fitsRegularPattern<int>(Mid, 1, End, 1, 2)) in lowerVECTOR_SHUFFLE_VPICKOD()
793 else if (fitsRegularPattern<int>(Mid, 1, End, Mask.size() + 1, 2)) in lowerVECTOR_SHUFFLE_VPICKOD()
817 // <0b00, 0b01> + <0b10, 0b11> -> <0b00, 0b01, 0b10, 0b11> in lowerVECTOR_SHUFFLE_VSHUF()
819 // <0b00, 0b01> + <0b10, 0b11> -> in lowerVECTOR_SHUFFLE_VSHUF()
820 // 0b0100 + 0b1110 -> 0b01001110 in lowerVECTOR_SHUFFLE_VSHUF()
826 /// Dispatching routine to lower various 128-bit LoongArch vector shuffles.
828 /// This routine breaks down the specific type of 128-bit shuffle and
887 int SplatIndex = -1; in lowerVECTOR_SHUFFLE_XVREPLVEI()
889 if (M != -1) { in lowerVECTOR_SHUFFLE_XVREPLVEI()
895 if (SplatIndex == -1) in lowerVECTOR_SHUFFLE_XVREPLVEI()
903 if (fitsRegularPattern<int>(Begin, 1, End - HalfSize, SplatIndex, 0) && in lowerVECTOR_SHUFFLE_XVREPLVEI()
950 if (fitsRegularPattern<int>(Begin, 2, End - HalfSize, HalfSize - LeftSize, in lowerVECTOR_SHUFFLE_XVILVH()
954 else if (fitsRegularPattern<int>(Begin, 2, End - HalfSize, in lowerVECTOR_SHUFFLE_XVILVH()
955 Mask.size() + HalfSize - LeftSize, 1) && in lowerVECTOR_SHUFFLE_XVILVH()
962 if (fitsRegularPattern<int>(Begin + 1, 2, End - HalfSize, HalfSize - LeftSize, in lowerVECTOR_SHUFFLE_XVILVH()
967 else if (fitsRegularPattern<int>(Begin + 1, 2, End - HalfSize, in lowerVECTOR_SHUFFLE_XVILVH()
968 Mask.size() + HalfSize - LeftSize, 1) && in lowerVECTOR_SHUFFLE_XVILVH()
988 if (fitsRegularPattern<int>(Begin, 2, End - HalfSize, 0, 1) && in lowerVECTOR_SHUFFLE_XVILVL()
991 else if (fitsRegularPattern<int>(Begin, 2, End - HalfSize, Mask.size(), 1) && in lowerVECTOR_SHUFFLE_XVILVL()
998 if (fitsRegularPattern<int>(Begin + 1, 2, End - HalfSize, 0, 1) && in lowerVECTOR_SHUFFLE_XVILVL()
1001 else if (fitsRegularPattern<int>(Begin + 1, 2, End - HalfSize, Mask.size(), in lowerVECTOR_SHUFFLE_XVILVL()
1019 const auto &Mid = Mask.begin() + Mask.size() / 2; in lowerVECTOR_SHUFFLE_XVPICKEV() local
1020 const auto &RightMid = Mask.end() - Mask.size() / 4; in lowerVECTOR_SHUFFLE_XVPICKEV()
1026 fitsRegularPattern<int>(Mid, 1, RightMid, HalfSize, 2)) in lowerVECTOR_SHUFFLE_XVPICKEV()
1029 fitsRegularPattern<int>(Mid, 1, RightMid, Mask.size() + HalfSize, 2)) in lowerVECTOR_SHUFFLE_XVPICKEV()
1034 if (fitsRegularPattern<int>(LeftMid, 1, Mid, 0, 2) && in lowerVECTOR_SHUFFLE_XVPICKEV()
1037 else if (fitsRegularPattern<int>(LeftMid, 1, Mid, Mask.size(), 2) && in lowerVECTOR_SHUFFLE_XVPICKEV()
1054 const auto &Mid = Mask.begin() + Mask.size() / 2; in lowerVECTOR_SHUFFLE_XVPICKOD() local
1055 const auto &RightMid = Mask.end() - Mask.size() / 4; in lowerVECTOR_SHUFFLE_XVPICKOD()
1061 fitsRegularPattern<int>(Mid, 1, RightMid, HalfSize + 1, 2)) in lowerVECTOR_SHUFFLE_XVPICKOD()
1064 fitsRegularPattern<int>(Mid, 1, RightMid, Mask.size() + HalfSize + 1, in lowerVECTOR_SHUFFLE_XVPICKOD()
1070 if (fitsRegularPattern<int>(LeftMid, 1, Mid, 1, 2) && in lowerVECTOR_SHUFFLE_XVPICKOD()
1073 else if (fitsRegularPattern<int>(LeftMid, 1, Mid, Mask.size() + 1, 2) && in lowerVECTOR_SHUFFLE_XVPICKOD()
1091 const auto &Mid = Mask.begin() + HalfSize; in lowerVECTOR_SHUFFLE_XVSHUF() local
1096 // shuffling -> in lowerVECTOR_SHUFFLE_XVSHUF()
1101 // shuffling -> in lowerVECTOR_SHUFFLE_XVSHUF()
1104 for (auto it = Begin; it < Mid; it++) { in lowerVECTOR_SHUFFLE_XVSHUF()
1109 int M = *it < HalfSize ? *it : *it - HalfSize; in lowerVECTOR_SHUFFLE_XVSHUF()
1116 for (auto it = Mid; it < End; it++) { in lowerVECTOR_SHUFFLE_XVSHUF()
1121 int M = *it < MaskSize ? *it - HalfSize : *it - MaskSize; in lowerVECTOR_SHUFFLE_XVSHUF()
1134 /// 256-bit shuffles are always considered as 2-lane 128-bit shuffles.
1137 /// as cross-lane shuffles, where optimization is relatively limited.
1139 /// - Shuffle high, low lanes of two inputs vector
1140 /// <0, 1, 2, 3> + <4, 5, 6, 7> --- <0, 5, 3, 6>
1141 /// - Shuffle low, high lanes of two inputs vector
1142 /// <0, 1, 2, 3> + <4, 5, 6, 7> --- <3, 6, 0, 5>
1143 /// - Shuffle low, low lanes of two inputs vector
1144 /// <0, 1, 2, 3> + <4, 5, 6, 7> --- <3, 6, 3, 6>
1145 /// - Shuffle high, high lanes of two inputs vector
1146 /// <0, 1, 2, 3> + <4, 5, 6, 7> --- <0, 5, 0, 5>
1186 // The pre-half of mask is high lane type, and the post-half of mask in canonicalizeShuffleVectorByLane()
1190 // to the lower 128-bit of vector register, and the low lane of mask in canonicalizeShuffleVectorByLane()
1191 // corresponds the higher 128-bit of vector register. in canonicalizeShuffleVectorByLane()
1209 *it = *it < 0 ? *it : *it - HalfSize; in canonicalizeShuffleVectorByLane()
1228 *it = *it < 0 ? *it : *it - HalfSize; in canonicalizeShuffleVectorByLane()
1246 } else { // cross-lane in canonicalizeShuffleVectorByLane()
1251 /// Dispatching routine to lower various 256-bit LoongArch vector shuffles.
1253 /// This routine breaks down the specific type of 256-bit shuffle and
1268 // canonicalize non cross-lane shuffle vector in lower256BitShuffle()
1308 ArrayRef<int> OrigMask = SVOp->getMask(); in lowerVECTOR_SHUFFLE()
1326 // Check for non-undef masks pointing at an undef vector and make the masks in lowerVECTOR_SHUFFLE()
1334 M = -1; in lowerVECTOR_SHUFFLE()
1342 [&](int M) { return -1 <= M && M < MaskUpperLimit; }) && in lowerVECTOR_SHUFFLE()
1356 if (Op->isUndef()) in isConstantOrUndef()
1366 for (unsigned i = 0; i < Op->getNumOperands(); ++i) in isConstantOrUndefBUILD_VECTOR()
1367 if (isConstantOrUndef(Op->getOperand(i))) in isConstantOrUndefBUILD_VECTOR()
1375 EVT ResTy = Op->getValueType(0); in lowerBUILD_VECTOR()
1387 if (Node->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, HasAnyUndefs, in lowerBUILD_VECTOR()
1390 // We can only cope with 8, 16, 32, or 64-bit elements. in lowerBUILD_VECTOR()
1431 EVT ResTy = Node->getValueType(0); in lowerBUILD_VECTOR()
1439 Node->getOperand(i), in lowerBUILD_VECTOR()
1451 EVT VecTy = Op->getOperand(0)->getValueType(0); in lowerEXTRACT_VECTOR_ELT()
1452 SDValue Idx = Op->getOperand(1); in lowerEXTRACT_VECTOR_ELT()
1458 EltTy == MVT::f64 || Idx->getAsZExtVal() < NumElts / 2)) in lowerEXTRACT_VECTOR_ELT()
1467 if (isa<ConstantSDNode>(Op->getOperand(2))) in lowerINSERT_VECTOR_ELT()
1482 // MEMBARRIER is a compiler barrier; it codegens to a no-op. in lowerATOMIC_FENCE()
1492 DAG.getContext()->emitError( in lowerWRITE_REGISTER()
1493 "On LA64, only 64-bit registers can be written."); in lowerWRITE_REGISTER()
1498 DAG.getContext()->emitError( in lowerWRITE_REGISTER()
1499 "On LA32, only 32-bit registers can be written."); in lowerWRITE_REGISTER()
1509 DAG.getContext()->emitError("argument to '__builtin_frame_address' must " in lowerFRAMEADDR()
1516 Register FrameReg = Subtarget.getRegisterInfo()->getFrameRegister(MF); in lowerFRAMEADDR()
1523 while (Depth--) { in lowerFRAMEADDR()
1524 int Offset = -(GRLenInBytes * 2); in lowerFRAMEADDR()
1540 DAG.getContext()->emitError( in lowerRETURNADDR()
1550 // live-in. in lowerRETURNADDR()
1551 Register Reg = MF.addLiveIn(Subtarget.getRegisterInfo()->getRARegister(), in lowerRETURNADDR()
1570 SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), in lowerVASTART()
1575 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); in lowerVASTART()
1587 if (Op0->getOpcode() == ISD::AND) { in lowerUINT_TO_FP()
1589 if (C && C->getZExtValue() < UINT64_C(0xFFFFFFFF)) in lowerUINT_TO_FP()
1593 if (Op0->getOpcode() == LoongArchISD::BSTRPICK && in lowerUINT_TO_FP()
1599 dyn_cast<VTSDNode>(Op0.getOperand(1))->getVT().bitsLT(MVT::i32)) in lowerUINT_TO_FP()
1624 dyn_cast<VTSDNode>(Op0.getOperand(1))->getVT().bitsLE(MVT::i32)) in lowerSINT_TO_FP()
1672 return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags); in getTargetNode()
1677 return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, N->getOffset(), in getTargetNode()
1683 return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlign(), in getTargetNode()
1684 N->getOffset(), Flags); in getTargetNode()
1689 return DAG.getTargetJumpTable(N->getIndex(), Ty, Flags); in getTargetNode()
1713 // eventually becomes the desired 5-insn code sequence. in getAddr()
1719 // eventually becomes the desired 5-insn code sequence. in getAddr()
1777 assert(N->getOffset() == 0 && "unexpected offset in global node"); in lowerGlobalAddress()
1779 const GlobalValue *GV = N->getGlobal(); in lowerGlobalAddress()
1781 if (GV->isDSOLocal() && isa<GlobalVariable>(GV)) { in lowerGlobalAddress()
1782 if (auto GCM = dyn_cast<GlobalVariable>(GV)->getCodeModel()) in lowerGlobalAddress()
1786 return getAddr(N, DAG, CM, GV->isDSOLocal()); in lowerGlobalAddress()
1800 SDValue Addr = DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, 0); in getStaticTLSAddr()
1832 // Use a PC-relative addressing mode to access the dynamic GOT address. in getDynamicTLSAddr()
1833 SDValue Addr = DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, 0); in getDynamicTLSAddr()
1860 const GlobalValue *GV = N->getGlobal(); in getTLSDescAddr()
1866 // Use a PC-relative addressing mode to access the global dynamic GOT address. in getTLSDescAddr()
1884 assert(N->getOffset() == 0 && "unexpected offset in global node"); in lowerGlobalTLSAddress()
1892 switch (getTargetMachine().getTLSModel(N->getGlobal())) { in lowerGlobalTLSAddress()
1936 auto *CImm = cast<ConstantSDNode>(Op->getOperand(ImmOp)); in checkIntrinsicImmArg()
1938 if ((IsSigned && !isInt<N>(CImm->getSExtValue())) || in checkIntrinsicImmArg()
1939 (!IsSigned && !isUInt<N>(CImm->getZExtValue()))) { in checkIntrinsicImmArg()
1940 DAG.getContext()->emitError(Op->getOperationName(0) + in checkIntrinsicImmArg()
2231 DAG.getContext()->emitError(Op->getOperationName(0) + ": " + ErrorMsg + "."); in emitIntrinsicWithChainErrorMessage()
2322 return !isInt<12>(cast<ConstantSDNode>(Op.getOperand(3))->getSExtValue()) in lowerINTRINSIC_W_CHAIN()
2328 cast<ConstantSDNode>(Op.getOperand(3))->getSExtValue()) in lowerINTRINSIC_W_CHAIN()
2335 cast<ConstantSDNode>(Op.getOperand(3))->getSExtValue()) in lowerINTRINSIC_W_CHAIN()
2342 cast<ConstantSDNode>(Op.getOperand(3))->getSExtValue()) in lowerINTRINSIC_W_CHAIN()
2354 DAG.getContext()->emitError(Op->getOperationName(0) + ": " + ErrorMsg + "."); in emitIntrinsicErrorMessage()
2381 unsigned Imm1 = Op2->getAsZExtVal(); in lowerINTRINSIC_VOID()
2382 int Imm2 = cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue(); in lowerINTRINSIC_VOID()
2388 unsigned Imm = Op2->getAsZExtVal(); in lowerINTRINSIC_VOID()
2395 unsigned Imm = Op2->getAsZExtVal(); in lowerINTRINSIC_VOID()
2402 unsigned Imm = Op2->getAsZExtVal(); in lowerINTRINSIC_VOID()
2411 unsigned Imm = Op2->getAsZExtVal(); in lowerINTRINSIC_VOID()
2420 unsigned Imm = Op2->getAsZExtVal(); in lowerINTRINSIC_VOID()
2466 return !isInt<12>(cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue()) in lowerINTRINSIC_VOID()
2470 return (!isInt<8>(cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue()) || in lowerINTRINSIC_VOID()
2475 return (!isInt<8>(cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue()) || in lowerINTRINSIC_VOID()
2481 cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue()) || in lowerINTRINSIC_VOID()
2488 cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue()) || in lowerINTRINSIC_VOID()
2495 cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue()) || in lowerINTRINSIC_VOID()
2502 cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue()) || in lowerINTRINSIC_VOID()
2509 cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue()) || in lowerINTRINSIC_VOID()
2516 cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue()) || in lowerINTRINSIC_VOID()
2532 // if Shamt-GRLen < 0: // Shamt < GRLen in lowerShiftLeftParts()
2534 // Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (GRLen-1 ^ Shamt)) in lowerShiftLeftParts()
2537 // Hi = Lo << (Shamt-GRLen) in lowerShiftLeftParts()
2541 SDValue MinusGRLen = DAG.getConstant(-(int)Subtarget.getGRLen(), DL, VT); in lowerShiftLeftParts()
2542 SDValue GRLenMinus1 = DAG.getConstant(Subtarget.getGRLen() - 1, DL, VT); in lowerShiftLeftParts()
2573 // if Shamt-GRLen < 0: // Shamt < GRLen in lowerShiftRightParts()
2574 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (ShAmt ^ GRLen-1)) in lowerShiftRightParts()
2577 // Lo = Hi >>s (Shamt-GRLen); in lowerShiftRightParts()
2578 // Hi = Hi >>s (GRLen-1) in lowerShiftRightParts()
2581 // if Shamt-GRLen < 0: // Shamt < GRLen in lowerShiftRightParts()
2582 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (ShAmt ^ GRLen-1)) in lowerShiftRightParts()
2585 // Lo = Hi >>u (Shamt-GRLen); in lowerShiftRightParts()
2592 SDValue MinusGRLen = DAG.getConstant(-(int)Subtarget.getGRLen(), DL, VT); in lowerShiftRightParts()
2593 SDValue GRLenMinus1 = DAG.getConstant(Subtarget.getGRLen() - 1, DL, VT); in lowerShiftRightParts()
2616 // Returns the opcode of the target-specific SDNode that implements the 32-bit
2642 // Converts the given i8/i16/i32 operation to a target-specific SelectionDAG
2650 LoongArchISD::NodeType WOpcode = getLoongArchWOpcode(N->getOpcode()); in customLegalizeToWOp()
2657 NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0)); in customLegalizeToWOp()
2662 NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0)); in customLegalizeToWOp()
2663 SDValue NewOp1 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(1)); in customLegalizeToWOp()
2664 if (N->getOpcode() == ISD::ROTL) { in customLegalizeToWOp()
2676 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes); in customLegalizeToWOp()
2679 // Converts the given 32-bit operation to a i64 operation with signed extension
2683 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0)); in customLegalizeToWOpWithSExt()
2684 SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1)); in customLegalizeToWOpWithSExt()
2685 SDValue NewWOp = DAG.getNode(N->getOpcode(), DL, MVT::i64, NewOp0, NewOp1); in customLegalizeToWOpWithSExt()
2696 DAG.getContext()->emitError(N->getOperationName(0) + ": " + ErrorMsg + "."); in emitErrorAndReplaceIntrinsicResults()
2697 Results.push_back(DAG.getUNDEF(N->getValueType(0))); in emitErrorAndReplaceIntrinsicResults()
2700 Results.push_back(N->getOperand(0)); in emitErrorAndReplaceIntrinsicResults()
2709 unsigned Imm = Node->getConstantOperandVal(2); in replaceVPICKVE2GRResults()
2716 SDValue Vec = Node->getOperand(1); in replaceVPICKVE2GRResults()
2722 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, Node->getValueType(0), in replaceVPICKVE2GRResults()
2732 SDValue Vec = N->getOperand(1); in replaceVecCondBranchResults()
2736 DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), CB.getValue(0))); in replaceVecCondBranchResults()
2743 switch (N->getConstantOperandVal(0)) { in replaceINTRINSIC_WO_CHAINResults()
2810 EVT VT = N->getValueType(0); in ReplaceNodeResults()
2811 switch (N->getOpcode()) { in ReplaceNodeResults()
2816 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && in ReplaceNodeResults()
2831 if (N->getOperand(1).getOpcode() != ISD::Constant) { in ReplaceNodeResults()
2845 SDValue Src = N->getOperand(0); in ReplaceNodeResults()
2846 EVT FVT = EVT::getFloatingPointVT(N->getValueSizeInBits(0)); in ReplaceNodeResults()
2868 SDValue Src = N->getOperand(0); in ReplaceNodeResults()
2888 SDValue Src = N->getOperand(0); in ReplaceNodeResults()
2910 SDValue Src = N->getOperand(0); in ReplaceNodeResults()
2937 SDValue Chain = N->getOperand(0); in ReplaceNodeResults()
2938 SDValue Op2 = N->getOperand(2); in ReplaceNodeResults()
2944 switch (N->getConstantOperandVal(1)) { in ReplaceNodeResults()
2952 unsigned Imm = Op2->getAsZExtVal(); in ReplaceNodeResults()
2970 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3))}); \ in ReplaceNodeResults()
2988 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3))}); \ in ReplaceNodeResults()
3008 unsigned Imm = Op2->getAsZExtVal(); in ReplaceNodeResults()
3022 unsigned Imm = N->getConstantOperandVal(3); in ReplaceNodeResults()
3037 unsigned Imm = N->getConstantOperandVal(4); in ReplaceNodeResults()
3045 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3)), in ReplaceNodeResults()
3087 DAG.getContext()->emitError( in ReplaceNodeResults()
3088 "On LA64, only 64-bit registers can be read."); in ReplaceNodeResults()
3090 DAG.getContext()->emitError( in ReplaceNodeResults()
3091 "On LA32, only 32-bit registers can be read."); in ReplaceNodeResults()
3093 Results.push_back(N->getOperand(0)); in ReplaceNodeResults()
3109 SDValue FirstOperand = N->getOperand(0); in performANDCombine()
3110 SDValue SecondOperand = N->getOperand(1); in performANDCombine()
3112 EVT ValTy = N->getValueType(0); in performANDCombine()
3122 !isShiftedMask_64(CN->getZExtValue(), SMIdx, SMLen)) in performANDCombine()
3127 // $dst = and ((sra or srl) $src , lsb), (2**len - 1) in performANDCombine()
3129 // where msb = lsb + len - 1 in performANDCombine()
3135 lsb = CN->getZExtValue(); in performANDCombine()
3145 // $dst = and $src, (2**len- 1) , if len > 12 in performANDCombine()
3147 // where lsb = 0 and msb = len - 1 in performANDCombine()
3150 if (CN->getZExtValue() <= 0xfff) in performANDCombine()
3161 if (CN->use_size() > 2) in performANDCombine()
3164 if ((CN->getZExtValue() & 0xfff) == 0) in performANDCombine()
3168 if (CN->getSExtValue() >= -2048 && CN->getSExtValue() < 0) in performANDCombine()
3176 msb = lsb + SMLen - 1; in performANDCombine()
3197 // BSTRPICK $dst, $src, MaskIdx+MaskLen-1, Shamt in performSRLCombine()
3198 // when Mask is a shifted mask, and MaskIdx <= Shamt <= MaskIdx+MaskLen-1 in performSRLCombine()
3201 SDValue FirstOperand = N->getOperand(0); in performSRLCombine()
3203 EVT ValTy = N->getValueType(0); in performSRLCombine()
3213 !isShiftedMask_64(CN->getZExtValue(), MaskIdx, MaskLen)) in performSRLCombine()
3217 if (!(CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))) in performSRLCombine()
3220 Shamt = CN->getZExtValue(); in performSRLCombine()
3221 if (MaskIdx <= Shamt && Shamt <= MaskIdx + MaskLen - 1) in performSRLCombine()
3223 FirstOperand->getOperand(0), in performSRLCombine()
3224 DAG.getConstant(MaskIdx + MaskLen - 1, DL, GRLenVT), in performSRLCombine()
3234 EVT ValTy = N->getValueType(0); in performORCombine()
3235 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1); in performORCombine()
3252 // where mask1 = (2**size - 1) << lsb, mask0 = ~mask1 in performORCombine()
3254 // R = BSTRINS X, Y, msb, lsb (where msb = lsb + size - 1) in performORCombine()
3257 isShiftedMask_64(~CN0->getSExtValue(), MaskIdx0, MaskLen0) && in performORCombine()
3260 isShiftedMask_64(CN1->getZExtValue(), MaskIdx1, MaskLen1) && in performORCombine()
3263 (Shamt = CN1->getZExtValue()) == MaskIdx0 && in performORCombine()
3268 DAG.getConstant((MaskIdx0 + MaskLen0 - 1), DL, GRLenVT), in performORCombine()
3274 // where mask1 = (2**size - 1), mask0 = ~(mask1 << lsb) in performORCombine()
3276 // R = BSTRINS X, Y, msb, lsb (where msb = lsb + size - 1) in performORCombine()
3279 isShiftedMask_64(~CN0->getSExtValue(), MaskIdx0, MaskLen0) && in performORCombine()
3282 (Shamt = CN1->getZExtValue()) == MaskIdx0 && in performORCombine()
3284 isShiftedMask_64(CN1->getZExtValue(), MaskIdx1, MaskLen1) && in performORCombine()
3290 DAG.getConstant((MaskIdx0 + MaskLen0 - 1), DL, GRLenVT), in performORCombine()
3296 // where ~mask0 = (2**size - 1) << lsb, mask0 & mask1 = 0 in performORCombine()
3299 // where msb = lsb + size - 1 in performORCombine()
3302 isShiftedMask_64(~CN0->getSExtValue(), MaskIdx0, MaskLen0) && in performORCombine()
3304 (CN1 = dyn_cast<ConstantSDNode>(N1->getOperand(1))) && in performORCombine()
3305 (CN1->getSExtValue() & CN0->getSExtValue()) == 0) { in performORCombine()
3308 DAG.getNode(ISD::SRL, DL, N1->getValueType(0), N1, in performORCombine()
3311 ? (MaskIdx0 + (MaskLen0 & 31) - 1) in performORCombine()
3312 : (MaskIdx0 + MaskLen0 - 1), in performORCombine()
3319 // where mask = (2**shamt - 1) in performORCombine()
3321 // R = BSTRINS X, Y, ValBits - 1, shamt in performORCombine()
3325 isShiftedMask_64(CN0->getZExtValue(), MaskIdx0, MaskLen0) && in performORCombine()
3327 (Shamt = CN1->getZExtValue()) == MaskLen0 && in performORCombine()
3332 DAG.getConstant((ValBits - 1), DL, GRLenVT), in performORCombine()
3338 // where ~mask = (2**size - 1) << lsb, mask & const = 0 in performORCombine()
3341 // where msb = lsb + size - 1 in performORCombine()
3344 isShiftedMask_64(~CN0->getSExtValue(), MaskIdx0, MaskLen0) && in performORCombine()
3346 (CN1->getSExtValue() & CN0->getSExtValue()) == 0) { in performORCombine()
3350 DAG.getConstant(CN1->getSExtValue() >> MaskIdx0, DL, ValTy), in performORCombine()
3351 DAG.getConstant(ValBits == 32 ? (MaskIdx0 + (MaskLen0 & 31) - 1) in performORCombine()
3352 : (MaskIdx0 + MaskLen0 - 1), in performORCombine()
3361 // a = BSTRINS b, c, shamt + MaskLen - 1, shamt in performORCombine()
3370 isShiftedMask_64(CNMask->getZExtValue(), MaskIdx, MaskLen) && in performORCombine()
3372 CNShamt->getZExtValue() + MaskLen <= ValBits) { in performORCombine()
3373 Shamt = CNShamt->getZExtValue(); in performORCombine()
3374 APInt ShMask(ValBits, CNMask->getZExtValue() << Shamt); in performORCombine()
3379 DAG.getConstant(Shamt + MaskLen - 1, DL, GRLenVT), in performORCombine()
3388 // a = BSTRINS b, c, MaskIdx + MaskLen - 1, MaskIdx in performORCombine()
3394 isShiftedMask_64(CNMask->getZExtValue(), MaskIdx, MaskLen) && in performORCombine()
3397 CNShamt->getZExtValue() == MaskIdx) { in performORCombine()
3398 APInt ShMask(ValBits, CNMask->getZExtValue()); in performORCombine()
3403 DAG.getConstant(MaskIdx + MaskLen - 1, DL, GRLenVT), in performORCombine()
3421 // a = BSTRINS b, c >> MaskIdx, MaskIdx + MaskLen - 1, MaskIdx in performORCombine()
3428 isShiftedMask_64(CNMask->getZExtValue(), MaskIdx, MaskLen)) { in performORCombine()
3429 APInt ShMask(ValBits, CNMask->getZExtValue()); in performORCombine()
3433 DAG.getNode(ISD::SRL, DL, N1->getValueType(0), in performORCombine()
3434 N1->getOperand(0), in performORCombine()
3436 DAG.getConstant(MaskIdx + MaskLen - 1, DL, GRLenVT), in performORCombine()
3453 switch (V.getNode()->getOpcode()) { in checkValueWidth()
3456 if ((LoadNode->getMemoryVT() == MVT::i8) || in checkValueWidth()
3457 (LoadNode->getMemoryVT() == MVT::i16)) { in checkValueWidth()
3458 ExtType = LoadNode->getExtensionType(); in checkValueWidth()
3464 VTSDNode *TypeNode = cast<VTSDNode>(V.getNode()->getOperand(1)); in checkValueWidth()
3465 if ((TypeNode->getVT() == MVT::i8) || (TypeNode->getVT() == MVT::i16)) { in checkValueWidth()
3472 VTSDNode *TypeNode = cast<VTSDNode>(V.getNode()->getOperand(1)); in checkValueWidth()
3473 if ((TypeNode->getVT() == MVT::i8) || (TypeNode->getVT() == MVT::i16)) { in checkValueWidth()
3486 // Eliminate redundant truncation and zero-extension nodes.
3488 // +------------+ +------------+ +------------+
3490 // +------------+ +------------+ +------------+
3492 // V V +----+
3493 // +------------+ +------------+ |
3495 // +------------+ +------------+ |
3498 // +------------+ +------------+ |
3500 // +------------+ +------------+ |
3502 // | +-------------+ |
3504 // +----------------+ | |
3506 // +----------------+ | |
3508 // +---------------+ | |
3511 // +-------------+
3513 // +-------------+
3515 // +------------+ +------------+ +-------------+ +------------+ +------------+
3516 // | Input1 | | Input2 | | Constant -1 | | Constant 0 | | CC |
3517 // +------------+ +------------+ +-------------+ +------------+ +------------+
3520 // +------------+ | | | |
3521 // | XOR |<---------------------+ | |
3522 // +------------+ | | |
3524 // V V +---------------+ |
3525 // +------------+ +------------+ | |
3526 // | TRUNCATE | | TRUNCATE | | +-------------------------+
3527 // +------------+ +------------+ | |
3530 // +------------+ +------------+ | |
3532 // +------------+ +------------+ | |
3535 // +----------------+ | |
3537 // +----------------+ | |
3539 // +---------------+ | |
3542 // +-------------+
3544 // +-------------+
3548 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get(); in performSETCCCombine()
3550 SDNode *AndNode = N->getOperand(0).getNode(); in performSETCCCombine()
3551 if (AndNode->getOpcode() != ISD::AND) in performSETCCCombine()
3554 SDValue AndInputValue2 = AndNode->getOperand(1); in performSETCCCombine()
3558 SDValue CmpInputValue = N->getOperand(1); in performSETCCCombine()
3559 SDValue AndInputValue1 = AndNode->getOperand(0); in performSETCCCombine()
3564 if (!CN || CN->getSExtValue() != -1) in performSETCCCombine()
3567 if (!CN || CN->getSExtValue() != 0) in performSETCCCombine()
3579 SDValue TruncValue1 = AndInputValue1.getNode()->getOperand(0); in performSETCCCombine()
3583 SDValue TruncValue2 = AndInputValue2.getNode()->getOperand(0); in performSETCCCombine()
3587 SDValue TruncInputValue1 = TruncValue1.getNode()->getOperand(0); in performSETCCCombine()
3588 SDValue TruncInputValue2 = TruncValue2.getNode()->getOperand(0); in performSETCCCombine()
3596 if (TruncInputValue1->getValueType(0) != TruncInputValue2->getValueType(0) || in performSETCCCombine()
3597 AndNode->getValueType(0) != TruncInputValue1->getValueType(0)) in performSETCCCombine()
3604 // These truncation and zero-extension nodes are not necessary, remove them. in performSETCCCombine()
3605 SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N), AndNode->getValueType(0), in performSETCCCombine()
3608 DAG.getSetCC(SDLoc(N), N->getValueType(0), NewAnd, TruncInputValue2, CC); in performSETCCCombine()
3620 SDValue Src = N->getOperand(0); in performBITREV_WCombine()
3624 return DAG.getNode(LoongArchISD::BITREV_4B, SDLoc(N), N->getValueType(0), in performBITREV_WCombine()
3634 auto *CImm = cast<ConstantSDNode>(Node->getOperand(ImmOp)); in legalizeIntrinsicImmArg()
3636 if ((IsSigned && !isInt<N>(CImm->getSExtValue())) || in legalizeIntrinsicImmArg()
3637 (!IsSigned && !isUInt<N>(CImm->getZExtValue()))) { in legalizeIntrinsicImmArg()
3638 DAG.getContext()->emitError(Node->getOperationName(0) + in legalizeIntrinsicImmArg()
3642 return DAG.getConstant(CImm->getZExtValue(), DL, Subtarget.getGRLenVT()); in legalizeIntrinsicImmArg()
3649 EVT ResTy = Node->getValueType(0); in lowerVectorSplatImm()
3650 auto *CImm = cast<ConstantSDNode>(Node->getOperand(ImmOp)); in lowerVectorSplatImm()
3653 if ((IsSigned && !isInt<N>(CImm->getSExtValue())) || in lowerVectorSplatImm()
3654 (!IsSigned && !isUInt<N>(CImm->getZExtValue()))) { in lowerVectorSplatImm()
3655 DAG.getContext()->emitError(Node->getOperationName(0) + in lowerVectorSplatImm()
3661 IsSigned ? CImm->getSExtValue() : CImm->getZExtValue(), IsSigned), in lowerVectorSplatImm()
3667 EVT ResTy = Node->getValueType(0); in truncateVecElts()
3668 SDValue Vec = Node->getOperand(2); in truncateVecElts()
3669 SDValue Mask = DAG.getConstant(Vec.getScalarValueSizeInBits() - 1, DL, ResTy); in truncateVecElts()
3675 EVT ResTy = Node->getValueType(0); in lowerVectorBitClear()
3680 return DAG.getNode(ISD::AND, DL, ResTy, Node->getOperand(1), in lowerVectorBitClear()
3687 EVT ResTy = Node->getValueType(0); in lowerVectorBitClearImm()
3688 auto *CImm = cast<ConstantSDNode>(Node->getOperand(2)); in lowerVectorBitClearImm()
3690 if (!isUInt<N>(CImm->getZExtValue())) { in lowerVectorBitClearImm()
3691 DAG.getContext()->emitError(Node->getOperationName(0) + in lowerVectorBitClearImm()
3696 APInt BitImm = APInt(ResTy.getScalarSizeInBits(), 1) << CImm->getAPIntValue(); in lowerVectorBitClearImm()
3699 return DAG.getNode(ISD::AND, DL, ResTy, Node->getOperand(1), Mask); in lowerVectorBitClearImm()
3705 EVT ResTy = Node->getValueType(0); in lowerVectorBitSetImm()
3706 auto *CImm = cast<ConstantSDNode>(Node->getOperand(2)); in lowerVectorBitSetImm()
3708 if (!isUInt<N>(CImm->getZExtValue())) { in lowerVectorBitSetImm()
3709 DAG.getContext()->emitError(Node->getOperationName(0) + in lowerVectorBitSetImm()
3714 APInt Imm = APInt(ResTy.getScalarSizeInBits(), 1) << CImm->getAPIntValue(); in lowerVectorBitSetImm()
3716 return DAG.getNode(ISD::OR, DL, ResTy, Node->getOperand(1), BitImm); in lowerVectorBitSetImm()
3722 EVT ResTy = Node->getValueType(0); in lowerVectorBitRevImm()
3723 auto *CImm = cast<ConstantSDNode>(Node->getOperand(2)); in lowerVectorBitRevImm()
3725 if (!isUInt<N>(CImm->getZExtValue())) { in lowerVectorBitRevImm()
3726 DAG.getContext()->emitError(Node->getOperationName(0) + in lowerVectorBitRevImm()
3731 APInt Imm = APInt(ResTy.getScalarSizeInBits(), 1) << CImm->getAPIntValue(); in lowerVectorBitRevImm()
3733 return DAG.getNode(ISD::XOR, DL, ResTy, Node->getOperand(1), BitImm); in lowerVectorBitRevImm()
3741 switch (N->getConstantOperandVal(0)) { in performINTRINSIC_WO_CHAINCombine()
3752 return DAG.getNode(ISD::ADD, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3753 N->getOperand(2)); in performINTRINSIC_WO_CHAINCombine()
3762 return DAG.getNode(ISD::ADD, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3772 return DAG.getNode(ISD::SUB, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3773 N->getOperand(2)); in performINTRINSIC_WO_CHAINCombine()
3782 return DAG.getNode(ISD::SUB, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3793 ISD::SUB, DL, N->getValueType(0), in performINTRINSIC_WO_CHAINCombine()
3795 APInt(N->getValueType(0).getScalarType().getSizeInBits(), 0, in performINTRINSIC_WO_CHAINCombine()
3797 SDLoc(N), N->getValueType(0)), in performINTRINSIC_WO_CHAINCombine()
3798 N->getOperand(1)); in performINTRINSIC_WO_CHAINCombine()
3807 return DAG.getNode(ISD::SMAX, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3808 N->getOperand(2)); in performINTRINSIC_WO_CHAINCombine()
3817 return DAG.getNode(ISD::UMAX, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3818 N->getOperand(2)); in performINTRINSIC_WO_CHAINCombine()
3827 return DAG.getNode(ISD::SMAX, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3837 return DAG.getNode(ISD::UMAX, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3847 return DAG.getNode(ISD::SMIN, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3848 N->getOperand(2)); in performINTRINSIC_WO_CHAINCombine()
3857 return DAG.getNode(ISD::UMIN, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3858 N->getOperand(2)); in performINTRINSIC_WO_CHAINCombine()
3867 return DAG.getNode(ISD::SMIN, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3877 return DAG.getNode(ISD::UMIN, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3887 return DAG.getNode(ISD::MUL, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3888 N->getOperand(2)); in performINTRINSIC_WO_CHAINCombine()
3897 EVT ResTy = N->getValueType(0); in performINTRINSIC_WO_CHAINCombine()
3898 return DAG.getNode(ISD::ADD, SDLoc(N), ResTy, N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3899 DAG.getNode(ISD::MUL, SDLoc(N), ResTy, N->getOperand(2), in performINTRINSIC_WO_CHAINCombine()
3900 N->getOperand(3))); in performINTRINSIC_WO_CHAINCombine()
3910 EVT ResTy = N->getValueType(0); in performINTRINSIC_WO_CHAINCombine()
3911 return DAG.getNode(ISD::SUB, SDLoc(N), ResTy, N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3912 DAG.getNode(ISD::MUL, SDLoc(N), ResTy, N->getOperand(2), in performINTRINSIC_WO_CHAINCombine()
3913 N->getOperand(3))); in performINTRINSIC_WO_CHAINCombine()
3923 return DAG.getNode(ISD::SDIV, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3924 N->getOperand(2)); in performINTRINSIC_WO_CHAINCombine()
3933 return DAG.getNode(ISD::UDIV, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3934 N->getOperand(2)); in performINTRINSIC_WO_CHAINCombine()
3943 return DAG.getNode(ISD::SREM, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3944 N->getOperand(2)); in performINTRINSIC_WO_CHAINCombine()
3953 return DAG.getNode(ISD::UREM, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3954 N->getOperand(2)); in performINTRINSIC_WO_CHAINCombine()
3957 return DAG.getNode(ISD::AND, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3958 N->getOperand(2)); in performINTRINSIC_WO_CHAINCombine()
3961 return DAG.getNode(ISD::OR, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3962 N->getOperand(2)); in performINTRINSIC_WO_CHAINCombine()
3965 return DAG.getNode(ISD::XOR, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3966 N->getOperand(2)); in performINTRINSIC_WO_CHAINCombine()
3969 SDValue Res = DAG.getNode(ISD::OR, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3970 N->getOperand(2)); in performINTRINSIC_WO_CHAINCombine()
3971 return DAG.getNOT(DL, Res, Res->getValueType(0)); in performINTRINSIC_WO_CHAINCombine()
3975 return DAG.getNode(ISD::AND, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3979 return DAG.getNode(ISD::OR, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3983 return DAG.getNode(ISD::XOR, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3993 return DAG.getNode(ISD::SHL, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
3997 return DAG.getNode(ISD::SHL, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
4001 return DAG.getNode(ISD::SHL, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
4005 return DAG.getNode(ISD::SHL, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
4009 return DAG.getNode(ISD::SHL, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
4019 return DAG.getNode(ISD::SRL, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
4023 return DAG.getNode(ISD::SRL, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
4027 return DAG.getNode(ISD::SRL, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
4031 return DAG.getNode(ISD::SRL, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
4035 return DAG.getNode(ISD::SRL, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
4045 return DAG.getNode(ISD::SRA, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
4049 return DAG.getNode(ISD::SRA, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
4053 return DAG.getNode(ISD::SRA, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
4057 return DAG.getNode(ISD::SRA, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
4061 return DAG.getNode(ISD::SRA, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
4071 return DAG.getNode(ISD::CTLZ, DL, N->getValueType(0), N->getOperand(1)); in performINTRINSIC_WO_CHAINCombine()
4080 return DAG.getNode(ISD::CTPOP, DL, N->getValueType(0), N->getOperand(1)); in performINTRINSIC_WO_CHAINCombine()
4110 EVT VecTy = N->getValueType(0); in performINTRINSIC_WO_CHAINCombine()
4113 ISD::OR, DL, VecTy, N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
4136 EVT VecTy = N->getValueType(0); in performINTRINSIC_WO_CHAINCombine()
4139 ISD::XOR, DL, VecTy, N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
4158 return DAG.getNode(ISD::FADD, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
4159 N->getOperand(2)); in performINTRINSIC_WO_CHAINCombine()
4164 return DAG.getNode(ISD::FSUB, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
4165 N->getOperand(2)); in performINTRINSIC_WO_CHAINCombine()
4170 return DAG.getNode(ISD::FMUL, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
4171 N->getOperand(2)); in performINTRINSIC_WO_CHAINCombine()
4176 return DAG.getNode(ISD::FDIV, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
4177 N->getOperand(2)); in performINTRINSIC_WO_CHAINCombine()
4182 return DAG.getNode(ISD::FMA, DL, N->getValueType(0), N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
4183 N->getOperand(2), N->getOperand(3)); in performINTRINSIC_WO_CHAINCombine()
4185 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N), N->getValueType(0), in performINTRINSIC_WO_CHAINCombine()
4186 N->getOperand(1), N->getOperand(2), in performINTRINSIC_WO_CHAINCombine()
4190 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N), N->getValueType(0), in performINTRINSIC_WO_CHAINCombine()
4191 N->getOperand(1), N->getOperand(2), in performINTRINSIC_WO_CHAINCombine()
4195 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N), N->getValueType(0), in performINTRINSIC_WO_CHAINCombine()
4196 N->getOperand(1), N->getOperand(2), in performINTRINSIC_WO_CHAINCombine()
4199 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N), N->getValueType(0), in performINTRINSIC_WO_CHAINCombine()
4200 N->getOperand(1), N->getOperand(2), in performINTRINSIC_WO_CHAINCombine()
4210 EVT ResTy = N->getValueType(0); in performINTRINSIC_WO_CHAINCombine()
4211 SmallVector<SDValue> Ops(ResTy.getVectorNumElements(), N->getOperand(1)); in performINTRINSIC_WO_CHAINCombine()
4222 return DAG.getNode(LoongArchISD::VREPLVE, DL, N->getValueType(0), in performINTRINSIC_WO_CHAINCombine()
4223 N->getOperand(1), in performINTRINSIC_WO_CHAINCombine()
4225 N->getOperand(2))); in performINTRINSIC_WO_CHAINCombine()
4233 switch (N->getOpcode()) { in PerformDAGCombine()
4265 const BasicBlock *LLVM_BB = MBB->getBasicBlock(); in insertDivByZeroTrap()
4266 MachineFunction::iterator It = ++MBB->getIterator(); in insertDivByZeroTrap()
4267 MachineFunction *MF = MBB->getParent(); in insertDivByZeroTrap()
4268 auto BreakMBB = MF->CreateMachineBasicBlock(LLVM_BB); in insertDivByZeroTrap()
4269 auto SinkMBB = MF->CreateMachineBasicBlock(LLVM_BB); in insertDivByZeroTrap()
4270 MF->insert(It, BreakMBB); in insertDivByZeroTrap()
4271 MF->insert(It, SinkMBB); in insertDivByZeroTrap()
4274 SinkMBB->splice(SinkMBB->end(), MBB, std::next(MI.getIterator()), MBB->end()); in insertDivByZeroTrap()
4275 SinkMBB->transferSuccessorsAndUpdatePHIs(MBB); in insertDivByZeroTrap()
4277 const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo(); in insertDivByZeroTrap()
4286 MBB->addSuccessor(BreakMBB); in insertDivByZeroTrap()
4287 MBB->addSuccessor(SinkMBB); in insertDivByZeroTrap()
4293 BreakMBB->addSuccessor(SinkMBB); in insertDivByZeroTrap()
4371 const BasicBlock *LLVM_BB = BB->getBasicBlock(); in emitVecCondBranchPseudo()
4373 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); in emitVecCondBranchPseudo()
4374 MachineFunction::iterator It = ++BB->getIterator(); in emitVecCondBranchPseudo()
4376 MachineFunction *F = BB->getParent(); in emitVecCondBranchPseudo()
4377 MachineBasicBlock *FalseBB = F->CreateMachineBasicBlock(LLVM_BB); in emitVecCondBranchPseudo()
4378 MachineBasicBlock *TrueBB = F->CreateMachineBasicBlock(LLVM_BB); in emitVecCondBranchPseudo()
4379 MachineBasicBlock *SinkBB = F->CreateMachineBasicBlock(LLVM_BB); in emitVecCondBranchPseudo()
4381 F->insert(It, FalseBB); in emitVecCondBranchPseudo()
4382 F->insert(It, TrueBB); in emitVecCondBranchPseudo()
4383 F->insert(It, SinkBB); in emitVecCondBranchPseudo()
4386 SinkBB->splice(SinkBB->end(), BB, std::next(MI.getIterator()), BB->end()); in emitVecCondBranchPseudo()
4387 SinkBB->transferSuccessorsAndUpdatePHIs(BB); in emitVecCondBranchPseudo()
4391 BuildMI(BB, DL, TII->get(CondOpc), FCC).addReg(MI.getOperand(1).getReg()); in emitVecCondBranchPseudo()
4394 BuildMI(BB, DL, TII->get(LoongArch::BCNEZ)).addReg(FCC).addMBB(TrueBB); in emitVecCondBranchPseudo()
4395 BB->addSuccessor(FalseBB); in emitVecCondBranchPseudo()
4396 BB->addSuccessor(TrueBB); in emitVecCondBranchPseudo()
4400 BuildMI(FalseBB, DL, TII->get(LoongArch::ADDI_W), RD1) in emitVecCondBranchPseudo()
4403 BuildMI(FalseBB, DL, TII->get(LoongArch::PseudoBR)).addMBB(SinkBB); in emitVecCondBranchPseudo()
4404 FalseBB->addSuccessor(SinkBB); in emitVecCondBranchPseudo()
4408 BuildMI(TrueBB, DL, TII->get(LoongArch::ADDI_W), RD2) in emitVecCondBranchPseudo()
4411 TrueBB->addSuccessor(SinkBB); in emitVecCondBranchPseudo()
4414 BuildMI(*SinkBB, SinkBB->begin(), DL, TII->get(LoongArch::PHI), in emitVecCondBranchPseudo()
4447 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); in emitPseudoXVINSGR2VR()
4457 BuildMI(*BB, MI, DL, TII->get(LoongArch::XVPERMI_Q), ScratchReg1) in emitPseudoXVINSGR2VR()
4465 BuildMI(*BB, MI, DL, TII->get(LoongArch::COPY), ScratchSubReg1) in emitPseudoXVINSGR2VR()
4467 BuildMI(*BB, MI, DL, TII->get(InsOp), ScratchSubReg2) in emitPseudoXVINSGR2VR()
4470 .addImm(Idx >= HalfSize ? Idx - HalfSize : Idx); in emitPseudoXVINSGR2VR()
4476 BuildMI(*BB, MI, DL, TII->get(LoongArch::SUBREG_TO_REG), ScratchReg2) in emitPseudoXVINSGR2VR()
4482 BuildMI(*BB, MI, DL, TII->get(LoongArch::XVPERMI_Q), XDst) in emitPseudoXVINSGR2VR()
4510 BuildMI(*BB, MI, DL, TII->get(LoongArch::MOVGR2FCSR), in EmitInstrWithCustomInserter()
4518 BuildMI(*BB, MI, DL, TII->get(LoongArch::MOVFCSR2GR), in EmitInstrWithCustomInserter()
4521 ReadFCSR->getOperand(1).setIsUndef(); in EmitInstrWithCustomInserter()
4573 // TODO: Add more target-dependent nodes later. in getTargetNodeName()
4649 //===----------------------------------------------------------------------===//
4651 //===----------------------------------------------------------------------===//
4653 // Eight general-purpose registers a0-a7 used for passing integer arguments,
4654 // with a0-a1 reused to return values. Generally, the GPRs are used to pass
4655 // fixed-point arguments, and floating-point arguments when no FPR is available
4660 // Eight floating-point registers fa0-fa7 used for passing floating-point
4661 // arguments, and fa0-fa1 are also used to return values.
4833 // If we reach this point and PendingLocs is non-empty, we must be at the in CC_LoongArch()
4857 // When a floating-point value is passed on the stack, no bit-cast is needed. in CC_LoongArch()
4876 ArgTy = FType->getReturnType(); in analyzeInputArgs()
4878 ArgTy = FType->getParamType(Ins[i].getOrigArgIndex()); in analyzeInputArgs()
4896 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty : nullptr; in analyzeOutputArgs()
4944 if (OrigArg->getType()->isIntegerTy()) { in unpackFromRegLoc()
4945 unsigned BitWidth = OrigArg->getType()->getIntegerBitWidth(); in unpackFromRegLoc()
4951 LAFI->addSExt32Register(VReg); in unpackFromRegLoc()
5100 unsigned PartOffset = Ins[i + 1].PartOffset - ArgPartOffset; in LowerFormalArguments()
5122 // large enough to hold a0-a7. in LowerFormalArguments()
5131 VarArgsSaveSize = GRLenInBytes * (ArgRegs.size() - Idx); in LowerFormalArguments()
5132 VaArgOffset = -VarArgsSaveSize; in LowerFormalArguments()
5138 LoongArchFI->setVarArgsFrameIndex(FI); in LowerFormalArguments()
5141 // ensure that the frame pointer is 2*GRLen-aligned, which in turn ensures in LowerFormalArguments()
5142 // offsets to even-numbered registered remain 2*GRLen-aligned. in LowerFormalArguments()
5144 MFI.CreateFixedObject(GRLenInBytes, VaArgOffset - (int)GRLenInBytes, in LowerFormalArguments()
5161 ->getMemOperand() in LowerFormalArguments()
5162 ->setValue((Value *)nullptr); in LowerFormalArguments()
5165 LoongArchFI->setVarArgsSaveSize(VarArgsSaveSize); in LowerFormalArguments()
5179 return CI->isTailCall(); in mayBeEmittedAsTailCall()
5183 // we can't perform a tail-call.
5186 if (N->getNumValues() != 1) in isUsedByReturnOnly()
5188 if (!N->hasNUsesOfValue(1, 0)) in isUsedByReturnOnly()
5191 SDNode *Copy = *N->use_begin(); in isUsedByReturnOnly()
5192 if (Copy->getOpcode() != ISD::CopyToReg) in isUsedByReturnOnly()
5197 if (Copy->getGluedNode()) in isUsedByReturnOnly()
5202 for (SDNode *Node : Copy->uses()) { in isUsedByReturnOnly()
5203 if (Node->getOpcode() != LoongArchISD::RET) in isUsedByReturnOnly()
5211 Chain = Copy->getOperand(0); in isUsedByReturnOnly()
5248 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC); in isEligibleForTailCallOptimization()
5250 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC); in isEligibleForTailCallOptimization()
5251 if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved)) in isEligibleForTailCallOptimization()
5297 else if (CLI.CB && CLI.CB->isMustTailCall()) in LowerCall()
5358 unsigned PartOffset = Outs[i + 1].PartOffset - ArgPartOffset; in LowerCall()
5368 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex(); in LowerCall()
5417 // Build a sequence of copy-to-reg nodes, chained and glued together. in LowerCall()
5427 const GlobalValue *GV = S->getGlobal(); in LowerCall()
5431 Callee = DAG.getTargetGlobalAddress(S->getGlobal(), DL, PtrVT, 0, OpFlags); in LowerCall()
5436 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, OpFlags); in LowerCall()
5450 // Add a register mask operand representing the call-preserved registers. in LowerCall()
5452 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv); in LowerCall()
5604 Type *Ty = I->getOperand(0)->getType(); in shouldInsertFencesForAtomic()
5605 if (isa<StoreInst>(I) && Ty->isIntegerTy()) { in shouldInsertFencesForAtomic()
5606 unsigned Size = Ty->getIntegerBitWidth(); in shouldInsertFencesForAtomic()
5653 // Since floating-point operation requires a non-trivial set of data in shouldExpandAtomicRMWInIR()
5655 if (AI->isFloatingPointOperation() || in shouldExpandAtomicRMWInIR()
5656 AI->getOperation() == AtomicRMWInst::UIncWrap || in shouldExpandAtomicRMWInIR()
5657 AI->getOperation() == AtomicRMWInst::UDecWrap) in shouldExpandAtomicRMWInIR()
5660 unsigned Size = AI->getType()->getPrimitiveSizeInBits(); in shouldExpandAtomicRMWInIR()
5715 unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits(); in shouldExpandAtomicCmpXchgInIR()
5724 AtomicOrdering FailOrd = CI->getFailureOrdering(); in emitMaskedAtomicCmpXchgIntrinsic()
5733 Type *Tys[] = {AlignedAddr->getType()}; in emitMaskedAtomicCmpXchgIntrinsic()
5735 Intrinsic::getDeclaration(CI->getModule(), CmpXchgIntrID, Tys); in emitMaskedAtomicCmpXchgIntrinsic()
5745 // In the case of an atomicrmw xchg with a constant 0/-1 operand, replace in emitMaskedAtomicRMWIntrinsic()
5749 if (AI->getOperation() == AtomicRMWInst::Xchg && in emitMaskedAtomicRMWIntrinsic()
5750 isa<ConstantInt>(AI->getValOperand())) { in emitMaskedAtomicRMWIntrinsic()
5751 ConstantInt *CVal = cast<ConstantInt>(AI->getValOperand()); in emitMaskedAtomicRMWIntrinsic()
5752 if (CVal->isZero()) in emitMaskedAtomicRMWIntrinsic()
5755 AI->getAlign(), Ord); in emitMaskedAtomicRMWIntrinsic()
5756 if (CVal->isMinusOne()) in emitMaskedAtomicRMWIntrinsic()
5758 AI->getAlign(), Ord); in emitMaskedAtomicRMWIntrinsic()
5763 Builder.getIntN(GRLen, static_cast<uint64_t>(AI->getOrdering())); in emitMaskedAtomicRMWIntrinsic()
5764 Type *Tys[] = {AlignedAddr->getType()}; in emitMaskedAtomicRMWIntrinsic()
5766 AI->getModule(), in emitMaskedAtomicRMWIntrinsic()
5767 getIntrinsicForMaskedAtomicRMWBinOp(GRLen, AI->getOperation()), Tys); in emitMaskedAtomicRMWIntrinsic()
5779 // bits to shift the value into position. Pass GRLen-ShiftAmt-ValWidth, which in emitMaskedAtomicRMWIntrinsic()
5781 // sign-extend. in emitMaskedAtomicRMWIntrinsic()
5782 if (AI->getOperation() == AtomicRMWInst::Min || in emitMaskedAtomicRMWIntrinsic()
5783 AI->getOperation() == AtomicRMWInst::Max) { in emitMaskedAtomicRMWIntrinsic()
5784 const DataLayout &DL = AI->getDataLayout(); in emitMaskedAtomicRMWIntrinsic()
5786 DL.getTypeStoreSizeInBits(AI->getValOperand()->getType()); in emitMaskedAtomicRMWIntrinsic()
5788 Builder.CreateSub(Builder.getIntN(GRLen, GRLen - ValWidth), ShiftAmt); in emitMaskedAtomicRMWIntrinsic()
5829 //===----------------------------------------------------------------------===//
5831 //===----------------------------------------------------------------------===//
5837 // 'f': A floating-point register (if available). in getConstraintType()
5840 // 'l': A signed 16-bit constant. in getConstraintType()
5844 // 'I': A signed 12-bit constant (for arithmetic instructions). in getConstraintType()
5846 // 'K': An unsigned 12-bit constant (for logic instructions). in getConstraintType()
5847 // "ZB": An address that is held in a general-purpose register. The offset is in getConstraintType()
5902 TRI->isTypeLegalForClass(LoongArch::LSX128RegClass, VT)) in getRegForInlineAsmConstraint()
5905 TRI->isTypeLegalForClass(LoongArch::LASX256RegClass, VT)) in getRegForInlineAsmConstraint()
5936 unsigned DReg = RegNo - LoongArch::F0 + LoongArch::F0_64; in getRegForInlineAsmConstraint()
5954 // Validate & create a 16-bit signed immediate operand. in LowerAsmOperandForConstraint()
5956 uint64_t CVal = C->getSExtValue(); in LowerAsmOperandForConstraint()
5963 // Validate & create a 12-bit signed immediate operand. in LowerAsmOperandForConstraint()
5965 uint64_t CVal = C->getSExtValue(); in LowerAsmOperandForConstraint()
5974 if (C->getZExtValue() == 0) in LowerAsmOperandForConstraint()
5979 // Validate & create a 12-bit unsigned immediate operand. in LowerAsmOperandForConstraint()
5981 uint64_t CVal = C->getZExtValue(); in LowerAsmOperandForConstraint()
6008 BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF); in getRegisterByName()
6010 report_fatal_error(Twine("Trying to obtain non-reserved register \"" + in getRegisterByName()
6026 const APInt &Imm = ConstNode->getAPIntValue(); in decomposeMulByConstant()
6028 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() || in decomposeMulByConstant()
6029 (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2()) in decomposeMulByConstant()
6032 if (ConstNode->hasOneUse() && in decomposeMulByConstant()
6033 ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() || in decomposeMulByConstant()
6034 (Imm - 8).isPowerOf2() || (Imm - 16).isPowerOf2())) in decomposeMulByConstant()
6039 // equals to (1 << s0) - (1 << s1). in decomposeMulByConstant()
6040 if (ConstNode->hasOneUse() && !(Imm.sge(-2048) && Imm.sle(4095))) { in decomposeMulByConstant()
6050 // We do not consider the case `(-Imm - ImmSmall).isPowerOf2()`, in decomposeMulByConstant()
6053 if ((Imm - ImmSmall).isPowerOf2() || (Imm + ImmSmall).isPowerOf2() || in decomposeMulByConstant()
6054 (ImmSmall - Imm).isPowerOf2()) in decomposeMulByConstant()
6068 // 2. reg + 12-bit signed offset in isLegalAddressingMode()
6069 // 3. reg + 14-bit signed offset left-shifted by 2 in isLegalAddressingMode()
6077 // Require a 12-bit signed offset or 14-bit signed offset left-shifted by 2 in isLegalAddressingMode()
6083 switch (AM.Scale) { in isLegalAddressingMode()
6116 // Don't advertise i32->i64 zextload as being free for LA64. It interacts in isZExtFree()
6119 EVT MemVT = LD->getMemoryVT(); in isZExtFree()
6121 (LD->getExtensionType() == ISD::NON_EXTLOAD || in isZExtFree()
6122 LD->getExtensionType() == ISD::ZEXTLOAD)) in isZExtFree()
6135 return Subtarget.is64Bit() && CI->getType()->isIntegerTy(32); in signExtendConstant()