Lines Matching defs:dl
499 const SDLoc &dl, SelectionDAG &DAG) const {
501 IntOps.push_back(DAG.getConstant(IntId, dl, MVT::i32));
503 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, ResTy, IntOps);
548 HexagonTargetLowering::opJoin(const VectorPair &Ops, const SDLoc &dl,
550 return DAG.getNode(ISD::CONCAT_VECTORS, dl, typeJoin(ty(Ops)),
555 HexagonTargetLowering::opSplit(SDValue Vec, const SDLoc &dl,
560 return DAG.SplitVector(Vec, dl, Tys.first, Tys.second);
727 const SDLoc &dl(ElemIdx);
728 return DAG.getNode(ISD::SHL, dl, MVT::i32,
729 {ElemIdx, DAG.getConstant(L, dl, MVT::i32)});
742 const SDLoc &dl(Idx);
743 SDValue Mask = DAG.getConstant(32/ElemWidth - 1, dl, MVT::i32);
744 SDValue SubIdx = DAG.getNode(ISD::AND, dl, MVT::i32, {Idx, Mask});
749 HexagonTargetLowering::getByteShuffle(const SDLoc &dl, SDValue Op0,
757 return DAG.getVectorShuffle(OpTy, dl, Op0, Op1, Mask);
775 return DAG.getVectorShuffle(ResTy, dl, opCastElem(Op0, MVT::i8, DAG),
781 const SDLoc &dl, MVT VecTy,
800 SDValue W = buildVector32(Values.slice(i, OpsPerWord), dl, PartVT, DAG);
833 return getZero(dl, VecTy, DAG);
835 SDValue S = DAG.getNode(ISD::SPLAT_VECTOR, dl, WordTy, SplatV);
850 return DAG.getLoad(VecTy, dl, DAG.getEntryNode(), CP,
916 SDValue S = DAG.getVectorShuffle(ExtTy, dl, ExtVec,
941 SDValue HalfV = getZero(dl, VecTy, DAG);
943 SDValue SplatV = DAG.getNode(ISD::SPLAT_VECTOR, dl, VecTy, Words[n]);
944 HalfV = DAG.getNode(HexagonISD::VALIGN, dl, VecTy,
945 {HalfV, SplatV, DAG.getConstant(HwLen/2, dl, MVT::i32)});
960 Sn = DAG.getConstant(Rn, dl, MVT::i32);
961 HalfV0 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {N, Sn});
962 N = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy,
967 Sm = DAG.getConstant(Rm, dl, MVT::i32);
968 HalfV1 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {M, Sm});
969 M = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy,
977 Sn = DAG.getConstant(Rn+HwLen/2, dl, MVT::i32);
978 Sm = DAG.getConstant(Rm, dl, MVT::i32);
979 HalfV0 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {N, Sn});
980 HalfV1 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {M, Sm});
985 SDValue DstV = DAG.getNode(ISD::OR, dl, ty(T0), {T0, T1});
993 HexagonTargetLowering::createHvxPrefixPred(SDValue PredV, const SDLoc &dl,
1008 SDValue T = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, PredV);
1019 SDValue S = DAG.getVectorShuffle(ByteTy, dl, T, DAG.getUNDEF(ByteTy), Mask);
1027 SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
1028 {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG);
1029 SDValue M = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, Q);
1030 return DAG.getNode(ISD::AND, dl, ByteTy, S, M);
1042 : DAG.getNode(HexagonISD::P2D, dl, MVT::i64, PredV);
1052 SDValue T = expandPredicate(W, dl, DAG);
1067 SDValue Vec = ZeroFill ? getZero(dl, ByteTy, DAG) : DAG.getUNDEF(ByteTy);
1068 SDValue S4 = DAG.getConstant(HwLen-4, dl, MVT::i32);
1070 Vec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, Vec, S4);
1071 Vec = DAG.getNode(HexagonISD::VINSERTW0, dl, ByteTy, Vec, W);
1079 const SDLoc &dl, MVT VecTy,
1110 SDValue Ext = !V.isUndef() ? DAG.getZExtOrTrunc(V, dl, MVT::i8)
1130 SDValue Ext = (B < 8) ? DAG.getZExtOrTrunc(F, dl, MVT::i8)
1141 return DAG.getNode(HexagonISD::QTRUE, dl, VecTy);
1143 return DAG.getNode(HexagonISD::QFALSE, dl, VecTy);
1146 SDValue ByteVec = buildHvxVectorReg(Bytes, dl, ByteTy, DAG);
1147 return DAG.getNode(HexagonISD::V2Q, dl, VecTy, ByteVec);
1152 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
1160 SDValue ExWord = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32,
1171 return extractVector(ExVec, SubIdx, dl, ElemTy, MVT::i32, DAG);
1176 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
1182 SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
1185 SDValue ScV = DAG.getConstant(Scale, dl, MVT::i32);
1186 IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, ScV);
1188 SDValue ExtB = extractHvxElementReg(ByteVec, IdxV, dl, MVT::i32, DAG);
1189 SDValue Zero = DAG.getTargetConstant(0, dl, MVT::i32);
1190 return getInstr(Hexagon::C2_cmpgtui, dl, MVT::i1, {ExtB, Zero}, DAG);
1195 SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
1202 auto InsertWord = [&DAG,&dl,this] (SDValue VecV, SDValue ValV,
1206 SDValue MaskV = DAG.getNode(ISD::AND, dl, MVT::i32,
1207 {ByteIdxV, DAG.getConstant(-4, dl, MVT::i32)});
1208 SDValue RotV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {VecV, MaskV});
1209 SDValue InsV = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy, {RotV, ValV});
1210 SDValue SubV = DAG.getNode(ISD::SUB, dl, MVT::i32,
1211 {DAG.getConstant(HwLen, dl, MVT::i32), MaskV});
1212 SDValue TorV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {InsV, SubV});
1222 SDValue WordIdx = DAG.getNode(ISD::SRL, dl, MVT::i32,
1223 {ByteIdx, DAG.getConstant(2, dl, MVT::i32)});
1225 dl, MVT::i32, DAG);
1232 ValV, SubIdx, dl, ElemTy, DAG);
1240 SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
1243 SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
1246 SDValue ScV = DAG.getConstant(Scale, dl, MVT::i32);
1247 IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, ScV);
1248 ValV = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i32, ValV);
1250 SDValue InsV = insertHvxElementReg(ByteVec, IdxV, ValV, dl, DAG);
1251 return DAG.getNode(HexagonISD::V2Q, dl, ty(VecV), InsV);
1256 SDValue IdxV, const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
1283 SDValue W0Idx = DAG.getConstant(WordIdx, dl, MVT::i32);
1284 SDValue W0 = extractHvxElementReg(WordVec, W0Idx, dl, MVT::i32, DAG);
1288 SDValue W1Idx = DAG.getConstant(WordIdx+1, dl, MVT::i32);
1289 SDValue W1 = extractHvxElementReg(WordVec, W1Idx, dl, MVT::i32, DAG);
1290 SDValue WW = getCombine(W1, W0, dl, MVT::i64, DAG);
1296 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
1300 SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
1321 SDValue ShuffV = DAG.getVectorShuffle(ByteTy, dl, ByteVec, Undef, Mask);
1322 return DAG.getNode(HexagonISD::V2Q, dl, ResTy, ShuffV);
1346 SDValue Zero = getZero(dl, MVT::i32, DAG);
1347 SDValue ShuffV = DAG.getVectorShuffle(ByteTy, dl, ByteVec, Undef, Mask);
1350 SDValue W0 = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32, {ShuffV, Zero});
1351 SDValue W1 = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32,
1352 {ShuffV, DAG.getConstant(4, dl, MVT::i32)});
1353 SDValue Vec64 = getCombine(W1, W0, dl, MVT::v8i8, DAG);
1354 return getInstr(Hexagon::A4_vcmpbgtui, dl, ResTy,
1355 {Vec64, DAG.getTargetConstant(0, dl, MVT::i32)}, DAG);
1360 SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
1379 dl, MVT::i32);
1380 PickHi = DAG.getSetCC(dl, MVT::i1, IdxV, HalfV, ISD::SETUGT);
1386 return DAG.getTargetInsertSubreg(SubIdx, dl, VecTy, VecV, SubV);
1391 SDValue InLo = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {SubV, V1});
1392 SDValue InHi = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {V0, SubV});
1393 return DAG.getNode(ISD::SELECT, dl, VecTy, PickHi, InHi, InLo);
1398 SDValue S = DAG.getNode(ISD::SUB, dl, MVT::i32, IdxV, HalfV);
1399 IdxV = DAG.getNode(ISD::SELECT, dl, MVT::i32, PickHi, S, IdxV);
1400 SingleV = DAG.getNode(ISD::SELECT, dl, SingleTy, PickHi, V1, V0);
1409 IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
1410 DAG.getConstant(ElemWidth/8, dl, MVT::i32));
1411 SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV, IdxV);
1419 SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, SingleV, V);
1424 SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, SingleV, R0);
1425 SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV,
1426 DAG.getConstant(4, dl, MVT::i32));
1427 SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, SingleV, R1);
1432 SDValue RolV = DAG.getNode(ISD::SUB, dl, MVT::i32,
1433 DAG.getConstant(RolBase, dl, MVT::i32), IdxV);
1434 SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV, RolV);
1438 SDValue InLo = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {SingleV, V1});
1439 SDValue InHi = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {V0, SingleV});
1440 return DAG.getNode(ISD::SELECT, dl, VecTy, PickHi, InHi, InLo);
1447 SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
1463 SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
1464 SDValue ByteSub = createHvxPrefixPred(SubV, dl, BitBytes, false, DAG);
1469 ByteIdx = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
1470 DAG.getConstant(BitBytes, dl, MVT::i32));
1471 ByteVec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, ByteVec, ByteIdx);
1479 SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
1480 {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG);
1481 ByteVec = getInstr(Hexagon::V6_vmux, dl, ByteTy, {Q, ByteSub, ByteVec}, DAG);
1484 SDValue HwLenV = DAG.getConstant(HwLen, dl, MVT::i32);
1485 SDValue ByteXdi = DAG.getNode(ISD::SUB, dl, MVT::i32, HwLenV, ByteIdx);
1486 ByteVec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, ByteVec, ByteXdi);
1488 return DAG.getNode(HexagonISD::V2Q, dl, VecTy, ByteVec);
1492 HexagonTargetLowering::extendHvxVectorPred(SDValue VecV, const SDLoc &dl,
1499 return DAG.getNode(HexagonISD::Q2V, dl, ResTy, VecV);
1502 SDValue True = DAG.getNode(ISD::SPLAT_VECTOR, dl, ResTy,
1503 DAG.getConstant(1, dl, MVT::i32));
1504 SDValue False = getZero(dl, ResTy, DAG);
1505 return DAG.getSelect(dl, ResTy, VecV, True, False);
1509 HexagonTargetLowering::compressHvxPred(SDValue VecQ, const SDLoc &dl,
1537 DAG.getLoad(ByteTy, dl, DAG.getEntryNode(), CP,
1541 SDValue Sel = DAG.getSelect(dl, VecTy, VecQ, DAG.getBitcast(VecTy, Bytes),
1542 getZero(dl, VecTy, DAG));
1547 DAG.getSplatBuildVector(MVT::v4i8, dl, DAG.getConstant(1, dl, MVT::i32));
1548 SDValue Vrmpy = getInstr(Hexagon::V6_vrmpyub, dl, ByteTy, {Sel, All1}, DAG);
1550 SDValue Rot = getInstr(Hexagon::V6_valignbi, dl, ByteTy,
1551 {Vrmpy, Vrmpy, DAG.getTargetConstant(4, dl, MVT::i32)}, DAG);
1552 SDValue Vor = DAG.getNode(ISD::OR, dl, ByteTy, {Vrmpy, Rot});
1561 DAG.getVectorShuffle(ByteTy, dl, Vor, DAG.getUNDEF(ByteTy), Mask);
1567 const SDLoc &dl, SelectionDAG &DAG) const {
1577 return InpWidth < ResWidth ? DAG.getNode(ISD::FP_EXTEND, dl, ResTy, VecV)
1578 : DAG.getNode(ISD::FP_ROUND, dl, ResTy, VecV,
1579 getZero(dl, MVT::i32, DAG));
1586 return DAG.getNode(ExtOpc, dl, ResTy, VecV);
1589 return DAG.getNode(NarOpc, dl, ResTy, VecV, DAG.getValueType(ResTy));
1598 const SDLoc &dl(Vec);
1600 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubTy,
1601 {Vec, DAG.getConstant(ElemIdx, dl, MVT::i32)});
1607 const SDLoc &dl(Op);
1621 SDValue V0 = buildHvxVectorReg(A.take_front(Size/2), dl, SingleTy, DAG);
1622 SDValue V1 = buildHvxVectorReg(A.drop_front(Size/2), dl, SingleTy, DAG);
1623 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, V0, V1);
1627 return buildHvxVectorPred(Ops, dl, VecTy, DAG);
1637 SDValue T0 = DAG.getNode(ISD::BUILD_VECTOR, dl,
1642 return buildHvxVectorReg(Ops, dl, VecTy, DAG);
1648 const SDLoc &dl(Op);
1655 SDValue ToInt32 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, ToInt16);
1656 SDValue Splat = DAG.getNode(ISD::SPLAT_VECTOR, dl, SplatTy, ToInt32);
1670 const SDLoc &dl(Op);
1690 Elems[i] = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NTy,
1691 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NTy,
1699 Elems[i] = DAG.getSExtOrTrunc(V, dl, NTy);
1712 return DAG.getBuildVector(VecTy, dl, Elems);
1725 return DAG.getNode(HexagonISD::QCAT, dl, VecTy, Op0, Op.getOperand(1));
1732 SDValue V0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfTy,
1734 SDValue V1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfTy,
1736 return DAG.getNode(HexagonISD::QCAT, dl, VecTy, V0, V1);
1745 SDValue P = createHvxPrefixPred(V, dl, BitBytes, true, DAG);
1751 SDValue S = DAG.getConstant(InpLen*BitBytes, dl, MVT::i32);
1752 SDValue Res = getZero(dl, ByteTy, DAG);
1754 Res = DAG.getNode(HexagonISD::VROR, dl, ByteTy, Res, S);
1755 Res = DAG.getNode(ISD::OR, dl, ByteTy, Res, Prefixes[e-i-1]);
1757 return DAG.getNode(HexagonISD::V2Q, dl, VecTy, Res);
1766 const SDLoc &dl(Op);
1769 return extractHvxElementPred(VecV, IdxV, dl, ty(Op), DAG);
1771 return extractHvxElementReg(VecV, IdxV, dl, ty(Op), DAG);
1777 const SDLoc &dl(Op);
1784 return insertHvxElementPred(VecV, IdxV, ValV, dl, DAG);
1787 SDValue T0 = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
1794 return insertHvxElementReg(VecV, IdxV, ValV, dl, DAG);
1807 const SDLoc &dl(Op);
1811 return extractHvxSubvectorPred(SrcV, IdxV, dl, DstTy, DAG);
1813 return extractHvxSubvectorReg(Op, SrcV, IdxV, dl, DstTy, DAG);
1824 const SDLoc &dl(Op);
1828 return insertHvxSubvectorPred(VecV, ValV, IdxV, dl, DAG);
1830 return insertHvxSubvectorReg(VecV, ValV, IdxV, dl, DAG);
1871 const SDLoc &dl(Op);
1880 SDValue Vec1 = DAG.getNode(ISD::SPLAT_VECTOR, dl, ResTy,
1881 DAG.getConstant(1, dl, MVT::i32));
1882 SDValue VecW = DAG.getNode(ISD::SPLAT_VECTOR, dl, ResTy,
1883 DAG.getConstant(ElemWidth, dl, MVT::i32));
1884 SDValue VecN1 = DAG.getNode(ISD::SPLAT_VECTOR, dl, ResTy,
1885 DAG.getConstant(-1, dl, MVT::i32));
1890 SDValue A = DAG.getNode(ISD::AND, dl, ResTy,
1891 {DAG.getNode(ISD::XOR, dl, ResTy, {InpV, VecN1}),
1892 DAG.getNode(ISD::SUB, dl, ResTy, {InpV, Vec1})});
1893 return DAG.getNode(ISD::SUB, dl, ResTy,
1894 {VecW, DAG.getNode(ISD::CTLZ, dl, ResTy, A)});
1899 const SDLoc &dl(Op);
1911 return DAG.getNode(HexagonISD::UMUL_LOHI, dl, ResTys, {Vs, Vt}).getValue(1);
1913 return DAG.getNode(HexagonISD::SMUL_LOHI, dl, ResTys, {Vs, Vt}).getValue(1);
1923 const SDLoc &dl(Op);
1932 SDValue Lo = DAG.getNode(ISD::MUL, dl, ty(Op), {Vu, Vv});
1933 return DAG.getMergeValues({Lo, Hi}, dl);
1942 return emitHvxMulLoHiV62(Vu, SignedVu, Vv, SignedVv, dl, DAG);
1948 SDValue Hi = emitHvxMulHsV60(Vu, Vv, dl, DAG);
1950 return DAG.getMergeValues({Lo, Hi}, dl);
1954 return emitHvxMulLoHiV60(Vu, SignedVu, Vv, SignedVv, dl, DAG);
1962 const SDLoc &dl(Op);
1967 SDValue VQ = compressHvxPred(Val, dl, WordTy, DAG);
1971 SDValue W0 = extractHvxElementReg(VQ, DAG.getConstant(0, dl, MVT::i32),
1972 dl, MVT::i32, DAG);
1976 return DAG.getZExtOrTrunc(W0, dl, ResTy);
1984 VQ, DAG.getConstant(i, dl, MVT::i32), dl, MVT::i32, DAG);
1990 SDValue C = getCombine(Words[i+1], Words[i], dl, MVT::i64, DAG);
1997 return DAG.getNode(ISD::BUILD_PAIR, dl, ResTy, Combines);
2015 SDValue Idx = DAG.getConstant(I, dl, MVT::i32);
2017 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i8, ValAsVec, Idx);
2020 Tmp.push_back(DAG.getConstant(1ull << J, dl, MVT::i8));
2025 SDValue ConstantVec = DAG.getBuildVector(ConstantVecTy, dl, Tmp);
2026 SDValue I2V = buildHvxVectorReg(Bytes, dl, ConstantVecTy, DAG);
2029 I2V = DAG.getNode(ISD::AND, dl, ConstantVecTy, {I2V, ConstantVec});
2030 return DAG.getNode(HexagonISD::V2Q, dl, ResTy, I2V);
2050 const SDLoc &dl(Op);
2058 DAG.getNode(ISD::SELECT, dl, VecTy, Op.getOperand(0),
2059 DAG.getNode(HexagonISD::Q2V, dl, VecTy, Op.getOperand(1)),
2060 DAG.getNode(HexagonISD::Q2V, dl, VecTy, Op.getOperand(2)));
2061 return DAG.getNode(HexagonISD::V2Q, dl, ResTy, S);
2086 const SDLoc &dl(Op);
2098 SDValue Mask = DAG.getConstant(ElemWidth - 1, dl, MVT::i32);
2100 DAG.getNode(ISD::AND, dl, MVT::i32,
2101 {DAG.getZExtOrTrunc(SplatV, dl, MVT::i32), Mask});
2103 DAG.getNode(ISD::SUB, dl, MVT::i32,
2104 {DAG.getConstant(ElemWidth, dl, MVT::i32), ModS});
2106 DAG.getSetCC(dl, MVT::i1, ModS, getZero(dl, MVT::i32, DAG), ISD::SETEQ);
2110 DAG.getNode(HexagonISD::VASL, dl, InpTy, {A, IsLeft ? ModS : NegS});
2112 DAG.getNode(HexagonISD::VLSR, dl, InpTy, {B, IsLeft ? NegS : ModS});
2113 SDValue Or = DAG.getNode(ISD::OR, dl, InpTy, {Part1, Part2});
2116 return DAG.getNode(ISD::SELECT, dl, InpTy, {IsZero, (IsLeft ? A : B), Or});
2120 InpTy, dl, DAG.getConstant(ElemWidth - 1, dl, ElemTy));
2123 return DAG.getNode(MOpc, dl, ty(Op),
2124 {A, B, DAG.getNode(ISD::AND, dl, InpTy, {S, Mask})});
2129 const SDLoc &dl(Op);
2134 return DAG.getMergeValues({P.getValue(1), P.getValue(0)}, dl);
2144 return DAG.getNode(HexagonISD::TYPECAST, dl, ResTy, Ops[1]);
2150 return Swap(DAG.getNode(HexagonISD::SMUL_LOHI, dl, Op->getVTList(),
2154 return Swap(DAG.getNode(HexagonISD::UMUL_LOHI, dl, Op->getVTList(),
2158 return Swap(DAG.getNode(HexagonISD::USMUL_LOHI, dl, Op->getVTList(),
2168 const SDLoc &dl(Op);
2182 SDValue Load = DAG.getLoad(ValTy, dl, Chain, Base, MemOp);
2186 SDValue VSel = DAG.getNode(ISD::VSELECT, dl, ValTy, Mask, Load, Thru);
2187 return DAG.getMergeValues({VSel, Load.getValue(1)}, dl);
2196 SDValue Offset0 = DAG.getTargetConstant(0, dl, ty(Base));
2199 SDValue Store = getInstr(StoreOpc, dl, MVT::Other,
2207 SDValue Z = getZero(dl, ty(V), DAG);
2211 SDValue LoV = getInstr(Hexagon::V6_vlalignb, dl, ty(V), {V, Z, A}, DAG);
2212 SDValue HiV = getInstr(Hexagon::V6_vlalignb, dl, ty(V), {Z, V, A}, DAG);
2218 SDValue MaskV = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, Mask);
2220 VectorPair MaskU = {DAG.getNode(HexagonISD::V2Q, dl, BoolTy, Tmp.first),
2221 DAG.getNode(HexagonISD::V2Q, dl, BoolTy, Tmp.second)};
2224 SDValue Offset1 = DAG.getTargetConstant(HwLen, dl, MVT::i32);
2226 getInstr(StoreOpc, dl, MVT::Other,
2229 getInstr(StoreOpc, dl, MVT::Other,
2233 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, {StoreLo, StoreHi});
2246 const SDLoc &dl(Op);
2254 SDValue Fp16Ones = DAG.getConstantFP(FloatVal, dl, ArgTy);
2256 getInstr(Hexagon::V6_vmpy_qf32_hf, dl, VecTy, {F16Vec, Fp16Ones}, DAG);
2259 VectorPair Pair = opSplit(VmpyVec, dl, DAG);
2261 getInstr(Hexagon::V6_vconv_sf_qf32, dl, HalfTy, {Pair.first}, DAG);
2263 getInstr(Hexagon::V6_vconv_sf_qf32, dl, HalfTy, {Pair.second}, DAG);
2266 getInstr(Hexagon::V6_vshuffvdd, dl, VecTy,
2267 {HiVec, LoVec, DAG.getConstant(-4, dl, MVT::i32)}, DAG);
2399 const SDLoc &dl, bool Signed, SelectionDAG &DAG) const {
2409 SDValue Add = DAG.getNode(ISD::ADD, dl, ResTy, {A, B});
2410 SDValue Ovf = DAG.getSetCC(dl, PredTy, Add, A, ISD::SETULT);
2417 SDValue Add = DAG.getNode(ISD::ADD, dl, ResTy, {A, B});
2419 DAG.getNode(ISD::XOR, dl, ResTy, {A, DAG.getConstant(-1, dl, ResTy)});
2420 SDValue Xor0 = DAG.getNode(ISD::XOR, dl, ResTy, {NotA, B});
2421 SDValue Xor1 = DAG.getNode(ISD::XOR, dl, ResTy, {Add, B});
2422 SDValue And = DAG.getNode(ISD::AND, dl, ResTy, {Xor0, Xor1});
2424 DAG.getSetCC(dl, PredTy, And, getZero(dl, ResTy, DAG), ISD::SETLT);
2434 const SDLoc &dl(Val);
2454 SDValue LowBits = DAG.getConstant((1ull << (Amt - 1)) - 1, dl, IntTy);
2456 SDValue AmtP1 = DAG.getConstant(1ull << Amt, dl, IntTy);
2457 SDValue And = DAG.getNode(ISD::AND, dl, IntTy, {Inp, AmtP1});
2458 SDValue Zero = getZero(dl, IntTy, DAG);
2459 SDValue Bit = DAG.getSetCC(dl, PredTy, And, Zero, ISD::SETNE);
2460 SDValue Rup = DAG.getZExtOrTrunc(Bit, dl, IntTy);
2461 auto [Tmp0, Ovf] = emitHvxAddWithOverflow(Inp, LowBits, dl, Signed, DAG);
2463 SDValue AmtM1 = DAG.getConstant(Amt - 1, dl, IntTy);
2464 SDValue Tmp1 = DAG.getNode(ShRight, dl, IntTy, Inp, AmtM1);
2465 SDValue Tmp2 = DAG.getNode(ShRight, dl, IntTy, Tmp0, AmtM1);
2466 SDValue Tmp3 = DAG.getNode(ISD::ADD, dl, IntTy, Tmp2, Rup);
2468 SDValue Eq = DAG.getSetCC(dl, PredTy, Tmp1, Tmp2, ISD::SETEQ);
2469 SDValue One = DAG.getConstant(1, dl, IntTy);
2470 SDValue Tmp4 = DAG.getNode(ShRight, dl, IntTy, {Tmp2, One});
2471 SDValue Tmp5 = DAG.getNode(ShRight, dl, IntTy, {Tmp3, One});
2472 SDValue Mux = DAG.getNode(ISD::VSELECT, dl, IntTy, {Eq, Tmp5, Tmp4});
2477 HexagonTargetLowering::emitHvxMulHsV60(SDValue A, SDValue B, const SDLoc &dl,
2483 SDValue S16 = DAG.getConstant(16, dl, MVT::i32);
2506 SDValue T0 = getInstr(Hexagon::V6_vmpyewuh, dl, VecTy, {B, A}, DAG);
2508 SDValue T1 = getInstr(Hexagon::V6_vasrw, dl, VecTy, {A, S16}, DAG);
2510 SDValue P0 = getInstr(Hexagon::V6_vmpyhus, dl, PairTy, {T1, B}, DAG);
2516 SDValue P1 = getInstr(Hexagon::V6_vadduhw, dl, PairTy, {T0, T2}, DAG);
2518 SDValue P2 = getInstr(Hexagon::V6_vaddhw, dl, PairTy, {T0, T2}, DAG);
2521 SDValue T3 = getInstr(Hexagon::V6_vasrw_acc, dl, VecTy,
2523 SDValue T4 = getInstr(Hexagon::V6_vasrw, dl, VecTy, {B, S16}, DAG);
2526 SDValue P3 = getInstr(Hexagon::V6_vmpyhv, dl, PairTy, {T1, T4}, DAG);
2529 SDValue T6 = DAG.getNode(ISD::ADD, dl, VecTy, {T3, T5});
2535 bool SignedB, const SDLoc &dl,
2541 SDValue S16 = DAG.getConstant(16, dl, MVT::i32);
2556 SDValue P0 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {A, B}, DAG);
2559 SDValue T0 = getInstr(Hexagon::V6_lvsplatw, dl, VecTy,
2560 {DAG.getConstant(0x02020202, dl, MVT::i32)}, DAG);
2561 SDValue T1 = getInstr(Hexagon::V6_vdelta, dl, VecTy, {B, T0}, DAG);
2565 SDValue P1 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {A, T1}, DAG);
2569 SDValue P2 = getInstr(Hexagon::V6_vadduhw, dl, PairTy,
2573 getInstr(Hexagon::V6_vlsrw, dl, VecTy, {LoHalf(P0, DAG), S16}, DAG);
2574 SDValue T3 = DAG.getNode(ISD::ADD, dl, VecTy, {LoHalf(P2, DAG), T2});
2577 SDValue T4 = getInstr(Hexagon::V6_vasrw_acc, dl, VecTy,
2581 Lo = getInstr(Hexagon::V6_vaslw_acc, dl, VecTy,
2583 Hi = DAG.getNode(ISD::ADD, dl, VecTy, {HiHalf(P0, DAG), T4});
2589 SDValue Zero = getZero(dl, VecTy, DAG);
2590 SDValue Q0 = DAG.getSetCC(dl, PredTy, A, Zero, ISD::SETLT);
2591 SDValue Q1 = DAG.getSetCC(dl, PredTy, B, Zero, ISD::SETLT);
2592 SDValue X0 = DAG.getNode(ISD::VSELECT, dl, VecTy, {Q0, B, Zero});
2593 SDValue X1 = getInstr(Hexagon::V6_vaddwq, dl, VecTy, {Q1, X0, A}, DAG);
2594 Hi = getInstr(Hexagon::V6_vsubw, dl, VecTy, {Hi, X1}, DAG);
2599 SDValue Zero = getZero(dl, VecTy, DAG);
2600 SDValue Q1 = DAG.getSetCC(dl, PredTy, B, Zero, ISD::SETLT);
2601 Hi = getInstr(Hexagon::V6_vsubwq, dl, VecTy, {Q1, Hi, A}, DAG);
2606 return DAG.getMergeValues({Lo, Hi}, dl);
2612 const SDLoc &dl,
2625 SDValue P0 = getInstr(Hexagon::V6_vmpyewuh_64, dl, PairTy, {A, B}, DAG);
2627 getInstr(Hexagon::V6_vmpyowh_64_acc, dl, PairTy, {P0, A, B}, DAG);
2633 SDValue Zero = getZero(dl, VecTy, DAG);
2642 SDValue Q0 = DAG.getSetCC(dl, PredTy, A, Zero, ISD::SETLT);
2643 SDValue Q1 = DAG.getSetCC(dl, PredTy, B, Zero, ISD::SETLT);
2644 SDValue T0 = getInstr(Hexagon::V6_vandvqv, dl, VecTy, {Q0, B}, DAG);
2645 SDValue T1 = getInstr(Hexagon::V6_vaddwq, dl, VecTy, {Q1, T0, A}, DAG);
2646 Hi = getInstr(Hexagon::V6_vaddw, dl, VecTy, {Hi, T1}, DAG);
2648 SDValue Zero = getZero(dl, VecTy, DAG);
2656 SDValue Q0 = DAG.getSetCC(dl, PredTy, A, Zero, ISD::SETLT);
2657 Hi = getInstr(Hexagon::V6_vaddwq, dl, VecTy, {Q0, Hi, B}, DAG);
2660 return DAG.getMergeValues({Lo, Hi}, dl);
2686 const SDLoc &dl(Op);
2690 SDValue WInp = resizeToWidth(Inp, WInpTy, Signed, dl, DAG);
2691 SDValue Conv = DAG.getNode(Opc, dl, WResTy, WInp);
2692 SDValue Res = resizeToWidth(Conv, ResTy, Signed, dl, DAG);
2701 const SDLoc &dl(Op);
2787 SDValue Zero = getZero(dl, ResTy, DAG);
2788 SDValue Neg = DAG.getSetCC(dl, PredTy, Inp, Zero, ISD::SETLT);
2789 SDValue M80 = DAG.getConstant(1ull << (ElemWidth - 1), dl, ResTy);
2790 SDValue M7F = DAG.getConstant((1ull << (ElemWidth - 1)) - 1, dl, ResTy);
2791 SDValue One = DAG.getConstant(1, dl, ResTy);
2792 SDValue Exp00 = DAG.getNode(ISD::SHL, dl, ResTy, {Inp, One});
2793 SDValue Exp01 = DAG.getNode(ISD::SUB, dl, ResTy, {Exp00, M80});
2794 SDValue MNE = DAG.getConstant(ElemWidth - ExpWidth, dl, ResTy);
2795 SDValue ExpM1 = DAG.getNode(ISD::SRA, dl, ResTy, {Exp01, MNE});
2797 SDValue ExpW = DAG.getConstant(ExpWidth, dl, ResTy);
2798 SDValue Frc00 = DAG.getNode(ISD::SHL, dl, ResTy, {Inp, ExpW});
2799 SDValue Frc01 = DAG.getNode(ISD::OR, dl, ResTy, {Frc00, M80});
2801 SDValue MN2 = DAG.getConstant(ElemWidth - 2, dl, ResTy);
2802 SDValue Rsh00 = DAG.getNode(ISD::SUB, dl, ResTy, {MN2, ExpM1});
2803 SDValue MW = DAG.getConstant(ElemWidth, dl, ResTy);
2804 SDValue Rsh01 = DAG.getNode(ISD::SMIN, dl, ResTy, {Rsh00, MW});
2805 SDValue Frc02 = DAG.getNode(ISD::SRL, dl, ResTy, {Frc01, Rsh01});
2810 SDValue Bnd = DAG.getNode(ISD::VSELECT, dl, ResTy, {Neg, M80, M7F});
2811 SDValue Pos = DAG.getSetCC(dl, PredTy, Rsh01, Zero, ISD::SETGT);
2812 SDValue Frc13 = DAG.getNode(ISD::SUB, dl, ResTy, {Zero, Frc02});
2813 SDValue Frc14 = DAG.getNode(ISD::VSELECT, dl, ResTy, {Neg, Frc13, Frc02});
2814 Int = DAG.getNode(ISD::VSELECT, dl, ResTy, {Pos, Frc14, Bnd});
2817 SDValue Rsn = DAG.getSetCC(dl, PredTy, Rsh01, Zero, ISD::SETLT);
2818 SDValue Frc23 = DAG.getNode(ISD::VSELECT, dl, ResTy, Rsn, M7F, Frc02);
2819 Int = DAG.getNode(ISD::VSELECT, dl, ResTy, Neg, Zero, Frc23);
2830 const SDLoc &dl(Op);
2867 SDValue Zero = getZero(dl, InpTy, DAG);
2868 SDValue One = DAG.getConstant(1, dl, InpTy);
2869 SDValue IsZero = DAG.getSetCC(dl, PredTy, Op0, Zero, ISD::SETEQ);
2870 SDValue Abs = Signed ? DAG.getNode(ISD::ABS, dl, InpTy, Op0) : Op0;
2871 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, InpTy, Abs);
2872 SDValue NLeft = DAG.getNode(ISD::ADD, dl, InpTy, {Clz, One});
2873 SDValue Frac0 = DAG.getNode(ISD::SHL, dl, InpTy, {Abs, NLeft});
2877 SDValue IsNeg = DAG.getSetCC(dl, PredTy, Op0, Zero, ISD::SETLT);
2878 SDValue M80 = DAG.getConstant(1ull << (ElemWidth - 1), dl, InpTy);
2879 SDValue Sign = DAG.getNode(ISD::VSELECT, dl, InpTy, {IsNeg, M80, Zero});
2880 Frac = DAG.getNode(ISD::OR, dl, InpTy, {Sign, Frac});
2883 SDValue Rnrm = DAG.getZExtOrTrunc(Ovf, dl, InpTy);
2884 SDValue Exp0 = DAG.getConstant(ElemWidth + ExpBias, dl, InpTy);
2885 SDValue Exp1 = DAG.getNode(ISD::ADD, dl, InpTy, {Rnrm, Exp0});
2886 SDValue Exp2 = DAG.getNode(ISD::SUB, dl, InpTy, {Exp1, NLeft});
2887 SDValue Exp3 = DAG.getNode(ISD::SHL, dl, InpTy,
2888 {Exp2, DAG.getConstant(FracWidth, dl, InpTy)});
2889 SDValue Flt0 = DAG.getNode(ISD::OR, dl, InpTy, {Frac, Exp3});
2890 SDValue Flt1 = DAG.getNode(ISD::VSELECT, dl, InpTy, {IsZero, Zero, Flt0});
2915 const SDLoc &dl(Op);
2916 return DAG.getNode(TLOpc, dl, ty(Op), Op.getOperand(0),
2918 DAG.getConstant(Opc, dl, MVT::i32));
2933 const SDLoc &dl(Op);
2943 ty(A).isVector() ? opSplit(A, dl, DAG) : std::make_pair(A, A);
2959 SDValue L = DAG.getNode(Op.getOpcode(), dl, HalfTy, OpsL);
2960 SDValue H = DAG.getNode(Op.getOpcode(), dl, HalfTy, OpsH);
2972 const SDLoc &dl(Op);
2978 DAG.getMemBasePlusOffset(Base0, TypeSize::getFixed(HwLen), dl);
2993 SDValue Load0 = DAG.getLoad(SingleTy, dl, Chain, Base0, MOp0);
2994 SDValue Load1 = DAG.getLoad(SingleTy, dl, Chain, Base1, MOp1);
2996 { DAG.getNode(ISD::CONCAT_VECTORS, dl, MemTy, Load0, Load1),
2997 DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
2998 Load0.getValue(1), Load1.getValue(1)) }, dl);
3002 VectorPair Vals = opSplit(cast<StoreSDNode>(Op)->getValue(), dl, DAG);
3003 SDValue Store0 = DAG.getStore(Chain, dl, Vals.first, Base0, MOp0);
3004 SDValue Store1 = DAG.getStore(Chain, dl, Vals.second, Base1, MOp1);
3005 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store0, Store1);
3012 VectorPair Masks = opSplit(MaskN->getMask(), dl, DAG);
3017 opSplit(cast<MaskedLoadSDNode>(Op)->getPassThru(), dl, DAG);
3019 DAG.getMaskedLoad(SingleTy, dl, Chain, Base0, Offset, Masks.first,
3023 DAG.getMaskedLoad(SingleTy, dl, Chain, Base1, Offset, Masks.second,
3027 { DAG.getNode(ISD::CONCAT_VECTORS, dl, MemTy, MLoad0, MLoad1),
3028 DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3029 MLoad0.getValue(1), MLoad1.getValue(1)) }, dl);
3032 VectorPair Vals = opSplit(cast<MaskedStoreSDNode>(Op)->getValue(), dl, DAG);
3033 SDValue MStore0 = DAG.getMaskedStore(Chain, dl, Vals.first, Base0, Offset,
3036 SDValue MStore1 = DAG.getMaskedStore(Chain, dl, Vals.second, Base1, Offset,
3039 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MStore0, MStore1);
3048 const SDLoc &dl(Op);
3064 SDValue Mask = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
3065 {DAG.getConstant(ResLen, dl, MVT::i32)}, DAG);
3071 SDValue Load = DAG.getMaskedLoad(LoadTy, dl, Chain, Base, Offset, Mask,
3075 return DAG.getMergeValues({Value, Load.getValue(1)}, dl);
3080 const SDLoc &dl(Op);
3097 Value = opJoin({Value, DAG.getUNDEF(ty(Value))}, dl, DAG);
3104 SDValue Mask = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
3105 {DAG.getConstant(ValueLen, dl, MVT::i32)}, DAG);
3108 return DAG.getMaskedStore(Chain, dl, Value, Base, Offset, Mask, ty(Value),
3114 const SDLoc &dl(Op);
3129 SDValue SetCC = DAG.getNode(ISD::SETCC, dl, ResTy,
3133 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, RetTy,
3134 {SetCC, getZero(dl, MVT::i32, DAG)});
3289 const SDLoc &dl(Op);
3298 return DAG.getNode(Opc, dl, Ty, {Arg, DAG.getValueType(Ty)});
3301 return DAG.getNode(Opc, dl, Ty, {Arg, Op.getOperand(1), Op.getOperand(2)});
3489 const SDLoc &dl(Op);
3514 DAG.getVectorShuffle(SrcTy, dl, Src, DAG.getUNDEF(SrcTy), Mask);
3515 return opSplit(Deal, dl, DAG).first;
3529 const SDLoc &dl(Op);
3580 SDValue Cat = DAG.getNode(ISD::CONCAT_VECTORS, dl, LongTy, {C0, C1});
3581 return DAG.getVectorShuffle(LongTy, dl, Cat, DAG.getUNDEF(LongTy), LongMask);
3587 const SDLoc &dl(N);
3609 return DAG.getNode(ISD::VSELECT, dl, ty(Op), C0, Ops[2], Ops[1]);
3616 return C->isZero() ? DAG.getNode(HexagonISD::QFALSE, dl, ty(Op))
3617 : DAG.getNode(HexagonISD::QTRUE, dl, ty(Op));
3622 return DAG.getNode(ISD::SPLAT_VECTOR, dl, ty(Op),
3623 DAG.getConstant(-1, dl, MVT::i32));
3625 return getZero(dl, ty(Op), DAG);
3635 SDValue Rot = DAG.getNode(ISD::ADD, dl, ty(Rot0), {Rot0, Rot1});
3636 return DAG.getNode(HexagonISD::VROR, dl, ty(Op), {Vec, Rot});