Lines Matching +full:test +full:- +full:part1
1 //===-- HexagonISelLoweringHVX.cpp --- Lowering HVX operations ------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
30 static cl::opt<unsigned> HvxWidenThreshold("hexagon-hvx-widen",
40 // For a float scalar type, return (exp-bits, exp-bias, fraction-bits)
70 // The idea is to rely on target-independent operations as much
108 // Note: v16i1 -> i16 is handled in type legalization instead of op
141 // Custom-lower BUILD_VECTOR. The standard (target-independent)
172 // Custom-lower BUILD_VECTOR. The standard (target-independent)
176 // Make concat-vectors custom to handle concats of more than 2 vectors.
230 // Make concat-vectors custom to handle concats of more than 2 vectors.
270 // Custom-lower BUILD_VECTOR for vector pairs. The standard (target-
274 // Make concat-vectors custom to handle concats of more than 2 vectors.
277 // Custom-lower these operations for pairs. Expand them into a concat
335 setOperationAction(ISD::MULHS, WordV, Custom); // -> _LOHI
336 setOperationAction(ISD::MULHU, WordV, Custom); // -> _LOHI
371 // v64i8 -> v64i1 (single)
372 // v64i16 -> v64i1 (pair)
519 assert((NumElem % 2) == 0 && "Expecting even-sized vector type");
743 SDValue Mask = DAG.getConstant(32/ElemWidth - 1, dl, MVT::i32);
767 ByteMask.push_back(-1);
860 // should be of a sub-vector type of the source vector type.
866 SrcIdx.push_back(-1);
879 int I = C->getSExtValue();
912 if (!Used.test(I))
952 // performed post-loop to position the last element.
1002 // uses. The scaling down will pick every 2nd or 4th (every Scale-th
1068 SDValue S4 = DAG.getConstant(HwLen-4, dl, MVT::i32);
1091 return !N->isZero();
1096 return N->isZero();
1121 // Find the first non-undef value in this group.
1167 // of the desired element in the 32-bit word.
1207 {ByteIdxV, DAG.getConstant(-4, dl, MVT::i32)});
1220 // If this is not inserting a 32-bit word, convert it into such a thing.
1227 // 2. Treating the extracted word as a 32-bit vector, insert the given
1234 // 3. Insert the 32-bit word back into the original vector.
1259 unsigned Idx = IdxV.getNode()->getAsZExtVal();
1268 Idx -= VecTy.getVectorNumElements() / 2;
1302 unsigned Idx = IdxV.getNode()->getAsZExtVal();
1332 // 8-byte vector. To avoid repeated extracts from ByteVec, shuffle the
1337 // 8-byte groups as many times as necessary.
1348 // Combine the two low words from ShuffV into a v8i8, and byte-compare
1383 unsigned Idx = CN->getZExtValue();
1408 if (!IdxN || !IdxN->isZero()) {
1414 // would be by HwLen-Idx, but if two words are inserted, it will need to be
1415 // by (HwLen-4)-Idx.
1428 RolBase = HwLen-4;
1431 if (RolBase != 4 || !IdxN || !IdxN->isZero()) {
1468 if (!IdxN || !IdxN->isZero()) {
1483 if (!IdxN || !IdxN->isZero()) {
1494 // Sign- and any-extending of a vector predicate to a vector register is
1495 // equivalent to Q2V. For zero-extensions, generate a vmux between 0 and
1511 // Given a predicate register VecQ, transfer bits VecQ[0..HwLen-1]
1512 // (i.e. the entire predicate register) to bits [0..HwLen-1] of a
1666 // Vector concatenation of two integer (non-bool) vectors does not need
1667 // special lowering. Custom-lower concats of bool vectors and expand
1675 // Expand the other cases into a build-vector.
1677 for (SDValue V : Op.getNode()->ops())
1681 // all operations are expected to be type-legalized, and i16 is not
1683 // type, sign-extend it to a valid one.
1727 ArrayRef<SDUse> U(Op.getNode()->ops());
1744 for (SDValue V : Op.getNode()->op_values()) {
1755 Res = DAG.getNode(ISD::OR, dl, ByteTy, Res, Prefixes[e-i-1]);
1804 unsigned Idx = IdxV.getNode()->getAsZExtVal();
1835 // Lower any-extends of boolean vectors to sign-extends, since they
1836 // translate directly to Q2V. Zero-extending could also be done equally
1838 // For all other vectors, use zero-extend.
1870 // cttz(x) = bitwidth(x) - ctlz(~x & (x-1))
1885 DAG.getConstant(-1, dl, MVT::i32));
1939 // Legal on HVX v62+, but lower it here because patterns can't handle multi-
2000 // Handle bitcast from i128 -> v128i1 and i64 -> v64i1.
2038 // Sign- and zero-extends are legal.
2098 SDValue Mask = DAG.getConstant(ElemWidth - 1, dl, MVT::i32);
2109 SDValue Part1 =
2113 SDValue Or = DAG.getNode(ISD::OR, dl, InpTy, {Part1, Part2});
2120 InpTy, dl, DAG.getConstant(ElemWidth - 1, dl, ElemTy));
2131 SmallVector<SDValue> Ops(Op->ops().begin(), Op->ops().end());
2150 return Swap(DAG.getNode(HexagonISD::SMUL_LOHI, dl, Op->getVTList(),
2154 return Swap(DAG.getNode(HexagonISD::UMUL_LOHI, dl, Op->getVTList(),
2158 return Swap(DAG.getNode(HexagonISD::USMUL_LOHI, dl, Op->getVTList(),
2172 SDValue Mask = MaskN->getMask();
2173 SDValue Chain = MaskN->getChain();
2174 SDValue Base = MaskN->getBasePtr();
2175 auto *MemOp = MF.getMachineMemOperand(MaskN->getMemOperand(), 0, HwLen);
2177 unsigned Opc = Op->getOpcode();
2183 SDValue Thru = cast<MaskedLoadSDNode>(MaskN)->getPassThru();
2195 SDValue Value = cast<MaskedStoreSDNode>(MaskN)->getValue();
2198 if (MaskN->getAlign().value() % HwLen == 0) {
2242 assert(Op->getOpcode() == ISD::FP_EXTEND);
2267 {HiVec, LoVec, DAG.getConstant(-4, dl, MVT::i32)}, DAG);
2314 // i8, i16 -> f16 is legal.
2419 DAG.getNode(ISD::XOR, dl, ResTy, {A, DAG.getConstant(-1, dl, ResTy)});
2432 // tie-break by rounding halves to even integer.
2439 // uint tmp0 = inp + ((1 << (Amt-1)) - 1);
2443 // uint tmp1 = inp >> (Amt-1); // tmp1 == tmp2 iff
2444 // uint tmp2 = tmp0 >> (Amt-1); // the Amt-1 lower bits were all 0
2454 SDValue LowBits = DAG.getConstant((1ull << (Amt - 1)) - 1, dl, IntTy);
2463 SDValue AmtM1 = DAG.getConstant(Amt - 1, dl, IntTy);
2495 // The final additions need to make sure to properly maintain any carry-
2500 // --------------
2513 // We need to add T0+T2, recording the carry-out, which will be 1<<16
2515 // P1 = interleaved even/odd 32-bit (unsigned) sums of 16-bit halves
2517 // P2 = interleaved even/odd 32-bit (signed) sums of 16-bit halves
2519 // T3 = full-precision(T0+T2) >> 16
2520 // The low halves are added-unsigned, the high ones are added-signed.
2549 // Do halfword-wise multiplications for unsigned*unsigned product, then
2597 // mulhus(A.uw,B.w) = mulhu(A.uw,B.uw) - (A.w if B < 0)
2666 // Rewrite conversion between integer and floating-point in such a way that
2668 // floating-point type, combined with additional integer-integer extensions
2670 // E.g. f32 -> i8 ==> f32 -> i32 -> i8
2713 // // "bias+1". When the encoded exp is "all-1" (i.e. inf/nan), this will
2716 // // represented in bias-width bits (i.e. 011..1).
2717 // int32_t expm1 = (inp << 1) - 0x80000000;
2731 // // amount as a 6-bit signed value (so 33 is same as -31, i.e. shift
2733 // int32_t rsh = min(31 - (expm1 + 1), 32);
2741 // frac = -frac; //
2784 assert((1ull << (ExpWidth - 1)) == (1 + ExpBias));
2789 SDValue M80 = DAG.getConstant(1ull << (ElemWidth - 1), dl, ResTy);
2790 SDValue M7F = DAG.getConstant((1ull << (ElemWidth - 1)) - 1, dl, ResTy);
2794 SDValue MNE = DAG.getConstant(ElemWidth - ExpWidth, dl, ResTy);
2801 SDValue MN2 = DAG.getConstant(ElemWidth - 2, dl, ResTy);
2845 // uint32_t frac1 = frac0 + ((1 << 8) - 1);
2853 // int32_t exp = 32 - norm_left + renorm + 127;
2878 SDValue M80 = DAG.getConstant(1ull << (ElemWidth - 1), dl, InpTy);
2936 MVT Ty = typeSplit(N->getVT().getSimpleVT()).first;
2941 for (SDValue A : Op.getNode()->ops()) {
2968 MVT MemTy = MemN->getMemoryVT().getSimpleVT();
2975 SDValue Chain = MemN->getChain();
2976 SDValue Base0 = MemN->getBasePtr();
2979 unsigned MemOpc = MemN->getOpcode();
2982 if (MachineMemOperand *MMO = MemN->getMemOperand()) {
2992 assert(cast<LoadSDNode>(Op)->isUnindexed());
3001 assert(cast<StoreSDNode>(Op)->isUnindexed());
3002 VectorPair Vals = opSplit(cast<StoreSDNode>(Op)->getValue(), dl, DAG);
3011 assert(MaskN->isUnindexed());
3012 VectorPair Masks = opSplit(MaskN->getMask(), dl, DAG);
3017 opSplit(cast<MaskedLoadSDNode>(Op)->getPassThru(), dl, DAG);
3032 VectorPair Vals = opSplit(cast<MaskedStoreSDNode>(Op)->getValue(), dl, DAG);
3042 std::string Name = "Unexpected operation: " + Op->getOperationName(&DAG);
3050 assert(LoadN->isUnindexed() && "Not widening indexed loads yet");
3051 assert(LoadN->getMemoryVT().getVectorElementType() != MVT::i1 &&
3054 SDValue Chain = LoadN->getChain();
3055 SDValue Base = LoadN->getBasePtr();
3069 auto *MemOp = MF.getMachineMemOperand(LoadN->getMemOperand(), 0, HwLen);
3082 assert(StoreN->isUnindexed() && "Not widening indexed stores yet");
3083 assert(StoreN->getMemoryVT().getVectorElementType() != MVT::i1 &&
3086 SDValue Chain = StoreN->getChain();
3087 SDValue Base = StoreN->getBasePtr();
3090 SDValue Value = opCastElem(StoreN->getValue(), MVT::i8, DAG);
3107 auto *MemOp = MF.getMachineMemOperand(StoreN->getMemOperand(), 0, HwLen);
3141 llvm::any_of(Op.getNode()->ops(), [this] (SDValue V) {
3192 // In general, sign- and zero-extends can't be split and still
3255 // E.g. i8 -> i16 remains unchanged, but i8 -> i32 ==> i8 -> i16 -> i32.
3270 llvm_unreachable("ISD:: ops will be auto-folded");
3360 unsigned Opc = N->getOpcode();
3363 if (N->getNumOperands() > 0)
3383 if (shouldWidenToHvx(ty(cast<StoreSDNode>(N)->getValue()), DAG)) {
3392 assert(S->getOpcode() == ISD::MERGE_VALUES);
3398 if (isHvxPairTy(ty(Op->getOperand(1)))) { // Stored value
3426 unsigned Opc = N->getOpcode();
3429 if (N->getNumOperands() > 0)
3451 assert(Load->getOpcode() == ISD::MERGE_VALUES);
3484 // Simplify V:v2NiB --(bitcast)--> vNi2B --(truncate)--> vNiB
3485 // to extract-subvector (shuffle V, pick even, pick odd)
3556 ArrayRef<int> Mask = SV->getMask();
3560 if (M == -1) {
3566 M -= InpLen;
3592 SmallVector<SDValue, 4> Ops(N->ops().begin(), N->ops().end());
3604 // (vselect (xor x, qtrue), v0, v1) -> (vselect x, v1, v0)
3606 if (Cond->getOpcode() == ISD::XOR) {
3608 if (C1->getOpcode() == HexagonISD::QTRUE)
3616 return C->isZero() ? DAG.getNode(HexagonISD::QFALSE, dl, ty(Op))
3623 DAG.getConstant(-1, dl, MVT::i32));
3678 if (llvm::any_of(N->values(), IsHvxTy) || llvm::any_of(N->ops(), IsHvxOp))
3689 for (int i = 0, e = N->getNumValues(); i != e; ++i) {
3693 return llvm::any_of(N->ops(), IsWidenedToHvx);