1 //===-- HexagonISelLoweringHVX.cpp --- Lowering HVX operations ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "HexagonISelLowering.h" 10 #include "HexagonRegisterInfo.h" 11 #include "HexagonSubtarget.h" 12 #include "llvm/IR/IntrinsicsHexagon.h" 13 #include "llvm/Support/CommandLine.h" 14 15 using namespace llvm; 16 17 static const MVT LegalV64[] = { MVT::v64i8, MVT::v32i16, MVT::v16i32 }; 18 static const MVT LegalW64[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 }; 19 static const MVT LegalV128[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 }; 20 static const MVT LegalW128[] = { MVT::v256i8, MVT::v128i16, MVT::v64i32 }; 21 22 23 void 24 HexagonTargetLowering::initializeHVXLowering() { 25 if (Subtarget.useHVX64BOps()) { 26 addRegisterClass(MVT::v64i8, &Hexagon::HvxVRRegClass); 27 addRegisterClass(MVT::v32i16, &Hexagon::HvxVRRegClass); 28 addRegisterClass(MVT::v16i32, &Hexagon::HvxVRRegClass); 29 addRegisterClass(MVT::v128i8, &Hexagon::HvxWRRegClass); 30 addRegisterClass(MVT::v64i16, &Hexagon::HvxWRRegClass); 31 addRegisterClass(MVT::v32i32, &Hexagon::HvxWRRegClass); 32 // These "short" boolean vector types should be legal because 33 // they will appear as results of vector compares. If they were 34 // not legal, type legalization would try to make them legal 35 // and that would require using operations that do not use or 36 // produce such types. That, in turn, would imply using custom 37 // nodes, which would be unoptimizable by the DAG combiner. 38 // The idea is to rely on target-independent operations as much 39 // as possible. 40 addRegisterClass(MVT::v16i1, &Hexagon::HvxQRRegClass); 41 addRegisterClass(MVT::v32i1, &Hexagon::HvxQRRegClass); 42 addRegisterClass(MVT::v64i1, &Hexagon::HvxQRRegClass); 43 } else if (Subtarget.useHVX128BOps()) { 44 addRegisterClass(MVT::v128i8, &Hexagon::HvxVRRegClass); 45 addRegisterClass(MVT::v64i16, &Hexagon::HvxVRRegClass); 46 addRegisterClass(MVT::v32i32, &Hexagon::HvxVRRegClass); 47 addRegisterClass(MVT::v256i8, &Hexagon::HvxWRRegClass); 48 addRegisterClass(MVT::v128i16, &Hexagon::HvxWRRegClass); 49 addRegisterClass(MVT::v64i32, &Hexagon::HvxWRRegClass); 50 addRegisterClass(MVT::v32i1, &Hexagon::HvxQRRegClass); 51 addRegisterClass(MVT::v64i1, &Hexagon::HvxQRRegClass); 52 addRegisterClass(MVT::v128i1, &Hexagon::HvxQRRegClass); 53 } 54 55 // Set up operation actions. 56 57 bool Use64b = Subtarget.useHVX64BOps(); 58 ArrayRef<MVT> LegalV = Use64b ? LegalV64 : LegalV128; 59 ArrayRef<MVT> LegalW = Use64b ? LegalW64 : LegalW128; 60 MVT ByteV = Use64b ? MVT::v64i8 : MVT::v128i8; 61 MVT ByteW = Use64b ? MVT::v128i8 : MVT::v256i8; 62 63 auto setPromoteTo = [this] (unsigned Opc, MVT FromTy, MVT ToTy) { 64 setOperationAction(Opc, FromTy, Promote); 65 AddPromotedToType(Opc, FromTy, ToTy); 66 }; 67 68 // Handle bitcasts of vector predicates to scalars (e.g. v32i1 to i32). 69 // Note: v16i1 -> i16 is handled in type legalization instead of op 70 // legalization. 71 setOperationAction(ISD::BITCAST, MVT::i16, Custom); 72 setOperationAction(ISD::BITCAST, MVT::i32, Custom); 73 setOperationAction(ISD::BITCAST, MVT::i64, Custom); 74 setOperationAction(ISD::BITCAST, MVT::v16i1, Custom); 75 setOperationAction(ISD::BITCAST, MVT::v128i1, Custom); 76 setOperationAction(ISD::BITCAST, MVT::i128, Custom); 77 setOperationAction(ISD::VECTOR_SHUFFLE, ByteV, Legal); 78 setOperationAction(ISD::VECTOR_SHUFFLE, ByteW, Legal); 79 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 80 81 for (MVT T : LegalV) { 82 setIndexedLoadAction(ISD::POST_INC, T, Legal); 83 setIndexedStoreAction(ISD::POST_INC, T, Legal); 84 85 setOperationAction(ISD::AND, T, Legal); 86 setOperationAction(ISD::OR, T, Legal); 87 setOperationAction(ISD::XOR, T, Legal); 88 setOperationAction(ISD::ADD, T, Legal); 89 setOperationAction(ISD::SUB, T, Legal); 90 setOperationAction(ISD::CTPOP, T, Legal); 91 setOperationAction(ISD::CTLZ, T, Legal); 92 if (T != ByteV) { 93 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, T, Legal); 94 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, T, Legal); 95 setOperationAction(ISD::BSWAP, T, Legal); 96 } 97 98 setOperationAction(ISD::CTTZ, T, Custom); 99 setOperationAction(ISD::LOAD, T, Custom); 100 setOperationAction(ISD::MUL, T, Custom); 101 setOperationAction(ISD::MULHS, T, Custom); 102 setOperationAction(ISD::MULHU, T, Custom); 103 setOperationAction(ISD::BUILD_VECTOR, T, Custom); 104 // Make concat-vectors custom to handle concats of more than 2 vectors. 105 setOperationAction(ISD::CONCAT_VECTORS, T, Custom); 106 setOperationAction(ISD::INSERT_SUBVECTOR, T, Custom); 107 setOperationAction(ISD::INSERT_VECTOR_ELT, T, Custom); 108 setOperationAction(ISD::EXTRACT_SUBVECTOR, T, Custom); 109 setOperationAction(ISD::EXTRACT_VECTOR_ELT, T, Custom); 110 setOperationAction(ISD::ANY_EXTEND, T, Custom); 111 setOperationAction(ISD::SIGN_EXTEND, T, Custom); 112 setOperationAction(ISD::ZERO_EXTEND, T, Custom); 113 if (T != ByteV) { 114 setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, T, Custom); 115 // HVX only has shifts of words and halfwords. 116 setOperationAction(ISD::SRA, T, Custom); 117 setOperationAction(ISD::SHL, T, Custom); 118 setOperationAction(ISD::SRL, T, Custom); 119 120 // Promote all shuffles to operate on vectors of bytes. 121 setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteV); 122 } 123 124 setCondCodeAction(ISD::SETNE, T, Expand); 125 setCondCodeAction(ISD::SETLE, T, Expand); 126 setCondCodeAction(ISD::SETGE, T, Expand); 127 setCondCodeAction(ISD::SETLT, T, Expand); 128 setCondCodeAction(ISD::SETULE, T, Expand); 129 setCondCodeAction(ISD::SETUGE, T, Expand); 130 setCondCodeAction(ISD::SETULT, T, Expand); 131 } 132 133 for (MVT T : LegalW) { 134 // Custom-lower BUILD_VECTOR for vector pairs. The standard (target- 135 // independent) handling of it would convert it to a load, which is 136 // not always the optimal choice. 137 setOperationAction(ISD::BUILD_VECTOR, T, Custom); 138 // Make concat-vectors custom to handle concats of more than 2 vectors. 139 setOperationAction(ISD::CONCAT_VECTORS, T, Custom); 140 141 // Custom-lower these operations for pairs. Expand them into a concat 142 // of the corresponding operations on individual vectors. 143 setOperationAction(ISD::ANY_EXTEND, T, Custom); 144 setOperationAction(ISD::SIGN_EXTEND, T, Custom); 145 setOperationAction(ISD::ZERO_EXTEND, T, Custom); 146 setOperationAction(ISD::SIGN_EXTEND_INREG, T, Custom); 147 setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, T, Custom); 148 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, T, Legal); 149 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, T, Legal); 150 151 setOperationAction(ISD::LOAD, T, Custom); 152 setOperationAction(ISD::STORE, T, Custom); 153 setOperationAction(ISD::CTLZ, T, Custom); 154 setOperationAction(ISD::CTTZ, T, Custom); 155 setOperationAction(ISD::CTPOP, T, Custom); 156 157 setOperationAction(ISD::ADD, T, Legal); 158 setOperationAction(ISD::SUB, T, Legal); 159 setOperationAction(ISD::MUL, T, Custom); 160 setOperationAction(ISD::MULHS, T, Custom); 161 setOperationAction(ISD::MULHU, T, Custom); 162 setOperationAction(ISD::AND, T, Custom); 163 setOperationAction(ISD::OR, T, Custom); 164 setOperationAction(ISD::XOR, T, Custom); 165 setOperationAction(ISD::SETCC, T, Custom); 166 setOperationAction(ISD::VSELECT, T, Custom); 167 if (T != ByteW) { 168 setOperationAction(ISD::SRA, T, Custom); 169 setOperationAction(ISD::SHL, T, Custom); 170 setOperationAction(ISD::SRL, T, Custom); 171 172 // Promote all shuffles to operate on vectors of bytes. 173 setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteW); 174 } 175 } 176 177 // Boolean vectors. 178 179 for (MVT T : LegalW) { 180 // Boolean types for vector pairs will overlap with the boolean 181 // types for single vectors, e.g. 182 // v64i8 -> v64i1 (single) 183 // v64i16 -> v64i1 (pair) 184 // Set these actions first, and allow the single actions to overwrite 185 // any duplicates. 186 MVT BoolW = MVT::getVectorVT(MVT::i1, T.getVectorNumElements()); 187 setOperationAction(ISD::SETCC, BoolW, Custom); 188 setOperationAction(ISD::AND, BoolW, Custom); 189 setOperationAction(ISD::OR, BoolW, Custom); 190 setOperationAction(ISD::XOR, BoolW, Custom); 191 } 192 193 for (MVT T : LegalV) { 194 MVT BoolV = MVT::getVectorVT(MVT::i1, T.getVectorNumElements()); 195 setOperationAction(ISD::BUILD_VECTOR, BoolV, Custom); 196 setOperationAction(ISD::CONCAT_VECTORS, BoolV, Custom); 197 setOperationAction(ISD::INSERT_SUBVECTOR, BoolV, Custom); 198 setOperationAction(ISD::INSERT_VECTOR_ELT, BoolV, Custom); 199 setOperationAction(ISD::EXTRACT_SUBVECTOR, BoolV, Custom); 200 setOperationAction(ISD::EXTRACT_VECTOR_ELT, BoolV, Custom); 201 setOperationAction(ISD::AND, BoolV, Legal); 202 setOperationAction(ISD::OR, BoolV, Legal); 203 setOperationAction(ISD::XOR, BoolV, Legal); 204 } 205 206 if (Use64b) { 207 for (MVT T: {MVT::v32i8, MVT::v32i16, MVT::v16i8, MVT::v16i16, MVT::v16i32}) 208 setOperationAction(ISD::SIGN_EXTEND_INREG, T, Legal); 209 } else { 210 for (MVT T: {MVT::v64i8, MVT::v64i16, MVT::v32i8, MVT::v32i16, MVT::v32i32}) 211 setOperationAction(ISD::SIGN_EXTEND_INREG, T, Legal); 212 } 213 214 setTargetDAGCombine(ISD::VSELECT); 215 } 216 217 SDValue 218 HexagonTargetLowering::getInt(unsigned IntId, MVT ResTy, ArrayRef<SDValue> Ops, 219 const SDLoc &dl, SelectionDAG &DAG) const { 220 SmallVector<SDValue,4> IntOps; 221 IntOps.push_back(DAG.getConstant(IntId, dl, MVT::i32)); 222 for (const SDValue &Op : Ops) 223 IntOps.push_back(Op); 224 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, ResTy, IntOps); 225 } 226 227 MVT 228 HexagonTargetLowering::typeJoin(const TypePair &Tys) const { 229 assert(Tys.first.getVectorElementType() == Tys.second.getVectorElementType()); 230 231 MVT ElemTy = Tys.first.getVectorElementType(); 232 return MVT::getVectorVT(ElemTy, Tys.first.getVectorNumElements() + 233 Tys.second.getVectorNumElements()); 234 } 235 236 HexagonTargetLowering::TypePair 237 HexagonTargetLowering::typeSplit(MVT VecTy) const { 238 assert(VecTy.isVector()); 239 unsigned NumElem = VecTy.getVectorNumElements(); 240 assert((NumElem % 2) == 0 && "Expecting even-sized vector type"); 241 MVT HalfTy = MVT::getVectorVT(VecTy.getVectorElementType(), NumElem/2); 242 return { HalfTy, HalfTy }; 243 } 244 245 MVT 246 HexagonTargetLowering::typeExtElem(MVT VecTy, unsigned Factor) const { 247 MVT ElemTy = VecTy.getVectorElementType(); 248 MVT NewElemTy = MVT::getIntegerVT(ElemTy.getSizeInBits() * Factor); 249 return MVT::getVectorVT(NewElemTy, VecTy.getVectorNumElements()); 250 } 251 252 MVT 253 HexagonTargetLowering::typeTruncElem(MVT VecTy, unsigned Factor) const { 254 MVT ElemTy = VecTy.getVectorElementType(); 255 MVT NewElemTy = MVT::getIntegerVT(ElemTy.getSizeInBits() / Factor); 256 return MVT::getVectorVT(NewElemTy, VecTy.getVectorNumElements()); 257 } 258 259 SDValue 260 HexagonTargetLowering::opCastElem(SDValue Vec, MVT ElemTy, 261 SelectionDAG &DAG) const { 262 if (ty(Vec).getVectorElementType() == ElemTy) 263 return Vec; 264 MVT CastTy = tyVector(Vec.getValueType().getSimpleVT(), ElemTy); 265 return DAG.getBitcast(CastTy, Vec); 266 } 267 268 SDValue 269 HexagonTargetLowering::opJoin(const VectorPair &Ops, const SDLoc &dl, 270 SelectionDAG &DAG) const { 271 return DAG.getNode(ISD::CONCAT_VECTORS, dl, typeJoin(ty(Ops)), 272 Ops.second, Ops.first); 273 } 274 275 HexagonTargetLowering::VectorPair 276 HexagonTargetLowering::opSplit(SDValue Vec, const SDLoc &dl, 277 SelectionDAG &DAG) const { 278 TypePair Tys = typeSplit(ty(Vec)); 279 if (Vec.getOpcode() == HexagonISD::QCAT) 280 return VectorPair(Vec.getOperand(0), Vec.getOperand(1)); 281 return DAG.SplitVector(Vec, dl, Tys.first, Tys.second); 282 } 283 284 bool 285 HexagonTargetLowering::isHvxSingleTy(MVT Ty) const { 286 return Subtarget.isHVXVectorType(Ty) && 287 Ty.getSizeInBits() == 8 * Subtarget.getVectorLength(); 288 } 289 290 bool 291 HexagonTargetLowering::isHvxPairTy(MVT Ty) const { 292 return Subtarget.isHVXVectorType(Ty) && 293 Ty.getSizeInBits() == 16 * Subtarget.getVectorLength(); 294 } 295 296 bool 297 HexagonTargetLowering::isHvxBoolTy(MVT Ty) const { 298 return Subtarget.isHVXVectorType(Ty, true) && 299 Ty.getVectorElementType() == MVT::i1; 300 } 301 302 bool HexagonTargetLowering::allowsHvxMemoryAccess( 303 MVT VecTy, MachineMemOperand::Flags Flags, bool *Fast) const { 304 // Bool vectors are excluded by default, but make it explicit to 305 // emphasize that bool vectors cannot be loaded or stored. 306 // Also, disallow double vector stores (to prevent unnecessary 307 // store widening in DAG combiner). 308 if (VecTy.getSizeInBits() > 8*Subtarget.getVectorLength()) 309 return false; 310 if (!Subtarget.isHVXVectorType(VecTy, /*IncludeBool=*/false)) 311 return false; 312 if (Fast) 313 *Fast = true; 314 return true; 315 } 316 317 bool HexagonTargetLowering::allowsHvxMisalignedMemoryAccesses( 318 MVT VecTy, MachineMemOperand::Flags Flags, bool *Fast) const { 319 if (!Subtarget.isHVXVectorType(VecTy)) 320 return false; 321 // XXX Should this be false? vmemu are a bit slower than vmem. 322 if (Fast) 323 *Fast = true; 324 return true; 325 } 326 327 SDValue 328 HexagonTargetLowering::convertToByteIndex(SDValue ElemIdx, MVT ElemTy, 329 SelectionDAG &DAG) const { 330 if (ElemIdx.getValueType().getSimpleVT() != MVT::i32) 331 ElemIdx = DAG.getBitcast(MVT::i32, ElemIdx); 332 333 unsigned ElemWidth = ElemTy.getSizeInBits(); 334 if (ElemWidth == 8) 335 return ElemIdx; 336 337 unsigned L = Log2_32(ElemWidth/8); 338 const SDLoc &dl(ElemIdx); 339 return DAG.getNode(ISD::SHL, dl, MVT::i32, 340 {ElemIdx, DAG.getConstant(L, dl, MVT::i32)}); 341 } 342 343 SDValue 344 HexagonTargetLowering::getIndexInWord32(SDValue Idx, MVT ElemTy, 345 SelectionDAG &DAG) const { 346 unsigned ElemWidth = ElemTy.getSizeInBits(); 347 assert(ElemWidth >= 8 && ElemWidth <= 32); 348 if (ElemWidth == 32) 349 return Idx; 350 351 if (ty(Idx) != MVT::i32) 352 Idx = DAG.getBitcast(MVT::i32, Idx); 353 const SDLoc &dl(Idx); 354 SDValue Mask = DAG.getConstant(32/ElemWidth - 1, dl, MVT::i32); 355 SDValue SubIdx = DAG.getNode(ISD::AND, dl, MVT::i32, {Idx, Mask}); 356 return SubIdx; 357 } 358 359 SDValue 360 HexagonTargetLowering::getByteShuffle(const SDLoc &dl, SDValue Op0, 361 SDValue Op1, ArrayRef<int> Mask, 362 SelectionDAG &DAG) const { 363 MVT OpTy = ty(Op0); 364 assert(OpTy == ty(Op1)); 365 366 MVT ElemTy = OpTy.getVectorElementType(); 367 if (ElemTy == MVT::i8) 368 return DAG.getVectorShuffle(OpTy, dl, Op0, Op1, Mask); 369 assert(ElemTy.getSizeInBits() >= 8); 370 371 MVT ResTy = tyVector(OpTy, MVT::i8); 372 unsigned ElemSize = ElemTy.getSizeInBits() / 8; 373 374 SmallVector<int,128> ByteMask; 375 for (int M : Mask) { 376 if (M < 0) { 377 for (unsigned I = 0; I != ElemSize; ++I) 378 ByteMask.push_back(-1); 379 } else { 380 int NewM = M*ElemSize; 381 for (unsigned I = 0; I != ElemSize; ++I) 382 ByteMask.push_back(NewM+I); 383 } 384 } 385 assert(ResTy.getVectorNumElements() == ByteMask.size()); 386 return DAG.getVectorShuffle(ResTy, dl, opCastElem(Op0, MVT::i8, DAG), 387 opCastElem(Op1, MVT::i8, DAG), ByteMask); 388 } 389 390 SDValue 391 HexagonTargetLowering::buildHvxVectorReg(ArrayRef<SDValue> Values, 392 const SDLoc &dl, MVT VecTy, 393 SelectionDAG &DAG) const { 394 unsigned VecLen = Values.size(); 395 MachineFunction &MF = DAG.getMachineFunction(); 396 MVT ElemTy = VecTy.getVectorElementType(); 397 unsigned ElemWidth = ElemTy.getSizeInBits(); 398 unsigned HwLen = Subtarget.getVectorLength(); 399 400 unsigned ElemSize = ElemWidth / 8; 401 assert(ElemSize*VecLen == HwLen); 402 SmallVector<SDValue,32> Words; 403 404 if (VecTy.getVectorElementType() != MVT::i32) { 405 assert((ElemSize == 1 || ElemSize == 2) && "Invalid element size"); 406 unsigned OpsPerWord = (ElemSize == 1) ? 4 : 2; 407 MVT PartVT = MVT::getVectorVT(VecTy.getVectorElementType(), OpsPerWord); 408 for (unsigned i = 0; i != VecLen; i += OpsPerWord) { 409 SDValue W = buildVector32(Values.slice(i, OpsPerWord), dl, PartVT, DAG); 410 Words.push_back(DAG.getBitcast(MVT::i32, W)); 411 } 412 } else { 413 Words.assign(Values.begin(), Values.end()); 414 } 415 416 unsigned NumWords = Words.size(); 417 bool IsSplat = true, IsUndef = true; 418 SDValue SplatV; 419 for (unsigned i = 0; i != NumWords && IsSplat; ++i) { 420 if (isUndef(Words[i])) 421 continue; 422 IsUndef = false; 423 if (!SplatV.getNode()) 424 SplatV = Words[i]; 425 else if (SplatV != Words[i]) 426 IsSplat = false; 427 } 428 if (IsUndef) 429 return DAG.getUNDEF(VecTy); 430 if (IsSplat) { 431 assert(SplatV.getNode()); 432 auto *IdxN = dyn_cast<ConstantSDNode>(SplatV.getNode()); 433 if (IdxN && IdxN->isNullValue()) 434 return getZero(dl, VecTy, DAG); 435 return DAG.getNode(HexagonISD::VSPLATW, dl, VecTy, SplatV); 436 } 437 438 // Delay recognizing constant vectors until here, so that we can generate 439 // a vsplat. 440 SmallVector<ConstantInt*, 128> Consts(VecLen); 441 bool AllConst = getBuildVectorConstInts(Values, VecTy, DAG, Consts); 442 if (AllConst) { 443 ArrayRef<Constant*> Tmp((Constant**)Consts.begin(), 444 (Constant**)Consts.end()); 445 Constant *CV = ConstantVector::get(Tmp); 446 Align Alignment(HwLen); 447 SDValue CP = 448 LowerConstantPool(DAG.getConstantPool(CV, VecTy, Alignment), DAG); 449 return DAG.getLoad(VecTy, dl, DAG.getEntryNode(), CP, 450 MachinePointerInfo::getConstantPool(MF), Alignment); 451 } 452 453 // A special case is a situation where the vector is built entirely from 454 // elements extracted from another vector. This could be done via a shuffle 455 // more efficiently, but typically, the size of the source vector will not 456 // match the size of the vector being built (which precludes the use of a 457 // shuffle directly). 458 // This only handles a single source vector, and the vector being built 459 // should be of a sub-vector type of the source vector type. 460 auto IsBuildFromExtracts = [this,&Values] (SDValue &SrcVec, 461 SmallVectorImpl<int> &SrcIdx) { 462 SDValue Vec; 463 for (SDValue V : Values) { 464 if (isUndef(V)) { 465 SrcIdx.push_back(-1); 466 continue; 467 } 468 if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT) 469 return false; 470 // All extracts should come from the same vector. 471 SDValue T = V.getOperand(0); 472 if (Vec.getNode() != nullptr && T.getNode() != Vec.getNode()) 473 return false; 474 Vec = T; 475 ConstantSDNode *C = dyn_cast<ConstantSDNode>(V.getOperand(1)); 476 if (C == nullptr) 477 return false; 478 int I = C->getSExtValue(); 479 assert(I >= 0 && "Negative element index"); 480 SrcIdx.push_back(I); 481 } 482 SrcVec = Vec; 483 return true; 484 }; 485 486 SmallVector<int,128> ExtIdx; 487 SDValue ExtVec; 488 if (IsBuildFromExtracts(ExtVec, ExtIdx)) { 489 MVT ExtTy = ty(ExtVec); 490 unsigned ExtLen = ExtTy.getVectorNumElements(); 491 if (ExtLen == VecLen || ExtLen == 2*VecLen) { 492 // Construct a new shuffle mask that will produce a vector with the same 493 // number of elements as the input vector, and such that the vector we 494 // want will be the initial subvector of it. 495 SmallVector<int,128> Mask; 496 BitVector Used(ExtLen); 497 498 for (int M : ExtIdx) { 499 Mask.push_back(M); 500 if (M >= 0) 501 Used.set(M); 502 } 503 // Fill the rest of the mask with the unused elements of ExtVec in hopes 504 // that it will result in a permutation of ExtVec's elements. It's still 505 // fine if it doesn't (e.g. if undefs are present, or elements are 506 // repeated), but permutations can always be done efficiently via vdelta 507 // and vrdelta. 508 for (unsigned I = 0; I != ExtLen; ++I) { 509 if (Mask.size() == ExtLen) 510 break; 511 if (!Used.test(I)) 512 Mask.push_back(I); 513 } 514 515 SDValue S = DAG.getVectorShuffle(ExtTy, dl, ExtVec, 516 DAG.getUNDEF(ExtTy), Mask); 517 if (ExtLen == VecLen) 518 return S; 519 return DAG.getTargetExtractSubreg(Hexagon::vsub_lo, dl, VecTy, S); 520 } 521 } 522 523 // Construct two halves in parallel, then or them together. 524 assert(4*Words.size() == Subtarget.getVectorLength()); 525 SDValue HalfV0 = getInstr(Hexagon::V6_vd0, dl, VecTy, {}, DAG); 526 SDValue HalfV1 = getInstr(Hexagon::V6_vd0, dl, VecTy, {}, DAG); 527 SDValue S = DAG.getConstant(4, dl, MVT::i32); 528 for (unsigned i = 0; i != NumWords/2; ++i) { 529 SDValue N = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy, 530 {HalfV0, Words[i]}); 531 SDValue M = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy, 532 {HalfV1, Words[i+NumWords/2]}); 533 HalfV0 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {N, S}); 534 HalfV1 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {M, S}); 535 } 536 537 HalfV0 = DAG.getNode(HexagonISD::VROR, dl, VecTy, 538 {HalfV0, DAG.getConstant(HwLen/2, dl, MVT::i32)}); 539 SDValue DstV = DAG.getNode(ISD::OR, dl, VecTy, {HalfV0, HalfV1}); 540 return DstV; 541 } 542 543 SDValue 544 HexagonTargetLowering::createHvxPrefixPred(SDValue PredV, const SDLoc &dl, 545 unsigned BitBytes, bool ZeroFill, SelectionDAG &DAG) const { 546 MVT PredTy = ty(PredV); 547 unsigned HwLen = Subtarget.getVectorLength(); 548 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); 549 550 if (Subtarget.isHVXVectorType(PredTy, true)) { 551 // Move the vector predicate SubV to a vector register, and scale it 552 // down to match the representation (bytes per type element) that VecV 553 // uses. The scaling down will pick every 2nd or 4th (every Scale-th 554 // in general) element and put them at the front of the resulting 555 // vector. This subvector will then be inserted into the Q2V of VecV. 556 // To avoid having an operation that generates an illegal type (short 557 // vector), generate a full size vector. 558 // 559 SDValue T = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, PredV); 560 SmallVector<int,128> Mask(HwLen); 561 // Scale = BitBytes(PredV) / Given BitBytes. 562 unsigned Scale = HwLen / (PredTy.getVectorNumElements() * BitBytes); 563 unsigned BlockLen = PredTy.getVectorNumElements() * BitBytes; 564 565 for (unsigned i = 0; i != HwLen; ++i) { 566 unsigned Num = i % Scale; 567 unsigned Off = i / Scale; 568 Mask[BlockLen*Num + Off] = i; 569 } 570 SDValue S = DAG.getVectorShuffle(ByteTy, dl, T, DAG.getUNDEF(ByteTy), Mask); 571 if (!ZeroFill) 572 return S; 573 // Fill the bytes beyond BlockLen with 0s. 574 MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen); 575 SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy, 576 {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG); 577 SDValue M = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, Q); 578 return DAG.getNode(ISD::AND, dl, ByteTy, S, M); 579 } 580 581 // Make sure that this is a valid scalar predicate. 582 assert(PredTy == MVT::v2i1 || PredTy == MVT::v4i1 || PredTy == MVT::v8i1); 583 584 unsigned Bytes = 8 / PredTy.getVectorNumElements(); 585 SmallVector<SDValue,4> Words[2]; 586 unsigned IdxW = 0; 587 588 auto Lo32 = [&DAG, &dl] (SDValue P) { 589 return DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, P); 590 }; 591 auto Hi32 = [&DAG, &dl] (SDValue P) { 592 return DAG.getTargetExtractSubreg(Hexagon::isub_hi, dl, MVT::i32, P); 593 }; 594 595 SDValue W0 = isUndef(PredV) 596 ? DAG.getUNDEF(MVT::i64) 597 : DAG.getNode(HexagonISD::P2D, dl, MVT::i64, PredV); 598 Words[IdxW].push_back(Hi32(W0)); 599 Words[IdxW].push_back(Lo32(W0)); 600 601 while (Bytes < BitBytes) { 602 IdxW ^= 1; 603 Words[IdxW].clear(); 604 605 if (Bytes < 4) { 606 for (const SDValue &W : Words[IdxW ^ 1]) { 607 SDValue T = expandPredicate(W, dl, DAG); 608 Words[IdxW].push_back(Hi32(T)); 609 Words[IdxW].push_back(Lo32(T)); 610 } 611 } else { 612 for (const SDValue &W : Words[IdxW ^ 1]) { 613 Words[IdxW].push_back(W); 614 Words[IdxW].push_back(W); 615 } 616 } 617 Bytes *= 2; 618 } 619 620 assert(Bytes == BitBytes); 621 622 SDValue Vec = ZeroFill ? getZero(dl, ByteTy, DAG) : DAG.getUNDEF(ByteTy); 623 SDValue S4 = DAG.getConstant(HwLen-4, dl, MVT::i32); 624 for (const SDValue &W : Words[IdxW]) { 625 Vec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, Vec, S4); 626 Vec = DAG.getNode(HexagonISD::VINSERTW0, dl, ByteTy, Vec, W); 627 } 628 629 return Vec; 630 } 631 632 SDValue 633 HexagonTargetLowering::buildHvxVectorPred(ArrayRef<SDValue> Values, 634 const SDLoc &dl, MVT VecTy, 635 SelectionDAG &DAG) const { 636 // Construct a vector V of bytes, such that a comparison V >u 0 would 637 // produce the required vector predicate. 638 unsigned VecLen = Values.size(); 639 unsigned HwLen = Subtarget.getVectorLength(); 640 assert(VecLen <= HwLen || VecLen == 8*HwLen); 641 SmallVector<SDValue,128> Bytes; 642 bool AllT = true, AllF = true; 643 644 auto IsTrue = [] (SDValue V) { 645 if (const auto *N = dyn_cast<ConstantSDNode>(V.getNode())) 646 return !N->isNullValue(); 647 return false; 648 }; 649 auto IsFalse = [] (SDValue V) { 650 if (const auto *N = dyn_cast<ConstantSDNode>(V.getNode())) 651 return N->isNullValue(); 652 return false; 653 }; 654 655 if (VecLen <= HwLen) { 656 // In the hardware, each bit of a vector predicate corresponds to a byte 657 // of a vector register. Calculate how many bytes does a bit of VecTy 658 // correspond to. 659 assert(HwLen % VecLen == 0); 660 unsigned BitBytes = HwLen / VecLen; 661 for (SDValue V : Values) { 662 AllT &= IsTrue(V); 663 AllF &= IsFalse(V); 664 665 SDValue Ext = !V.isUndef() ? DAG.getZExtOrTrunc(V, dl, MVT::i8) 666 : DAG.getUNDEF(MVT::i8); 667 for (unsigned B = 0; B != BitBytes; ++B) 668 Bytes.push_back(Ext); 669 } 670 } else { 671 // There are as many i1 values, as there are bits in a vector register. 672 // Divide the values into groups of 8 and check that each group consists 673 // of the same value (ignoring undefs). 674 for (unsigned I = 0; I != VecLen; I += 8) { 675 unsigned B = 0; 676 // Find the first non-undef value in this group. 677 for (; B != 8; ++B) { 678 if (!Values[I+B].isUndef()) 679 break; 680 } 681 SDValue F = Values[I+B]; 682 AllT &= IsTrue(F); 683 AllF &= IsFalse(F); 684 685 SDValue Ext = (B < 8) ? DAG.getZExtOrTrunc(F, dl, MVT::i8) 686 : DAG.getUNDEF(MVT::i8); 687 Bytes.push_back(Ext); 688 // Verify that the rest of values in the group are the same as the 689 // first. 690 for (; B != 8; ++B) 691 assert(Values[I+B].isUndef() || Values[I+B] == F); 692 } 693 } 694 695 if (AllT) 696 return DAG.getNode(HexagonISD::QTRUE, dl, VecTy); 697 if (AllF) 698 return DAG.getNode(HexagonISD::QFALSE, dl, VecTy); 699 700 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); 701 SDValue ByteVec = buildHvxVectorReg(Bytes, dl, ByteTy, DAG); 702 return DAG.getNode(HexagonISD::V2Q, dl, VecTy, ByteVec); 703 } 704 705 SDValue 706 HexagonTargetLowering::extractHvxElementReg(SDValue VecV, SDValue IdxV, 707 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const { 708 MVT ElemTy = ty(VecV).getVectorElementType(); 709 710 unsigned ElemWidth = ElemTy.getSizeInBits(); 711 assert(ElemWidth >= 8 && ElemWidth <= 32); 712 (void)ElemWidth; 713 714 SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG); 715 SDValue ExWord = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32, 716 {VecV, ByteIdx}); 717 if (ElemTy == MVT::i32) 718 return ExWord; 719 720 // Have an extracted word, need to extract the smaller element out of it. 721 // 1. Extract the bits of (the original) IdxV that correspond to the index 722 // of the desired element in the 32-bit word. 723 SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG); 724 // 2. Extract the element from the word. 725 SDValue ExVec = DAG.getBitcast(tyVector(ty(ExWord), ElemTy), ExWord); 726 return extractVector(ExVec, SubIdx, dl, ElemTy, MVT::i32, DAG); 727 } 728 729 SDValue 730 HexagonTargetLowering::extractHvxElementPred(SDValue VecV, SDValue IdxV, 731 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const { 732 // Implement other return types if necessary. 733 assert(ResTy == MVT::i1); 734 735 unsigned HwLen = Subtarget.getVectorLength(); 736 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); 737 SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV); 738 739 unsigned Scale = HwLen / ty(VecV).getVectorNumElements(); 740 SDValue ScV = DAG.getConstant(Scale, dl, MVT::i32); 741 IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, ScV); 742 743 SDValue ExtB = extractHvxElementReg(ByteVec, IdxV, dl, MVT::i32, DAG); 744 SDValue Zero = DAG.getTargetConstant(0, dl, MVT::i32); 745 return getInstr(Hexagon::C2_cmpgtui, dl, MVT::i1, {ExtB, Zero}, DAG); 746 } 747 748 SDValue 749 HexagonTargetLowering::insertHvxElementReg(SDValue VecV, SDValue IdxV, 750 SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const { 751 MVT ElemTy = ty(VecV).getVectorElementType(); 752 753 unsigned ElemWidth = ElemTy.getSizeInBits(); 754 assert(ElemWidth >= 8 && ElemWidth <= 32); 755 (void)ElemWidth; 756 757 auto InsertWord = [&DAG,&dl,this] (SDValue VecV, SDValue ValV, 758 SDValue ByteIdxV) { 759 MVT VecTy = ty(VecV); 760 unsigned HwLen = Subtarget.getVectorLength(); 761 SDValue MaskV = DAG.getNode(ISD::AND, dl, MVT::i32, 762 {ByteIdxV, DAG.getConstant(-4, dl, MVT::i32)}); 763 SDValue RotV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {VecV, MaskV}); 764 SDValue InsV = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy, {RotV, ValV}); 765 SDValue SubV = DAG.getNode(ISD::SUB, dl, MVT::i32, 766 {DAG.getConstant(HwLen, dl, MVT::i32), MaskV}); 767 SDValue TorV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {InsV, SubV}); 768 return TorV; 769 }; 770 771 SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG); 772 if (ElemTy == MVT::i32) 773 return InsertWord(VecV, ValV, ByteIdx); 774 775 // If this is not inserting a 32-bit word, convert it into such a thing. 776 // 1. Extract the existing word from the target vector. 777 SDValue WordIdx = DAG.getNode(ISD::SRL, dl, MVT::i32, 778 {ByteIdx, DAG.getConstant(2, dl, MVT::i32)}); 779 SDValue Ext = extractHvxElementReg(opCastElem(VecV, MVT::i32, DAG), WordIdx, 780 dl, MVT::i32, DAG); 781 782 // 2. Treating the extracted word as a 32-bit vector, insert the given 783 // value into it. 784 SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG); 785 MVT SubVecTy = tyVector(ty(Ext), ElemTy); 786 SDValue Ins = insertVector(DAG.getBitcast(SubVecTy, Ext), 787 ValV, SubIdx, dl, ElemTy, DAG); 788 789 // 3. Insert the 32-bit word back into the original vector. 790 return InsertWord(VecV, Ins, ByteIdx); 791 } 792 793 SDValue 794 HexagonTargetLowering::insertHvxElementPred(SDValue VecV, SDValue IdxV, 795 SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const { 796 unsigned HwLen = Subtarget.getVectorLength(); 797 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); 798 SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV); 799 800 unsigned Scale = HwLen / ty(VecV).getVectorNumElements(); 801 SDValue ScV = DAG.getConstant(Scale, dl, MVT::i32); 802 IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, ScV); 803 ValV = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i32, ValV); 804 805 SDValue InsV = insertHvxElementReg(ByteVec, IdxV, ValV, dl, DAG); 806 return DAG.getNode(HexagonISD::V2Q, dl, ty(VecV), InsV); 807 } 808 809 SDValue 810 HexagonTargetLowering::extractHvxSubvectorReg(SDValue VecV, SDValue IdxV, 811 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const { 812 MVT VecTy = ty(VecV); 813 unsigned HwLen = Subtarget.getVectorLength(); 814 unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue(); 815 MVT ElemTy = VecTy.getVectorElementType(); 816 unsigned ElemWidth = ElemTy.getSizeInBits(); 817 818 // If the source vector is a vector pair, get the single vector containing 819 // the subvector of interest. The subvector will never overlap two single 820 // vectors. 821 if (isHvxPairTy(VecTy)) { 822 unsigned SubIdx; 823 if (Idx * ElemWidth >= 8*HwLen) { 824 SubIdx = Hexagon::vsub_hi; 825 Idx -= VecTy.getVectorNumElements() / 2; 826 } else { 827 SubIdx = Hexagon::vsub_lo; 828 } 829 VecTy = typeSplit(VecTy).first; 830 VecV = DAG.getTargetExtractSubreg(SubIdx, dl, VecTy, VecV); 831 if (VecTy == ResTy) 832 return VecV; 833 } 834 835 // The only meaningful subvectors of a single HVX vector are those that 836 // fit in a scalar register. 837 assert(ResTy.getSizeInBits() == 32 || ResTy.getSizeInBits() == 64); 838 839 MVT WordTy = tyVector(VecTy, MVT::i32); 840 SDValue WordVec = DAG.getBitcast(WordTy, VecV); 841 unsigned WordIdx = (Idx*ElemWidth) / 32; 842 843 SDValue W0Idx = DAG.getConstant(WordIdx, dl, MVT::i32); 844 SDValue W0 = extractHvxElementReg(WordVec, W0Idx, dl, MVT::i32, DAG); 845 if (ResTy.getSizeInBits() == 32) 846 return DAG.getBitcast(ResTy, W0); 847 848 SDValue W1Idx = DAG.getConstant(WordIdx+1, dl, MVT::i32); 849 SDValue W1 = extractHvxElementReg(WordVec, W1Idx, dl, MVT::i32, DAG); 850 SDValue WW = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64, {W1, W0}); 851 return DAG.getBitcast(ResTy, WW); 852 } 853 854 SDValue 855 HexagonTargetLowering::extractHvxSubvectorPred(SDValue VecV, SDValue IdxV, 856 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const { 857 MVT VecTy = ty(VecV); 858 unsigned HwLen = Subtarget.getVectorLength(); 859 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); 860 SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV); 861 // IdxV is required to be a constant. 862 unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue(); 863 864 unsigned ResLen = ResTy.getVectorNumElements(); 865 unsigned BitBytes = HwLen / VecTy.getVectorNumElements(); 866 unsigned Offset = Idx * BitBytes; 867 SDValue Undef = DAG.getUNDEF(ByteTy); 868 SmallVector<int,128> Mask; 869 870 if (Subtarget.isHVXVectorType(ResTy, true)) { 871 // Converting between two vector predicates. Since the result is shorter 872 // than the source, it will correspond to a vector predicate with the 873 // relevant bits replicated. The replication count is the ratio of the 874 // source and target vector lengths. 875 unsigned Rep = VecTy.getVectorNumElements() / ResLen; 876 assert(isPowerOf2_32(Rep) && HwLen % Rep == 0); 877 for (unsigned i = 0; i != HwLen/Rep; ++i) { 878 for (unsigned j = 0; j != Rep; ++j) 879 Mask.push_back(i + Offset); 880 } 881 SDValue ShuffV = DAG.getVectorShuffle(ByteTy, dl, ByteVec, Undef, Mask); 882 return DAG.getNode(HexagonISD::V2Q, dl, ResTy, ShuffV); 883 } 884 885 // Converting between a vector predicate and a scalar predicate. In the 886 // vector predicate, a group of BitBytes bits will correspond to a single 887 // i1 element of the source vector type. Those bits will all have the same 888 // value. The same will be true for ByteVec, where each byte corresponds 889 // to a bit in the vector predicate. 890 // The algorithm is to traverse the ByteVec, going over the i1 values from 891 // the source vector, and generate the corresponding representation in an 892 // 8-byte vector. To avoid repeated extracts from ByteVec, shuffle the 893 // elements so that the interesting 8 bytes will be in the low end of the 894 // vector. 895 unsigned Rep = 8 / ResLen; 896 // Make sure the output fill the entire vector register, so repeat the 897 // 8-byte groups as many times as necessary. 898 for (unsigned r = 0; r != HwLen/ResLen; ++r) { 899 // This will generate the indexes of the 8 interesting bytes. 900 for (unsigned i = 0; i != ResLen; ++i) { 901 for (unsigned j = 0; j != Rep; ++j) 902 Mask.push_back(Offset + i*BitBytes); 903 } 904 } 905 906 SDValue Zero = getZero(dl, MVT::i32, DAG); 907 SDValue ShuffV = DAG.getVectorShuffle(ByteTy, dl, ByteVec, Undef, Mask); 908 // Combine the two low words from ShuffV into a v8i8, and byte-compare 909 // them against 0. 910 SDValue W0 = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32, {ShuffV, Zero}); 911 SDValue W1 = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32, 912 {ShuffV, DAG.getConstant(4, dl, MVT::i32)}); 913 SDValue Vec64 = DAG.getNode(HexagonISD::COMBINE, dl, MVT::v8i8, {W1, W0}); 914 return getInstr(Hexagon::A4_vcmpbgtui, dl, ResTy, 915 {Vec64, DAG.getTargetConstant(0, dl, MVT::i32)}, DAG); 916 } 917 918 SDValue 919 HexagonTargetLowering::insertHvxSubvectorReg(SDValue VecV, SDValue SubV, 920 SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const { 921 MVT VecTy = ty(VecV); 922 MVT SubTy = ty(SubV); 923 unsigned HwLen = Subtarget.getVectorLength(); 924 MVT ElemTy = VecTy.getVectorElementType(); 925 unsigned ElemWidth = ElemTy.getSizeInBits(); 926 927 bool IsPair = isHvxPairTy(VecTy); 928 MVT SingleTy = MVT::getVectorVT(ElemTy, (8*HwLen)/ElemWidth); 929 // The two single vectors that VecV consists of, if it's a pair. 930 SDValue V0, V1; 931 SDValue SingleV = VecV; 932 SDValue PickHi; 933 934 if (IsPair) { 935 V0 = DAG.getTargetExtractSubreg(Hexagon::vsub_lo, dl, SingleTy, VecV); 936 V1 = DAG.getTargetExtractSubreg(Hexagon::vsub_hi, dl, SingleTy, VecV); 937 938 SDValue HalfV = DAG.getConstant(SingleTy.getVectorNumElements(), 939 dl, MVT::i32); 940 PickHi = DAG.getSetCC(dl, MVT::i1, IdxV, HalfV, ISD::SETUGT); 941 if (isHvxSingleTy(SubTy)) { 942 if (const auto *CN = dyn_cast<const ConstantSDNode>(IdxV.getNode())) { 943 unsigned Idx = CN->getZExtValue(); 944 assert(Idx == 0 || Idx == VecTy.getVectorNumElements()/2); 945 unsigned SubIdx = (Idx == 0) ? Hexagon::vsub_lo : Hexagon::vsub_hi; 946 return DAG.getTargetInsertSubreg(SubIdx, dl, VecTy, VecV, SubV); 947 } 948 // If IdxV is not a constant, generate the two variants: with the 949 // SubV as the high and as the low subregister, and select the right 950 // pair based on the IdxV. 951 SDValue InLo = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {SubV, V1}); 952 SDValue InHi = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {V0, SubV}); 953 return DAG.getNode(ISD::SELECT, dl, VecTy, PickHi, InHi, InLo); 954 } 955 // The subvector being inserted must be entirely contained in one of 956 // the vectors V0 or V1. Set SingleV to the correct one, and update 957 // IdxV to be the index relative to the beginning of that vector. 958 SDValue S = DAG.getNode(ISD::SUB, dl, MVT::i32, IdxV, HalfV); 959 IdxV = DAG.getNode(ISD::SELECT, dl, MVT::i32, PickHi, S, IdxV); 960 SingleV = DAG.getNode(ISD::SELECT, dl, SingleTy, PickHi, V1, V0); 961 } 962 963 // The only meaningful subvectors of a single HVX vector are those that 964 // fit in a scalar register. 965 assert(SubTy.getSizeInBits() == 32 || SubTy.getSizeInBits() == 64); 966 // Convert IdxV to be index in bytes. 967 auto *IdxN = dyn_cast<ConstantSDNode>(IdxV.getNode()); 968 if (!IdxN || !IdxN->isNullValue()) { 969 IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, 970 DAG.getConstant(ElemWidth/8, dl, MVT::i32)); 971 SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV, IdxV); 972 } 973 // When inserting a single word, the rotation back to the original position 974 // would be by HwLen-Idx, but if two words are inserted, it will need to be 975 // by (HwLen-4)-Idx. 976 unsigned RolBase = HwLen; 977 if (VecTy.getSizeInBits() == 32) { 978 SDValue V = DAG.getBitcast(MVT::i32, SubV); 979 SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, V); 980 } else { 981 SDValue V = DAG.getBitcast(MVT::i64, SubV); 982 SDValue R0 = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, V); 983 SDValue R1 = DAG.getTargetExtractSubreg(Hexagon::isub_hi, dl, MVT::i32, V); 984 SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, SingleV, R0); 985 SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV, 986 DAG.getConstant(4, dl, MVT::i32)); 987 SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, SingleV, R1); 988 RolBase = HwLen-4; 989 } 990 // If the vector wasn't ror'ed, don't ror it back. 991 if (RolBase != 4 || !IdxN || !IdxN->isNullValue()) { 992 SDValue RolV = DAG.getNode(ISD::SUB, dl, MVT::i32, 993 DAG.getConstant(RolBase, dl, MVT::i32), IdxV); 994 SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV, RolV); 995 } 996 997 if (IsPair) { 998 SDValue InLo = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {SingleV, V1}); 999 SDValue InHi = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {V0, SingleV}); 1000 return DAG.getNode(ISD::SELECT, dl, VecTy, PickHi, InHi, InLo); 1001 } 1002 return SingleV; 1003 } 1004 1005 SDValue 1006 HexagonTargetLowering::insertHvxSubvectorPred(SDValue VecV, SDValue SubV, 1007 SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const { 1008 MVT VecTy = ty(VecV); 1009 MVT SubTy = ty(SubV); 1010 assert(Subtarget.isHVXVectorType(VecTy, true)); 1011 // VecV is an HVX vector predicate. SubV may be either an HVX vector 1012 // predicate as well, or it can be a scalar predicate. 1013 1014 unsigned VecLen = VecTy.getVectorNumElements(); 1015 unsigned HwLen = Subtarget.getVectorLength(); 1016 assert(HwLen % VecLen == 0 && "Unexpected vector type"); 1017 1018 unsigned Scale = VecLen / SubTy.getVectorNumElements(); 1019 unsigned BitBytes = HwLen / VecLen; 1020 unsigned BlockLen = HwLen / Scale; 1021 1022 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); 1023 SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV); 1024 SDValue ByteSub = createHvxPrefixPred(SubV, dl, BitBytes, false, DAG); 1025 SDValue ByteIdx; 1026 1027 auto *IdxN = dyn_cast<ConstantSDNode>(IdxV.getNode()); 1028 if (!IdxN || !IdxN->isNullValue()) { 1029 ByteIdx = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, 1030 DAG.getConstant(BitBytes, dl, MVT::i32)); 1031 ByteVec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, ByteVec, ByteIdx); 1032 } 1033 1034 // ByteVec is the target vector VecV rotated in such a way that the 1035 // subvector should be inserted at index 0. Generate a predicate mask 1036 // and use vmux to do the insertion. 1037 MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen); 1038 SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy, 1039 {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG); 1040 ByteVec = getInstr(Hexagon::V6_vmux, dl, ByteTy, {Q, ByteSub, ByteVec}, DAG); 1041 // Rotate ByteVec back, and convert to a vector predicate. 1042 if (!IdxN || !IdxN->isNullValue()) { 1043 SDValue HwLenV = DAG.getConstant(HwLen, dl, MVT::i32); 1044 SDValue ByteXdi = DAG.getNode(ISD::SUB, dl, MVT::i32, HwLenV, ByteIdx); 1045 ByteVec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, ByteVec, ByteXdi); 1046 } 1047 return DAG.getNode(HexagonISD::V2Q, dl, VecTy, ByteVec); 1048 } 1049 1050 SDValue 1051 HexagonTargetLowering::extendHvxVectorPred(SDValue VecV, const SDLoc &dl, 1052 MVT ResTy, bool ZeroExt, SelectionDAG &DAG) const { 1053 // Sign- and any-extending of a vector predicate to a vector register is 1054 // equivalent to Q2V. For zero-extensions, generate a vmux between 0 and 1055 // a vector of 1s (where the 1s are of type matching the vector type). 1056 assert(Subtarget.isHVXVectorType(ResTy)); 1057 if (!ZeroExt) 1058 return DAG.getNode(HexagonISD::Q2V, dl, ResTy, VecV); 1059 1060 assert(ty(VecV).getVectorNumElements() == ResTy.getVectorNumElements()); 1061 SDValue True = DAG.getNode(HexagonISD::VSPLAT, dl, ResTy, 1062 DAG.getConstant(1, dl, MVT::i32)); 1063 SDValue False = getZero(dl, ResTy, DAG); 1064 return DAG.getSelect(dl, ResTy, VecV, True, False); 1065 } 1066 1067 SDValue 1068 HexagonTargetLowering::compressHvxPred(SDValue VecQ, const SDLoc &dl, 1069 MVT ResTy, SelectionDAG &DAG) const { 1070 // Given a predicate register VecQ, transfer bits VecQ[0..HwLen-1] 1071 // (i.e. the entire predicate register) to bits [0..HwLen-1] of a 1072 // vector register. The remaining bits of the vector register are 1073 // unspecified. 1074 1075 MachineFunction &MF = DAG.getMachineFunction(); 1076 unsigned HwLen = Subtarget.getVectorLength(); 1077 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); 1078 MVT PredTy = ty(VecQ); 1079 unsigned PredLen = PredTy.getVectorNumElements(); 1080 assert(HwLen % PredLen == 0); 1081 MVT VecTy = MVT::getVectorVT(MVT::getIntegerVT(8*HwLen/PredLen), PredLen); 1082 1083 Type *Int8Ty = Type::getInt8Ty(*DAG.getContext()); 1084 SmallVector<Constant*, 128> Tmp; 1085 // Create an array of bytes (hex): 01,02,04,08,10,20,40,80, 01,02,04,08,... 1086 // These are bytes with the LSB rotated left with respect to their index. 1087 for (unsigned i = 0; i != HwLen/8; ++i) { 1088 for (unsigned j = 0; j != 8; ++j) 1089 Tmp.push_back(ConstantInt::get(Int8Ty, 1ull << j)); 1090 } 1091 Constant *CV = ConstantVector::get(Tmp); 1092 Align Alignment(HwLen); 1093 SDValue CP = 1094 LowerConstantPool(DAG.getConstantPool(CV, ByteTy, Alignment), DAG); 1095 SDValue Bytes = 1096 DAG.getLoad(ByteTy, dl, DAG.getEntryNode(), CP, 1097 MachinePointerInfo::getConstantPool(MF), Alignment); 1098 1099 // Select the bytes that correspond to true bits in the vector predicate. 1100 SDValue Sel = DAG.getSelect(dl, VecTy, VecQ, DAG.getBitcast(VecTy, Bytes), 1101 getZero(dl, VecTy, DAG)); 1102 // Calculate the OR of all bytes in each group of 8. That will compress 1103 // all the individual bits into a single byte. 1104 // First, OR groups of 4, via vrmpy with 0x01010101. 1105 SDValue All1 = 1106 DAG.getSplatBuildVector(MVT::v4i8, dl, DAG.getConstant(1, dl, MVT::i32)); 1107 SDValue Vrmpy = getInstr(Hexagon::V6_vrmpyub, dl, ByteTy, {Sel, All1}, DAG); 1108 // Then rotate the accumulated vector by 4 bytes, and do the final OR. 1109 SDValue Rot = getInstr(Hexagon::V6_valignbi, dl, ByteTy, 1110 {Vrmpy, Vrmpy, DAG.getTargetConstant(4, dl, MVT::i32)}, DAG); 1111 SDValue Vor = DAG.getNode(ISD::OR, dl, ByteTy, {Vrmpy, Rot}); 1112 1113 // Pick every 8th byte and coalesce them at the beginning of the output. 1114 // For symmetry, coalesce every 1+8th byte after that, then every 2+8th 1115 // byte and so on. 1116 SmallVector<int,128> Mask; 1117 for (unsigned i = 0; i != HwLen; ++i) 1118 Mask.push_back((8*i) % HwLen + i/(HwLen/8)); 1119 SDValue Collect = 1120 DAG.getVectorShuffle(ByteTy, dl, Vor, DAG.getUNDEF(ByteTy), Mask); 1121 return DAG.getBitcast(ResTy, Collect); 1122 } 1123 1124 SDValue 1125 HexagonTargetLowering::LowerHvxBuildVector(SDValue Op, SelectionDAG &DAG) 1126 const { 1127 const SDLoc &dl(Op); 1128 MVT VecTy = ty(Op); 1129 1130 unsigned Size = Op.getNumOperands(); 1131 SmallVector<SDValue,128> Ops; 1132 for (unsigned i = 0; i != Size; ++i) 1133 Ops.push_back(Op.getOperand(i)); 1134 1135 if (VecTy.getVectorElementType() == MVT::i1) 1136 return buildHvxVectorPred(Ops, dl, VecTy, DAG); 1137 1138 if (VecTy.getSizeInBits() == 16*Subtarget.getVectorLength()) { 1139 ArrayRef<SDValue> A(Ops); 1140 MVT SingleTy = typeSplit(VecTy).first; 1141 SDValue V0 = buildHvxVectorReg(A.take_front(Size/2), dl, SingleTy, DAG); 1142 SDValue V1 = buildHvxVectorReg(A.drop_front(Size/2), dl, SingleTy, DAG); 1143 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, V0, V1); 1144 } 1145 1146 return buildHvxVectorReg(Ops, dl, VecTy, DAG); 1147 } 1148 1149 SDValue 1150 HexagonTargetLowering::LowerHvxConcatVectors(SDValue Op, SelectionDAG &DAG) 1151 const { 1152 // Vector concatenation of two integer (non-bool) vectors does not need 1153 // special lowering. Custom-lower concats of bool vectors and expand 1154 // concats of more than 2 vectors. 1155 MVT VecTy = ty(Op); 1156 const SDLoc &dl(Op); 1157 unsigned NumOp = Op.getNumOperands(); 1158 if (VecTy.getVectorElementType() != MVT::i1) { 1159 if (NumOp == 2) 1160 return Op; 1161 // Expand the other cases into a build-vector. 1162 SmallVector<SDValue,8> Elems; 1163 for (SDValue V : Op.getNode()->ops()) 1164 DAG.ExtractVectorElements(V, Elems); 1165 // A vector of i16 will be broken up into a build_vector of i16's. 1166 // This is a problem, since at the time of operation legalization, 1167 // all operations are expected to be type-legalized, and i16 is not 1168 // a legal type. If any of the extracted elements is not of a valid 1169 // type, sign-extend it to a valid one. 1170 for (unsigned i = 0, e = Elems.size(); i != e; ++i) { 1171 SDValue V = Elems[i]; 1172 MVT Ty = ty(V); 1173 if (!isTypeLegal(Ty)) { 1174 EVT NTy = getTypeToTransformTo(*DAG.getContext(), Ty); 1175 if (V.getOpcode() == ISD::EXTRACT_VECTOR_ELT) { 1176 Elems[i] = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NTy, 1177 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NTy, 1178 V.getOperand(0), V.getOperand(1)), 1179 DAG.getValueType(Ty)); 1180 continue; 1181 } 1182 // A few less complicated cases. 1183 if (V.getOpcode() == ISD::Constant) 1184 Elems[i] = DAG.getSExtOrTrunc(V, dl, NTy); 1185 else if (V.isUndef()) 1186 Elems[i] = DAG.getUNDEF(NTy); 1187 else 1188 llvm_unreachable("Unexpected vector element"); 1189 } 1190 } 1191 return DAG.getBuildVector(VecTy, dl, Elems); 1192 } 1193 1194 assert(VecTy.getVectorElementType() == MVT::i1); 1195 unsigned HwLen = Subtarget.getVectorLength(); 1196 assert(isPowerOf2_32(NumOp) && HwLen % NumOp == 0); 1197 1198 SDValue Op0 = Op.getOperand(0); 1199 1200 // If the operands are HVX types (i.e. not scalar predicates), then 1201 // defer the concatenation, and create QCAT instead. 1202 if (Subtarget.isHVXVectorType(ty(Op0), true)) { 1203 if (NumOp == 2) 1204 return DAG.getNode(HexagonISD::QCAT, dl, VecTy, Op0, Op.getOperand(1)); 1205 1206 ArrayRef<SDUse> U(Op.getNode()->ops()); 1207 SmallVector<SDValue,4> SV(U.begin(), U.end()); 1208 ArrayRef<SDValue> Ops(SV); 1209 1210 MVT HalfTy = typeSplit(VecTy).first; 1211 SDValue V0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfTy, 1212 Ops.take_front(NumOp/2)); 1213 SDValue V1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfTy, 1214 Ops.take_back(NumOp/2)); 1215 return DAG.getNode(HexagonISD::QCAT, dl, VecTy, V0, V1); 1216 } 1217 1218 // Count how many bytes (in a vector register) each bit in VecTy 1219 // corresponds to. 1220 unsigned BitBytes = HwLen / VecTy.getVectorNumElements(); 1221 1222 SmallVector<SDValue,8> Prefixes; 1223 for (SDValue V : Op.getNode()->op_values()) { 1224 SDValue P = createHvxPrefixPred(V, dl, BitBytes, true, DAG); 1225 Prefixes.push_back(P); 1226 } 1227 1228 unsigned InpLen = ty(Op.getOperand(0)).getVectorNumElements(); 1229 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); 1230 SDValue S = DAG.getConstant(InpLen*BitBytes, dl, MVT::i32); 1231 SDValue Res = getZero(dl, ByteTy, DAG); 1232 for (unsigned i = 0, e = Prefixes.size(); i != e; ++i) { 1233 Res = DAG.getNode(HexagonISD::VROR, dl, ByteTy, Res, S); 1234 Res = DAG.getNode(ISD::OR, dl, ByteTy, Res, Prefixes[e-i-1]); 1235 } 1236 return DAG.getNode(HexagonISD::V2Q, dl, VecTy, Res); 1237 } 1238 1239 SDValue 1240 HexagonTargetLowering::LowerHvxExtractElement(SDValue Op, SelectionDAG &DAG) 1241 const { 1242 // Change the type of the extracted element to i32. 1243 SDValue VecV = Op.getOperand(0); 1244 MVT ElemTy = ty(VecV).getVectorElementType(); 1245 const SDLoc &dl(Op); 1246 SDValue IdxV = Op.getOperand(1); 1247 if (ElemTy == MVT::i1) 1248 return extractHvxElementPred(VecV, IdxV, dl, ty(Op), DAG); 1249 1250 return extractHvxElementReg(VecV, IdxV, dl, ty(Op), DAG); 1251 } 1252 1253 SDValue 1254 HexagonTargetLowering::LowerHvxInsertElement(SDValue Op, SelectionDAG &DAG) 1255 const { 1256 const SDLoc &dl(Op); 1257 SDValue VecV = Op.getOperand(0); 1258 SDValue ValV = Op.getOperand(1); 1259 SDValue IdxV = Op.getOperand(2); 1260 MVT ElemTy = ty(VecV).getVectorElementType(); 1261 if (ElemTy == MVT::i1) 1262 return insertHvxElementPred(VecV, IdxV, ValV, dl, DAG); 1263 1264 return insertHvxElementReg(VecV, IdxV, ValV, dl, DAG); 1265 } 1266 1267 SDValue 1268 HexagonTargetLowering::LowerHvxExtractSubvector(SDValue Op, SelectionDAG &DAG) 1269 const { 1270 SDValue SrcV = Op.getOperand(0); 1271 MVT SrcTy = ty(SrcV); 1272 MVT DstTy = ty(Op); 1273 SDValue IdxV = Op.getOperand(1); 1274 unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue(); 1275 assert(Idx % DstTy.getVectorNumElements() == 0); 1276 (void)Idx; 1277 const SDLoc &dl(Op); 1278 1279 MVT ElemTy = SrcTy.getVectorElementType(); 1280 if (ElemTy == MVT::i1) 1281 return extractHvxSubvectorPred(SrcV, IdxV, dl, DstTy, DAG); 1282 1283 return extractHvxSubvectorReg(SrcV, IdxV, dl, DstTy, DAG); 1284 } 1285 1286 SDValue 1287 HexagonTargetLowering::LowerHvxInsertSubvector(SDValue Op, SelectionDAG &DAG) 1288 const { 1289 // Idx does not need to be a constant. 1290 SDValue VecV = Op.getOperand(0); 1291 SDValue ValV = Op.getOperand(1); 1292 SDValue IdxV = Op.getOperand(2); 1293 1294 const SDLoc &dl(Op); 1295 MVT VecTy = ty(VecV); 1296 MVT ElemTy = VecTy.getVectorElementType(); 1297 if (ElemTy == MVT::i1) 1298 return insertHvxSubvectorPred(VecV, ValV, IdxV, dl, DAG); 1299 1300 return insertHvxSubvectorReg(VecV, ValV, IdxV, dl, DAG); 1301 } 1302 1303 SDValue 1304 HexagonTargetLowering::LowerHvxAnyExt(SDValue Op, SelectionDAG &DAG) const { 1305 // Lower any-extends of boolean vectors to sign-extends, since they 1306 // translate directly to Q2V. Zero-extending could also be done equally 1307 // fast, but Q2V is used/recognized in more places. 1308 // For all other vectors, use zero-extend. 1309 MVT ResTy = ty(Op); 1310 SDValue InpV = Op.getOperand(0); 1311 MVT ElemTy = ty(InpV).getVectorElementType(); 1312 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy)) 1313 return LowerHvxSignExt(Op, DAG); 1314 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Op), ResTy, InpV); 1315 } 1316 1317 SDValue 1318 HexagonTargetLowering::LowerHvxSignExt(SDValue Op, SelectionDAG &DAG) const { 1319 MVT ResTy = ty(Op); 1320 SDValue InpV = Op.getOperand(0); 1321 MVT ElemTy = ty(InpV).getVectorElementType(); 1322 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy)) 1323 return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), false, DAG); 1324 return Op; 1325 } 1326 1327 SDValue 1328 HexagonTargetLowering::LowerHvxZeroExt(SDValue Op, SelectionDAG &DAG) const { 1329 MVT ResTy = ty(Op); 1330 SDValue InpV = Op.getOperand(0); 1331 MVT ElemTy = ty(InpV).getVectorElementType(); 1332 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy)) 1333 return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), true, DAG); 1334 return Op; 1335 } 1336 1337 SDValue 1338 HexagonTargetLowering::LowerHvxCttz(SDValue Op, SelectionDAG &DAG) const { 1339 // Lower vector CTTZ into a computation using CTLZ (Hacker's Delight): 1340 // cttz(x) = bitwidth(x) - ctlz(~x & (x-1)) 1341 const SDLoc &dl(Op); 1342 MVT ResTy = ty(Op); 1343 SDValue InpV = Op.getOperand(0); 1344 assert(ResTy == ty(InpV)); 1345 1346 // Calculate the vectors of 1 and bitwidth(x). 1347 MVT ElemTy = ty(InpV).getVectorElementType(); 1348 unsigned ElemWidth = ElemTy.getSizeInBits(); 1349 // Using uint64_t because a shift by 32 can happen. 1350 uint64_t Splat1 = 0, SplatW = 0; 1351 assert(isPowerOf2_32(ElemWidth) && ElemWidth <= 32); 1352 for (unsigned i = 0; i != 32/ElemWidth; ++i) { 1353 Splat1 = (Splat1 << ElemWidth) | 1; 1354 SplatW = (SplatW << ElemWidth) | ElemWidth; 1355 } 1356 SDValue Vec1 = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy, 1357 DAG.getConstant(uint32_t(Splat1), dl, MVT::i32)); 1358 SDValue VecW = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy, 1359 DAG.getConstant(uint32_t(SplatW), dl, MVT::i32)); 1360 SDValue VecN1 = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy, 1361 DAG.getConstant(-1, dl, MVT::i32)); 1362 // Do not use DAG.getNOT, because that would create BUILD_VECTOR with 1363 // a BITCAST. Here we can skip the BITCAST (so we don't have to handle 1364 // it separately in custom combine or selection). 1365 SDValue A = DAG.getNode(ISD::AND, dl, ResTy, 1366 {DAG.getNode(ISD::XOR, dl, ResTy, {InpV, VecN1}), 1367 DAG.getNode(ISD::SUB, dl, ResTy, {InpV, Vec1})}); 1368 return DAG.getNode(ISD::SUB, dl, ResTy, 1369 {VecW, DAG.getNode(ISD::CTLZ, dl, ResTy, A)}); 1370 } 1371 1372 SDValue 1373 HexagonTargetLowering::LowerHvxMul(SDValue Op, SelectionDAG &DAG) const { 1374 MVT ResTy = ty(Op); 1375 assert(ResTy.isVector() && isHvxSingleTy(ResTy)); 1376 const SDLoc &dl(Op); 1377 SmallVector<int,256> ShuffMask; 1378 1379 MVT ElemTy = ResTy.getVectorElementType(); 1380 unsigned VecLen = ResTy.getVectorNumElements(); 1381 SDValue Vs = Op.getOperand(0); 1382 SDValue Vt = Op.getOperand(1); 1383 1384 switch (ElemTy.SimpleTy) { 1385 case MVT::i8: { 1386 // For i8 vectors Vs = (a0, a1, ...), Vt = (b0, b1, ...), 1387 // V6_vmpybv Vs, Vt produces a pair of i16 vectors Hi:Lo, 1388 // where Lo = (a0*b0, a2*b2, ...), Hi = (a1*b1, a3*b3, ...). 1389 MVT ExtTy = typeExtElem(ResTy, 2); 1390 unsigned MpyOpc = ElemTy == MVT::i8 ? Hexagon::V6_vmpybv 1391 : Hexagon::V6_vmpyhv; 1392 SDValue M = getInstr(MpyOpc, dl, ExtTy, {Vs, Vt}, DAG); 1393 1394 // Discard high halves of the resulting values, collect the low halves. 1395 for (unsigned I = 0; I < VecLen; I += 2) { 1396 ShuffMask.push_back(I); // Pick even element. 1397 ShuffMask.push_back(I+VecLen); // Pick odd element. 1398 } 1399 VectorPair P = opSplit(opCastElem(M, ElemTy, DAG), dl, DAG); 1400 SDValue BS = getByteShuffle(dl, P.first, P.second, ShuffMask, DAG); 1401 return DAG.getBitcast(ResTy, BS); 1402 } 1403 case MVT::i16: 1404 // For i16 there is V6_vmpyih, which acts exactly like the MUL opcode. 1405 // (There is also V6_vmpyhv, which behaves in an analogous way to 1406 // V6_vmpybv.) 1407 return getInstr(Hexagon::V6_vmpyih, dl, ResTy, {Vs, Vt}, DAG); 1408 case MVT::i32: { 1409 // Use the following sequence for signed word multiply: 1410 // T0 = V6_vmpyiowh Vs, Vt 1411 // T1 = V6_vaslw T0, 16 1412 // T2 = V6_vmpyiewuh_acc T1, Vs, Vt 1413 SDValue S16 = DAG.getConstant(16, dl, MVT::i32); 1414 SDValue T0 = getInstr(Hexagon::V6_vmpyiowh, dl, ResTy, {Vs, Vt}, DAG); 1415 SDValue T1 = getInstr(Hexagon::V6_vaslw, dl, ResTy, {T0, S16}, DAG); 1416 SDValue T2 = getInstr(Hexagon::V6_vmpyiewuh_acc, dl, ResTy, 1417 {T1, Vs, Vt}, DAG); 1418 return T2; 1419 } 1420 default: 1421 break; 1422 } 1423 return SDValue(); 1424 } 1425 1426 SDValue 1427 HexagonTargetLowering::LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const { 1428 MVT ResTy = ty(Op); 1429 assert(ResTy.isVector()); 1430 const SDLoc &dl(Op); 1431 SmallVector<int,256> ShuffMask; 1432 1433 MVT ElemTy = ResTy.getVectorElementType(); 1434 unsigned VecLen = ResTy.getVectorNumElements(); 1435 SDValue Vs = Op.getOperand(0); 1436 SDValue Vt = Op.getOperand(1); 1437 bool IsSigned = Op.getOpcode() == ISD::MULHS; 1438 1439 if (ElemTy == MVT::i8 || ElemTy == MVT::i16) { 1440 // For i8 vectors Vs = (a0, a1, ...), Vt = (b0, b1, ...), 1441 // V6_vmpybv Vs, Vt produces a pair of i16 vectors Hi:Lo, 1442 // where Lo = (a0*b0, a2*b2, ...), Hi = (a1*b1, a3*b3, ...). 1443 // For i16, use V6_vmpyhv, which behaves in an analogous way to 1444 // V6_vmpybv: results Lo and Hi are products of even/odd elements 1445 // respectively. 1446 MVT ExtTy = typeExtElem(ResTy, 2); 1447 unsigned MpyOpc = ElemTy == MVT::i8 1448 ? (IsSigned ? Hexagon::V6_vmpybv : Hexagon::V6_vmpyubv) 1449 : (IsSigned ? Hexagon::V6_vmpyhv : Hexagon::V6_vmpyuhv); 1450 SDValue M = getInstr(MpyOpc, dl, ExtTy, {Vs, Vt}, DAG); 1451 1452 // Discard low halves of the resulting values, collect the high halves. 1453 for (unsigned I = 0; I < VecLen; I += 2) { 1454 ShuffMask.push_back(I+1); // Pick even element. 1455 ShuffMask.push_back(I+VecLen+1); // Pick odd element. 1456 } 1457 VectorPair P = opSplit(opCastElem(M, ElemTy, DAG), dl, DAG); 1458 SDValue BS = getByteShuffle(dl, P.first, P.second, ShuffMask, DAG); 1459 return DAG.getBitcast(ResTy, BS); 1460 } 1461 1462 assert(ElemTy == MVT::i32); 1463 SDValue S16 = DAG.getConstant(16, dl, MVT::i32); 1464 1465 if (IsSigned) { 1466 // mulhs(Vs,Vt) = 1467 // = [(Hi(Vs)*2^16 + Lo(Vs)) *s (Hi(Vt)*2^16 + Lo(Vt))] >> 32 1468 // = [Hi(Vs)*2^16 *s Hi(Vt)*2^16 + Hi(Vs) *su Lo(Vt)*2^16 1469 // + Lo(Vs) *us (Hi(Vt)*2^16 + Lo(Vt))] >> 32 1470 // = [Hi(Vs) *s Hi(Vt)*2^32 + Hi(Vs) *su Lo(Vt)*2^16 1471 // + Lo(Vs) *us Vt] >> 32 1472 // The low half of Lo(Vs)*Lo(Vt) will be discarded (it's not added to 1473 // anything, so it cannot produce any carry over to higher bits), 1474 // so everything in [] can be shifted by 16 without loss of precision. 1475 // = [Hi(Vs) *s Hi(Vt)*2^16 + Hi(Vs)*su Lo(Vt) + Lo(Vs)*Vt >> 16] >> 16 1476 // = [Hi(Vs) *s Hi(Vt)*2^16 + Hi(Vs)*su Lo(Vt) + V6_vmpyewuh(Vs,Vt)] >> 16 1477 // Denote Hi(Vs) = Vs': 1478 // = [Vs'*s Hi(Vt)*2^16 + Vs' *su Lo(Vt) + V6_vmpyewuh(Vt,Vs)] >> 16 1479 // = Vs'*s Hi(Vt) + (V6_vmpyiewuh(Vs',Vt) + V6_vmpyewuh(Vt,Vs)) >> 16 1480 SDValue T0 = getInstr(Hexagon::V6_vmpyewuh, dl, ResTy, {Vt, Vs}, DAG); 1481 // Get Vs': 1482 SDValue S0 = getInstr(Hexagon::V6_vasrw, dl, ResTy, {Vs, S16}, DAG); 1483 SDValue T1 = getInstr(Hexagon::V6_vmpyiewuh_acc, dl, ResTy, 1484 {T0, S0, Vt}, DAG); 1485 // Shift by 16: 1486 SDValue S2 = getInstr(Hexagon::V6_vasrw, dl, ResTy, {T1, S16}, DAG); 1487 // Get Vs'*Hi(Vt): 1488 SDValue T2 = getInstr(Hexagon::V6_vmpyiowh, dl, ResTy, {S0, Vt}, DAG); 1489 // Add: 1490 SDValue T3 = DAG.getNode(ISD::ADD, dl, ResTy, {S2, T2}); 1491 return T3; 1492 } 1493 1494 // Unsigned mulhw. (Would expansion using signed mulhw be better?) 1495 1496 auto LoVec = [&DAG,ResTy,dl] (SDValue Pair) { 1497 return DAG.getTargetExtractSubreg(Hexagon::vsub_lo, dl, ResTy, Pair); 1498 }; 1499 auto HiVec = [&DAG,ResTy,dl] (SDValue Pair) { 1500 return DAG.getTargetExtractSubreg(Hexagon::vsub_hi, dl, ResTy, Pair); 1501 }; 1502 1503 MVT PairTy = typeJoin({ResTy, ResTy}); 1504 SDValue P = getInstr(Hexagon::V6_lvsplatw, dl, ResTy, 1505 {DAG.getConstant(0x02020202, dl, MVT::i32)}, DAG); 1506 // Multiply-unsigned halfwords: 1507 // LoVec = Vs.uh[2i] * Vt.uh[2i], 1508 // HiVec = Vs.uh[2i+1] * Vt.uh[2i+1] 1509 SDValue T0 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {Vs, Vt}, DAG); 1510 // The low halves in the LoVec of the pair can be discarded. They are 1511 // not added to anything (in the full-precision product), so they cannot 1512 // produce a carry into the higher bits. 1513 SDValue T1 = getInstr(Hexagon::V6_vlsrw, dl, ResTy, {LoVec(T0), S16}, DAG); 1514 // Swap low and high halves in Vt, and do the halfword multiplication 1515 // to get products Vs.uh[2i] * Vt.uh[2i+1] and Vs.uh[2i+1] * Vt.uh[2i]. 1516 SDValue D0 = getInstr(Hexagon::V6_vdelta, dl, ResTy, {Vt, P}, DAG); 1517 SDValue T2 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {Vs, D0}, DAG); 1518 // T2 has mixed products of halfwords: Lo(Vt)*Hi(Vs) and Hi(Vt)*Lo(Vs). 1519 // These products are words, but cannot be added directly because the 1520 // sums could overflow. Add these products, by halfwords, where each sum 1521 // of a pair of halfwords gives a word. 1522 SDValue T3 = getInstr(Hexagon::V6_vadduhw, dl, PairTy, 1523 {LoVec(T2), HiVec(T2)}, DAG); 1524 // Add the high halfwords from the products of the low halfwords. 1525 SDValue T4 = DAG.getNode(ISD::ADD, dl, ResTy, {T1, LoVec(T3)}); 1526 SDValue T5 = getInstr(Hexagon::V6_vlsrw, dl, ResTy, {T4, S16}, DAG); 1527 SDValue T6 = DAG.getNode(ISD::ADD, dl, ResTy, {HiVec(T0), HiVec(T3)}); 1528 SDValue T7 = DAG.getNode(ISD::ADD, dl, ResTy, {T5, T6}); 1529 return T7; 1530 } 1531 1532 SDValue 1533 HexagonTargetLowering::LowerHvxBitcast(SDValue Op, SelectionDAG &DAG) const { 1534 SDValue ValQ = Op.getOperand(0); 1535 MVT ResTy = ty(Op); 1536 MVT VecTy = ty(ValQ); 1537 const SDLoc &dl(Op); 1538 1539 if (isHvxBoolTy(VecTy) && ResTy.isScalarInteger()) { 1540 unsigned HwLen = Subtarget.getVectorLength(); 1541 MVT WordTy = MVT::getVectorVT(MVT::i32, HwLen/4); 1542 SDValue VQ = compressHvxPred(ValQ, dl, WordTy, DAG); 1543 unsigned BitWidth = ResTy.getSizeInBits(); 1544 1545 if (BitWidth < 64) { 1546 SDValue W0 = extractHvxElementReg(VQ, DAG.getConstant(0, dl, MVT::i32), 1547 dl, MVT::i32, DAG); 1548 if (BitWidth == 32) 1549 return W0; 1550 assert(BitWidth < 32u); 1551 return DAG.getZExtOrTrunc(W0, dl, ResTy); 1552 } 1553 1554 // The result is >= 64 bits. The only options are 64 or 128. 1555 assert(BitWidth == 64 || BitWidth == 128); 1556 SmallVector<SDValue,4> Words; 1557 for (unsigned i = 0; i != BitWidth/32; ++i) { 1558 SDValue W = extractHvxElementReg( 1559 VQ, DAG.getConstant(i, dl, MVT::i32), dl, MVT::i32, DAG); 1560 Words.push_back(W); 1561 } 1562 SmallVector<SDValue,2> Combines; 1563 assert(Words.size() % 2 == 0); 1564 for (unsigned i = 0, e = Words.size(); i < e; i += 2) { 1565 SDValue C = DAG.getNode( 1566 HexagonISD::COMBINE, dl, MVT::i64, {Words[i+1], Words[i]}); 1567 Combines.push_back(C); 1568 } 1569 1570 if (BitWidth == 64) 1571 return Combines[0]; 1572 1573 return DAG.getNode(ISD::BUILD_PAIR, dl, ResTy, Combines); 1574 } 1575 1576 return Op; 1577 } 1578 1579 SDValue 1580 HexagonTargetLowering::LowerHvxExtend(SDValue Op, SelectionDAG &DAG) const { 1581 // Sign- and zero-extends are legal. 1582 assert(Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG); 1583 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(Op), ty(Op), 1584 Op.getOperand(0)); 1585 } 1586 1587 SDValue 1588 HexagonTargetLowering::LowerHvxShift(SDValue Op, SelectionDAG &DAG) const { 1589 if (SDValue S = getVectorShiftByInt(Op, DAG)) 1590 return S; 1591 return Op; 1592 } 1593 1594 SDValue 1595 HexagonTargetLowering::LowerHvxIntrinsic(SDValue Op, SelectionDAG &DAG) const { 1596 const SDLoc &dl(Op); 1597 MVT ResTy = ty(Op); 1598 1599 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 1600 bool Use64b = Subtarget.useHVX64BOps(); 1601 unsigned IntPredCast = Use64b ? Intrinsic::hexagon_V6_pred_typecast 1602 : Intrinsic::hexagon_V6_pred_typecast_128B; 1603 if (IntNo == IntPredCast) { 1604 SDValue Vs = Op.getOperand(1); 1605 MVT OpTy = ty(Vs); 1606 if (isHvxBoolTy(ResTy) && isHvxBoolTy(OpTy)) { 1607 if (ResTy == OpTy) 1608 return Vs; 1609 return DAG.getNode(HexagonISD::TYPECAST, dl, ResTy, Vs); 1610 } 1611 } 1612 1613 return Op; 1614 } 1615 1616 SDValue 1617 HexagonTargetLowering::SplitHvxPairOp(SDValue Op, SelectionDAG &DAG) const { 1618 assert(!Op.isMachineOpcode()); 1619 SmallVector<SDValue,2> OpsL, OpsH; 1620 const SDLoc &dl(Op); 1621 1622 auto SplitVTNode = [&DAG,this] (const VTSDNode *N) { 1623 MVT Ty = typeSplit(N->getVT().getSimpleVT()).first; 1624 SDValue TV = DAG.getValueType(Ty); 1625 return std::make_pair(TV, TV); 1626 }; 1627 1628 for (SDValue A : Op.getNode()->ops()) { 1629 VectorPair P = Subtarget.isHVXVectorType(ty(A), true) 1630 ? opSplit(A, dl, DAG) 1631 : std::make_pair(A, A); 1632 // Special case for type operand. 1633 if (Op.getOpcode() == ISD::SIGN_EXTEND_INREG) { 1634 if (const auto *N = dyn_cast<const VTSDNode>(A.getNode())) 1635 P = SplitVTNode(N); 1636 } 1637 OpsL.push_back(P.first); 1638 OpsH.push_back(P.second); 1639 } 1640 1641 MVT ResTy = ty(Op); 1642 MVT HalfTy = typeSplit(ResTy).first; 1643 SDValue L = DAG.getNode(Op.getOpcode(), dl, HalfTy, OpsL); 1644 SDValue H = DAG.getNode(Op.getOpcode(), dl, HalfTy, OpsH); 1645 SDValue S = DAG.getNode(ISD::CONCAT_VECTORS, dl, ResTy, L, H); 1646 return S; 1647 } 1648 1649 SDValue 1650 HexagonTargetLowering::SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const { 1651 LSBaseSDNode *BN = cast<LSBaseSDNode>(Op.getNode()); 1652 assert(BN->isUnindexed()); 1653 MVT MemTy = BN->getMemoryVT().getSimpleVT(); 1654 if (!isHvxPairTy(MemTy)) 1655 return Op; 1656 1657 const SDLoc &dl(Op); 1658 unsigned HwLen = Subtarget.getVectorLength(); 1659 MVT SingleTy = typeSplit(MemTy).first; 1660 SDValue Chain = BN->getChain(); 1661 SDValue Base0 = BN->getBasePtr(); 1662 SDValue Base1 = DAG.getMemBasePlusOffset(Base0, HwLen, dl); 1663 1664 MachineMemOperand *MOp0 = nullptr, *MOp1 = nullptr; 1665 if (MachineMemOperand *MMO = BN->getMemOperand()) { 1666 MachineFunction &MF = DAG.getMachineFunction(); 1667 MOp0 = MF.getMachineMemOperand(MMO, 0, HwLen); 1668 MOp1 = MF.getMachineMemOperand(MMO, HwLen, HwLen); 1669 } 1670 1671 unsigned MemOpc = BN->getOpcode(); 1672 SDValue NewOp; 1673 1674 if (MemOpc == ISD::LOAD) { 1675 SDValue Load0 = DAG.getLoad(SingleTy, dl, Chain, Base0, MOp0); 1676 SDValue Load1 = DAG.getLoad(SingleTy, dl, Chain, Base1, MOp1); 1677 NewOp = DAG.getMergeValues( 1678 { DAG.getNode(ISD::CONCAT_VECTORS, dl, MemTy, Load0, Load1), 1679 DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 1680 Load0.getValue(1), Load1.getValue(1)) }, dl); 1681 } else { 1682 assert(MemOpc == ISD::STORE); 1683 VectorPair Vals = opSplit(cast<StoreSDNode>(Op)->getValue(), dl, DAG); 1684 SDValue Store0 = DAG.getStore(Chain, dl, Vals.first, Base0, MOp0); 1685 SDValue Store1 = DAG.getStore(Chain, dl, Vals.second, Base1, MOp1); 1686 NewOp = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store0, Store1); 1687 } 1688 1689 return NewOp; 1690 } 1691 1692 SDValue 1693 HexagonTargetLowering::LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const { 1694 unsigned Opc = Op.getOpcode(); 1695 bool IsPairOp = isHvxPairTy(ty(Op)) || 1696 llvm::any_of(Op.getNode()->ops(), [this] (SDValue V) { 1697 return isHvxPairTy(ty(V)); 1698 }); 1699 1700 if (IsPairOp) { 1701 switch (Opc) { 1702 default: 1703 break; 1704 case ISD::LOAD: 1705 case ISD::STORE: 1706 return SplitHvxMemOp(Op, DAG); 1707 case ISD::CTPOP: 1708 case ISD::CTLZ: 1709 case ISD::CTTZ: 1710 case ISD::MUL: 1711 case ISD::MULHS: 1712 case ISD::MULHU: 1713 case ISD::AND: 1714 case ISD::OR: 1715 case ISD::XOR: 1716 case ISD::SRA: 1717 case ISD::SHL: 1718 case ISD::SRL: 1719 case ISD::SETCC: 1720 case ISD::VSELECT: 1721 case ISD::SIGN_EXTEND: 1722 case ISD::ZERO_EXTEND: 1723 case ISD::SIGN_EXTEND_INREG: 1724 return SplitHvxPairOp(Op, DAG); 1725 } 1726 } 1727 1728 switch (Opc) { 1729 default: 1730 break; 1731 case ISD::BUILD_VECTOR: return LowerHvxBuildVector(Op, DAG); 1732 case ISD::CONCAT_VECTORS: return LowerHvxConcatVectors(Op, DAG); 1733 case ISD::INSERT_SUBVECTOR: return LowerHvxInsertSubvector(Op, DAG); 1734 case ISD::INSERT_VECTOR_ELT: return LowerHvxInsertElement(Op, DAG); 1735 case ISD::EXTRACT_SUBVECTOR: return LowerHvxExtractSubvector(Op, DAG); 1736 case ISD::EXTRACT_VECTOR_ELT: return LowerHvxExtractElement(Op, DAG); 1737 case ISD::BITCAST: return LowerHvxBitcast(Op, DAG); 1738 case ISD::ANY_EXTEND: return LowerHvxAnyExt(Op, DAG); 1739 case ISD::SIGN_EXTEND: return LowerHvxSignExt(Op, DAG); 1740 case ISD::ZERO_EXTEND: return LowerHvxZeroExt(Op, DAG); 1741 case ISD::CTTZ: return LowerHvxCttz(Op, DAG); 1742 case ISD::SRA: 1743 case ISD::SHL: 1744 case ISD::SRL: return LowerHvxShift(Op, DAG); 1745 case ISD::MUL: return LowerHvxMul(Op, DAG); 1746 case ISD::MULHS: 1747 case ISD::MULHU: return LowerHvxMulh(Op, DAG); 1748 case ISD::ANY_EXTEND_VECTOR_INREG: return LowerHvxExtend(Op, DAG); 1749 case ISD::SETCC: 1750 case ISD::INTRINSIC_VOID: return Op; 1751 case ISD::INTRINSIC_WO_CHAIN: return LowerHvxIntrinsic(Op, DAG); 1752 // Unaligned loads will be handled by the default lowering. 1753 case ISD::LOAD: return SDValue(); 1754 } 1755 #ifndef NDEBUG 1756 Op.dumpr(&DAG); 1757 #endif 1758 llvm_unreachable("Unhandled HVX operation"); 1759 } 1760 1761 void 1762 HexagonTargetLowering::LowerHvxOperationWrapper(SDNode *N, 1763 SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const { 1764 } 1765 1766 void 1767 HexagonTargetLowering::ReplaceHvxNodeResults(SDNode *N, 1768 SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const { 1769 unsigned Opc = N->getOpcode(); 1770 switch (Opc) { 1771 case ISD::BITCAST: 1772 if (isHvxBoolTy(ty(N->getOperand(0)))) { 1773 SDValue Op(N, 0); 1774 SDValue C = LowerHvxBitcast(Op, DAG); 1775 Results.push_back(C); 1776 } 1777 break; 1778 default: 1779 break; 1780 } 1781 } 1782 1783 SDValue 1784 HexagonTargetLowering::PerformHvxDAGCombine(SDNode *N, DAGCombinerInfo &DCI) 1785 const { 1786 const SDLoc &dl(N); 1787 SDValue Op(N, 0); 1788 1789 unsigned Opc = Op.getOpcode(); 1790 if (Opc == ISD::VSELECT) { 1791 // (vselect (xor x, qtrue), v0, v1) -> (vselect x, v1, v0) 1792 SDValue Cond = Op.getOperand(0); 1793 if (Cond->getOpcode() == ISD::XOR) { 1794 SDValue C0 = Cond.getOperand(0), C1 = Cond.getOperand(1); 1795 if (C1->getOpcode() == HexagonISD::QTRUE) { 1796 SDValue VSel = DCI.DAG.getNode(ISD::VSELECT, dl, ty(Op), C0, 1797 Op.getOperand(2), Op.getOperand(1)); 1798 return VSel; 1799 } 1800 } 1801 } 1802 return SDValue(); 1803 } 1804 1805 bool 1806 HexagonTargetLowering::isHvxOperation(SDValue Op) const { 1807 // If the type of the result, or any operand type are HVX vector types, 1808 // this is an HVX operation. 1809 return Subtarget.isHVXVectorType(ty(Op), true) || 1810 llvm::any_of(Op.getNode()->ops(), 1811 [this] (SDValue V) { 1812 return Subtarget.isHVXVectorType(ty(V), true); 1813 }); 1814 } 1815 1816 bool 1817 HexagonTargetLowering::isHvxOperation(SDNode *N) const { 1818 // If the type of any result, or any operand type are HVX vector types, 1819 // this is an HVX operation. 1820 auto IsHvxTy = [this] (EVT Ty) { 1821 return Ty.isSimple() && Subtarget.isHVXVectorType(Ty.getSimpleVT(), true); 1822 }; 1823 auto IsHvxOp = [this] (SDValue Op) { 1824 return Subtarget.isHVXVectorType(ty(Op), true); 1825 }; 1826 return llvm::any_of(N->values(), IsHvxTy) || llvm::any_of(N->ops(), IsHvxOp); 1827 } 1828