Lines Matching full:vs
133 const VectorSplit &VS, ValueVector *cachePtr = nullptr);
139 unsigned size() const { return VS.NumFragments; } in size()
145 VectorSplit VS; member in __anon7628c1430111::Scatterer
212 VectorSplit VS; member
222 /// described in @p VS.
224 const VectorSplit &VS, Twine Name) { in concatenate() argument
225 unsigned NumElements = VS.VecTy->getNumElements(); in concatenate()
229 if (VS.NumPacked > 1) { in concatenate()
233 for (unsigned I = 0; I < VS.NumPacked; ++I) in concatenate()
241 Value *Res = PoisonValue::get(VS.VecTy); in concatenate()
242 for (unsigned I = 0; I < VS.NumFragments; ++I) { in concatenate()
245 unsigned NumPacked = VS.NumPacked; in concatenate()
246 if (I == VS.NumFragments - 1 && VS.RemainderTy) { in concatenate()
247 if (auto *RemVecTy = dyn_cast<FixedVectorType>(VS.RemainderTy)) in concatenate()
254 Res = Builder.CreateInsertElement(Res, Fragment, I * VS.NumPacked, in concatenate()
262 InsertMask[I * VS.NumPacked + J] = NumElements + J; in concatenate()
266 InsertMask[I * VS.NumPacked + J] = I * VS.NumPacked + J; in concatenate()
315 Scatterer scatter(Instruction *Point, Value *V, const VectorSplit &VS);
316 void gather(Instruction *Op, const ValueVector &CV, const VectorSplit &VS);
346 const VectorSplit &VS, ValueVector *cachePtr) in Scatterer() argument
347 : BB(bb), BBI(bbi), V(v), VS(VS), CachePtr(cachePtr) { in Scatterer()
350 Tmp.resize(VS.NumFragments, nullptr); in Scatterer()
352 assert((CachePtr->empty() || VS.NumFragments == CachePtr->size() || in Scatterer()
355 if (VS.NumFragments > CachePtr->size()) in Scatterer()
356 CachePtr->resize(VS.NumFragments, nullptr); in Scatterer()
371 CV[Frag] = Builder.CreateConstGEP1_32(VS.SplitTy, V, Frag, in operator []()
376 Type *FragmentTy = VS.getFragmentType(Frag); in operator []()
381 Mask.push_back(Frag * VS.NumPacked + J); in operator []()
398 if (Frag * VS.NumPacked == J) { in operator []()
403 if (VS.NumPacked == 1 && !CV[J]) { in operator []()
410 CV[Frag] = Builder.CreateExtractElement(V, Frag * VS.NumPacked, in operator []()
440 const VectorSplit &VS) { in scatter() argument
446 return Scatterer(BB, BB->begin(), V, VS, &Scattered[{V, VS.SplitTy}]); in scatter()
457 PoisonValue::get(V->getType()), VS); in scatter()
462 BB, skipPastPhiNodesAndDbg(std::next(BasicBlock::iterator(VOp))), V, VS, in scatter()
463 &Scattered[{V, VS.SplitTy}]); in scatter()
467 return Scatterer(Point->getParent(), Point->getIterator(), V, VS); in scatter()
475 const VectorSplit &VS) { in gather() argument
480 ValueVector &SV = Scattered[{Op, VS.SplitTy}]; in gather()
577 std::optional<VectorSplit> VS = getVectorSplit(Ty); in getVectorLayout() local
578 if (!VS) in getVectorLayout()
582 Layout.VS = *VS; in getVectorLayout()
584 if (!DL.typeSizeEqualsStoreSize(VS->SplitTy) || in getVectorLayout()
585 (VS->RemainderTy && !DL.typeSizeEqualsStoreSize(VS->RemainderTy))) in getVectorLayout()
588 Layout.SplitSize = DL.getTypeStoreSize(VS->SplitTy); in getVectorLayout()
596 std::optional<VectorSplit> VS = getVectorSplit(I.getType()); in splitUnary() local
597 if (!VS) in splitUnary()
602 OpVS = VS; in splitUnary()
605 if (!OpVS || VS->NumPacked != OpVS->NumPacked) in splitUnary()
611 assert(Op.size() == VS->NumFragments && "Mismatched unary operation"); in splitUnary()
613 Res.resize(VS->NumFragments); in splitUnary()
614 for (unsigned Frag = 0; Frag < VS->NumFragments; ++Frag) in splitUnary()
616 gather(&I, Res, *VS); in splitUnary()
624 std::optional<VectorSplit> VS = getVectorSplit(I.getType()); in splitBinary() local
625 if (!VS) in splitBinary()
630 OpVS = VS; in splitBinary()
633 if (!OpVS || VS->NumPacked != OpVS->NumPacked) in splitBinary()
640 assert(VOp0.size() == VS->NumFragments && "Mismatched binary operation"); in splitBinary()
641 assert(VOp1.size() == VS->NumFragments && "Mismatched binary operation"); in splitBinary()
643 Res.resize(VS->NumFragments); in splitBinary()
644 for (unsigned Frag = 0; Frag < VS->NumFragments; ++Frag) { in splitBinary()
649 gather(&I, Res, *VS); in splitBinary()
660 std::optional<VectorSplit> VS = getVectorSplit(CI.getType()); in splitCall() local
661 if (!VS) in splitCall()
682 Tys.push_back(VS->SplitTy); in splitCall()
690 assert(OpVecTy->getNumElements() == VS->VecTy->getNumElements()); in splitCall()
692 if (!OpVS || OpVS->NumPacked != VS->NumPacked) { in splitCall()
715 ValueVector Res(VS->NumFragments); in splitCall()
722 for (unsigned I = 0; I < VS->NumFragments; ++I) { in splitCall()
723 bool IsRemainder = I == VS->NumFragments - 1 && VS->RemainderTy; in splitCall()
727 Tys[0] = VS->RemainderTy; in splitCall()
746 gather(&CI, Res, *VS); in splitCall()
751 std::optional<VectorSplit> VS = getVectorSplit(SI.getType()); in visitSelectInst() local
752 if (!VS) in visitSelectInst()
758 if (!CondVS || CondVS->NumPacked != VS->NumPacked) { in visitSelectInst()
765 Scatterer VOp1 = scatter(&SI, SI.getOperand(1), *VS); in visitSelectInst()
766 Scatterer VOp2 = scatter(&SI, SI.getOperand(2), *VS); in visitSelectInst()
767 assert(VOp1.size() == VS->NumFragments && "Mismatched select"); in visitSelectInst()
768 assert(VOp2.size() == VS->NumFragments && "Mismatched select"); in visitSelectInst()
770 Res.resize(VS->NumFragments); in visitSelectInst()
775 for (unsigned I = 0; I < VS->NumFragments; ++I) { in visitSelectInst()
784 for (unsigned I = 0; I < VS->NumFragments; ++I) { in visitSelectInst()
791 gather(&SI, Res, *VS); in visitSelectInst()
812 std::optional<VectorSplit> VS = getVectorSplit(GEPI.getType()); in visitGetElementPtrInst() local
813 if (!VS) in visitGetElementPtrInst()
827 if (!OpVS || OpVS->NumPacked != VS->NumPacked) { in visitGetElementPtrInst()
838 Res.resize(VS->NumFragments); in visitGetElementPtrInst()
839 for (unsigned I = 0; I < VS->NumFragments; ++I) { in visitGetElementPtrInst()
855 gather(&GEPI, Res, *VS); in visitGetElementPtrInst()
963 std::optional<VectorSplit> VS = getVectorSplit(IEI.getType()); in visitInsertElementInst() local
964 if (!VS) in visitInsertElementInst()
968 Scatterer Op0 = scatter(&IEI, IEI.getOperand(0), *VS); in visitInsertElementInst()
973 Res.resize(VS->NumFragments); in visitInsertElementInst()
977 unsigned Fragment = Idx / VS->NumPacked; in visitInsertElementInst()
978 for (unsigned I = 0; I < VS->NumFragments; ++I) { in visitInsertElementInst()
980 bool IsPacked = VS->NumPacked > 1; in visitInsertElementInst()
981 if (Fragment == VS->NumFragments - 1 && VS->RemainderTy && in visitInsertElementInst()
982 !VS->RemainderTy->isVectorTy()) in visitInsertElementInst()
986 Builder.CreateInsertElement(Op0[I], NewElt, Idx % VS->NumPacked); in visitInsertElementInst()
996 if (!ScalarizeVariableInsertExtract || VS->NumPacked > 1) in visitInsertElementInst()
999 for (unsigned I = 0; I < VS->NumFragments; ++I) { in visitInsertElementInst()
1009 gather(&IEI, Res, *VS); in visitInsertElementInst()
1014 std::optional<VectorSplit> VS = getVectorSplit(EEI.getOperand(0)->getType()); in visitExtractElementInst() local
1015 if (!VS) in visitExtractElementInst()
1019 Scatterer Op0 = scatter(&EEI, EEI.getOperand(0), *VS); in visitExtractElementInst()
1024 unsigned Fragment = Idx / VS->NumPacked; in visitExtractElementInst()
1026 bool IsPacked = VS->NumPacked > 1; in visitExtractElementInst()
1027 if (Fragment == VS->NumFragments - 1 && VS->RemainderTy && in visitExtractElementInst()
1028 !VS->RemainderTy->isVectorTy()) in visitExtractElementInst()
1031 Res = Builder.CreateExtractElement(Res, Idx % VS->NumPacked); in visitExtractElementInst()
1037 if (!ScalarizeVariableInsertExtract || VS->NumPacked > 1) in visitExtractElementInst()
1040 Value *Res = PoisonValue::get(VS->VecTy->getElementType()); in visitExtractElementInst()
1041 for (unsigned I = 0; I < VS->NumFragments; ++I) { in visitExtractElementInst()
1054 std::optional<VectorSplit> VS = getVectorSplit(SVI.getType()); in visitShuffleVectorInst() local
1057 if (!VS || !VSOp || VS->NumPacked > 1 || VSOp->NumPacked > 1) in visitShuffleVectorInst()
1063 Res.resize(VS->NumFragments); in visitShuffleVectorInst()
1065 for (unsigned I = 0; I < VS->NumFragments; ++I) { in visitShuffleVectorInst()
1068 Res[I] = PoisonValue::get(VS->VecTy->getElementType()); in visitShuffleVectorInst()
1074 gather(&SVI, Res, *VS); in visitShuffleVectorInst()
1079 std::optional<VectorSplit> VS = getVectorSplit(PHI.getType()); in visitPHINode() local
1080 if (!VS) in visitPHINode()
1085 Res.resize(VS->NumFragments); in visitPHINode()
1088 for (unsigned I = 0; I < VS->NumFragments; ++I) { in visitPHINode()
1089 Res[I] = Builder.CreatePHI(VS->getFragmentType(I), NumOps, in visitPHINode()
1094 Scatterer Op = scatter(&PHI, PHI.getIncomingValue(I), *VS); in visitPHINode()
1096 for (unsigned J = 0; J < VS->NumFragments; ++J) in visitPHINode()
1099 gather(&PHI, Res, *VS); in visitPHINode()
1115 Scatterer Ptr = scatter(&LI, LI.getPointerOperand(), Layout->VS); in visitLoadInst()
1117 Res.resize(Layout->VS.NumFragments); in visitLoadInst()
1119 for (unsigned I = 0; I < Layout->VS.NumFragments; ++I) { in visitLoadInst()
1120 Res[I] = Builder.CreateAlignedLoad(Layout->VS.getFragmentType(I), Ptr[I], in visitLoadInst()
1124 gather(&LI, Res, Layout->VS); in visitLoadInst()
1141 Scatterer VPtr = scatter(&SI, SI.getPointerOperand(), Layout->VS); in visitStoreInst()
1142 Scatterer VVal = scatter(&SI, FullValue, Layout->VS); in visitStoreInst()
1145 Stores.resize(Layout->VS.NumFragments); in visitStoreInst()
1146 for (unsigned I = 0; I < Layout->VS.NumFragments; ++I) { in visitStoreInst()
1186 VectorSplit VS = *getVectorSplit(Ty); in finish() local
1187 assert(VS.NumFragments == CV.size()); in finish()
1189 Res = concatenate(Builder, CV, VS, Op->getName()); in finish()