Lines Matching +full:n +full:- +full:factor
1 //===- InterleavedAccessPass.cpp ------------------------------------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
14 // DE-interleaving the data on a factor. An interleaved store writes several
15 // vectors to memory with RE-interleaving the data on a factor.
23 // E.g. An interleaved load (Factor = 2):
34 // E.g. An interleaved store (Factor = 3):
45 //===----------------------------------------------------------------------===//
77 #define DEBUG_TYPE "interleaved-access"
80 "lower-interleaved-accesses",
92 : DT(DT), TLI(TLI), MaxFactor(TLI->getMaxSupportedInterleaveFactor()) {} in InterleavedAccessImpl()
99 /// The maximum supported interleave factor.
162 auto *TLI = TM->getSubtargetImpl(F)->getTargetLowering(); in run()
181 LLVM_DEBUG(dbgs() << "*** " << getPassName() << ": " << F.getName() << "\n"); in runOnFunction()
184 auto &TM = TPC->getTM<TargetMachine>(); in runOnFunction()
185 Impl.TLI = TM.getSubtargetImpl(F)->getTargetLowering(); in runOnFunction()
186 Impl.MaxFactor = Impl.TLI->getMaxSupportedInterleaveFactor(); in runOnFunction()
203 /// Check if the mask is a DE-interleave mask for an interleaved load.
205 /// E.g. DE-interleave masks (Factor = 2) could be:
208 static bool isDeInterleaveMask(ArrayRef<int> Mask, unsigned &Factor, in isDeInterleaveMask() argument
215 for (Factor = 2; Factor <= MaxFactor; Factor++) { in isDeInterleaveMask()
217 if (Mask.size() * Factor > NumLoadElements) in isDeInterleaveMask()
219 if (ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor, Index)) in isDeInterleaveMask()
228 /// It checks for a more general pattern than the RE-interleave mask.
230 /// E.g. For a Factor of 2 (LaneLen=4): <4, 32, 5, 33, 6, 34, 7, 35>
231 /// E.g. For a Factor of 3 (LaneLen=4): <4, 32, 16, 5, 33, 17, 6, 34, 18, 7, 35, 19>
232 /// E.g. For a Factor of 4 (LaneLen=2): <8, 2, 12, 4, 9, 3, 13, 5>
234 /// The particular case of an RE-interleave mask is:
235 /// I.e. <0, LaneLen, ... , LaneLen*(Factor - 1), 1, LaneLen + 1, ...>
236 /// E.g. For a Factor of 2 (LaneLen=4): <0, 4, 1, 5, 2, 6, 3, 7>
237 static bool isReInterleaveMask(ShuffleVectorInst *SVI, unsigned &Factor, in isReInterleaveMask() argument
239 unsigned NumElts = SVI->getShuffleMask().size(); in isReInterleaveMask()
244 for (Factor = 2; Factor <= MaxFactor; Factor++) { in isReInterleaveMask()
245 if (SVI->isInterleave(Factor)) in isReInterleaveMask()
254 if (!LI->isSimple() || isa<ScalableVectorType>(LI->getType())) in lowerInterleavedLoad()
268 for (auto *User : LI->users()) { in lowerInterleavedLoad()
270 if (Extract && isa<ConstantInt>(Extract->getIndexOperand())) { in lowerInterleavedLoad()
275 if (!BI->user_empty() && all_of(BI->users(), [](auto *U) { in lowerInterleavedLoad()
277 return SVI && isa<UndefValue>(SVI->getOperand(1)); in lowerInterleavedLoad()
279 for (auto *SVI : BI->users()) in lowerInterleavedLoad()
285 if (!SVI || !isa<UndefValue>(SVI->getOperand(1))) in lowerInterleavedLoad()
294 unsigned Factor, Index; in lowerInterleavedLoad() local
297 cast<FixedVectorType>(LI->getType())->getNumElements(); in lowerInterleavedLoad()
299 // Check if the first shufflevector is DE-interleave shuffle. in lowerInterleavedLoad()
300 if (!isDeInterleaveMask(FirstSVI->getShuffleMask(), Factor, Index, MaxFactor, in lowerInterleavedLoad()
304 // Holds the corresponding index for each DE-interleave shuffle. in lowerInterleavedLoad()
307 Type *VecTy = FirstSVI->getType(); in lowerInterleavedLoad()
309 // Check if other shufflevectors are also DE-interleaved of the same type in lowerInterleavedLoad()
310 // and factor as the first shufflevector. in lowerInterleavedLoad()
312 if (Shuffle->getType() != VecTy) in lowerInterleavedLoad()
315 Shuffle->getShuffleMask(), Factor, Index)) in lowerInterleavedLoad()
318 assert(Shuffle->getShuffleMask().size() <= NumLoadElements); in lowerInterleavedLoad()
322 if (Shuffle->getType() != VecTy) in lowerInterleavedLoad()
325 Shuffle->getShuffleMask(), Factor, Index)) in lowerInterleavedLoad()
328 assert(Shuffle->getShuffleMask().size() <= NumLoadElements); in lowerInterleavedLoad()
330 if (cast<Instruction>(Shuffle->getOperand(0))->getOperand(0) == LI) in lowerInterleavedLoad()
332 if (cast<Instruction>(Shuffle->getOperand(0))->getOperand(1) == LI) in lowerInterleavedLoad()
344 LLVM_DEBUG(dbgs() << "IA: Found an interleaved load: " << *LI << "\n"); in lowerInterleavedLoad()
347 if (!TLI->lowerInterleavedLoad(LI, Shuffles, Indices, Factor)) { in lowerInterleavedLoad()
362 BinaryOperator *BI = cast<BinaryOperator>(SVI->getOperand(0)); in replaceBinOpShuffles()
363 Type *BIOp0Ty = BI->getOperand(0)->getType(); in replaceBinOpShuffles()
364 ArrayRef<int> Mask = SVI->getShuffleMask(); in replaceBinOpShuffles()
366 return Idx < (int)cast<FixedVectorType>(BIOp0Ty)->getNumElements(); in replaceBinOpShuffles()
369 BasicBlock::iterator insertPos = SVI->getIterator(); in replaceBinOpShuffles()
371 new ShuffleVectorInst(BI->getOperand(0), PoisonValue::get(BIOp0Ty), in replaceBinOpShuffles()
372 Mask, SVI->getName(), insertPos); in replaceBinOpShuffles()
374 BI->getOperand(1), PoisonValue::get(BI->getOperand(1)->getType()), Mask, in replaceBinOpShuffles()
375 SVI->getName(), insertPos); in replaceBinOpShuffles()
377 BI->getOpcode(), NewSVI1, NewSVI2, BI, BI->getName(), insertPos); in replaceBinOpShuffles()
378 SVI->replaceAllUsesWith(NewBI); in replaceBinOpShuffles()
379 LLVM_DEBUG(dbgs() << " Replaced: " << *BI << "\n And : " << *SVI in replaceBinOpShuffles()
380 << "\n With : " << *NewSVI1 << "\n And : " in replaceBinOpShuffles()
381 << *NewSVI2 << "\n And : " << *NewBI << "\n"); in replaceBinOpShuffles()
383 if (NewSVI1->getOperand(0) == LI) in replaceBinOpShuffles()
385 if (NewSVI2->getOperand(0) == LI) in replaceBinOpShuffles()
400 // Maps extractelement instructions to vector-index pairs. The extractlement in tryReplaceExtracts()
406 auto *IndexOperand = cast<ConstantInt>(Extract->getIndexOperand()); in tryReplaceExtracts()
407 auto Index = IndexOperand->getSExtValue(); in tryReplaceExtracts()
415 if (!DT->dominates(Shuffle, Extract)) in tryReplaceExtracts()
422 Shuffle->getShuffleMask(Indices); in tryReplaceExtracts()
425 assert(Extract->getOperand(0) == Shuffle->getOperand(0) && in tryReplaceExtracts()
443 IRBuilder<> Builder(Extracts[0]->getContext()); in tryReplaceExtracts()
449 Extract->replaceAllUsesWith(Builder.CreateExtractElement(Vector, Index)); in tryReplaceExtracts()
450 Extract->eraseFromParent(); in tryReplaceExtracts()
458 if (!SI->isSimple()) in lowerInterleavedStore()
461 auto *SVI = dyn_cast<ShuffleVectorInst>(SI->getValueOperand()); in lowerInterleavedStore()
462 if (!SVI || !SVI->hasOneUse() || isa<ScalableVectorType>(SVI->getType())) in lowerInterleavedStore()
465 // Check if the shufflevector is RE-interleave shuffle. in lowerInterleavedStore()
466 unsigned Factor; in lowerInterleavedStore() local
467 if (!isReInterleaveMask(SVI, Factor, MaxFactor)) in lowerInterleavedStore()
470 LLVM_DEBUG(dbgs() << "IA: Found an interleaved store: " << *SI << "\n"); in lowerInterleavedStore()
473 if (!TLI->lowerInterleavedStore(SI, SVI, Factor)) in lowerInterleavedStore()
484 LoadInst *LI = dyn_cast<LoadInst>(DI->getOperand(0)); in lowerDeinterleaveIntrinsic()
486 if (!LI || !LI->hasOneUse() || !LI->isSimple()) in lowerDeinterleaveIntrinsic()
489 LLVM_DEBUG(dbgs() << "IA: Found a deinterleave intrinsic: " << *DI << "\n"); in lowerDeinterleaveIntrinsic()
492 if (!TLI->lowerDeinterleaveIntrinsicToLoad(DI, LI)) in lowerDeinterleaveIntrinsic()
495 // We now have a target-specific load, so delete the old one. in lowerDeinterleaveIntrinsic()
503 if (!II->hasOneUse()) in lowerInterleaveIntrinsic()
506 StoreInst *SI = dyn_cast<StoreInst>(*(II->users().begin())); in lowerInterleaveIntrinsic()
508 if (!SI || !SI->isSimple()) in lowerInterleaveIntrinsic()
511 LLVM_DEBUG(dbgs() << "IA: Found an interleave intrinsic: " << *II << "\n"); in lowerInterleaveIntrinsic()
514 if (!TLI->lowerInterleaveIntrinsicToStore(II, SI)) in lowerInterleaveIntrinsic()
517 // We now have a target-specific store, so delete the old one. in lowerInterleaveIntrinsic()
537 // with a factor of 2. in runOnFunction()
538 if (II->getIntrinsicID() == Intrinsic::vector_deinterleave2) in runOnFunction()
540 if (II->getIntrinsicID() == Intrinsic::vector_interleave2) in runOnFunction()
546 I->eraseFromParent(); in runOnFunction()