1 //===- InterleavedAccessPass.cpp ------------------------------------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
14 // DE-interleaving the data on a factor. An interleaved store writes several
15 // vectors to memory with RE-interleaving the data on a factor.
45 //===----------------------------------------------------------------------===//
77 #define DEBUG_TYPE "interleaved-access"
80     "lower-interleaved-accesses",
92       : DT(DT), TLI(TLI), MaxFactor(TLI->getMaxSupportedInterleaveFactor()) {}  in InterleavedAccessImpl()
162   auto *TLI = TM->getSubtargetImpl(F)->getTargetLowering();  in run()
184   auto &TM = TPC->getTM<TargetMachine>();  in runOnFunction()
185   Impl.TLI = TM.getSubtargetImpl(F)->getTargetLowering();  in runOnFunction()
186   Impl.MaxFactor = Impl.TLI->getMaxSupportedInterleaveFactor();  in runOnFunction()
203 /// Check if the mask is a DE-interleave mask for an interleaved load.
205 /// E.g. DE-interleave masks (Factor = 2) could be:
228 /// It checks for a more general pattern than the RE-interleave mask.
234 /// The particular case of an RE-interleave mask is:
235 /// I.e. <0, LaneLen, ... , LaneLen*(Factor - 1), 1, LaneLen + 1, ...>
239   unsigned NumElts = SVI->getShuffleMask().size();  in isReInterleaveMask()
245     if (SVI->isInterleave(Factor))  in isReInterleaveMask()
254   if (!LI->isSimple() || isa<ScalableVectorType>(LI->getType()))  in lowerInterleavedLoad()
268   for (auto *User : LI->users()) {  in lowerInterleavedLoad()
270     if (Extract && isa<ConstantInt>(Extract->getIndexOperand())) {  in lowerInterleavedLoad()
275       if (!BI->user_empty() && all_of(BI->users(), [](auto *U) {  in lowerInterleavedLoad()
277             return SVI && isa<UndefValue>(SVI->getOperand(1));  in lowerInterleavedLoad()
279         for (auto *SVI : BI->users())  in lowerInterleavedLoad()
285     if (!SVI || !isa<UndefValue>(SVI->getOperand(1)))  in lowerInterleavedLoad()
297       cast<FixedVectorType>(LI->getType())->getNumElements();  in lowerInterleavedLoad()
299   // Check if the first shufflevector is DE-interleave shuffle.  in lowerInterleavedLoad()
300   if (!isDeInterleaveMask(FirstSVI->getShuffleMask(), Factor, Index, MaxFactor,  in lowerInterleavedLoad()
304   // Holds the corresponding index for each DE-interleave shuffle.  in lowerInterleavedLoad()
307   Type *VecTy = FirstSVI->getType();  in lowerInterleavedLoad()
309   // Check if other shufflevectors are also DE-interleaved of the same type  in lowerInterleavedLoad()
312     if (Shuffle->getType() != VecTy)  in lowerInterleavedLoad()
315             Shuffle->getShuffleMask(), Factor, Index))  in lowerInterleavedLoad()
318     assert(Shuffle->getShuffleMask().size() <= NumLoadElements);  in lowerInterleavedLoad()
322     if (Shuffle->getType() != VecTy)  in lowerInterleavedLoad()
325             Shuffle->getShuffleMask(), Factor, Index))  in lowerInterleavedLoad()
328     assert(Shuffle->getShuffleMask().size() <= NumLoadElements);  in lowerInterleavedLoad()
330     if (cast<Instruction>(Shuffle->getOperand(0))->getOperand(0) == LI)  in lowerInterleavedLoad()
332     if (cast<Instruction>(Shuffle->getOperand(0))->getOperand(1) == LI)  in lowerInterleavedLoad()
347   if (!TLI->lowerInterleavedLoad(LI, Shuffles, Indices, Factor)) {  in lowerInterleavedLoad()
362     BinaryOperator *BI = cast<BinaryOperator>(SVI->getOperand(0));  in replaceBinOpShuffles()
363     Type *BIOp0Ty = BI->getOperand(0)->getType();  in replaceBinOpShuffles()
364     ArrayRef<int> Mask = SVI->getShuffleMask();  in replaceBinOpShuffles()
365     assert(all_of(Mask, [&](int Idx) {  in replaceBinOpShuffles()
366       return Idx < (int)cast<FixedVectorType>(BIOp0Ty)->getNumElements();  in replaceBinOpShuffles()
369     BasicBlock::iterator insertPos = SVI->getIterator();  in replaceBinOpShuffles()
371         new ShuffleVectorInst(BI->getOperand(0), PoisonValue::get(BIOp0Ty),  in replaceBinOpShuffles()
372                               Mask, SVI->getName(), insertPos);  in replaceBinOpShuffles()
374         BI->getOperand(1), PoisonValue::get(BI->getOperand(1)->getType()), Mask,  in replaceBinOpShuffles()
375         SVI->getName(), insertPos);  in replaceBinOpShuffles()
377         BI->getOpcode(), NewSVI1, NewSVI2, BI, BI->getName(), insertPos);  in replaceBinOpShuffles()
378     SVI->replaceAllUsesWith(NewBI);  in replaceBinOpShuffles()
383     if (NewSVI1->getOperand(0) == LI)  in replaceBinOpShuffles()
385     if (NewSVI2->getOperand(0) == LI)  in replaceBinOpShuffles()
400   // Maps extractelement instructions to vector-index pairs. The extractlement  in tryReplaceExtracts()
406     auto *IndexOperand = cast<ConstantInt>(Extract->getIndexOperand());  in tryReplaceExtracts()
407     auto Index = IndexOperand->getSExtValue();  in tryReplaceExtracts()
415       if (!DT->dominates(Shuffle, Extract))  in tryReplaceExtracts()
422       Shuffle->getShuffleMask(Indices);  in tryReplaceExtracts()
425           assert(Extract->getOperand(0) == Shuffle->getOperand(0) &&  in tryReplaceExtracts()
443   IRBuilder<> Builder(Extracts[0]->getContext());  in tryReplaceExtracts()
449     Extract->replaceAllUsesWith(Builder.CreateExtractElement(Vector, Index));  in tryReplaceExtracts()
450     Extract->eraseFromParent();  in tryReplaceExtracts()
458   if (!SI->isSimple())  in lowerInterleavedStore()
461   auto *SVI = dyn_cast<ShuffleVectorInst>(SI->getValueOperand());  in lowerInterleavedStore()
462   if (!SVI || !SVI->hasOneUse() || isa<ScalableVectorType>(SVI->getType()))  in lowerInterleavedStore()
465   // Check if the shufflevector is RE-interleave shuffle.  in lowerInterleavedStore()
473   if (!TLI->lowerInterleavedStore(SI, SVI, Factor))  in lowerInterleavedStore()
484   LoadInst *LI = dyn_cast<LoadInst>(DI->getOperand(0));  in lowerDeinterleaveIntrinsic()
486   if (!LI || !LI->hasOneUse() || !LI->isSimple())  in lowerDeinterleaveIntrinsic()
492   if (!TLI->lowerDeinterleaveIntrinsicToLoad(DI, LI))  in lowerDeinterleaveIntrinsic()
495   // We now have a target-specific load, so delete the old one.  in lowerDeinterleaveIntrinsic()
503   if (!II->hasOneUse())  in lowerInterleaveIntrinsic()
506   StoreInst *SI = dyn_cast<StoreInst>(*(II->users().begin()));  in lowerInterleaveIntrinsic()
508   if (!SI || !SI->isSimple())  in lowerInterleaveIntrinsic()
514   if (!TLI->lowerInterleaveIntrinsicToStore(II, SI))  in lowerInterleaveIntrinsic()
517   // We now have a target-specific store, so delete the old one.  in lowerInterleaveIntrinsic()
536       // At present, we only have intrinsics to represent (de)interleaving  in runOnFunction()
538       if (II->getIntrinsicID() == Intrinsic::vector_deinterleave2)  in runOnFunction()
540       if (II->getIntrinsicID() == Intrinsic::vector_interleave2)  in runOnFunction()
546     I->eraseFromParent();  in runOnFunction()