Lines Matching +full:auto +full:- +full:load
1 //===- InterleavedAccessPass.cpp ------------------------------------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
13 // An interleaved load reads data from memory into several vectors, with
14 // DE-interleaving the data on a factor. An interleaved store writes several
15 // vectors to memory with RE-interleaving the data on a factor.
23 // E.g. An interleaved load (Factor = 2):
24 // %wide.vec = load <8 x i32>, <8 x i32>* %ptr
45 //===----------------------------------------------------------------------===//
77 #define DEBUG_TYPE "interleaved-access"
80 "lower-interleaved-accesses",
92 : DT(DT), TLI(TLI), MaxFactor(TLI->getMaxSupportedInterleaveFactor()) {} in InterleavedAccessImpl()
102 /// Transform an interleaved load into target specific intrinsics.
110 /// Transform a load and a deinterleave intrinsic into target specific
120 /// Returns true if the uses of an interleaved load by the
129 /// interleaving load. Any newly created shuffles that operate on \p LI will
161 auto *DT = &FAM.getResult<DominatorTreeAnalysis>(F); in run()
162 auto *TLI = TM->getSubtargetImpl(F)->getTargetLowering(); in run()
177 auto *TPC = getAnalysisIfAvailable<TargetPassConfig>(); in runOnFunction()
184 auto &TM = TPC->getTM<TargetMachine>(); in runOnFunction()
185 Impl.TLI = TM.getSubtargetImpl(F)->getTargetLowering(); in runOnFunction()
186 Impl.MaxFactor = Impl.TLI->getMaxSupportedInterleaveFactor(); in runOnFunction()
203 /// Check if the mask is a DE-interleave mask for an interleaved load.
205 /// E.g. DE-interleave masks (Factor = 2) could be:
216 // Make sure we don't produce a load wider than the input load. in isDeInterleaveMask()
228 /// It checks for a more general pattern than the RE-interleave mask.
234 /// The particular case of an RE-interleave mask is:
235 /// I.e. <0, LaneLen, ... , LaneLen*(Factor - 1), 1, LaneLen + 1, ...>
239 unsigned NumElts = SVI->getShuffleMask().size(); in isReInterleaveMask()
245 if (SVI->isInterleave(Factor)) in isReInterleaveMask()
254 if (!LI->isSimple() || isa<ScalableVectorType>(LI->getType())) in lowerInterleavedLoad()
257 // Check if all users of this load are shufflevectors. If we encounter any in lowerInterleavedLoad()
260 // shufflevectors instead of the load. in lowerInterleavedLoad()
265 // binop are the same load. in lowerInterleavedLoad()
268 for (auto *User : LI->users()) { in lowerInterleavedLoad()
269 auto *Extract = dyn_cast<ExtractElementInst>(User); in lowerInterleavedLoad()
270 if (Extract && isa<ConstantInt>(Extract->getIndexOperand())) { in lowerInterleavedLoad()
274 if (auto *BI = dyn_cast<BinaryOperator>(User)) { in lowerInterleavedLoad()
275 if (!BI->user_empty() && all_of(BI->users(), [](auto *U) { in lowerInterleavedLoad()
276 auto *SVI = dyn_cast<ShuffleVectorInst>(U); in lowerInterleavedLoad()
277 return SVI && isa<UndefValue>(SVI->getOperand(1)); in lowerInterleavedLoad()
279 for (auto *SVI : BI->users()) in lowerInterleavedLoad()
284 auto *SVI = dyn_cast<ShuffleVectorInst>(User); in lowerInterleavedLoad()
285 if (!SVI || !isa<UndefValue>(SVI->getOperand(1))) in lowerInterleavedLoad()
297 cast<FixedVectorType>(LI->getType())->getNumElements(); in lowerInterleavedLoad()
298 auto *FirstSVI = Shuffles.size() > 0 ? Shuffles[0] : BinOpShuffles[0]; in lowerInterleavedLoad()
299 // Check if the first shufflevector is DE-interleave shuffle. in lowerInterleavedLoad()
300 if (!isDeInterleaveMask(FirstSVI->getShuffleMask(), Factor, Index, MaxFactor, in lowerInterleavedLoad()
304 // Holds the corresponding index for each DE-interleave shuffle. in lowerInterleavedLoad()
307 Type *VecTy = FirstSVI->getType(); in lowerInterleavedLoad()
309 // Check if other shufflevectors are also DE-interleaved of the same type in lowerInterleavedLoad()
311 for (auto *Shuffle : Shuffles) { in lowerInterleavedLoad()
312 if (Shuffle->getType() != VecTy) in lowerInterleavedLoad()
315 Shuffle->getShuffleMask(), Factor, Index)) in lowerInterleavedLoad()
318 assert(Shuffle->getShuffleMask().size() <= NumLoadElements); in lowerInterleavedLoad()
321 for (auto *Shuffle : BinOpShuffles) { in lowerInterleavedLoad()
322 if (Shuffle->getType() != VecTy) in lowerInterleavedLoad()
325 Shuffle->getShuffleMask(), Factor, Index)) in lowerInterleavedLoad()
328 assert(Shuffle->getShuffleMask().size() <= NumLoadElements); in lowerInterleavedLoad()
330 if (cast<Instruction>(Shuffle->getOperand(0))->getOperand(0) == LI) in lowerInterleavedLoad()
332 if (cast<Instruction>(Shuffle->getOperand(0))->getOperand(1) == LI) in lowerInterleavedLoad()
336 // Try and modify users of the load that are extractelement instructions to in lowerInterleavedLoad()
337 // use the shufflevector instructions instead of the load. in lowerInterleavedLoad()
344 LLVM_DEBUG(dbgs() << "IA: Found an interleaved load: " << *LI << "\n"); in lowerInterleavedLoad()
346 // Try to create target specific intrinsics to replace the load and shuffles. in lowerInterleavedLoad()
347 if (!TLI->lowerInterleavedLoad(LI, Shuffles, Indices, Factor)) { in lowerInterleavedLoad()
361 for (auto *SVI : BinOpShuffles) { in replaceBinOpShuffles()
362 BinaryOperator *BI = cast<BinaryOperator>(SVI->getOperand(0)); in replaceBinOpShuffles()
363 Type *BIOp0Ty = BI->getOperand(0)->getType(); in replaceBinOpShuffles()
364 ArrayRef<int> Mask = SVI->getShuffleMask(); in replaceBinOpShuffles()
366 return Idx < (int)cast<FixedVectorType>(BIOp0Ty)->getNumElements(); in replaceBinOpShuffles()
369 BasicBlock::iterator insertPos = SVI->getIterator(); in replaceBinOpShuffles()
370 auto *NewSVI1 = in replaceBinOpShuffles()
371 new ShuffleVectorInst(BI->getOperand(0), PoisonValue::get(BIOp0Ty), in replaceBinOpShuffles()
372 Mask, SVI->getName(), insertPos); in replaceBinOpShuffles()
373 auto *NewSVI2 = new ShuffleVectorInst( in replaceBinOpShuffles()
374 BI->getOperand(1), PoisonValue::get(BI->getOperand(1)->getType()), Mask, in replaceBinOpShuffles()
375 SVI->getName(), insertPos); in replaceBinOpShuffles()
377 BI->getOpcode(), NewSVI1, NewSVI2, BI, BI->getName(), insertPos); in replaceBinOpShuffles()
378 SVI->replaceAllUsesWith(NewBI); in replaceBinOpShuffles()
383 if (NewSVI1->getOperand(0) == LI) in replaceBinOpShuffles()
385 if (NewSVI2->getOperand(0) == LI) in replaceBinOpShuffles()
400 // Maps extractelement instructions to vector-index pairs. The extractlement in tryReplaceExtracts()
404 for (auto *Extract : Extracts) { in tryReplaceExtracts()
406 auto *IndexOperand = cast<ConstantInt>(Extract->getIndexOperand()); in tryReplaceExtracts()
407 auto Index = IndexOperand->getSExtValue(); in tryReplaceExtracts()
410 // extractelement instruction (which uses an interleaved load) to use one in tryReplaceExtracts()
411 // of the shufflevector instructions instead of the load. in tryReplaceExtracts()
412 for (auto *Shuffle : Shuffles) { in tryReplaceExtracts()
415 if (!DT->dominates(Shuffle, Extract)) in tryReplaceExtracts()
422 Shuffle->getShuffleMask(Indices); in tryReplaceExtracts()
425 assert(Extract->getOperand(0) == Shuffle->getOperand(0) && in tryReplaceExtracts()
443 IRBuilder<> Builder(Extracts[0]->getContext()); in tryReplaceExtracts()
444 for (auto &Replacement : ReplacementMap) { in tryReplaceExtracts()
445 auto *Extract = Replacement.first; in tryReplaceExtracts()
446 auto *Vector = Replacement.second.first; in tryReplaceExtracts()
447 auto Index = Replacement.second.second; in tryReplaceExtracts()
449 Extract->replaceAllUsesWith(Builder.CreateExtractElement(Vector, Index)); in tryReplaceExtracts()
450 Extract->eraseFromParent(); in tryReplaceExtracts()
458 if (!SI->isSimple()) in lowerInterleavedStore()
461 auto *SVI = dyn_cast<ShuffleVectorInst>(SI->getValueOperand()); in lowerInterleavedStore()
462 if (!SVI || !SVI->hasOneUse() || isa<ScalableVectorType>(SVI->getType())) in lowerInterleavedStore()
465 // Check if the shufflevector is RE-interleave shuffle. in lowerInterleavedStore()
473 if (!TLI->lowerInterleavedStore(SI, SVI, Factor)) in lowerInterleavedStore()
484 LoadInst *LI = dyn_cast<LoadInst>(DI->getOperand(0)); in lowerDeinterleaveIntrinsic()
486 if (!LI || !LI->hasOneUse() || !LI->isSimple()) in lowerDeinterleaveIntrinsic()
492 if (!TLI->lowerDeinterleaveIntrinsicToLoad(DI, LI)) in lowerDeinterleaveIntrinsic()
495 // We now have a target-specific load, so delete the old one. in lowerDeinterleaveIntrinsic()
503 if (!II->hasOneUse()) in lowerInterleaveIntrinsic()
506 StoreInst *SI = dyn_cast<StoreInst>(*(II->users().begin())); in lowerInterleaveIntrinsic()
508 if (!SI || !SI->isSimple()) in lowerInterleaveIntrinsic()
514 if (!TLI->lowerInterleaveIntrinsicToStore(II, SI)) in lowerInterleaveIntrinsic()
517 // We now have a target-specific store, so delete the old one. in lowerInterleaveIntrinsic()
528 for (auto &I : instructions(F)) { in runOnFunction()
529 if (auto *LI = dyn_cast<LoadInst>(&I)) in runOnFunction()
532 if (auto *SI = dyn_cast<StoreInst>(&I)) in runOnFunction()
535 if (auto *II = dyn_cast<IntrinsicInst>(&I)) { in runOnFunction()
538 if (II->getIntrinsicID() == Intrinsic::vector_deinterleave2) in runOnFunction()
540 if (II->getIntrinsicID() == Intrinsic::vector_interleave2) in runOnFunction()
545 for (auto *I : DeadInsts) in runOnFunction()
546 I->eraseFromParent(); in runOnFunction()