Lines Matching refs:SI
231 isSafeSelectToSpeculate(SelectInst &SI, bool PreserveCFG);
735 iterator SI, SJ; member in __anondf5662880411::Partition
742 Partition(iterator SI) : SI(SI), SJ(SI) {} in Partition() argument
765 bool empty() const { return SI == SJ; } in empty()
776 iterator begin() const { return SI; } in begin()
817 partition_iterator(AllocaSlices::iterator SI, AllocaSlices::iterator SE) in partition_iterator() argument
818 : P(SI), SE(SE) { in partition_iterator()
821 if (SI != SE) in partition_iterator()
829 assert((P.SI != SE || !P.SplitTails.empty()) && in advance()
859 if (P.SI == SE) { in advance()
866 if (P.SI != P.SJ) { in advance()
877 P.SI = P.SJ; in advance()
880 if (P.SI == SE) { in advance()
889 if (!P.SplitTails.empty() && P.SI->beginOffset() != P.EndOffset && in advance()
890 !P.SI->isSplittable()) { in advance()
892 P.EndOffset = P.SI->beginOffset(); in advance()
902 P.BeginOffset = P.SplitTails.empty() ? P.SI->beginOffset() : P.EndOffset; in advance()
903 P.EndOffset = P.SI->endOffset(); in advance()
908 if (!P.SI->isSplittable()) { in advance()
911 assert(P.BeginOffset == P.SI->beginOffset()); in advance()
929 assert(P.SI->isSplittable() && "Forming a splittable partition!"); in advance()
957 if (P.SI == RHS.P.SI && P.SplitTails.empty() == RHS.P.SplitTails.empty()) { in operator ==()
988 static Value *foldSelectInst(SelectInst &SI) { in foldSelectInst() argument
992 if (ConstantInt *CI = dyn_cast<ConstantInt>(SI.getCondition())) in foldSelectInst()
993 return SI.getOperand(1 + CI->isZero()); in foldSelectInst()
994 if (SI.getOperand(1) == SI.getOperand(2)) in foldSelectInst()
995 return SI.getOperand(1); in foldSelectInst()
1123 void visitStoreInst(StoreInst &SI) { in visitStoreInst() argument
1124 Value *ValOp = SI.getValueOperand(); in visitStoreInst()
1126 return PI.setEscapedAndAborted(&SI); in visitStoreInst()
1128 return PI.setAborted(&SI); in visitStoreInst()
1132 return PI.setAborted(&SI); in visitStoreInst()
1148 << " use: " << SI << "\n"); in visitStoreInst()
1149 return markAsDead(SI); in visitStoreInst()
1152 assert((!SI.isSimple() || ValOp->getType()->isSingleValueType()) && in visitStoreInst()
1154 handleLoadOrStore(ValOp->getType(), SI, Offset, Size, SI.isVolatile()); in visitStoreInst()
1300 if (StoreInst *SI = dyn_cast<StoreInst>(I)) { in hasUnsafePHIOrSelectUse() local
1301 Value *Op = SI->getOperand(0); in hasUnsafePHIOrSelectUse()
1303 return SI; in hasUnsafePHIOrSelectUse()
1306 PI.setAborted(SI); in hasUnsafePHIOrSelectUse()
1389 void visitSelectInst(SelectInst &SI) { visitPHINodeOrSelectInst(SI); } in visitSelectInst() argument
1481 } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) { in findCommonType() local
1482 UserTy = SI->getValueOperand()->getType(); in findCommonType()
1684 isSafeLoadOfSelectToSpeculate(LoadInst &LI, SelectInst &SI, bool PreserveCFG) { in isSafeLoadOfSelectToSpeculate() argument
1688 const DataLayout &DL = SI.getDataLayout(); in isSafeLoadOfSelectToSpeculate()
1689 for (Value *Value : {SI.getTrueValue(), SI.getFalseValue()}) in isSafeLoadOfSelectToSpeculate()
1692 Spec.setAsSpeculatable(/*isTrueVal=*/Value == SI.getTrueValue()); in isSafeLoadOfSelectToSpeculate()
1700 SROA::isSafeSelectToSpeculate(SelectInst &SI, bool PreserveCFG) { in isSafeSelectToSpeculate() argument
1703 for (User *U : SI.users()) { in isSafeSelectToSpeculate()
1735 isSafeLoadOfSelectToSpeculate(*LI, SI, PreserveCFG); in isSafeSelectToSpeculate()
1746 static void speculateSelectInstLoads(SelectInst &SI, LoadInst &LI, in speculateSelectInstLoads() argument
1748 LLVM_DEBUG(dbgs() << " original load: " << SI << "\n"); in speculateSelectInstLoads()
1750 Value *TV = SI.getTrueValue(); in speculateSelectInstLoads()
1751 Value *FV = SI.getFalseValue(); in speculateSelectInstLoads()
1776 Value *V = IRB.CreateSelect(SI.getCondition(), TL, FL, in speculateSelectInstLoads()
1784 static void rewriteMemOpOfSelect(SelectInst &SI, T &I, in rewriteMemOpOfSelect() argument
1793 SplitBlockAndInsertIfThenElse(SI.getCondition(), &I, &ThenTerm, &ElseTerm, in rewriteMemOpOfSelect()
1794 SI.getMetadata(LLVMContext::MD_prof), &DTU); in rewriteMemOpOfSelect()
1796 SplitBlockAndInsertIfThen(SI.getCondition(), &I, /*Unreachable=*/false, in rewriteMemOpOfSelect()
1797 SI.getMetadata(LLVMContext::MD_prof), &DTU, in rewriteMemOpOfSelect()
1825 Value *Ptr = SI.getOperand(1 + SuccIdx); in rewriteMemOpOfSelect()
1845 else if (auto *SI = dyn_cast<StoreInst>(&I)) in rewriteMemOpOfSelect() local
1846 rewriteMemOpOfSelect(SelInst, *SI, Spec, DTU); in rewriteMemOpOfSelect()
1851 static bool rewriteSelectInstMemOps(SelectInst &SI, in rewriteSelectInstMemOps() argument
1855 LLVM_DEBUG(dbgs() << " original select: " << SI << "\n"); in rewriteSelectInstMemOps()
1868 speculateSelectInstLoads(SI, cast<LoadInst>(*I), IRB); in rewriteSelectInstMemOps()
1871 rewriteMemOpOfSelect(SI, *I, Spec, *DTU); in rewriteSelectInstMemOps()
1877 for (User *U : make_early_inc_range(SI.users())) in rewriteSelectInstMemOps()
1879 SI.eraseFromParent(); in rewriteSelectInstMemOps()
2071 } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) { in isVectorPromotionViableForSlice() local
2072 if (SI->isVolatile()) in isVectorPromotionViableForSlice()
2074 Type *STy = SI->getValueOperand()->getType(); in isVectorPromotionViableForSlice()
2300 else if (auto *SI = dyn_cast<StoreInst>(S.getUse()->getUser())) in isVectorPromotionViable() local
2301 Ty = SI->getValueOperand()->getType(); in isVectorPromotionViable()
2386 } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) { in isIntegerWideningViableForSlice() local
2387 Type *ValueTy = SI->getValueOperand()->getType(); in isIntegerWideningViableForSlice()
2388 if (SI->isVolatile()) in isIntegerWideningViableForSlice()
2968 bool rewriteVectorizedStoreInst(Value *V, StoreInst &SI, Value *OldOp, in rewriteVectorizedStoreInst() argument
2992 Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access, in rewriteVectorizedStoreInst()
2997 Pass.DeadInsts.push_back(&SI); in rewriteVectorizedStoreInst()
3000 migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &SI, in rewriteVectorizedStoreInst()
3006 bool rewriteIntegerStore(Value *V, StoreInst &SI, AAMDNodes AATags) { in rewriteIntegerStore() argument
3008 assert(!SI.isVolatile()); in rewriteIntegerStore()
3016 V = insertInteger(DL, IRB, Old, SI.getValueOperand(), Offset, "insert"); in rewriteIntegerStore()
3020 Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access, in rewriteIntegerStore()
3026 migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &SI, in rewriteIntegerStore()
3030 Pass.DeadInsts.push_back(&SI); in rewriteIntegerStore()
3035 bool visitStoreInst(StoreInst &SI) { in visitStoreInst() argument
3036 LLVM_DEBUG(dbgs() << " original: " << SI << "\n"); in visitStoreInst()
3037 Value *OldOp = SI.getOperand(1); in visitStoreInst()
3040 AAMDNodes AATags = SI.getAAMetadata(); in visitStoreInst()
3041 Value *V = SI.getValueOperand(); in visitStoreInst()
3050 assert(!SI.isVolatile()); in visitStoreInst()
3055 IntegerType *NarrowTy = Type::getIntNTy(SI.getContext(), SliceSize * 8); in visitStoreInst()
3061 return rewriteVectorizedStoreInst(V, SI, OldOp, AATags); in visitStoreInst()
3063 return rewriteIntegerStore(V, SI, AATags); in visitStoreInst()
3071 getPtrToNewAI(SI.getPointerAddressSpace(), SI.isVolatile()); in visitStoreInst()
3074 IRB.CreateAlignedStore(V, NewPtr, NewAI.getAlign(), SI.isVolatile()); in visitStoreInst()
3076 unsigned AS = SI.getPointerAddressSpace(); in visitStoreInst()
3079 IRB.CreateAlignedStore(V, NewPtr, getSliceAlign(), SI.isVolatile()); in visitStoreInst()
3081 NewSI->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access, in visitStoreInst()
3086 if (SI.isVolatile()) in visitStoreInst()
3087 NewSI->setAtomic(SI.getOrdering(), SI.getSyncScopeID()); in visitStoreInst()
3089 NewSI->setAlignment(SI.getAlign()); in visitStoreInst()
3091 migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &SI, in visitStoreInst()
3095 Pass.DeadInsts.push_back(&SI); in visitStoreInst()
3101 !SI.isVolatile(); in visitStoreInst()
3582 if (StoreInst *SI = dyn_cast<StoreInst>(I)) { in fixLoadStoreAlign() local
3583 SI->setAlignment(std::min(SI->getAlign(), getSliceAlign())); in fixLoadStoreAlign()
3630 bool visitSelectInst(SelectInst &SI) { in visitSelectInst() argument
3631 LLVM_DEBUG(dbgs() << " original: " << SI << "\n"); in visitSelectInst()
3632 assert((SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr) && in visitSelectInst()
3639 if (SI.getOperand(1) == OldPtr) in visitSelectInst()
3640 SI.setOperand(1, NewPtr); in visitSelectInst()
3641 if (SI.getOperand(2) == OldPtr) in visitSelectInst()
3642 SI.setOperand(2, NewPtr); in visitSelectInst()
3644 LLVM_DEBUG(dbgs() << " to: " << SI << "\n"); in visitSelectInst()
3648 fixLoadStoreAlign(SI); in visitSelectInst()
3653 SelectUsers.insert(&SI); in visitSelectInst()
3906 bool visitStoreInst(StoreInst &SI) { in visitStoreInst() argument
3907 if (!SI.isSimple() || SI.getPointerOperand() != *U) in visitStoreInst()
3909 Value *V = SI.getValueOperand(); in visitStoreInst()
3914 LLVM_DEBUG(dbgs() << " original: " << SI << "\n"); in visitStoreInst()
3915 StoreOpSplitter Splitter(&SI, *U, V->getType(), SI.getAAMetadata(), &SI, in visitStoreInst()
3916 getAdjustedAlignment(&SI, 0), DL, IRB); in visitStoreInst()
3918 Visited.erase(&SI); in visitStoreInst()
3921 at::deleteAssignmentMarkers(&SI); in visitStoreInst()
3922 SI.eraseFromParent(); in visitStoreInst()
3945 if (auto *SI = dyn_cast<SelectInst>(Op)) { in unfoldGEPSelect() local
3949 Sel = SI; in unfoldGEPSelect()
4035 if (auto *SI = dyn_cast<PHINode>(Op)) { in unfoldGEPPhi() local
4039 Phi = SI; in unfoldGEPPhi()
4121 bool visitSelectInst(SelectInst &SI) { in visitSelectInst() argument
4122 enqueueUsers(SI); in visitSelectInst()
4348 else if (auto *SI = dyn_cast<StoreInst>(I)) in presplitLoadsAndStores() local
4349 if (auto *LI = dyn_cast<LoadInst>(SI->getValueOperand())) in presplitLoadsAndStores()
4365 auto *SI = dyn_cast<StoreInst>(LU); in presplitLoadsAndStores() local
4366 if (!SI || !SI->isSimple()) in presplitLoadsAndStores()
4377 } else if (auto *SI = dyn_cast<StoreInst>(I)) { in presplitLoadsAndStores() local
4378 if (S.getUse() != &SI->getOperandUse(SI->getPointerOperandIndex())) in presplitLoadsAndStores()
4381 auto *StoredLoad = dyn_cast<LoadInst>(SI->getValueOperand()); in presplitLoadsAndStores()
4384 assert(!SI->isVolatile() && "Cannot split volatile stores!"); in presplitLoadsAndStores()
4386 Stores.push_back(SI); in presplitLoadsAndStores()
4428 llvm::erase_if(Stores, [&UnsplittableLoads, &SplitOffsetsMap](StoreInst *SI) { in presplitLoadsAndStores() argument
4431 auto *LI = cast<LoadInst>(SI->getValueOperand()); in presplitLoadsAndStores()
4443 auto &StoreOffsets = SplitOffsetsMap[SI]; in presplitLoadsAndStores()
4453 << " " << *SI << "\n"); in presplitLoadsAndStores()
4466 llvm::erase_if(Stores, [&UnsplittableLoads](StoreInst *SI) { in presplitLoadsAndStores() argument
4467 auto *LI = cast<LoadInst>(SI->getValueOperand()); in presplitLoadsAndStores()
4566 StoreInst *SI = cast<StoreInst>(LU); in presplitLoadsAndStores() local
4567 if (!Stores.empty() && SplitOffsetsMap.count(SI)) { in presplitLoadsAndStores()
4569 LLVM_DEBUG(dbgs() << " Deferred splitting of store: " << *SI in presplitLoadsAndStores()
4574 Value *StoreBasePtr = SI->getPointerOperand(); in presplitLoadsAndStores()
4575 IRB.SetInsertPoint(SI); in presplitLoadsAndStores()
4576 AAMDNodes AATags = SI->getAAMetadata(); in presplitLoadsAndStores()
4578 LLVM_DEBUG(dbgs() << " Splitting store of load: " << *SI << "\n"); in presplitLoadsAndStores()
4583 auto *PartPtrTy = SI->getPointerOperandType(); in presplitLoadsAndStores()
4585 auto AS = SI->getPointerAddressSpace(); in presplitLoadsAndStores()
4591 getAdjustedAlignment(SI, PartOffset), in presplitLoadsAndStores()
4593 PStore->copyMetadata(*SI, {LLVMContext::MD_mem_parallel_loop_access, in presplitLoadsAndStores()
4616 DeadInsts.push_back(SI); in presplitLoadsAndStores()
4633 for (StoreInst *SI : Stores) { in presplitLoadsAndStores()
4634 auto *LI = cast<LoadInst>(SI->getValueOperand()); in presplitLoadsAndStores()
4640 auto &Offsets = SplitOffsetsMap[SI]; in presplitLoadsAndStores()
4648 Instruction *StoreBasePtr = cast<Instruction>(SI->getPointerOperand()); in presplitLoadsAndStores()
4650 LLVM_DEBUG(dbgs() << " Splitting store: " << *SI << "\n"); in presplitLoadsAndStores()
4668 auto *StorePartPtrTy = SI->getPointerOperandType(); in presplitLoadsAndStores()
4689 IRB.SetInsertPoint(SI); in presplitLoadsAndStores()
4690 auto AS = SI->getPointerAddressSpace(); in presplitLoadsAndStores()
4696 getAdjustedAlignment(SI, PartOffset), in presplitLoadsAndStores()
4698 PStore->copyMetadata(*SI, {LLVMContext::MD_mem_parallel_loop_access, in presplitLoadsAndStores()
4750 assert(*LI->user_begin() == SI && "Single use isn't this store!"); in presplitLoadsAndStores()
4753 DeadInsts.push_back(SI); in presplitLoadsAndStores()