Lines Matching +full:cpu +full:- +full:viewed
1 //===- CodeGenPrepare.cpp - Prepare a function for code generation --------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
10 // SelectionDAG-based code generation. This works around limitations in it's
11 // basic-block-at-a-time approach. It should eventually be removed.
13 //===----------------------------------------------------------------------===//
43 #include "llvm/Config/llvm-config.h"
137 "disable-cgp-branch-opts", cl::Hidden, cl::init(false),
141 DisableGCOpts("disable-cgp-gc-opts", cl::Hidden, cl::init(false),
145 DisableSelectToBranch("disable-cgp-select2branch", cl::Hidden,
150 AddrSinkUsingGEPs("addr-sink-using-gep", cl::Hidden, cl::init(true),
154 EnableAndCmpSinking("enable-andcmp-sinking", cl::Hidden, cl::init(true),
158 "disable-cgp-store-extract", cl::Hidden, cl::init(false),
162 "stress-cgp-store-extract", cl::Hidden, cl::init(false),
166 "disable-cgp-ext-ld-promotion", cl::Hidden, cl::init(false),
167 cl::desc("Disable ext(promotable(ld)) -> promoted(ext(ld)) optimization in "
171 "stress-cgp-ext-ld-promotion", cl::Hidden, cl::init(false),
172 cl::desc("Stress test ext(promotable(ld)) -> promoted(ext(ld)) "
176 "disable-preheader-prot", cl::Hidden, cl::init(false),
180 "profile-guided-section-prefix", cl::Hidden, cl::init(true),
184 "profile-unknown-in-special-section", cl::Hidden,
194 "bbsections-guided-section-prefix", cl::Hidden, cl::init(true),
195 cl::desc("Use the basic-block-sections profile to determine the text "
197 "basic-block-sections profile will be placed in `.text.hot` "
203 "cgp-freq-ratio-to-skip-merge", cl::Hidden, cl::init(2),
208 "force-split-store", cl::Hidden, cl::init(false),
212 "cgp-type-promotion-merge", cl::Hidden,
218 "disable-complex-addr-modes", cl::Hidden, cl::init(false),
223 AddrSinkNewPhis("addr-sink-new-phis", cl::Hidden, cl::init(false),
227 "addr-sink-new-select", cl::Hidden, cl::init(true),
231 "addr-sink-combine-base-reg", cl::Hidden, cl::init(true),
235 "addr-sink-combine-base-gv", cl::Hidden, cl::init(true),
239 "addr-sink-combine-base-offs", cl::Hidden, cl::init(true),
243 "addr-sink-combine-scaled-reg", cl::Hidden, cl::init(true),
247 EnableGEPOffsetSplit("cgp-split-large-offset-gep", cl::Hidden,
252 "cgp-icmp-eq2icmp-st", cl::Hidden, cl::init(false),
256 VerifyBFIUpdates("cgp-verify-bfi-updates", cl::Hidden, cl::init(false),
261 OptimizePhiTypes("cgp-optimize-phi-types", cl::Hidden, cl::init(true),
265 HugeFuncThresholdInCGPP("cgpp-huge-func", cl::init(10000), cl::Hidden,
269 MaxAddressUsersToScan("cgp-max-address-users-to-scan", cl::init(100),
274 DisableDeletePHIs("disable-cgp-delete-phis", cl::Hidden, cl::init(false),
293 // in a Basic Block. So we should re-iterate instructions
323 /// Keeps track of non-local addresses that have been sunk into a block.
406 CurInstIterator = BB->begin(); in resetIteratorIfInvalidatedWhileCalling()
513 CGP.SubtargetInfo = TM->getSubtargetImpl(F); in runOnFunction()
514 CGP.TLI = CGP.SubtargetInfo->getTargetLowering(); in runOnFunction()
515 CGP.TRI = CGP.SubtargetInfo->getRegisterInfo(); in runOnFunction()
524 CGP.BBSectionsProfileReader = BBSPRWP ? &BBSPRWP->getBBSPR() : nullptr; in runOnFunction()
561 SubtargetInfo = TM->getSubtargetImpl(F); in run()
562 TLI = SubtargetInfo->getTargetLowering(); in run()
563 TRI = SubtargetInfo->getRegisterInfo(); in run()
580 // Use the basic-block-sections profile to promote hot functions to .text.hot in _run()
583 BBSectionsProfileReader->isFunctionHot(F.getName())) { in _run()
590 PSI->isFunctionHotInCallGraph(&F, *BFI)) in _run()
595 else if (PSI->isFunctionColdInCallGraph(&F, *BFI) || in _run()
598 else if (ProfileUnknownInSpecialSection && PSI->hasPartialSampleProfile() && in _run()
599 PSI->isFunctionHotnessUnknown(F)) in _run()
605 if (!OptSize && !PSI->hasHugeWorkingSetSize() && TLI->isSlowDivBypassed()) { in _run()
607 TLI->getBypassSlowDivWidths(); in _run()
612 BasicBlock *Next = BB->getNextNode(); in _run()
644 LI->releaseMemory(); in _run()
645 LI->analyze(getDT(F)); in _run()
698 LI->verify(getDT(F)); in _run()
703 I->deleteValue(); in _run()
764 // Do this last to clean up use-before-def scenarios introduced by other in _run()
785 Value *Operand = Assume->getOperand(0); in eliminateAssumptions()
786 Assume->eraseFromParent(); in eliminateAssumptions()
798 /// GEP-tracking data strcutures.
809 auto VecI = LargeOffsetGEPMap.find(GEP->getPointerOperand()); in removeAllAssertingVHReferences()
813 auto &GEPVector = VecI->second; in removeAllAssertingVHReferences()
848 BasicBlock *SinglePred = BB->getSinglePredecessor(); in eliminateFallThrough()
851 if (!SinglePred || SinglePred == BB || BB->hasAddressTaken()) in eliminateFallThrough()
855 if (DT && !DT->isReachableFromEntry(BB)) in eliminateFallThrough()
858 BranchInst *Term = dyn_cast<BranchInst>(SinglePred->getTerminator()); in eliminateFallThrough()
859 if (Term && !Term->isConditional()) { in eliminateFallThrough()
889 BranchInst *BI = dyn_cast<BranchInst>(BB->getTerminator()); in findDestBlockOfMergeableEmptyBlock()
890 if (!BI || !BI->isUnconditional()) in findDestBlockOfMergeableEmptyBlock()
895 BasicBlock::iterator BBI = BI->getIterator(); in findDestBlockOfMergeableEmptyBlock()
896 if (BBI != BB->begin()) { in findDestBlockOfMergeableEmptyBlock()
897 --BBI; in findDestBlockOfMergeableEmptyBlock()
899 if (BBI == BB->begin()) in findDestBlockOfMergeableEmptyBlock()
901 --BBI; in findDestBlockOfMergeableEmptyBlock()
908 BasicBlock *DestBB = BI->getSuccessor(0); in findDestBlockOfMergeableEmptyBlock()
920 /// edges in ways that are non-optimal for isel. Start by eliminating these
924 SmallVector<Loop *, 16> LoopList(LI->begin(), LI->end()); in eliminateMostlyEmptyBlocks()
928 if (BasicBlock *Preheader = L->getLoopPreheader()) in eliminateMostlyEmptyBlocks()
967 !(BB->getSinglePredecessor() && in isMergingEmptyBlockProfitable()
968 BB->getSinglePredecessor()->getSingleSuccessor())) in isMergingEmptyBlockProfitable()
975 if (isa<CallBrInst>(Pred->getTerminator()) && in isMergingEmptyBlockProfitable()
988 BasicBlock *Pred = BB->getUniquePredecessor(); in isMergingEmptyBlockProfitable()
989 if (!Pred || !(isa<SwitchInst>(Pred->getTerminator()) || in isMergingEmptyBlockProfitable()
990 isa<IndirectBrInst>(Pred->getTerminator()))) in isMergingEmptyBlockProfitable()
993 if (BB->getTerminator() != BB->getFirstNonPHIOrDbg()) in isMergingEmptyBlockProfitable()
1007 if (!isa<PHINode>(DestBB->begin())) in isMergingEmptyBlockProfitable()
1018 if (llvm::all_of(DestBB->phis(), [&](const PHINode &DestPN) { in isMergingEmptyBlockProfitable()
1031 BlockFrequency PredFreq = BFI->getBlockFreq(Pred); in isMergingEmptyBlockProfitable()
1032 BlockFrequency BBFreq = BFI->getBlockFreq(BB); in isMergingEmptyBlockProfitable()
1035 if (SameValueBB->getUniquePredecessor() == Pred && in isMergingEmptyBlockProfitable()
1037 BBFreq += BFI->getBlockFreq(SameValueBB); in isMergingEmptyBlockProfitable()
1044 /// unconditional branch between them, and BB contains no other non-phi
1051 for (const PHINode &PN : BB->phis()) { in canMergeBlocks()
1054 if (UI->getParent() != DestBB || !isa<PHINode>(UI)) in canMergeBlocks()
1059 if (UI->getParent() == DestBB) { in canMergeBlocks()
1061 for (unsigned I = 0, E = UPN->getNumIncomingValues(); I != E; ++I) { in canMergeBlocks()
1062 Instruction *Insn = dyn_cast<Instruction>(UPN->getIncomingValue(I)); in canMergeBlocks()
1063 if (Insn && Insn->getParent() == BB && in canMergeBlocks()
1064 Insn->getParent() != UPN->getIncomingBlock(I)) in canMergeBlocks()
1074 const PHINode *DestBBPN = dyn_cast<PHINode>(DestBB->begin()); in canMergeBlocks()
1080 if (const PHINode *BBPN = dyn_cast<PHINode>(BB->begin())) { in canMergeBlocks()
1082 for (unsigned i = 0, e = BBPN->getNumIncomingValues(); i != e; ++i) in canMergeBlocks()
1083 BBPreds.insert(BBPN->getIncomingBlock(i)); in canMergeBlocks()
1089 for (unsigned i = 0, e = DestBBPN->getNumIncomingValues(); i != e; ++i) { in canMergeBlocks()
1090 BasicBlock *Pred = DestBBPN->getIncomingBlock(i); in canMergeBlocks()
1092 for (const PHINode &PN : DestBB->phis()) { in canMergeBlocks()
1098 if (V2PN->getParent() == BB) in canMergeBlocks()
1099 V2 = V2PN->getIncomingValueForBlock(Pred); in canMergeBlocks()
1117 for (Value::user_iterator UI = OldI->user_begin(), E = OldI->user_end(); in replaceAllUsesWith()
1121 FreshBBs.insert(User->getParent()); in replaceAllUsesWith()
1124 Old->replaceAllUsesWith(New); in replaceAllUsesWith()
1130 BranchInst *BI = cast<BranchInst>(BB->getTerminator()); in eliminateMostlyEmptyBlock()
1131 BasicBlock *DestBB = BI->getSuccessor(0); in eliminateMostlyEmptyBlock()
1133 LLVM_DEBUG(dbgs() << "MERGING MOSTLY EMPTY BLOCKS - BEFORE:\n" in eliminateMostlyEmptyBlock()
1138 if (BasicBlock *SinglePred = DestBB->getSinglePredecessor()) { in eliminateMostlyEmptyBlock()
1159 for (PHINode &PN : DestBB->phis()) { in eliminateMostlyEmptyBlock()
1166 if (InValPhi && InValPhi->getParent() == BB) { in eliminateMostlyEmptyBlock()
1168 for (unsigned i = 0, e = InValPhi->getNumIncomingValues(); i != e; ++i) in eliminateMostlyEmptyBlock()
1169 PN.addIncoming(InValPhi->getIncomingValue(i), in eliminateMostlyEmptyBlock()
1170 InValPhi->getIncomingBlock(i)); in eliminateMostlyEmptyBlock()
1174 if (PHINode *BBPN = dyn_cast<PHINode>(BB->begin())) { in eliminateMostlyEmptyBlock()
1175 for (unsigned i = 0, e = BBPN->getNumIncomingValues(); i != e; ++i) in eliminateMostlyEmptyBlock()
1176 PN.addIncoming(InVal, BBPN->getIncomingBlock(i)); in eliminateMostlyEmptyBlock()
1186 BB->replaceAllUsesWith(DestBB); in eliminateMostlyEmptyBlock()
1187 BB->eraseFromParent(); in eliminateMostlyEmptyBlock()
1204 auto K = std::make_pair(ThisRelocate->getBasePtrIndex(), in computeBaseDerivedRelocateMap()
1205 ThisRelocate->getDerivedPtrIndex()); in computeBaseDerivedRelocateMap()
1224 RelocateInstMap[MaybeBase->second].push_back(I); in computeBaseDerivedRelocateMap()
1232 for (unsigned i = 1; i < GEP->getNumOperands(); i++) { in getGEPSmallConstantIntOffsetV()
1234 auto *Op = dyn_cast<ConstantInt>(GEP->getOperand(i)); in getGEPSmallConstantIntOffsetV()
1235 if (!Op || Op->getZExtValue() > 20) in getGEPSmallConstantIntOffsetV()
1239 for (unsigned i = 1; i < GEP->getNumOperands(); i++) in getGEPSmallConstantIntOffsetV()
1240 OffsetV.push_back(GEP->getOperand(i)); in getGEPSmallConstantIntOffsetV()
1256 for (auto R = RelocatedBase->getParent()->getFirstInsertionPt(); in simplifyRelocatesOffABase()
1259 if (RI->getStatepoint() == RelocatedBase->getStatepoint()) in simplifyRelocatesOffABase()
1260 if (RI->getBasePtrIndex() == RelocatedBase->getBasePtrIndex()) { in simplifyRelocatesOffABase()
1261 RelocatedBase->moveBefore(RI); in simplifyRelocatesOffABase()
1267 assert(ToReplace->getBasePtrIndex() == RelocatedBase->getBasePtrIndex() && in simplifyRelocatesOffABase()
1269 if (ToReplace->getBasePtrIndex() == ToReplace->getDerivedPtrIndex()) { in simplifyRelocatesOffABase()
1274 if (RelocatedBase->getParent() != ToReplace->getParent()) { in simplifyRelocatesOffABase()
1282 Value *Base = ToReplace->getBasePtr(); in simplifyRelocatesOffABase()
1283 auto *Derived = dyn_cast<GetElementPtrInst>(ToReplace->getDerivedPtr()); in simplifyRelocatesOffABase()
1284 if (!Derived || Derived->getPointerOperand() != Base) in simplifyRelocatesOffABase()
1292 assert(RelocatedBase->getNextNode() && in simplifyRelocatesOffABase()
1296 IRBuilder<> Builder(RelocatedBase->getNextNode()); in simplifyRelocatesOffABase()
1297 Builder.SetCurrentDebugLocation(ToReplace->getDebugLoc()); in simplifyRelocatesOffABase()
1322 if (RelocatedBase->getType() != Base->getType()) { in simplifyRelocatesOffABase()
1324 Builder.CreateBitCast(RelocatedBase, Base->getType()); in simplifyRelocatesOffABase()
1327 Builder.CreateGEP(Derived->getSourceElementType(), ActualRelocatedBase, in simplifyRelocatesOffABase()
1329 Replacement->takeName(ToReplace); in simplifyRelocatesOffABase()
1334 if (Replacement->getType() != ToReplace->getType()) { in simplifyRelocatesOffABase()
1336 Builder.CreateBitCast(Replacement, ToReplace->getType()); in simplifyRelocatesOffABase()
1338 ToReplace->replaceAllUsesWith(ActualReplacement); in simplifyRelocatesOffABase()
1339 ToReplace->eraseFromParent(); in simplifyRelocatesOffABase()
1392 BasicBlock *DefBB = CI->getParent(); in SinkCast()
1394 /// InsertedCasts - Only insert a cast in each block once. in SinkCast()
1398 for (Value::user_iterator UI = CI->user_begin(), E = CI->user_end(); in SinkCast()
1405 BasicBlock *UserBB = User->getParent(); in SinkCast()
1407 UserBB = PN->getIncomingBlock(TheUse); in SinkCast()
1415 if (User->isEHPad()) in SinkCast()
1419 // allow non-PHI instructions before the terminator, we can't sink the in SinkCast()
1421 if (UserBB->getTerminator()->isEHPad()) in SinkCast()
1432 BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt(); in SinkCast()
1433 assert(InsertPt != UserBB->end()); in SinkCast()
1434 InsertedCast = cast<CastInst>(CI->clone()); in SinkCast()
1435 InsertedCast->insertBefore(*UserBB, InsertPt); in SinkCast()
1445 if (CI->use_empty()) { in SinkCast()
1447 CI->eraseFromParent(); in SinkCast()
1455 /// one pointer type to another, i32->i8 on PPC), sink it into user blocks to
1461 // Sink only "cheap" (or nop) address-space casts. This is a weaker condition in OptimizeNoopCopyExpression()
1464 if (!TLI.isFreeAddrSpaceCast(ASC->getSrcAddressSpace(), in OptimizeNoopCopyExpression()
1465 ASC->getDestAddressSpace())) in OptimizeNoopCopyExpression()
1470 EVT SrcVT = TLI.getValueType(DL, CI->getOperand(0)->getType()); in OptimizeNoopCopyExpression()
1471 EVT DstVT = TLI.getValueType(DL, CI->getType()); in OptimizeNoopCopyExpression()
1473 // This is an fp<->int conversion? in OptimizeNoopCopyExpression()
1485 if (TLI.getTypeAction(CI->getContext(), SrcVT) == in OptimizeNoopCopyExpression()
1487 SrcVT = TLI.getTypeToTransformTo(CI->getContext(), SrcVT); in OptimizeNoopCopyExpression()
1488 if (TLI.getTypeAction(CI->getContext(), DstVT) == in OptimizeNoopCopyExpression()
1490 DstVT = TLI.getTypeToTransformTo(CI->getContext(), DstVT); in OptimizeNoopCopyExpression()
1522 const Loop *L = LI->getLoopFor(PN->getParent()); in getIVIncrement()
1523 if (!L || L->getHeader() != PN->getParent() || !L->getLoopLatch()) in getIVIncrement()
1526 dyn_cast<Instruction>(PN->getIncomingValueForBlock(L->getLoopLatch())); in getIVIncrement()
1527 if (!IVInc || LI->getLoopFor(IVInc->getParent()) != L) in getIVIncrement()
1546 return IVInc->first == I; in isIVIncrement()
1557 const Loop *L = LI->getLoopFor(BO->getParent()); in replaceMathCmpWithIntrinsic()
1560 if (LI->getLoopFor(Cmp->getParent()) != L) in replaceMathCmpWithIntrinsic()
1566 auto &DT = getDT(*BO->getParent()->getParent()); in replaceMathCmpWithIntrinsic()
1567 if (DT.dominates(Cmp->getParent(), BO->getParent())) in replaceMathCmpWithIntrinsic()
1573 return BO->hasOneUse() && DT.dominates(Cmp->getParent(), L->getLoopLatch()); in replaceMathCmpWithIntrinsic()
1575 if (BO->getParent() != Cmp->getParent() && !IsReplacableIVIncrement(BO)) { in replaceMathCmpWithIntrinsic()
1576 // We used to use a dominator tree here to allow multi-block optimization. in replaceMathCmpWithIntrinsic()
1582 // 3. Use of a dominator tree could cause large compile-time regression. in replaceMathCmpWithIntrinsic()
1584 // run-loop. The recomputing is probably unnecessary in many cases, so if in replaceMathCmpWithIntrinsic()
1589 // - We can speculate IV increment anywhere in the loop (as long as the in replaceMathCmpWithIntrinsic()
1591 // - Upon computing Cmp, we effectively compute something equivalent to the in replaceMathCmpWithIntrinsic()
1597 // We allow matching the canonical IR (add X, C) back to (usubo X, -C). in replaceMathCmpWithIntrinsic()
1598 if (BO->getOpcode() == Instruction::Add && in replaceMathCmpWithIntrinsic()
1606 for (Instruction &Iter : *Cmp->getParent()) { in replaceMathCmpWithIntrinsic()
1609 if ((BO->getOpcode() != Instruction::Xor && &Iter == BO) || &Iter == Cmp) { in replaceMathCmpWithIntrinsic()
1618 if (BO->getOpcode() != Instruction::Xor) { in replaceMathCmpWithIntrinsic()
1622 assert(BO->hasOneUse() && in replaceMathCmpWithIntrinsic()
1626 Cmp->eraseFromParent(); in replaceMathCmpWithIntrinsic()
1627 BO->eraseFromParent(); in replaceMathCmpWithIntrinsic()
1631 /// Match special-case patterns that check for unsigned add overflow.
1634 // Add = add A, 1; Cmp = icmp eq A,-1 (overflow if A is max val) in matchUAddWithOverflowConstantEdgeCases()
1635 // Add = add A,-1; Cmp = icmp ne A, 0 (overflow if A is non-zero) in matchUAddWithOverflowConstantEdgeCases()
1636 Value *A = Cmp->getOperand(0), *B = Cmp->getOperand(1); in matchUAddWithOverflowConstantEdgeCases()
1638 // We are not expecting non-canonical/degenerate code. Just bail out. in matchUAddWithOverflowConstantEdgeCases()
1642 ICmpInst::Predicate Pred = Cmp->getPredicate(); in matchUAddWithOverflowConstantEdgeCases()
1644 B = ConstantInt::get(B->getType(), 1); in matchUAddWithOverflowConstantEdgeCases()
1646 B = ConstantInt::get(B->getType(), -1); in matchUAddWithOverflowConstantEdgeCases()
1652 for (User *U : A->users()) { in matchUAddWithOverflowConstantEdgeCases()
1672 A = Add->getOperand(0); in combineToUAddWithOverflow()
1673 B = Add->getOperand(1); in combineToUAddWithOverflow()
1677 if (!TLI->shouldFormOverflowOp(ISD::UADDO, in combineToUAddWithOverflow()
1678 TLI->getValueType(*DL, Add->getType()), in combineToUAddWithOverflow()
1679 Add->hasNUsesOrMore(EdgeCase ? 1 : 2))) in combineToUAddWithOverflow()
1685 if (Add->getParent() != Cmp->getParent() && !Add->hasOneUse()) in combineToUAddWithOverflow()
1692 // Reset callers - do not crash by iterating over a dead instruction. in combineToUAddWithOverflow()
1699 // We are not expecting non-canonical/degenerate code. Just bail out. in combineToUSubWithOverflow()
1700 Value *A = Cmp->getOperand(0), *B = Cmp->getOperand(1); in combineToUSubWithOverflow()
1705 ICmpInst::Predicate Pred = Cmp->getPredicate(); in combineToUSubWithOverflow()
1710 // Convert special-case: (A == 0) is the same as (A u< 1). in combineToUSubWithOverflow()
1712 B = ConstantInt::get(B->getType(), 1); in combineToUSubWithOverflow()
1715 // Convert special-case: (A != 0) is the same as (0 u< A). in combineToUSubWithOverflow()
1728 for (User *U : CmpVariableOperand->users()) { in combineToUSubWithOverflow()
1729 // A - B, A u< B --> usubo(A, B) in combineToUSubWithOverflow()
1735 // A + (-C), A u< C (canonicalized form of (sub A, C)) in combineToUSubWithOverflow()
1738 match(B, m_APInt(CmpC)) && *AddC == -(*CmpC)) { in combineToUSubWithOverflow()
1746 if (!TLI->shouldFormOverflowOp(ISD::USUBO, in combineToUSubWithOverflow()
1747 TLI->getValueType(*DL, Sub->getType()), in combineToUSubWithOverflow()
1748 Sub->hasNUsesOrMore(1))) in combineToUSubWithOverflow()
1751 if (!replaceMathCmpWithIntrinsic(Sub, Sub->getOperand(0), Sub->getOperand(1), in combineToUSubWithOverflow()
1755 // Reset callers - do not crash by iterating over a dead instruction. in combineToUSubWithOverflow()
1770 // Avoid sinking soft-FP comparisons, since this can move them into a loop. in sinkCmpExpression()
1778 for (Value::user_iterator UI = Cmp->user_begin(), E = Cmp->user_end(); in sinkCmpExpression()
1791 BasicBlock *UserBB = User->getParent(); in sinkCmpExpression()
1792 BasicBlock *DefBB = Cmp->getParent(); in sinkCmpExpression()
1802 BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt(); in sinkCmpExpression()
1803 assert(InsertPt != UserBB->end()); in sinkCmpExpression()
1804 InsertedCmp = CmpInst::Create(Cmp->getOpcode(), Cmp->getPredicate(), in sinkCmpExpression()
1805 Cmp->getOperand(0), Cmp->getOperand(1), ""); in sinkCmpExpression()
1806 InsertedCmp->insertBefore(*UserBB, InsertPt); in sinkCmpExpression()
1808 InsertedCmp->setDebugLoc(Cmp->getDebugLoc()); in sinkCmpExpression()
1818 if (Cmp->use_empty()) { in sinkCmpExpression()
1819 Cmp->eraseFromParent(); in sinkCmpExpression()
1850 ICmpInst::Predicate Pred = Cmp->getPredicate(); in foldICmpWithDominatingICmp()
1856 for (User *U : Cmp->users()) { in foldICmpWithDominatingICmp()
1859 if (isa<SelectInst>(U) && cast<SelectInst>(U)->getCondition() == Cmp) in foldICmpWithDominatingICmp()
1864 // This is a cheap/incomplete check for dominance - just match a single in foldICmpWithDominatingICmp()
1866 BasicBlock *CmpBB = Cmp->getParent(); in foldICmpWithDominatingICmp()
1867 BasicBlock *DomBB = CmpBB->getSinglePredecessor(); in foldICmpWithDominatingICmp()
1876 if (!match(DomBB->getTerminator(), m_Br(m_Value(DomCond), TrueBB, FalseBB))) in foldICmpWithDominatingICmp()
1881 Value *CmpOp0 = Cmp->getOperand(0), *CmpOp1 = Cmp->getOperand(1); in foldICmpWithDominatingICmp()
1895 for (User *U : Cmp->users()) { in foldICmpWithDominatingICmp()
1897 assert(BI->isConditional() && "Must be conditional"); in foldICmpWithDominatingICmp()
1898 BI->swapSuccessors(); in foldICmpWithDominatingICmp()
1903 SI->swapValues(); in foldICmpWithDominatingICmp()
1904 SI->swapProfMetadata(); in foldICmpWithDominatingICmp()
1909 Cmp->setPredicate(CmpInst::getSwappedPredicate(DomPred)); in foldICmpWithDominatingICmp()
1916 Value *Op0 = Cmp->getOperand(0); in swapICmpOperandsToExposeCSEOpportunities()
1917 Value *Op1 = Cmp->getOperand(1); in swapICmpOperandsToExposeCSEOpportunities()
1918 if (!Op0->getType()->isIntegerTy() || isa<Constant>(Op0) || in swapICmpOperandsToExposeCSEOpportunities()
1927 for (const User *U : Op0->users()) { in swapICmpOperandsToExposeCSEOpportunities()
1934 GoodToSwap--; in swapICmpOperandsToExposeCSEOpportunities()
1938 Cmp->swapOperands(); in swapICmpOperandsToExposeCSEOpportunities()
1951 EVT VT = TLI.getValueType(DL, Cmp->getOperand(0)->getType()); in foldFCmpToFPClassTest()
1953 TLI.isCondCodeLegal(getFCmpCondCode(FCmp->getPredicate()), in foldFCmpToFPClassTest()
1962 fcmpToClassTest(FCmp->getPredicate(), *FCmp->getParent()->getParent(), in foldFCmpToFPClassTest()
1963 FCmp->getOperand(0), FCmp->getOperand(1)); in foldFCmpToFPClassTest()
1972 Cmp->replaceAllUsesWith(IsFPClass); in foldFCmpToFPClassTest()
2006 // Double-check that we're not trying to optimize an instruction that was in sinkAndCmp0Expression()
2013 if (AndI->hasOneUse() && in sinkAndCmp0Expression()
2014 AndI->getParent() == cast<Instruction>(*AndI->user_begin())->getParent()) in sinkAndCmp0Expression()
2019 if (!isa<ConstantInt>(AndI->getOperand(0)) && in sinkAndCmp0Expression()
2020 !isa<ConstantInt>(AndI->getOperand(1)) && in sinkAndCmp0Expression()
2021 AndI->getOperand(0)->hasOneUse() && AndI->getOperand(1)->hasOneUse()) in sinkAndCmp0Expression()
2024 for (auto *U : AndI->users()) { in sinkAndCmp0Expression()
2031 auto *CmpC = dyn_cast<ConstantInt>(User->getOperand(1)); in sinkAndCmp0Expression()
2032 if (!CmpC || !CmpC->isZero()) in sinkAndCmp0Expression()
2040 LLVM_DEBUG(AndI->getParent()->dump()); in sinkAndCmp0Expression()
2045 for (Value::user_iterator UI = AndI->user_begin(), E = AndI->user_end(); in sinkAndCmp0Expression()
2057 User->getParent() == AndI->getParent() ? AndI : User; in sinkAndCmp0Expression()
2059 Instruction::And, AndI->getOperand(0), AndI->getOperand(1), "", in sinkAndCmp0Expression()
2060 InsertPt->getIterator()); in sinkAndCmp0Expression()
2062 InsertedAnd->setDebugLoc(AndI->getDebugLoc()); in sinkAndCmp0Expression()
2067 LLVM_DEBUG(User->getParent()->dump()); in sinkAndCmp0Expression()
2071 AndI->eraseFromParent(); in sinkAndCmp0Expression()
2082 if (User->getOpcode() != Instruction::And || in isExtractBitsCandidateUse()
2083 !isa<ConstantInt>(User->getOperand(1))) in isExtractBitsCandidateUse()
2086 const APInt &Cimm = cast<ConstantInt>(User->getOperand(1))->getValue(); in isExtractBitsCandidateUse()
2099 BasicBlock *UserBB = User->getParent(); in SinkShiftAndTruncate()
2104 for (Value::user_iterator TruncUI = TruncI->user_begin(), in SinkShiftAndTruncate()
2105 TruncE = TruncI->user_end(); in SinkShiftAndTruncate()
2114 int ISDOpcode = TLI.InstructionOpcodeToISD(TruncUser->getOpcode()); in SinkShiftAndTruncate()
2124 ISDOpcode, TLI.getValueType(DL, TruncUser->getType(), true))) in SinkShiftAndTruncate()
2131 BasicBlock *TruncUserBB = TruncUser->getParent(); in SinkShiftAndTruncate()
2140 BasicBlock::iterator InsertPt = TruncUserBB->getFirstInsertionPt(); in SinkShiftAndTruncate()
2141 assert(InsertPt != TruncUserBB->end()); in SinkShiftAndTruncate()
2143 if (ShiftI->getOpcode() == Instruction::AShr) in SinkShiftAndTruncate()
2145 BinaryOperator::CreateAShr(ShiftI->getOperand(0), CI, ""); in SinkShiftAndTruncate()
2148 BinaryOperator::CreateLShr(ShiftI->getOperand(0), CI, ""); in SinkShiftAndTruncate()
2149 InsertedShift->setDebugLoc(ShiftI->getDebugLoc()); in SinkShiftAndTruncate()
2150 InsertedShift->insertBefore(*TruncUserBB, InsertPt); in SinkShiftAndTruncate()
2153 BasicBlock::iterator TruncInsertPt = TruncUserBB->getFirstInsertionPt(); in SinkShiftAndTruncate()
2155 // It will go ahead of any debug-info. in SinkShiftAndTruncate()
2157 assert(TruncInsertPt != TruncUserBB->end()); in SinkShiftAndTruncate()
2159 InsertedTrunc = CastInst::Create(TruncI->getOpcode(), InsertedShift, in SinkShiftAndTruncate()
2160 TruncI->getType(), ""); in SinkShiftAndTruncate()
2161 InsertedTrunc->insertBefore(*TruncUserBB, TruncInsertPt); in SinkShiftAndTruncate()
2162 InsertedTrunc->setDebugLoc(TruncI->getDebugLoc()); in SinkShiftAndTruncate()
2192 BasicBlock *DefBB = ShiftI->getParent(); in OptimizeExtractBits()
2197 bool shiftIsLegal = TLI.isTypeLegal(TLI.getValueType(DL, ShiftI->getType())); in OptimizeExtractBits()
2200 for (Value::user_iterator UI = ShiftI->user_begin(), E = ShiftI->user_end(); in OptimizeExtractBits()
2214 BasicBlock *UserBB = User->getParent(); in OptimizeExtractBits()
2227 // ----> We will have an implicit truncate here if the architecture does in OptimizeExtractBits()
2235 && (!TLI.isTypeLegal(TLI.getValueType(DL, User->getType())))) in OptimizeExtractBits()
2245 BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt(); in OptimizeExtractBits()
2246 assert(InsertPt != UserBB->end()); in OptimizeExtractBits()
2248 if (ShiftI->getOpcode() == Instruction::AShr) in OptimizeExtractBits()
2250 BinaryOperator::CreateAShr(ShiftI->getOperand(0), CI, ""); in OptimizeExtractBits()
2253 BinaryOperator::CreateLShr(ShiftI->getOperand(0), CI, ""); in OptimizeExtractBits()
2254 InsertedShift->insertBefore(*UserBB, InsertPt); in OptimizeExtractBits()
2255 InsertedShift->setDebugLoc(ShiftI->getDebugLoc()); in OptimizeExtractBits()
2265 if (ShiftI->use_empty()) { in OptimizeExtractBits()
2267 ShiftI->eraseFromParent(); in OptimizeExtractBits()
2298 if (match(CountZeros->getOperand(1), m_One())) in despeculateCountZeros()
2302 Type *Ty = CountZeros->getType(); in despeculateCountZeros()
2303 auto IntrinsicID = CountZeros->getIntrinsicID(); in despeculateCountZeros()
2304 if ((IntrinsicID == Intrinsic::cttz && TLI->isCheapToSpeculateCttz(Ty)) || in despeculateCountZeros()
2305 (IntrinsicID == Intrinsic::ctlz && TLI->isCheapToSpeculateCtlz(Ty))) in despeculateCountZeros()
2309 unsigned SizeInBits = Ty->getScalarSizeInBits(); in despeculateCountZeros()
2310 if (Ty->isVectorTy() || SizeInBits > DL->getLargestLegalIntTypeSizeInBits()) in despeculateCountZeros()
2314 Use &Op = CountZeros->getOperandUse(0); in despeculateCountZeros()
2319 BasicBlock *StartBlock = CountZeros->getParent(); in despeculateCountZeros()
2320 BasicBlock *CallBlock = StartBlock->splitBasicBlock(CountZeros, "cond.false"); in despeculateCountZeros()
2325 // in this block to select the result of the intrinsic or the bit-width in despeculateCountZeros()
2328 // Any debug-info after CountZeros should not be included. in despeculateCountZeros()
2330 BasicBlock *EndBlock = CallBlock->splitBasicBlock(SplitPt, "cond.end"); in despeculateCountZeros()
2337 L->addBasicBlockToLoop(CallBlock, LI); in despeculateCountZeros()
2338 L->addBasicBlockToLoop(EndBlock, LI); in despeculateCountZeros()
2342 IRBuilder<> Builder(CountZeros->getContext()); in despeculateCountZeros()
2343 Builder.SetInsertPoint(StartBlock->getTerminator()); in despeculateCountZeros()
2344 Builder.SetCurrentDebugLocation(CountZeros->getDebugLoc()); in despeculateCountZeros()
2351 Op = Builder.CreateFreeze(Op, Op->getName() + ".fr"); in despeculateCountZeros()
2354 StartBlock->getTerminator()->eraseFromParent(); in despeculateCountZeros()
2358 Builder.SetInsertPoint(EndBlock, EndBlock->begin()); in despeculateCountZeros()
2362 PN->addIncoming(BitWidth, StartBlock); in despeculateCountZeros()
2363 PN->addIncoming(CountZeros, CallBlock); in despeculateCountZeros()
2368 CountZeros->setArgOperand(1, Builder.getTrue()); in despeculateCountZeros()
2374 BasicBlock *BB = CI->getParent(); in optimizeCallInst()
2379 if (CI->isInlineAsm()) { in optimizeCallInst()
2380 if (TLI->ExpandInlineAsm(CI)) { in optimizeCallInst()
2382 CurInstIterator = BB->begin(); in optimizeCallInst()
2397 if (TLI->shouldAlignPointerArgs(CI, MinSize, PrefAlign)) { in optimizeCallInst()
2398 for (auto &Arg : CI->args()) { in optimizeCallInst()
2402 // if size - offset meets the size threshold. in optimizeCallInst()
2403 if (!Arg->getType()->isPointerTy()) in optimizeCallInst()
2405 APInt Offset(DL->getIndexSizeInBits( in optimizeCallInst()
2406 cast<PointerType>(Arg->getType())->getAddressSpace()), in optimizeCallInst()
2408 Value *Val = Arg->stripAndAccumulateInBoundsConstantOffsets(*DL, Offset); in optimizeCallInst()
2413 if ((AI = dyn_cast<AllocaInst>(Val)) && AI->getAlign() < PrefAlign && in optimizeCallInst()
2414 DL->getTypeAllocSize(AI->getAllocatedType()) >= MinSize + Offset2) in optimizeCallInst()
2415 AI->setAlignment(PrefAlign); in optimizeCallInst()
2418 // over-aligning global variables that have an explicit section is in optimizeCallInst()
2421 if ((GV = dyn_cast<GlobalVariable>(Val)) && GV->canIncreaseAlignment() && in optimizeCallInst()
2422 GV->getPointerAlignment(*DL) < PrefAlign && in optimizeCallInst()
2423 DL->getTypeAllocSize(GV->getValueType()) >= MinSize + Offset2) in optimizeCallInst()
2424 GV->setAlignment(PrefAlign); in optimizeCallInst()
2430 Align DestAlign = getKnownAlignment(MI->getDest(), *DL); in optimizeCallInst()
2431 MaybeAlign MIDestAlign = MI->getDestAlign(); in optimizeCallInst()
2433 MI->setDestAlignment(DestAlign); in optimizeCallInst()
2435 MaybeAlign MTISrcAlign = MTI->getSourceAlign(); in optimizeCallInst()
2436 Align SrcAlign = getKnownAlignment(MTI->getSource(), *DL); in optimizeCallInst()
2438 MTI->setSourceAlignment(SrcAlign); in optimizeCallInst()
2446 if (CI->hasFnAttr(Attribute::Cold) && !OptSize && in optimizeCallInst()
2448 for (auto &Arg : CI->args()) { in optimizeCallInst()
2449 if (!Arg->getType()->isPointerTy()) in optimizeCallInst()
2451 unsigned AS = Arg->getType()->getPointerAddressSpace(); in optimizeCallInst()
2452 if (optimizeMemoryInst(CI, Arg, Arg->getType(), AS)) in optimizeCallInst()
2458 switch (II->getIntrinsicID()) { in optimizeCallInst()
2467 // paths and merge blocks before going into block-local instruction in optimizeCallInst()
2469 if (II->use_empty()) { in optimizeCallInst()
2470 II->eraseFromParent(); in optimizeCallInst()
2473 Constant *RetVal = ConstantInt::getTrue(II->getContext()); in optimizeCallInst()
2485 ZExtInst *ExtVal = dyn_cast<ZExtInst>(CI->getArgOperand(0)); in optimizeCallInst()
2486 if (!ExtVal || !ExtVal->hasOneUse() || in optimizeCallInst()
2487 ExtVal->getParent() == CI->getParent()) in optimizeCallInst()
2490 ExtVal->moveBefore(CI); in optimizeCallInst()
2499 Value *ArgVal = II->getArgOperand(0); in optimizeCallInst()
2505 auto GEPs = std::move(it->second); in optimizeCallInst()
2511 II->eraseFromParent(); in optimizeCallInst()
2526 return optimizeGatherScatterInst(II, II->getArgOperand(0)); in optimizeCallInst()
2528 return optimizeGatherScatterInst(II, II->getArgOperand(1)); in optimizeCallInst()
2533 if (TLI->getAddrModeArguments(II, PtrOps, AccessTy)) in optimizeCallInst()
2536 unsigned AS = PtrVal->getType()->getPointerAddressSpace(); in optimizeCallInst()
2543 if (!CI->getCalledFunction()) in optimizeCallInst()
2554 CI->eraseFromParent(); in optimizeCallInst()
2563 assert(CI && CI->use_empty()); in isIntrinsicOrLFToBeTailCalled()
2566 switch (II->getIntrinsicID()) { in isIntrinsicOrLFToBeTailCalled()
2576 Function *Callee = CI->getCalledFunction(); in isIntrinsicOrLFToBeTailCalled()
2577 if (Callee && TLInfo && TLInfo->getLibFunc(*Callee, LF)) in isIntrinsicOrLFToBeTailCalled()
2625 if (!BB->getTerminator()) in dupRetToEnableTailCallOpts()
2628 ReturnInst *RetI = dyn_cast<ReturnInst>(BB->getTerminator()); in dupRetToEnableTailCallOpts()
2632 assert(LI->getLoopFor(BB) == nullptr && "A return block cannot be in a loop"); in dupRetToEnableTailCallOpts()
2637 Value *V = RetI->getReturnValue(); in dupRetToEnableTailCallOpts()
2641 V = BCI->getOperand(0); in dupRetToEnableTailCallOpts()
2645 V = EVI->getOperand(0); in dupRetToEnableTailCallOpts()
2646 if (!llvm::all_of(EVI->indices(), [](unsigned idx) { return idx == 0; })) in dupRetToEnableTailCallOpts()
2653 if (PN && PN->getParent() != BB) in dupRetToEnableTailCallOpts()
2658 if (BC && BC->hasOneUse()) in dupRetToEnableTailCallOpts()
2659 Inst = BC->user_back(); in dupRetToEnableTailCallOpts()
2662 return II->getIntrinsicID() == Intrinsic::lifetime_end; in dupRetToEnableTailCallOpts()
2668 const Instruction *BI = BB->getFirstNonPHI(); in dupRetToEnableTailCallOpts()
2672 BI = BI->getNextNode(); in dupRetToEnableTailCallOpts()
2678 const Function *F = BB->getParent(); in dupRetToEnableTailCallOpts()
2681 for (unsigned I = 0, E = PN->getNumIncomingValues(); I != E; ++I) { in dupRetToEnableTailCallOpts()
2683 Value *IncomingVal = PN->getIncomingValue(I)->stripPointerCasts(); in dupRetToEnableTailCallOpts()
2685 BasicBlock *PredBB = PN->getIncomingBlock(I); in dupRetToEnableTailCallOpts()
2687 if (CI && CI->hasOneUse() && CI->getParent() == PredBB && in dupRetToEnableTailCallOpts()
2688 TLI->mayBeEmittedAsTailCall(CI) && in dupRetToEnableTailCallOpts()
2702 if (PredBB && PredBB->getSingleSuccessor() == BB) in dupRetToEnableTailCallOpts()
2704 PredBB->getTerminator()->getPrevNonDebugInstruction(true)); in dupRetToEnableTailCallOpts()
2706 if (CI && CI->use_empty() && in dupRetToEnableTailCallOpts()
2708 IncomingVal == CI->getArgOperand(0) && in dupRetToEnableTailCallOpts()
2709 TLI->mayBeEmittedAsTailCall(CI) && in dupRetToEnableTailCallOpts()
2719 if (Instruction *I = Pred->rbegin()->getPrevNonDebugInstruction(true)) { in dupRetToEnableTailCallOpts()
2721 if (CI && CI->use_empty() && TLI->mayBeEmittedAsTailCall(CI) && in dupRetToEnableTailCallOpts()
2727 V == CI->getArgOperand(0))) { in dupRetToEnableTailCallOpts()
2739 BranchInst *BI = dyn_cast<BranchInst>(TailCallBB->getTerminator()); in dupRetToEnableTailCallOpts()
2740 if (!BI || !BI->isUnconditional() || BI->getSuccessor(0) != BB) in dupRetToEnableTailCallOpts()
2746 BFI->getBlockFreq(BB) >= BFI->getBlockFreq(TailCallBB)); in dupRetToEnableTailCallOpts()
2747 BFI->setBlockFreq(BB, in dupRetToEnableTailCallOpts()
2748 (BFI->getBlockFreq(BB) - BFI->getBlockFreq(TailCallBB))); in dupRetToEnableTailCallOpts()
2755 if (Changed && !BB->hasAddressTaken() && pred_empty(BB)) in dupRetToEnableTailCallOpts()
2756 BB->eraseFromParent(); in dupRetToEnableTailCallOpts()
2761 //===----------------------------------------------------------------------===//
2763 //===----------------------------------------------------------------------===//
2794 BaseReg->getType() != other.BaseReg->getType()) in compare()
2796 if (BaseGV && other.BaseGV && BaseGV->getType() != other.BaseGV->getType()) in compare()
2799 ScaledReg->getType() != other.ScaledReg->getType()) in compare()
2833 // consider to be 'non-zero' here. in isTrivial()
2906 BaseGV->printAsOperand(OS, /*PrintType=*/false); in print()
2917 BaseReg->printAsOperand(OS, /*PrintType=*/false); in print()
2922 ScaledReg->printAsOperand(OS, /*PrintType=*/false); in print()
2979 /// - Is the first in a basic block: BB is used.
2980 /// - Has a previous instruction: PrevInst is used.
2993 HasPrevInstruction = (Inst != &*(Inst->getParent()->begin())); in InsertionHandler()
2994 BasicBlock *BB = Inst->getParent(); in InsertionHandler()
2996 // Record where we would have to re-insert the instruction in the sequence in InsertionHandler()
2998 if (BB->IsNewDbgInfoFormat) in InsertionHandler()
2999 BeforeDbgRecord = Inst->getDbgReinsertionPosition(); in InsertionHandler()
3002 Point.PrevInst = &*std::prev(Inst->getIterator()); in InsertionHandler()
3011 if (Inst->getParent()) in insert()
3012 Inst->removeFromParent(); in insert()
3013 Inst->insertAfter(&*Point.PrevInst); in insert()
3015 BasicBlock::iterator Position = Point.BB->getFirstInsertionPt(); in insert()
3016 if (Inst->getParent()) in insert()
3017 Inst->moveBefore(*Point.BB, Position); in insert()
3019 Inst->insertBefore(*Point.BB, Position); in insert()
3022 Inst->getParent()->reinsertInstInDbgRecords(Inst, BeforeDbgRecord); in insert()
3037 Inst->moveBefore(Before); in InstructionMoveBefore()
3062 Origin = Inst->getOperand(Idx); in OperandSetter()
3063 Inst->setOperand(Idx, NewVal); in OperandSetter()
3071 Inst->setOperand(Idx, Origin); in undo()
3085 unsigned NumOpnds = Inst->getNumOperands(); in OperandsHider()
3089 Value *Val = Inst->getOperand(It); in OperandsHider()
3094 Inst->setOperand(It, UndefValue::get(Val->getType())); in OperandsHider()
3102 Inst->setOperand(It, OriginalValues[It]); in undo()
3128 IVal->eraseFromParent(); in undo()
3154 IVal->eraseFromParent(); in undo()
3181 IVal->eraseFromParent(); in undo()
3193 : TypePromotionAction(Inst), OrigTy(Inst->getType()) { in TypeMutator()
3196 Inst->mutateType(NewTy); in TypeMutator()
3203 Inst->mutateType(OrigTy); in undo()
3225 /// And non-instruction debug-users too.
3241 for (Use &U : Inst->uses()) { in UsesReplacer()
3250 Inst->replaceAllUsesWith(New); in UsesReplacer()
3257 Use.Inst->setOperand(Use.Idx, Inst); in undo()
3263 DVI->replaceVariableLocationOp(New, Inst); in undo()
3264 // Similar story with DbgVariableRecords, the non-instruction in undo()
3267 DVR->replaceVariableLocationOp(New, Inst); in undo()
3290 /// \pre If !Inst->use_empty(), then New != nullptr
3302 Inst->removeFromParent(); in InstructionRemover()
3316 Replacer->undo(); in undo()
3402 Value *Val = Ptr->getBuiltValue(); in createTrunc()
3410 Value *Val = Ptr->getBuiltValue(); in createSExt()
3418 Value *Val = Ptr->getBuiltValue(); in createZExt()
3430 Action->commit(); in commit()
3440 Curr->undo(); in rollback()
3448 /// This encapsulates the logic for matching the target-legal addressing modes.
3457 /// AccessTy/MemoryInst - This is the type for the access (e.g. double) and
3499 DL(MI->getDataLayout()), LI(LI), getDTFn(getDTFn), in AddressingModeMatcher()
3654 if (it != NodeMap.end() && it->second == CurrentIndex) in SkipRemovedElements()
3665 assert(CurrentIndex < Set->NodeList.size() && in operator *()
3667 return Set->NodeList[CurrentIndex]; in operator *()
3671 assert(CurrentIndex < Set->NodeList.size() && in operator ++()
3674 Set->SkipRemovedElements(CurrentIndex); in operator ++()
3706 V = SV->second; in Get()
3720 for (auto *U : PI->users()) in Simplify()
3723 PI->replaceAllUsesWith(V); in Simplify()
3728 PI->eraseFromParent(); in Simplify()
3745 From->replaceAllUsesWith(To); in ReplacePhi()
3747 From->eraseFromParent(); in ReplacePhi()
3764 I->replaceAllUsesWith(Dummy); in destroyNewNodes()
3765 I->eraseFromParent(); in destroyNewNodes()
3769 I->replaceAllUsesWith(Dummy); in destroyNewNodes()
3770 I->eraseFromParent(); in destroyNewNodes()
3816 // Take note of if we have any non-trivial AddrModes, as we need to detect in addNewAddrMode()
3903 if (CommonValue && CommonValue->getNumUses() == 0) in eraseCommonValueIfDead()
3905 CommonInst->eraseFromParent(); in eraseCommonValueIfDead()
3917 Type *IntPtrTy = SQ.DL.getIntPtrType(AddrModes[0].OriginalValue->getType()); in initializeMap()
3921 auto *Type = DV->getType(); in initializeMap()
3930 assert(CommonType && "At least one non-null value must be!"); in initializeMap()
3954 // p1 -> b1
3955 // p2 -> b2
3957 // p -> ?
3989 auto *Result = ST.Get(Map.find(Original)->second); in findCommon()
4017 for (auto *B : Item.first->blocks()) { in MatchPhiNode()
4018 Value *FirstValue = Item.first->getIncomingValueForBlock(B); in MatchPhiNode()
4019 Value *SecondValue = Item.second->getIncomingValueForBlock(B); in MatchPhiNode()
4031 FirstPhi->getParent() != SecondPhi->getParent()) in MatchPhiNode()
4069 for (auto &P : PHI->getParent()->phis()) { in MatchPhiSet()
4111 auto *TrueValue = CurrentSelect->getTrueValue(); in FillPlaceholders()
4113 Select->setTrueValue(ST.Get(Map[TrueValue])); in FillPlaceholders()
4114 auto *FalseValue = CurrentSelect->getFalseValue(); in FillPlaceholders()
4116 Select->setFalseValue(ST.Get(Map[FalseValue])); in FillPlaceholders()
4121 for (auto *B : predecessors(PHI->getParent())) { in FillPlaceholders()
4122 Value *PV = cast<PHINode>(Current)->getIncomingValueForBlock(B); in FillPlaceholders()
4124 PHI->addIncoming(ST.Get(Map[PV]), B); in FillPlaceholders()
4131 /// Starting from original value recursively iterates over def-use chain up to
4157 SelectInst::Create(CurrentSelect->getCondition(), Dummy, Dummy, in InsertPlaceholders()
4158 CurrentSelect->getName(), in InsertPlaceholders()
4159 CurrentSelect->getIterator(), CurrentSelect); in InsertPlaceholders()
4163 Worklist.push_back(CurrentSelect->getTrueValue()); in InsertPlaceholders()
4164 Worklist.push_back(CurrentSelect->getFalseValue()); in InsertPlaceholders()
4168 unsigned PredCount = CurrentPhi->getNumIncomingValues(); in InsertPlaceholders()
4170 PHINode::Create(CommonType, PredCount, "sunk_phi", CurrentPhi->getIterator()); in InsertPlaceholders()
4173 append_range(Worklist, CurrentPhi->incoming_values()); in InsertPlaceholders()
4218 // Add scale to turn X*4+X*3 -> X*7. This could also do things like in matchScaledValue()
4219 // [A+B + A*7] -> [B+A*8]. in matchScaledValue()
4238 !isIVIncrement(ScaleReg, &LI) && CI->getValue().isSignedIntN(64)) { in matchScaledValue()
4241 TestAddrMode.BaseOffs += CI->getSExtValue() * TestAddrMode.Scale; in matchScaledValue()
4257 [this](const Value *V) -> std::optional<std::pair<Instruction *, APInt>> { in matchScaledValue()
4264 // TODO: The result of the intrinsics above is two-complement. However when in matchScaledValue()
4268 // well-defined two-complement computation with poison. Currently, to avoid in matchScaledValue()
4270 if (auto *OIVInc = dyn_cast<OverflowingBinaryOperator>(IVInc->first)) in matchScaledValue()
4271 if (OIVInc->hasNoSignedWrap() || OIVInc->hasNoUnsignedWrap()) in matchScaledValue()
4273 if (auto *ConstantStep = dyn_cast<ConstantInt>(IVInc->second)) in matchScaledValue()
4274 return std::make_pair(IVInc->first, ConstantStep->getValue()); in matchScaledValue()
4280 // 2. We use it with non-zero offset; in matchScaledValue()
4291 Instruction *IVInc = IVStep->first; in matchScaledValue()
4297 APInt Step = IVStep->second; in matchScaledValue()
4302 TestAddrMode.BaseOffs -= Offset.getLimitedValue(); in matchScaledValue()
4327 switch (I->getOpcode()) { in MightBeFoldableInst()
4331 if (I->getType() == I->getOperand(0)->getType()) in MightBeFoldableInst()
4333 return I->getType()->isIntOrPtrTy(); in MightBeFoldableInst()
4345 return isa<ConstantInt>(I->getOperand(1)); in MightBeFoldableInst()
4356 /// to be legal, as the non-promoted value would have had the same state.
4362 int ISDOpcode = TLI.InstructionOpcodeToISD(PromotedInst->getOpcode()); in isPromotedInstructionLegal()
4368 ISDOpcode, TLI.getValueType(DL, PromotedInst->getType())); in isPromotedInstructionLegal()
4384 if (It->second.getInt() == ExtTy) in addPromotedInst()
4392 PromotedInsts[ExtOpnd] = TypeIsSExt(ExtOpnd->getType(), ExtTy); in addPromotedInst()
4403 if (It != PromotedInsts.end() && It->second.getInt() == ExtTy) in getOrigType()
4404 return It->second.getPointer(); in getOrigType()
4512 if (Inst->getType()->isVectorTy()) in canGetThrough()
4527 ((!IsSExt && BinOp->hasNoUnsignedWrap()) || in canGetThrough()
4528 (IsSExt && BinOp->hasNoSignedWrap()))) in canGetThrough()
4531 // ext(and(opnd, cst)) --> and(ext(opnd), ext(cst)) in canGetThrough()
4532 if ((Inst->getOpcode() == Instruction::And || in canGetThrough()
4533 Inst->getOpcode() == Instruction::Or)) in canGetThrough()
4536 // ext(xor(opnd, cst)) --> xor(ext(opnd), ext(cst)) in canGetThrough()
4537 if (Inst->getOpcode() == Instruction::Xor) { in canGetThrough()
4539 if (const auto *Cst = dyn_cast<ConstantInt>(Inst->getOperand(1))) in canGetThrough()
4540 if (!Cst->getValue().isAllOnes()) in canGetThrough()
4544 // zext(shrl(opnd, cst)) --> shrl(zext(opnd), zext(cst)) in canGetThrough()
4546 // zext i32 (shrl i8 %val, 12) --> shrl i32 (zext i8 %val), 12 in canGetThrough()
4549 if (Inst->getOpcode() == Instruction::LShr && !IsSExt) in canGetThrough()
4552 // and(ext(shl(opnd, cst)), cst) --> and(shl(ext(opnd), ext(cst)), cst) in canGetThrough()
4554 // zext i32 (shl i8 %val, 12) --> shl i32 (zext i8 %val), 12 in canGetThrough()
4557 if (Inst->getOpcode() == Instruction::Shl && Inst->hasOneUse()) { in canGetThrough()
4558 const auto *ExtInst = cast<const Instruction>(*Inst->user_begin()); in canGetThrough()
4559 if (ExtInst->hasOneUse()) { in canGetThrough()
4560 const auto *AndInst = dyn_cast<const Instruction>(*ExtInst->user_begin()); in canGetThrough()
4561 if (AndInst && AndInst->getOpcode() == Instruction::And) { in canGetThrough()
4562 const auto *Cst = dyn_cast<ConstantInt>(AndInst->getOperand(1)); in canGetThrough()
4564 Cst->getValue().isIntN(Inst->getType()->getIntegerBitWidth())) in canGetThrough()
4571 // ext(trunc(opnd)) --> ext(opnd) in canGetThrough()
4575 Value *OpndVal = Inst->getOperand(0); in canGetThrough()
4578 if (!OpndVal->getType()->isIntegerTy() || in canGetThrough()
4579 OpndVal->getType()->getIntegerBitWidth() > in canGetThrough()
4580 ConsideredExtType->getIntegerBitWidth()) in canGetThrough()
4598 OpndType = Opnd->getOperand(0)->getType(); in canGetThrough()
4603 return Inst->getType()->getIntegerBitWidth() >= in canGetThrough()
4604 OpndType->getIntegerBitWidth(); in canGetThrough()
4612 Instruction *ExtOpnd = dyn_cast<Instruction>(Ext->getOperand(0)); in getAction()
4613 Type *ExtTy = Ext->getType(); in getAction()
4634 // Abort early if we will have to insert non-free instructions. in getAction()
4635 if (!ExtOpnd->hasOneUse() && !TLI.isTruncateFree(ExtTy, ExtOpnd->getType())) in getAction()
4647 Instruction *SExtOpnd = cast<Instruction>(SExt->getOperand(0)); in promoteOperandForTruncAndAnyExt()
4655 TPT.createZExt(SExt, SExtOpnd->getOperand(0), SExt->getType()); in promoteOperandForTruncAndAnyExt()
4662 TPT.setOperand(SExt, 0, SExtOpnd->getOperand(0)); in promoteOperandForTruncAndAnyExt()
4667 if (SExtOpnd->use_empty()) in promoteOperandForTruncAndAnyExt()
4672 if (!ExtInst || ExtInst->getType() != ExtInst->getOperand(0)->getType()) { in promoteOperandForTruncAndAnyExt()
4675 Exts->push_back(ExtInst); in promoteOperandForTruncAndAnyExt()
4683 Value *NextVal = ExtInst->getOperand(0); in promoteOperandForTruncAndAnyExt()
4696 Instruction *ExtOpnd = cast<Instruction>(Ext->getOperand(0)); in promoteOperandForOther()
4698 if (!ExtOpnd->hasOneUse()) { in promoteOperandForOther()
4703 Value *Trunc = TPT.createTrunc(Ext, ExtOpnd->getType()); in promoteOperandForOther()
4706 ITrunc->moveAfter(ExtOpnd); in promoteOperandForOther()
4708 Truncs->push_back(ITrunc); in promoteOperandForOther()
4713 // to replaceAllUsesWith) to avoid creating a cycle trunc <-> sext. in promoteOperandForOther()
4726 TPT.mutateType(ExtOpnd, Ext->getType()); in promoteOperandForOther()
4731 for (int OpIdx = 0, EndOpIdx = ExtOpnd->getNumOperands(); OpIdx != EndOpIdx; in promoteOperandForOther()
4733 LLVM_DEBUG(dbgs() << "Operand:\n" << *(ExtOpnd->getOperand(OpIdx)) << '\n'); in promoteOperandForOther()
4734 if (ExtOpnd->getOperand(OpIdx)->getType() == Ext->getType() || in promoteOperandForOther()
4740 Value *Opnd = ExtOpnd->getOperand(OpIdx); in promoteOperandForOther()
4743 unsigned BitWidth = Ext->getType()->getIntegerBitWidth(); in promoteOperandForOther()
4744 APInt CstVal = IsSExt ? Cst->getValue().sext(BitWidth) in promoteOperandForOther()
4745 : Cst->getValue().zext(BitWidth); in promoteOperandForOther()
4746 TPT.setOperand(ExtOpnd, OpIdx, ConstantInt::get(Ext->getType(), CstVal)); in promoteOperandForOther()
4752 TPT.setOperand(ExtOpnd, OpIdx, UndefValue::get(Ext->getType())); in promoteOperandForOther()
4758 ? TPT.createSExt(ExtOpnd, Opnd, Ext->getType()) in promoteOperandForOther()
4759 : TPT.createZExt(ExtOpnd, Opnd, Ext->getType()); in promoteOperandForOther()
4766 Exts->push_back(InstForExtOpnd); in promoteOperandForOther()
4825 return matchAddr(AddrInst->getOperand(0), Depth); in matchOperationAddr()
4827 auto AS = AddrInst->getType()->getPointerAddressSpace(); in matchOperationAddr()
4829 // This inttoptr is a no-op if the integer type is pointer sized. in matchOperationAddr()
4830 if (TLI.getValueType(DL, AddrInst->getOperand(0)->getType()) == PtrTy) in matchOperationAddr()
4831 return matchAddr(AddrInst->getOperand(0), Depth); in matchOperationAddr()
4836 // int->int or pointer->pointer (we don't want int<->fp or something). in matchOperationAddr()
4837 if (AddrInst->getOperand(0)->getType()->isIntOrPtrTy() && in matchOperationAddr()
4841 AddrInst->getOperand(0)->getType() != AddrInst->getType()) in matchOperationAddr()
4842 return matchAddr(AddrInst->getOperand(0), Depth); in matchOperationAddr()
4846 AddrInst->getOperand(0)->getType()->getPointerAddressSpace(); in matchOperationAddr()
4847 unsigned DestAS = AddrInst->getType()->getPointerAddressSpace(); in matchOperationAddr()
4849 return matchAddr(AddrInst->getOperand(0), Depth); in matchOperationAddr()
4867 if (isa<ConstantInt>(AddrInst->getOperand(First)) in matchOperationAddr()
4868 && !isa<ConstantInt>(AddrInst->getOperand(Second))) in matchOperationAddr()
4871 if (matchAddr(AddrInst->getOperand(First), Depth + 1) && in matchOperationAddr()
4872 matchAddr(AddrInst->getOperand(Second), Depth + 1)) in matchOperationAddr()
4880 // Otherwise this was over-aggressive. Try merging operands in the opposite in matchOperationAddr()
4882 if (matchAddr(AddrInst->getOperand(Second), Depth + 1) && in matchOperationAddr()
4883 matchAddr(AddrInst->getOperand(First), Depth + 1)) in matchOperationAddr()
4899 ConstantInt *RHS = dyn_cast<ConstantInt>(AddrInst->getOperand(1)); in matchOperationAddr()
4900 if (!RHS || RHS->getBitWidth() > 64) in matchOperationAddr()
4903 ? 1LL << RHS->getLimitedValue(RHS->getBitWidth() - 1) in matchOperationAddr()
4904 : RHS->getSExtValue(); in matchOperationAddr()
4906 return matchScaledValue(AddrInst->getOperand(0), Scale, Depth); in matchOperationAddr()
4911 int VariableOperand = -1; in matchOperationAddr()
4916 for (unsigned i = 1, e = AddrInst->getNumOperands(); i != e; ++i, ++GTI) { in matchOperationAddr()
4920 cast<ConstantInt>(AddrInst->getOperand(i))->getZExtValue(); in matchOperationAddr()
4921 ConstantOffset += SL->getElementOffset(Idx); in matchOperationAddr()
4930 dyn_cast<ConstantInt>(AddrInst->getOperand(i))) { in matchOperationAddr()
4931 const APInt &CVal = CI->getValue(); in matchOperationAddr()
4938 if (VariableOperand != -1) in matchOperationAddr()
4950 if (VariableOperand == -1) { in matchOperationAddr()
4952 if (matchAddr(AddrInst->getOperand(0), Depth + 1)) { in matchOperationAddr()
4953 if (!cast<GEPOperator>(AddrInst)->isInBounds()) in matchOperationAddr()
4957 AddrMode.BaseOffs -= ConstantOffset; in matchOperationAddr()
4962 // Record GEPs with non-zero offsets as candidates for splitting in in matchOperationAddr()
4966 Value *Base = AddrInst->getOperand(0); in matchOperationAddr()
4972 // Make sure the parent block allows inserting non-PHI instructions in matchOperationAddr()
4974 BasicBlock *Parent = BaseI ? BaseI->getParent() in matchOperationAddr()
4975 : &GEP->getFunction()->getEntryBlock(); in matchOperationAddr()
4976 if (!Parent->getTerminator()->isEHPad()) in matchOperationAddr()
4990 if (!cast<GEPOperator>(AddrInst)->isInBounds()) in matchOperationAddr()
4994 if (!matchAddr(AddrInst->getOperand(0), Depth + 1)) { in matchOperationAddr()
5002 AddrMode.BaseReg = AddrInst->getOperand(0); in matchOperationAddr()
5006 if (!matchScaledValue(AddrInst->getOperand(VariableOperand), VariableScale, in matchOperationAddr()
5015 AddrMode.BaseReg = AddrInst->getOperand(0); in matchOperationAddr()
5017 if (!matchScaledValue(AddrInst->getOperand(VariableOperand), in matchOperationAddr()
5055 // promotedOpnd = ext opnd <- no match here in matchOperationAddr()
5056 // op = promoted_add promotedOpnd, 1 <- match (later in recursive calls) in matchOperationAddr()
5057 // addr = gep base, op <- match in matchOperationAddr()
5073 ExtCost + (AddrModeInsts.size() - OldSize), in matchOperationAddr()
5085 if (II->getIntrinsicID() == Intrinsic::threadlocal_address) { in matchOperationAddr()
5086 GlobalValue &GV = cast<GlobalValue>(*II->getArgOperand(0)); in matchOperationAddr()
5088 return matchAddr(AddrInst->getOperand(0), Depth); in matchOperationAddr()
5107 if (CI->getValue().isSignedIntN(64)) { in matchAddr()
5109 AddrMode.BaseOffs += CI->getSExtValue(); in matchAddr()
5112 AddrMode.BaseOffs -= CI->getSExtValue(); in matchAddr()
5128 if (matchOperationAddr(I, I->getOpcode(), Depth, &MovedAway)) { in matchAddr()
5136 if (I->hasOneUse() || in matchAddr()
5148 if (matchOperationAddr(CE, CE->getOpcode(), Depth)) in matchAddr()
5186 const Function *F = CI->getFunction(); in IsOperandAMemoryOperand()
5188 TLI.ParseConstraints(F->getDataLayout(), &TRI, *CI); in IsOperandAMemoryOperand()
5206 /// If we find an obviously non-foldable instruction, return true.
5222 for (Use &U : I->uses()) { in FindAllMemoryUses()
5230 MemoryUses.push_back({&U, LI->getType()}); in FindAllMemoryUses()
5237 MemoryUses.push_back({&U, SI->getValueOperand()->getType()}); in FindAllMemoryUses()
5244 MemoryUses.push_back({&U, RMW->getValOperand()->getType()}); in FindAllMemoryUses()
5251 MemoryUses.push_back({&U, CmpX->getCompareOperand()->getType()}); in FindAllMemoryUses()
5256 if (CI->hasFnAttr(Attribute::Cold)) { in FindAllMemoryUses()
5260 OptSize || llvm::shouldOptimizeForSize(CI->getParent(), PSI, BFI); in FindAllMemoryUses()
5265 InlineAsm *IA = dyn_cast<InlineAsm>(CI->getCalledOperand()); in FindAllMemoryUses()
5301 // If Val is either of the known-live values, we know it is live! in valueAlreadyLiveAtInst()
5313 if (AI->isStaticAlloca()) in valueAlreadyLiveAtInst()
5319 return Val->isUsedInBasicBlock(MemoryInst->getParent()); in valueAlreadyLiveAtInst()
5330 /// use(Y) -> nonload/store
5377 return false; // Has a non-memory, non-foldable use! in isProfitableToFoldIntoAddressingMode()
5390 Value *Address = Pair.first->get(); in isProfitableToFoldIntoAddressingMode()
5391 Instruction *UserI = cast<Instruction>(Pair.first->getUser()); in isProfitableToFoldIntoAddressingMode()
5393 unsigned AS = Address->getType()->getPointerAddressSpace(); in isProfitableToFoldIntoAddressingMode()
5431 return I->getParent() != BB; in IsNonLocalValue()
5458 // Try to collapse single-value PHI nodes. This is necessary to undo in optimizeMemoryInst()
5465 // ensure that the addressing mode obtained from the non-PHI/select roots of in optimizeMemoryInst()
5491 append_range(worklist, P->incoming_values()); in optimizeMemoryInst()
5497 worklist.push_back(SI->getFalseValue()); in optimizeMemoryInst()
5498 worklist.push_back(SI->getTrueValue()); in optimizeMemoryInst()
5503 // For non-PHIs, determine the addressing mode being computed. Note that in optimizeMemoryInst()
5512 auto getDTFn = [MemoryInst, this]() -> const DominatorTree & { in optimizeMemoryInst()
5513 Function *F = MemoryInst->getParent()->getParent(); in optimizeMemoryInst()
5514 return this->getDT(*F); in optimizeMemoryInst()
5526 LargeOffsetGEPMap[GEP->getPointerOperand()].push_back(LargeOffsetGEP); in optimizeMemoryInst()
5552 return IsNonLocalValue(V, MemoryInst->getParent()); in optimizeMemoryInst()
5573 Type *IntPtrTy = DL->getIntPtrType(Addr->getType()); in optimizeMemoryInst()
5577 if (SunkAddr->getType() != Addr->getType()) { in optimizeMemoryInst()
5578 if (SunkAddr->getType()->getPointerAddressSpace() != in optimizeMemoryInst()
5579 Addr->getType()->getPointerAddressSpace() && in optimizeMemoryInst()
5580 !DL->isNonIntegralPointerType(Addr->getType())) { in optimizeMemoryInst()
5581 // There are two reasons the address spaces might not match: a no-op in optimizeMemoryInst()
5588 Builder.CreateIntToPtr(SunkAddr, Addr->getType(), "sunkaddr"); in optimizeMemoryInst()
5590 SunkAddr = Builder.CreatePointerCast(SunkAddr, Addr->getType()); in optimizeMemoryInst()
5593 SubtargetInfo->addrSinkUsingGEPs())) { in optimizeMemoryInst()
5594 // By default, we use the GEP-based method when AA is used later. This in optimizeMemoryInst()
5601 if (AddrMode.BaseReg && AddrMode.BaseReg->getType()->isPointerTy()) { in optimizeMemoryInst()
5606 if (AddrMode.Scale && AddrMode.ScaledReg->getType()->isPointerTy()) { in optimizeMemoryInst()
5624 Type *ScaledRegTy = AddrMode.ScaledReg->getType(); in optimizeMemoryInst()
5625 if (cast<IntegerType>(IntPtrTy)->getBitWidth() > in optimizeMemoryInst()
5626 cast<IntegerType>(ScaledRegTy)->getBitWidth()) in optimizeMemoryInst()
5635 if (BaseGV->isThreadLocal()) { in optimizeMemoryInst()
5645 if (!DL->isNonIntegralPointerType(Addr->getType())) { in optimizeMemoryInst()
5647 ResultPtr = Builder.CreateIntToPtr(AddrMode.BaseReg, Addr->getType(), in optimizeMemoryInst()
5651 ResultPtr = Builder.CreateIntToPtr(AddrMode.ScaledReg, Addr->getType(), in optimizeMemoryInst()
5659 SunkAddr = Constant::getNullValue(Addr->getType()); in optimizeMemoryInst()
5664 Builder.getPtrTy(Addr->getType()->getPointerAddressSpace()); in optimizeMemoryInst()
5673 if (V->getType() != IntPtrTy) in optimizeMemoryInst()
5682 if (V->getType() == IntPtrTy) { in optimizeMemoryInst()
5685 assert(cast<IntegerType>(IntPtrTy)->getBitWidth() < in optimizeMemoryInst()
5686 cast<IntegerType>(V->getType())->getBitWidth() && in optimizeMemoryInst()
5706 if (ResultPtr->getType() != I8PtrTy) in optimizeMemoryInst()
5718 if (ResultPtr->getType() != I8PtrTy) in optimizeMemoryInst()
5724 if (SunkAddr->getType() != Addr->getType()) { in optimizeMemoryInst()
5725 if (SunkAddr->getType()->getPointerAddressSpace() != in optimizeMemoryInst()
5726 Addr->getType()->getPointerAddressSpace() && in optimizeMemoryInst()
5727 !DL->isNonIntegralPointerType(Addr->getType())) { in optimizeMemoryInst()
5728 // There are two reasons the address spaces might not match: a no-op in optimizeMemoryInst()
5735 Builder.CreateIntToPtr(SunkAddr, Addr->getType(), "sunkaddr"); in optimizeMemoryInst()
5737 SunkAddr = Builder.CreatePointerCast(SunkAddr, Addr->getType()); in optimizeMemoryInst()
5742 // non-integral pointers, so in that case bail out now. in optimizeMemoryInst()
5743 Type *BaseTy = AddrMode.BaseReg ? AddrMode.BaseReg->getType() : nullptr; in optimizeMemoryInst()
5744 Type *ScaleTy = AddrMode.Scale ? AddrMode.ScaledReg->getType() : nullptr; in optimizeMemoryInst()
5747 if (DL->isNonIntegralPointerType(Addr->getType()) || in optimizeMemoryInst()
5748 (BasePtrTy && DL->isNonIntegralPointerType(BasePtrTy)) || in optimizeMemoryInst()
5749 (ScalePtrTy && DL->isNonIntegralPointerType(ScalePtrTy)) || in optimizeMemoryInst()
5751 DL->isNonIntegralPointerType(AddrMode.BaseGV->getType()))) in optimizeMemoryInst()
5756 Type *IntPtrTy = DL->getIntPtrType(Addr->getType()); in optimizeMemoryInst()
5766 if (V->getType()->isPointerTy()) in optimizeMemoryInst()
5768 if (V->getType() != IntPtrTy) in optimizeMemoryInst()
5776 if (V->getType() == IntPtrTy) { in optimizeMemoryInst()
5778 } else if (V->getType()->isPointerTy()) { in optimizeMemoryInst()
5780 } else if (cast<IntegerType>(IntPtrTy)->getBitWidth() < in optimizeMemoryInst()
5781 cast<IntegerType>(V->getType())->getBitWidth()) { in optimizeMemoryInst()
5791 I->eraseFromParent(); in optimizeMemoryInst()
5807 if (BaseGV->isThreadLocal()) { in optimizeMemoryInst()
5829 SunkAddr = Constant::getNullValue(Addr->getType()); in optimizeMemoryInst()
5831 SunkAddr = Builder.CreateIntToPtr(Result, Addr->getType(), "sunkaddr"); in optimizeMemoryInst()
5834 MemoryInst->replaceUsesOfWith(Repl, SunkAddr); in optimizeMemoryInst()
5841 if (Repl->use_empty()) { in optimizeMemoryInst()
5842 resetIteratorIfInvalidatedWhileCalling(CurInstIterator->getParent(), [&]() { in optimizeMemoryInst()
5875 if (!GEP->hasIndices()) in optimizeGatherScatterInst()
5880 if (MemoryInst->getParent() != GEP->getParent()) in optimizeGatherScatterInst()
5883 SmallVector<Value *, 2> Ops(GEP->operands()); in optimizeGatherScatterInst()
5887 if (Ops[0]->getType()->isVectorTy()) { in optimizeGatherScatterInst()
5894 unsigned FinalIndex = Ops.size() - 1; in optimizeGatherScatterInst()
5903 if (isa<VectorType>(C->getType())) in optimizeGatherScatterInst()
5904 C = C->getSplatValue(); in optimizeGatherScatterInst()
5906 if (!CI || !CI->isZero()) in optimizeGatherScatterInst()
5913 if (Ops[FinalIndex]->getType()->isVectorTy()) { in optimizeGatherScatterInst()
5917 if (!C || !C->isZero()) { in optimizeGatherScatterInst()
5929 auto NumElts = cast<VectorType>(Ptr->getType())->getElementCount(); in optimizeGatherScatterInst()
5933 Type *SourceTy = GEP->getSourceElementType(); in optimizeGatherScatterInst()
5934 Type *ScalarIndexTy = DL->getIndexType(Ops[0]->getType()->getScalarType()); in optimizeGatherScatterInst()
5938 if (!Ops[FinalIndex]->getType()->isVectorTy()) { in optimizeGatherScatterInst()
5953 Constant::getNullValue(Ops[FinalIndex]->getType()->getScalarType()); in optimizeGatherScatterInst()
5969 auto NumElts = cast<VectorType>(Ptr->getType())->getElementCount(); in optimizeGatherScatterInst()
5974 Type *ScalarIndexTy = DL->getIndexType(V->getType()->getScalarType()); in optimizeGatherScatterInst()
5977 if (cast<IntrinsicInst>(MemoryInst)->getIntrinsicID() == in optimizeGatherScatterInst()
5979 ScalarTy = MemoryInst->getType()->getScalarType(); in optimizeGatherScatterInst()
5981 assert(cast<IntrinsicInst>(MemoryInst)->getIntrinsicID() == in optimizeGatherScatterInst()
5983 ScalarTy = MemoryInst->getOperand(0)->getType()->getScalarType(); in optimizeGatherScatterInst()
5991 MemoryInst->replaceUsesOfWith(Ptr, NewAddr); in optimizeGatherScatterInst()
5995 if (Ptr->use_empty()) in optimizeGatherScatterInst()
6009 TM->getSubtargetImpl(*CS->getFunction())->getRegisterInfo(); in optimizeInlineAsmInst()
6011 TLI->ParseConstraints(*DL, TRI, *CS); in optimizeInlineAsmInst()
6015 TLI->ComputeConstraintToUse(OpInfo, SDValue()); in optimizeInlineAsmInst()
6020 Value *OpVal = CS->getArgOperand(ArgNo++); in optimizeInlineAsmInst()
6021 MadeChange |= optimizeMemoryInst(CS, OpVal, OpVal->getType(), ~0u); in optimizeInlineAsmInst()
6032 assert(!Val->use_empty() && "Input must have at least one use"); in hasSameExtUse()
6033 const Instruction *FirstUser = cast<Instruction>(*Val->user_begin()); in hasSameExtUse()
6035 Type *ExtTy = FirstUser->getType(); in hasSameExtUse()
6036 for (const User *U : Val->users()) { in hasSameExtUse()
6040 Type *CurTy = UI->getType(); in hasSameExtUse()
6061 if (ExtTy->getScalarType()->getIntegerBitWidth() > in hasSameExtUse()
6062 CurTy->getScalarType()->getIntegerBitWidth()) { in hasSameExtUse()
6093 if (isa<LoadInst>(I->getOperand(0))) { in tryToPromoteExts()
6102 if (!TLI->enableExtLdPromotion() || DisableExtLdPromotion) in tryToPromoteExts()
6120 unsigned ExtCost = !TLI->isExtFree(I); in tryToPromoteExts()
6139 std::max((long long)0, (TotalCreatedInstsCost - ExtCost)); in tryToPromoteExts()
6157 Value *ExtOperand = MovedExt->getOperand(0); in tryToPromoteExts()
6162 (ExtOperand->hasOneUse() || hasSameExtUse(ExtOperand, *TLI)))) in tryToPromoteExts()
6190 Inst->getOperand(0) != Entry.first) in mergeSExts()
6197 Pt->removeFromParent(); in mergeSExts()
6209 Inst->removeFromParent(); in mergeSExts()
6247 // %new_gep1 = gep %new_base, off1 - off0
6248 // %new_gep2 = gep %new_base, off2 - off0
6278 GetElementPtrInst *BaseGEP = LargeOffsetGEPs.begin()->first; in splitLargeGEPOffsets()
6279 int64_t BaseOffset = LargeOffsetGEPs.begin()->second; in splitLargeGEPOffsets()
6284 LLVMContext &Ctx = GEP->getContext(); in splitLargeGEPOffsets()
6285 Type *PtrIdxTy = DL->getIndexType(GEP->getType()); in splitLargeGEPOffsets()
6287 PointerType::get(Ctx, GEP->getType()->getPointerAddressSpace()); in splitLargeGEPOffsets()
6294 NewBaseInsertBB = BaseI->getParent(); in splitLargeGEPOffsets()
6296 NewBaseInsertPt = NewBaseInsertBB->getFirstInsertionPt(); in splitLargeGEPOffsets()
6299 SplitEdge(NewBaseInsertBB, Invoke->getNormalDest(), DT.get(), LI); in splitLargeGEPOffsets()
6300 NewBaseInsertPt = NewBaseInsertBB->getFirstInsertionPt(); in splitLargeGEPOffsets()
6302 NewBaseInsertPt = std::next(BaseI->getIterator()); in splitLargeGEPOffsets()
6306 NewBaseInsertBB = &BaseGEP->getFunction()->getEntryBlock(); in splitLargeGEPOffsets()
6307 NewBaseInsertPt = NewBaseInsertBB->getFirstInsertionPt(); in splitLargeGEPOffsets()
6313 if (NewBaseGEP->getType() != I8PtrTy) in splitLargeGEPOffsets()
6322 if (int64_t PreferBase = TLI->getPreferredLargeGEPBaseOffset( in splitLargeGEPOffsets()
6332 GetElementPtrInst *GEP = LargeOffsetGEP->first; in splitLargeGEPOffsets()
6333 int64_t Offset = LargeOffsetGEP->second; in splitLargeGEPOffsets()
6337 AddrMode.BaseOffs = Offset - BaseOffset; in splitLargeGEPOffsets()
6340 if (!TLI->isLegalAddressingMode(*DL, AddrMode, in splitLargeGEPOffsets()
6341 GEP->getResultElementType(), in splitLargeGEPOffsets()
6342 GEP->getAddressSpace())) { in splitLargeGEPOffsets()
6353 Type *PtrIdxTy = DL->getIndexType(GEP->getType()); in splitLargeGEPOffsets()
6365 Value *Index = ConstantInt::get(PtrIdxTy, Offset - BaseOffset); in splitLargeGEPOffsets()
6371 GEP->eraseFromParent(); in splitLargeGEPOffsets()
6385 Type *PhiTy = I->getType(); in optimizePhiType()
6388 (!I->getType()->isIntegerTy() && !I->getType()->isFloatingPointTy())) in optimizePhiType()
6411 for (Value *V : Phi->incoming_values()) { in optimizePhiType()
6420 if (!OpLoad->isSimple()) in optimizePhiType()
6429 ConvertTy = OpBC->getOperand(0)->getType(); in optimizePhiType()
6430 if (OpBC->getOperand(0)->getType() != ConvertTy) in optimizePhiType()
6434 AnyAnchored |= !isa<LoadInst>(OpBC->getOperand(0)) && in optimizePhiType()
6435 !isa<ExtractElementInst>(OpBC->getOperand(0)); in optimizePhiType()
6445 for (User *V : II->users()) { in optimizePhiType()
6455 if (!OpStore->isSimple() || OpStore->getOperand(0) != II) in optimizePhiType()
6460 ConvertTy = OpBC->getType(); in optimizePhiType()
6461 if (OpBC->getType() != ConvertTy) in optimizePhiType()
6465 any_of(OpBC->users(), [](User *U) { return !isa<StoreInst>(U); }); in optimizePhiType()
6473 !TLI->shouldConvertPhiType(PhiTy, ConvertTy)) in optimizePhiType()
6486 ValMap[D] = D->getOperand(0); in optimizePhiType()
6489 BasicBlock::iterator insertPt = std::next(D->getIterator()); in optimizePhiType()
6490 ValMap[D] = new BitCastInst(D, ConvertTy, D->getName() + ".bc", insertPt); in optimizePhiType()
6494 ValMap[Phi] = PHINode::Create(ConvertTy, Phi->getNumIncomingValues(), in optimizePhiType()
6495 Phi->getName() + ".tc", Phi->getIterator()); in optimizePhiType()
6499 for (int i = 0, e = Phi->getNumIncomingValues(); i < e; i++) in optimizePhiType()
6500 NewPhi->addIncoming(ValMap[Phi->getIncomingValue(i)], in optimizePhiType()
6501 Phi->getIncomingBlock(i)); in optimizePhiType()
6508 replaceAllUsesWith(U, ValMap[U->getOperand(0)], FreshBBs, IsHugeFunc); in optimizePhiType()
6510 U->setOperand(0, new BitCastInst(ValMap[U->getOperand(0)], PhiTy, "bc", in optimizePhiType()
6511 U->getIterator())); in optimizePhiType()
6536 replaceAllUsesWith(I, PoisonValue::get(I->getType()), FreshBBs, IsHugeFunc); in optimizePhiTypes()
6537 I->eraseFromParent(); in optimizePhiTypes()
6549 if (isa<LoadInst>(MovedExtInst->getOperand(0))) { in canFormExtLd()
6550 LI = cast<LoadInst>(MovedExtInst->getOperand(0)); in canFormExtLd()
6561 if (!HasPromoted && LI->getParent() == Inst->getParent()) in canFormExtLd()
6564 return TLI->isExtLoad(LI, Inst, *DL); in canFormExtLd()
6608 bool ATPConsiderable = TTI->shouldConsiderAddressTypePromotion( in optimizeExt()
6629 ExtFedByLoad->moveAfter(LI); in optimizeExt()
6658 Value *HeadOfChain = I->getOperand(0); in performAddressTypePromotion()
6664 if (AlreadySeen->second != nullptr) in performAddressTypePromotion()
6665 UnhandledExts.insert(AlreadySeen->second); in performAddressTypePromotion()
6676 Value *HeadOfChain = I->getOperand(0); in performAddressTypePromotion()
6687 Value *HeadOfChain = I->getOperand(0); in performAddressTypePromotion()
6706 Value *HeadOfChain = I->getOperand(0); in performAddressTypePromotion()
6716 BasicBlock *DefBB = I->getParent(); in optimizeExtUses()
6720 Value *Src = I->getOperand(0); in optimizeExtUses()
6721 if (Src->hasOneUse()) in optimizeExtUses()
6725 if (!TLI->isTruncateFree(I->getType(), Src->getType())) in optimizeExtUses()
6730 if (!isa<Instruction>(Src) || DefBB != cast<Instruction>(Src)->getParent()) in optimizeExtUses()
6734 for (User *U : I->users()) { in optimizeExtUses()
6738 BasicBlock *UserBB = UI->getParent(); in optimizeExtUses()
6748 for (User *U : Src->users()) { in optimizeExtUses()
6750 BasicBlock *UserBB = UI->getParent(); in optimizeExtUses()
6759 // InsertedTruncs - Only insert one trunc in each block once. in optimizeExtUses()
6763 for (Use &U : Src->uses()) { in optimizeExtUses()
6767 BasicBlock *UserBB = User->getParent(); in optimizeExtUses()
6775 BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt(); in optimizeExtUses()
6776 assert(InsertPt != UserBB->end()); in optimizeExtUses()
6777 InsertedTrunc = new TruncInst(I, Src->getType(), ""); in optimizeExtUses()
6778 InsertedTrunc->insertBefore(*UserBB, InsertPt); in optimizeExtUses()
6843 if (!Load->isSimple() || !Load->getType()->isIntOrPtrTy()) in optimizeLoadExt()
6847 if (Load->hasOneUse() && in optimizeLoadExt()
6848 InsertedInsts.count(cast<Instruction>(*Load->user_begin()))) in optimizeLoadExt()
6856 for (auto *U : Load->users()) in optimizeLoadExt()
6859 EVT LoadResultVT = TLI->getValueType(*DL, Load->getType()); in optimizeLoadExt()
6871 // Break use-def graph loops. in optimizeLoadExt()
6877 for (auto *U : Phi->users()) in optimizeLoadExt()
6882 switch (I->getOpcode()) { in optimizeLoadExt()
6884 auto *AndC = dyn_cast<ConstantInt>(I->getOperand(1)); in optimizeLoadExt()
6887 APInt AndBits = AndC->getValue(); in optimizeLoadExt()
6892 if (AndBits == WidestAndBits && I->getOperand(0) == Load) in optimizeLoadExt()
6898 auto *ShlC = dyn_cast<ConstantInt>(I->getOperand(1)); in optimizeLoadExt()
6901 uint64_t ShiftAmt = ShlC->getLimitedValue(BitWidth - 1); in optimizeLoadExt()
6902 DemandBits.setLowBits(BitWidth - ShiftAmt); in optimizeLoadExt()
6907 EVT TruncVT = TLI->getValueType(*DL, I->getType()); in optimizeLoadExt()
6934 LLVMContext &Ctx = Load->getType()->getContext(); in optimizeLoadExt()
6936 EVT TruncVT = TLI->getValueType(*DL, TruncTy); in optimizeLoadExt()
6940 !TLI->isLoadExtLegal(ISD::ZEXTLOAD, LoadResultVT, TruncVT)) in optimizeLoadExt()
6943 IRBuilder<> Builder(Load->getNextNonDebugInstruction()); in optimizeLoadExt()
6953 NewAnd->setOperand(0, Load); in optimizeLoadExt()
6959 if (cast<ConstantInt>(And->getOperand(1))->getValue() == DemandBits) { in optimizeLoadExt()
6962 CurInstIterator = std::next(And->getIterator()); in optimizeLoadExt()
6963 And->eraseFromParent(); in optimizeLoadExt()
6977 return I && I->hasOneUse() && isSafeToSpeculativelyExecute(I) && in sinkSelectOperand()
6978 TTI->isExpensiveToSpeculativelyExecute(I); in sinkSelectOperand()
6986 if (!TLI->isPredictableSelectExpensive()) in isFormingBranchFromSelectProfitable()
7000 if (Probability > TTI->getPredictableBranchThreshold()) in isFormingBranchFromSelectProfitable()
7005 CmpInst *Cmp = dyn_cast<CmpInst>(SI->getCondition()); in isFormingBranchFromSelectProfitable()
7007 // If a branch is predictable, an out-of-order CPU can avoid blocking on its in isFormingBranchFromSelectProfitable()
7010 if (!Cmp || !Cmp->hasOneUse()) in isFormingBranchFromSelectProfitable()
7015 if (sinkSelectOperand(TTI, SI->getTrueValue()) || in isFormingBranchFromSelectProfitable()
7016 sinkSelectOperand(TTI, SI->getFalseValue())) in isFormingBranchFromSelectProfitable()
7033 assert(DefSI->getCondition() == SI->getCondition() && in getTrueOrFalseValue()
7035 V = (isTrue ? DefSI->getTrueValue() : DefSI->getFalseValue()); in getTrueOrFalseValue()
7043 assert(Shift->isShift() && "Expected a shift"); in optimizeShiftInst()
7046 // general vector shifts, and (3) the shift amount is a select-of-splatted in optimizeShiftInst()
7048 // shift Op0, (select Cond, TVal, FVal) --> in optimizeShiftInst()
7052 // general vector shift is more than the cost of 2 shift-by-scalars. in optimizeShiftInst()
7055 Type *Ty = Shift->getType(); in optimizeShiftInst()
7056 if (!Ty->isVectorTy() || !TLI->isVectorShiftByScalarCheap(Ty)) in optimizeShiftInst()
7059 if (!match(Shift->getOperand(1), in optimizeShiftInst()
7066 BinaryOperator::BinaryOps Opcode = Shift->getOpcode(); in optimizeShiftInst()
7067 Value *NewTVal = Builder.CreateBinOp(Opcode, Shift->getOperand(0), TVal); in optimizeShiftInst()
7068 Value *NewFVal = Builder.CreateBinOp(Opcode, Shift->getOperand(0), FVal); in optimizeShiftInst()
7071 Shift->eraseFromParent(); in optimizeShiftInst()
7076 Intrinsic::ID Opcode = Fsh->getIntrinsicID(); in optimizeFunnelShift()
7081 // than general vector shifts, and (3) the shift amount is select-of-splatted in optimizeFunnelShift()
7083 // fsh Op0, Op1, (select Cond, TVal, FVal) --> in optimizeFunnelShift()
7087 // general vector shift is more than the cost of 2 shift-by-scalars. in optimizeFunnelShift()
7090 Type *Ty = Fsh->getType(); in optimizeFunnelShift()
7091 if (!Ty->isVectorTy() || !TLI->isVectorShiftByScalarCheap(Ty)) in optimizeFunnelShift()
7094 if (!match(Fsh->getOperand(2), in optimizeFunnelShift()
7101 Value *X = Fsh->getOperand(0), *Y = Fsh->getOperand(1); in optimizeFunnelShift()
7106 Fsh->eraseFromParent(); in optimizeFunnelShift()
7124 It != SI->getParent()->end(); ++It) { in optimizeSelectInst()
7126 if (I && SI->getCondition() == I->getCondition()) { in optimizeSelectInst()
7136 CurInstIterator = std::next(LastSI->getIterator()); in optimizeSelectInst()
7137 // Examine debug-info attached to the consecutive select instructions. They in optimizeSelectInst()
7143 bool VectorCond = !SI->getCondition()->getType()->isIntegerTy(1); in optimizeSelectInst()
7146 if (VectorCond || SI->getMetadata(LLVMContext::MD_unpredictable)) in optimizeSelectInst()
7150 if (SI->getType()->isVectorTy()) in optimizeSelectInst()
7155 if (TLI->isSelectSupported(SelectKind) && in optimizeSelectInst()
7157 llvm::shouldOptimizeForSize(SI->getParent(), PSI, BFI.get()))) in optimizeSelectInst()
7195 if (Value *V = SI->getTrueValue(); sinkSelectOperand(TTI, V)) in optimizeSelectInst()
7197 if (Value *V = SI->getFalseValue(); sinkSelectOperand(TTI, V)) in optimizeSelectInst()
7203 BasicBlock *StartBlock = SI->getParent(); in optimizeSelectInst()
7205 // We should split before any debug-info. in optimizeSelectInst()
7209 auto *CondFr = IB.CreateFreeze(SI->getCondition(), SI->getName() + ".frozen"); in optimizeSelectInst()
7219 FalseBlock = FalseBranch->getParent(); in optimizeSelectInst()
7220 EndBlock = cast<BasicBlock>(FalseBranch->getOperand(0)); in optimizeSelectInst()
7224 TrueBlock = TrueBranch->getParent(); in optimizeSelectInst()
7225 EndBlock = cast<BasicBlock>(TrueBranch->getOperand(0)); in optimizeSelectInst()
7233 TrueBlock = TrueBranch->getParent(); in optimizeSelectInst()
7234 FalseBlock = FalseBranch->getParent(); in optimizeSelectInst()
7235 EndBlock = cast<BasicBlock>(TrueBranch->getOperand(0)); in optimizeSelectInst()
7238 EndBlock->setName("select.end"); in optimizeSelectInst()
7240 TrueBlock->setName("select.true.sink"); in optimizeSelectInst()
7242 FalseBlock->setName(FalseInstrs.size() == 0 ? "select.false" in optimizeSelectInst()
7253 BFI->setBlockFreq(EndBlock, BFI->getBlockFreq(StartBlock)); in optimizeSelectInst()
7258 StartBlock->getTerminator()->copyMetadata(*SI, MD); in optimizeSelectInst()
7263 I->moveBefore(TrueBranch); in optimizeSelectInst()
7265 I->moveBefore(FalseBranch); in optimizeSelectInst()
7283 PHINode *PN = PHINode::Create(SI->getType(), 2, ""); in optimizeSelectInst()
7284 PN->insertBefore(EndBlock->begin()); in optimizeSelectInst()
7285 PN->takeName(SI); in optimizeSelectInst()
7286 PN->addIncoming(getTrueOrFalseValue(SI, true, INS), TrueBlock); in optimizeSelectInst()
7287 PN->addIncoming(getTrueOrFalseValue(SI, false, INS), FalseBlock); in optimizeSelectInst()
7288 PN->setDebugLoc(SI->getDebugLoc()); in optimizeSelectInst()
7291 SI->eraseFromParent(); in optimizeSelectInst()
7297 CurInstIterator = StartBlock->end(); in optimizeSelectInst()
7309 Type *NewType = TLI->shouldConvertSplatType(SVI); in optimizeShuffleVectorInst()
7313 auto *SVIVecType = cast<FixedVectorType>(SVI->getType()); in optimizeShuffleVectorInst()
7314 assert(!NewType->isVectorTy() && "Expected a scalar type!"); in optimizeShuffleVectorInst()
7315 assert(NewType->getScalarSizeInBits() == SVIVecType->getScalarSizeInBits() && in optimizeShuffleVectorInst()
7318 FixedVectorType::get(NewType, SVIVecType->getNumElements()); in optimizeShuffleVectorInst()
7321 IRBuilder<> Builder(SVI->getContext()); in optimizeShuffleVectorInst()
7324 cast<Instruction>(SVI->getOperand(0))->getOperand(1), NewType); in optimizeShuffleVectorInst()
7325 Value *Shuffle = Builder.CreateVectorSplat(NewVecType->getNumElements(), BC1); in optimizeShuffleVectorInst()
7336 if (auto *Op = dyn_cast<Instruction>(BCI->getOperand(0))) in optimizeShuffleVectorInst()
7337 if (BCI->getParent() != Op->getParent() && !isa<PHINode>(Op) && in optimizeShuffleVectorInst()
7338 !Op->isTerminator() && !Op->isEHPad()) in optimizeShuffleVectorInst()
7339 BCI->moveAfter(Op); in optimizeShuffleVectorInst()
7348 if (!TLI->shouldSinkOperands(I, OpsToSink)) in tryToSinkFreeOperands()
7355 BasicBlock *TargetBB = I->getParent(); in tryToSinkFreeOperands()
7365 auto *UI = cast<Instruction>(U->get()); in tryToSinkFreeOperands()
7368 if (UI->getParent() == TargetBB) { in tryToSinkFreeOperands()
7379 auto *UI = cast<Instruction>(U->get()); in tryToSinkFreeOperands()
7380 Instruction *NI = UI->clone(); in tryToSinkFreeOperands()
7385 for (unsigned I = 0; I < NI->getNumOperands(); ++I) { in tryToSinkFreeOperands()
7386 auto *OpDef = dyn_cast<Instruction>(NI->getOperand(I)); in tryToSinkFreeOperands()
7389 FreshBBs.insert(OpDef->getParent()); in tryToSinkFreeOperands()
7396 NI->insertBefore(InsertPoint); in tryToSinkFreeOperands()
7403 Instruction *OldI = cast<Instruction>(U->getUser()); in tryToSinkFreeOperands()
7405 NewInstructions[OldI]->setOperand(U->getOperandNo(), NI); in tryToSinkFreeOperands()
7407 U->set(NI); in tryToSinkFreeOperands()
7413 if (!I->hasNUsesOrMore(1)) { in tryToSinkFreeOperands()
7415 I->eraseFromParent(); in tryToSinkFreeOperands()
7423 Value *Cond = SI->getCondition(); in optimizeSwitchType()
7424 Type *OldType = Cond->getType(); in optimizeSwitchType()
7425 LLVMContext &Context = Cond->getContext(); in optimizeSwitchType()
7426 EVT OldVT = TLI->getValueType(*DL, OldType); in optimizeSwitchType()
7427 MVT RegType = TLI->getPreferredSwitchConditionType(Context, OldVT); in optimizeSwitchType()
7430 if (RegWidth <= cast<IntegerType>(OldType)->getBitWidth()) in optimizeSwitchType()
7437 // preferred register width, so we will potentially eliminate N-1 extends, in optimizeSwitchType()
7447 if (TLI->isSExtCheaperThanZExt(OldVT, RegType)) in optimizeSwitchType()
7451 if (Arg->hasSExtAttr()) in optimizeSwitchType()
7453 if (Arg->hasZExtAttr()) in optimizeSwitchType()
7458 ExtInst->insertBefore(SI); in optimizeSwitchType()
7459 ExtInst->setDebugLoc(SI->getDebugLoc()); in optimizeSwitchType()
7460 SI->setCondition(ExtInst); in optimizeSwitchType()
7461 for (auto Case : SI->cases()) { in optimizeSwitchType()
7462 const APInt &NarrowConst = Case.getCaseValue()->getValue(); in optimizeSwitchType()
7475 // Materializing the constant for the phi-argument needs instructions; So we in optimizeSwitchPhiConstants()
7479 Value *Condition = SI->getCondition(); in optimizeSwitchPhiConstants()
7485 BasicBlock *SwitchBB = SI->getParent(); in optimizeSwitchPhiConstants()
7486 Type *ConditionType = Condition->getType(); in optimizeSwitchPhiConstants()
7488 for (const SwitchInst::CaseHandle &Case : SI->cases()) { in optimizeSwitchPhiConstants()
7494 for (PHINode &PHI : CaseBB->phis()) { in optimizeSwitchPhiConstants()
7500 PHIType->isIntegerTy() && in optimizeSwitchPhiConstants()
7501 PHIType->getIntegerBitWidth() > ConditionType->getIntegerBitWidth() && in optimizeSwitchPhiConstants()
7502 TLI->isZExtFree(ConditionType, PHIType); in optimizeSwitchPhiConstants()
7514 PHIValueInt->getValue() != in optimizeSwitchPhiConstants()
7515 CaseValue->getValue().zext(PHIType->getIntegerBitWidth())) in optimizeSwitchPhiConstants()
7525 if (SI->findCaseDest(CaseBB) == nullptr) { in optimizeSwitchPhiConstants()
7627 return Transition->getOperand(getTransitionOriginalValueIdx())->getType(); in getTransitionType()
7642 Value *ValIdx = Transition->getOperand(getTransitionOriginalValueIdx()); in isProfitableToPromote()
7644 ? cast<ConstantInt>(ValIdx)->getZExtValue() in isProfitableToPromote()
7645 : -1; in isProfitableToPromote()
7649 unsigned AS = ST->getPointerAddressSpace(); in isProfitableToPromote()
7652 TLI.getValueType(DL, ST->getValueOperand()->getType()), AS, in isProfitableToPromote()
7653 ST->getAlign())) { in isProfitableToPromote()
7670 // Moreover, one argument is a constant that can be viewed as a splat in isProfitableToPromote()
7672 Value *Arg0 = Inst->getOperand(0); in isProfitableToPromote()
7682 Inst->getOpcode(), Inst->getType(), CostKind, Arg0Info, Arg1Info); in isProfitableToPromote()
7683 VectorCost += TTI.getArithmeticInstrCost(Inst->getOpcode(), PromotedType, in isProfitableToPromote()
7705 Value *ValExtractIdx = Transition->getOperand(getTransitionIdx()); in getConstantVector()
7707 ExtractIdx = CstVal->getSExtValue(); in getConstantVector()
7712 ElementCount EC = cast<VectorType>(getTransitionType())->getElementCount(); in getConstantVector()
7718 UndefValue *UndefVal = UndefValue::get(Val->getType()); in getConstantVector()
7728 "Generate scalable vector for non-splat is unimplemented"); in getConstantVector()
7736 // the right hand side of a division-like instruction. in canCauseUndefinedBehavior()
7739 switch (Use->getOpcode()) { in canCauseUndefinedBehavior()
7749 return !Use->hasNoNaNs(); in canCauseUndefinedBehavior()
7774 for (const Use &U : ToBePromoted->operands()) { in shouldPromote()
7789 int ISDOpcode = TLI.InstructionOpcodeToISD(ToBePromoted->getOpcode()); in shouldPromote()
7846 assert(ToBePromoted->getType() == Transition->getType() && in promoteImpl()
7849 ToBePromoted->replaceAllUsesWith(Transition); in promoteImpl()
7853 ToBePromoted->mutateType(TransitionTy); in promoteImpl()
7857 for (Use &U : ToBePromoted->operands()) { in promoteImpl()
7861 NewVal = Transition->getOperand(getTransitionOriginalValueIdx()); in promoteImpl()
7872 ToBePromoted->setOperand(U.getOperandNo(), NewVal); in promoteImpl()
7874 Transition->moveAfter(ToBePromoted); in promoteImpl()
7875 Transition->setOperand(getTransitionOriginalValueIdx(), ToBePromoted); in promoteImpl()
7885 !TLI->canCombineStoreAndExtract(Inst->getOperand(0)->getType(), in optimizeExtractElementInst()
7886 Inst->getOperand(1), CombineCost))) in optimizeExtractElementInst()
7890 // Try to move it down the def-use chain, until: in optimizeExtractElementInst()
7891 // - We can combine the transition with its single use in optimizeExtractElementInst()
7893 // - We escape the current basic block in optimizeExtractElementInst()
7896 BasicBlock *Parent = Inst->getParent(); in optimizeExtractElementInst()
7901 while (Inst->hasOneUse()) { in optimizeExtractElementInst()
7902 Instruction *ToBePromoted = cast<Instruction>(*Inst->user_begin()); in optimizeExtractElementInst()
7905 if (ToBePromoted->getParent() != Parent) { in optimizeExtractElementInst()
7907 << ToBePromoted->getParent()->getName() in optimizeExtractElementInst()
7908 << ") than the transition (" << Parent->getName() in optimizeExtractElementInst()
7940 /// (shl (zext I to i64), 32)), addr) -->
7944 /// For pair of {i32, i32}, i64 store --> two i32 stores.
7945 /// For pair of {i32, i16}, i64 store --> two i32 stores.
7946 /// For pair of {i16, i16}, i32 store --> two i16 stores.
7947 /// For pair of {i16, i8}, i32 store --> two i16 stores.
7948 /// For pair of {i8, i8}, i16 store --> two i8 stores.
7969 Type *StoreType = SI.getValueOperand()->getType(); in splitMergedValStore()
7975 if (StoreType->isScalableTy()) in splitMergedValStore()
8007 if (!LValue->getType()->isIntegerTy() || in splitMergedValStore()
8008 DL.getTypeSizeInBits(LValue->getType()) > HalfValBitSize || in splitMergedValStore()
8009 !HValue->getType()->isIntegerTy() || in splitMergedValStore()
8010 DL.getTypeSizeInBits(HValue->getType()) > HalfValBitSize) in splitMergedValStore()
8017 EVT LowTy = LBC ? EVT::getEVT(LBC->getOperand(0)->getType()) in splitMergedValStore()
8018 : EVT::getEVT(LValue->getType()); in splitMergedValStore()
8019 EVT HighTy = HBC ? EVT::getEVT(HBC->getOperand(0)->getType()) in splitMergedValStore()
8020 : EVT::getEVT(HValue->getType()); in splitMergedValStore()
8030 if (LBC && LBC->getParent() != SI.getParent()) in splitMergedValStore()
8031 LValue = Builder.CreateBitCast(LBC->getOperand(0), LBC->getType()); in splitMergedValStore()
8032 if (HBC && HBC->getParent() != SI.getParent()) in splitMergedValStore()
8033 HValue = Builder.CreateBitCast(HBC->getOperand(0), HBC->getType()); in splitMergedValStore()
8048 // over-aligned or not, while the other will require adjustment. in splitMergedValStore()
8066 return GEP->getNumOperands() == 2 && I.isSequential() && in GEPSequentialConstIndexed()
8067 isa<ConstantInt>(GEP->getOperand(1)); in GEPSequentialConstIndexed()
8077 // ---------- BEFORE ----------
8097 // ---------------------------
8099 // ---------- AFTER ----------
8109 // %UGEPI = gep %GEPI, (UIdx-Idx)
8111 // ---------------------------
8123 // between the register pressure and the length of data-flow critical
8129 BasicBlock *SrcBlock = GEPI->getParent(); in tryUnmergingGEPsAcrossIndirectBr()
8131 // (non-IndirectBr) cases exit early here. in tryUnmergingGEPsAcrossIndirectBr()
8132 if (!isa<IndirectBrInst>(SrcBlock->getTerminator())) in tryUnmergingGEPsAcrossIndirectBr()
8137 ConstantInt *GEPIIdx = cast<ConstantInt>(GEPI->getOperand(1)); in tryUnmergingGEPsAcrossIndirectBr()
8139 if (TTI->getIntImmCost(GEPIIdx->getValue(), GEPIIdx->getType(), in tryUnmergingGEPsAcrossIndirectBr()
8143 Value *GEPIOp = GEPI->getOperand(0); in tryUnmergingGEPsAcrossIndirectBr()
8148 if (GEPIOpI->getParent() != SrcBlock) in tryUnmergingGEPsAcrossIndirectBr()
8152 if (llvm::none_of(GEPI->users(), [&](User *Usr) { in tryUnmergingGEPsAcrossIndirectBr()
8154 if (I->getParent() != SrcBlock) { in tryUnmergingGEPsAcrossIndirectBr()
8165 for (User *Usr : GEPIOp->users()) { in tryUnmergingGEPsAcrossIndirectBr()
8173 if (UI->getParent() == SrcBlock) in tryUnmergingGEPsAcrossIndirectBr()
8184 if (UGEPI->getOperand(0) != GEPIOp) in tryUnmergingGEPsAcrossIndirectBr()
8186 if (UGEPI->getSourceElementType() != GEPI->getSourceElementType()) in tryUnmergingGEPsAcrossIndirectBr()
8188 if (GEPIIdx->getType() != in tryUnmergingGEPsAcrossIndirectBr()
8189 cast<ConstantInt>(UGEPI->getOperand(1))->getType()) in tryUnmergingGEPsAcrossIndirectBr()
8191 ConstantInt *UGEPIIdx = cast<ConstantInt>(UGEPI->getOperand(1)); in tryUnmergingGEPsAcrossIndirectBr()
8192 if (TTI->getIntImmCost(UGEPIIdx->getValue(), UGEPIIdx->getType(), in tryUnmergingGEPsAcrossIndirectBr()
8200 // Check the materializing cost of (Uidx-Idx). in tryUnmergingGEPsAcrossIndirectBr()
8202 ConstantInt *UGEPIIdx = cast<ConstantInt>(UGEPI->getOperand(1)); in tryUnmergingGEPsAcrossIndirectBr()
8203 APInt NewIdx = UGEPIIdx->getValue() - GEPIIdx->getValue(); in tryUnmergingGEPsAcrossIndirectBr()
8204 InstructionCost ImmCost = TTI->getIntImmCost( in tryUnmergingGEPsAcrossIndirectBr()
8205 NewIdx, GEPIIdx->getType(), TargetTransformInfo::TCK_SizeAndLatency); in tryUnmergingGEPsAcrossIndirectBr()
8211 UGEPI->setOperand(0, GEPI); in tryUnmergingGEPsAcrossIndirectBr()
8212 ConstantInt *UGEPIIdx = cast<ConstantInt>(UGEPI->getOperand(1)); in tryUnmergingGEPsAcrossIndirectBr()
8214 GEPIIdx->getType(), UGEPIIdx->getValue() - GEPIIdx->getValue()); in tryUnmergingGEPsAcrossIndirectBr()
8215 UGEPI->setOperand(1, NewUGEPIIdx); in tryUnmergingGEPsAcrossIndirectBr()
8218 if (!GEPI->isInBounds()) { in tryUnmergingGEPsAcrossIndirectBr()
8219 UGEPI->setIsInBounds(false); in tryUnmergingGEPsAcrossIndirectBr()
8224 assert(llvm::none_of(GEPIOp->users(), in tryUnmergingGEPsAcrossIndirectBr()
8226 return cast<Instruction>(Usr)->getParent() != SrcBlock; in tryUnmergingGEPsAcrossIndirectBr()
8245 if (!TLI.preferZeroCompareBranch() || !Branch->isConditional()) in optimizeBranch()
8248 ICmpInst *Cmp = dyn_cast<ICmpInst>(Branch->getCondition()); in optimizeBranch()
8249 if (!Cmp || !isa<ConstantInt>(Cmp->getOperand(1)) || !Cmp->hasOneUse()) in optimizeBranch()
8252 Value *X = Cmp->getOperand(0); in optimizeBranch()
8253 APInt CmpC = cast<ConstantInt>(Cmp->getOperand(1))->getValue(); in optimizeBranch()
8255 for (auto *U : X->users()) { in optimizeBranch()
8259 (UI->getParent() != Branch->getParent() && in optimizeBranch()
8260 UI->getParent() != Branch->getSuccessor(0) && in optimizeBranch()
8261 UI->getParent() != Branch->getSuccessor(1)) || in optimizeBranch()
8262 (UI->getParent() != Branch->getParent() && in optimizeBranch()
8263 !UI->getParent()->getSinglePredecessor())) in optimizeBranch()
8266 if (CmpC.isPowerOf2() && Cmp->getPredicate() == ICmpInst::ICMP_ULT && in optimizeBranch()
8269 if (UI->getParent() != Branch->getParent()) in optimizeBranch()
8270 UI->moveBefore(Branch); in optimizeBranch()
8271 UI->dropPoisonGeneratingFlags(); in optimizeBranch()
8273 ConstantInt::get(UI->getType(), 0)); in optimizeBranch()
8279 if (Cmp->isEquality() && in optimizeBranch()
8280 (match(UI, m_Add(m_Specific(X), m_SpecificInt(-CmpC))) || in optimizeBranch()
8283 if (UI->getParent() != Branch->getParent()) in optimizeBranch()
8284 UI->moveBefore(Branch); in optimizeBranch()
8285 UI->dropPoisonGeneratingFlags(); in optimizeBranch()
8286 Value *NewCmp = Builder.CreateCmp(Cmp->getPredicate(), UI, in optimizeBranch()
8287 ConstantInt::get(UI->getType(), 0)); in optimizeBranch()
8314 P->eraseFromParent(); in optimizeInst()
8327 // want to forward-subst the cast. in optimizeInst()
8328 if (isa<Constant>(CI->getOperand(0))) in optimizeInst()
8336 TLI->optimizeExtendOrTruncateConversion( in optimizeInst()
8337 I, LI->getLoopFor(I->getParent()), *TTI)) in optimizeInst()
8343 if (TLI->getTypeAction(CI->getContext(), in optimizeInst()
8344 TLI->getValueType(*DL, CI->getType())) == in optimizeInst()
8348 if (TLI->optimizeExtendOrTruncateConversion( in optimizeInst()
8349 I, LI->getLoopFor(I->getParent()), *TTI)) in optimizeInst()
8364 LI->setMetadata(LLVMContext::MD_invariant_group, nullptr); in optimizeInst()
8366 unsigned AS = LI->getPointerAddressSpace(); in optimizeInst()
8367 Modified |= optimizeMemoryInst(I, I->getOperand(0), LI->getType(), AS); in optimizeInst()
8374 SI->setMetadata(LLVMContext::MD_invariant_group, nullptr); in optimizeInst()
8375 unsigned AS = SI->getPointerAddressSpace(); in optimizeInst()
8376 return optimizeMemoryInst(I, SI->getOperand(1), in optimizeInst()
8377 SI->getOperand(0)->getType(), AS); in optimizeInst()
8381 unsigned AS = RMW->getPointerAddressSpace(); in optimizeInst()
8382 return optimizeMemoryInst(I, RMW->getPointerOperand(), RMW->getType(), AS); in optimizeInst()
8386 unsigned AS = CmpX->getPointerAddressSpace(); in optimizeInst()
8387 return optimizeMemoryInst(I, CmpX->getPointerOperand(), in optimizeInst()
8388 CmpX->getCompareOperand()->getType(), AS); in optimizeInst()
8393 if (BinOp && BinOp->getOpcode() == Instruction::And && EnableAndCmpSinking && in optimizeInst()
8397 // TODO: Move this into the switch on opcode - it handles shifts already. in optimizeInst()
8398 if (BinOp && (BinOp->getOpcode() == Instruction::AShr || in optimizeInst()
8399 BinOp->getOpcode() == Instruction::LShr)) { in optimizeInst()
8400 ConstantInt *CI = dyn_cast<ConstantInt>(BinOp->getOperand(1)); in optimizeInst()
8401 if (CI && TLI->hasExtractBitsInsn()) in optimizeInst()
8407 if (GEPI->hasAllZeroIndices()) { in optimizeInst()
8408 /// The GEP operand must be a pointer, so must its result -> BitCast in optimizeInst()
8409 Instruction *NC = new BitCastInst(GEPI->getOperand(0), GEPI->getType(), in optimizeInst()
8410 GEPI->getName(), GEPI->getIterator()); in optimizeInst()
8411 NC->setDebugLoc(GEPI->getDebugLoc()); in optimizeInst()
8426 // freeze(icmp a, const)) -> icmp (freeze a), const in optimizeInst()
8429 if (ICmpInst *II = dyn_cast<ICmpInst>(FI->getOperand(0))) in optimizeInst()
8431 else if (FCmpInst *F = dyn_cast<FCmpInst>(FI->getOperand(0))) in optimizeInst()
8432 CmpI = F->getFastMathFlags().none() ? F : nullptr; in optimizeInst()
8434 if (CmpI && CmpI->hasOneUse()) { in optimizeInst()
8435 auto Op0 = CmpI->getOperand(0), Op1 = CmpI->getOperand(1); in optimizeInst()
8442 auto *F = new FreezeInst(Const0 ? Op1 : Op0, "", CmpI->getIterator()); in optimizeInst()
8443 F->takeName(FI); in optimizeInst()
8444 CmpI->setOperand(Const0 ? 1 : 0, F); in optimizeInst()
8447 FI->eraseFromParent(); in optimizeInst()
8457 switch (I->getOpcode()) { in optimizeInst()
8482 if (!I.getType()->isIntegerTy() || in makeBitReverse()
8483 !TLI->isOperationLegalOrCustom(ISD::BITREVERSE, in makeBitReverse()
8484 TLI->getValueType(*DL, I.getType(), true))) in makeBitReverse()
8499 // across basic blocks and rewrite them to improve basic-block-at-a-time
8512 // opportunities in the BB. So we go back to the BB head to re-optimize in optimizeBlock()
8573 // FIXME: should updating debug-info really cause the "changed" flag to fire,
8601 DVI->removeFromParent(); in DbgInserterHelper()
8603 DVI->insertBefore(&*VI->getParent()->getFirstInsertionPt()); in DbgInserterHelper()
8605 DVI->insertAfter(VI); in DbgInserterHelper()
8609 DVR->removeFromParent(); in DbgInserterHelper()
8610 BasicBlock *VIBB = VI->getParent(); in DbgInserterHelper()
8612 VIBB->insertDbgRecordBefore(DVR, VIBB->getFirstInsertionPt()); in DbgInserterHelper()
8614 VIBB->insertDbgRecordAfter(DVR, VI); in DbgInserterHelper()
8621 // to re-order dbg.value intrinsics.
8628 for (Value *V : DbgItem->location_ops()) in placeDbgValues()
8637 if (VI->isTerminator()) in placeDbgValues()
8642 if (isa<PHINode>(VI) && VI->getParent()->getTerminator()->isEHPad()) in placeDbgValues()
8658 DbgItem->setKillLocation(); in placeDbgValues()
8701 while (FirstInst != Block.end() && FirstInst->isDebugOrPseudoInst()) in placePseudoProbes()
8707 II->moveBefore(&*FirstInst); in placePseudoProbes()
8746 if (!TM->Options.EnableFastISel || TLI->isJumpExpensive()) in splitBranchCondition()
8763 if (Br1->getMetadata(LLVMContext::MD_unpredictable)) in splitBranchCondition()
8801 Br1->setCondition(Cond1); in splitBranchCondition()
8802 LogicOp->eraseFromParent(); in splitBranchCondition()
8807 Br1->setSuccessor(0, TmpBB); in splitBranchCondition()
8809 Br1->setSuccessor(1, TmpBB); in splitBranchCondition()
8814 I->removeFromParent(); in splitBranchCondition()
8815 I->insertBefore(Br2); in splitBranchCondition()
8831 TBB->replacePhiUsesWith(&BB, TmpBB); in splitBranchCondition()
8834 for (PHINode &PN : FBB->phis()) { in splitBranchCondition()
8866 Br1->setMetadata(LLVMContext::MD_prof, in splitBranchCondition()
8867 MDBuilder(Br1->getContext()) in splitBranchCondition()
8874 Br2->setMetadata(LLVMContext::MD_prof, in splitBranchCondition()
8875 MDBuilder(Br2->getContext()) in splitBranchCondition()
8902 Br1->setMetadata(LLVMContext::MD_prof, in splitBranchCondition()
8903 MDBuilder(Br1->getContext()) in splitBranchCondition()
8909 Br2->setMetadata(LLVMContext::MD_prof, in splitBranchCondition()
8910 MDBuilder(Br2->getContext()) in splitBranchCondition()
8919 TmpBB->dump()); in splitBranchCondition()