Lines Matching +full:invert +full:- +full:ext
1 //===-- lib/CodeGen/GlobalISel/GICombinerHelper.cpp -----------------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
44 #define DEBUG_TYPE "gi-combiner"
52 ForceLegalIndexing("force-legal-indexing", cl::Hidden, cl::init(false),
64 (void)this->KB; in CombinerHelper()
71 /// \returns The little endian in-memory byte position of byte \p I in a
74 /// E.g. Given a 4-byte type x, x[0] -> byte 0
80 /// Determines the LogBase2 value for a non-null input value using the
81 /// transform: LogBase2(V) = (EltBits - 1) - ctlz(V).
86 auto Base = MIB.buildConstant(Ty, Ty.getScalarSizeInBits() - 1); in buildLogBase2()
90 /// \returns The big endian in-memory byte position of byte \p I in a
93 /// E.g. Given a 4-byte type x, x[0] -> byte 3
96 return ByteWidth - I - 1; in bigEndianByteAt()
108 /// E.g. given a 32-bit type x, and x[AddrOffset], the in-memory byte patterns
128 const int64_t Idx = MemOffsetAndIdx->second - LowestIdx; in isBigEndian()
129 assert(Idx >= 0 && "Expected non-negative byte offset?"); in isBigEndian()
145 return LI->getAction(Query).Action == LegalizeActions::Legal; in isLegal()
197 return RBI->getRegBank(Reg, MRI, *TRI); in getRegBank()
236 // Even if only a single operand of the PHI is not guaranteed non-poison, in matchFreezeOfSingleMaybePoisonOperand()
244 if (OrigDef->isPHI() || isa<GUnmerge>(OrigDef)) in matchFreezeOfSingleMaybePoisonOperand()
252 for (MachineOperand &Operand : OrigDef->uses()) { in matchFreezeOfSingleMaybePoisonOperand()
262 // We have more than one maybe-poison operand. Moving the freeze is in matchFreezeOfSingleMaybePoisonOperand()
268 // Eliminate freeze if all operands are guaranteed non-poison. in matchFreezeOfSingleMaybePoisonOperand()
272 cast<GenericMachineInstr>(OrigDef)->dropPoisonGeneratingFlags(); in matchFreezeOfSingleMaybePoisonOperand()
279 Register MaybePoisonOperandReg = MaybePoisonOperand->getReg(); in matchFreezeOfSingleMaybePoisonOperand()
284 cast<GenericMachineInstr>(OrigDef)->dropPoisonGeneratingFlags(); in matchFreezeOfSingleMaybePoisonOperand()
286 B.setInsertPt(*OrigDef->getParent(), OrigDef->getIterator()); in matchFreezeOfSingleMaybePoisonOperand()
289 MRI, *OrigDef->findRegisterUseOperand(MaybePoisonOperandReg, TRI), in matchFreezeOfSingleMaybePoisonOperand()
312 switch (Def->getOpcode()) { in matchCombineConcatVectors()
316 // them into the yet-to-build flattened concat vectors. in matchCombineConcatVectors()
317 for (const MachineOperand &BuildVecMO : Def->uses()) in matchCombineConcatVectors()
327 assert(MRI.getType(Undef->getOperand(0).getReg()) == in matchCombineConcatVectors()
334 Ops.push_back(Undef->getOperand(0).getReg()); in matchCombineConcatVectors()
387 if (MRI.getType(ConcatMI1->getSourceReg(0)) != in matchCombineShuffleConcat()
388 MRI.getType(ConcatMI2->getSourceReg(0))) in matchCombineShuffleConcat()
391 LLT ConcatSrcTy = MRI.getType(ConcatMI1->getReg(1)); in matchCombineShuffleConcat()
397 if (Mask[i] == -1) { in matchCombineShuffleConcat()
401 if (Mask[i + j] != -1) in matchCombineShuffleConcat()
418 Ops.push_back(ConcatMI1->getSourceReg(Mask[i] / ConcatSrcNumElt)); in matchCombineShuffleConcat()
420 Ops.push_back(ConcatMI2->getSourceReg(Mask[i] / ConcatSrcNumElt - in matchCombineShuffleConcat()
421 ConcatMI1->getNumSources())); in matchCombineShuffleConcat()
505 SmallVector<int, 8> ConcatSrcs(NumConcat, -1); in matchCombineShuffleVector()
575 I -= Src1NumElts; in applyShuffleToExtract()
619 // Prefer sign extensions to zero extensions as sign-extensions tend to be in ChoosePreferredUse()
620 // more expensive. Don't do this if the load is already a zero-extend load in ChoosePreferredUse()
621 // though, otherwise we'll rewrite a zero-extend load into a sign-extend in ChoosePreferredUse()
635 // registers and this choice potentially increases the live-range for the in ChoosePreferredUse()
661 InsertBB = PredBB->getMBB(); in InsertInsnsWithoutSideEffectsBeforeUse()
673 Inserter(InsertBB, InsertBB->getFirstNonPHI(), UseMO); in InsertInsnsWithoutSideEffectsBeforeUse()
716 Register LoadReg = LoadMI->getDstReg(); in matchCombineExtendingLoads()
730 // For non power-of-2 types, they will very likely be legalized into multiple in matchCombineExtendingLoads()
735 // Find the preferred type aside from the any-extends (unless it's the only in matchCombineExtendingLoads()
736 // one) and non-extending ops. We'll emit an extending load to that type and in matchCombineExtendingLoads()
749 const auto &MMO = LoadMI->getMMO(); in matchCombineExtendingLoads()
758 LLT SrcTy = MRI.getType(LoadMI->getPointerReg()); in matchCombineExtendingLoads()
759 if (LI->getAction({CandidateLoadOpc, {UseTy, SrcTy}, {MMDesc}}) in matchCombineExtendingLoads()
783 Register ChosenDstReg = Preferred.MI->getOperand(0).getReg(); in applyCombineExtendingLoads()
794 UseMO.setReg(PreviouslyEmitted->getOperand(0).getReg()); in applyCombineExtendingLoads()
817 MachineInstr *UseMI = UseMO->getParent(); in applyCombineExtendingLoads()
821 if (UseMI->getOpcode() == Preferred.ExtendOpcode || in applyCombineExtendingLoads()
822 UseMI->getOpcode() == TargetOpcode::G_ANYEXT) { in applyCombineExtendingLoads()
823 Register UseDstReg = UseMI->getOperand(0).getReg(); in applyCombineExtendingLoads()
824 MachineOperand &UseSrcMO = UseMI->getOperand(1); in applyCombineExtendingLoads()
838 Observer.erasingInstr(*UseMO->getParent()); in applyCombineExtendingLoads()
839 UseMO->getParent()->eraseFromParent(); in applyCombineExtendingLoads()
872 Observer.erasingInstr(*UseMO->getParent()); in applyCombineExtendingLoads()
873 UseMO->getParent()->eraseFromParent(); in applyCombineExtendingLoads()
907 APInt MaskVal = MaybeMask->Value; in matchCombineLoadWithAndMask()
916 if (!LoadMI || !MRI.hasOneNonDBGUse(LoadMI->getDstReg())) in matchCombineLoadWithAndMask()
919 Register LoadReg = LoadMI->getDstReg(); in matchCombineLoadWithAndMask()
921 Register PtrReg = LoadMI->getPointerReg(); in matchCombineLoadWithAndMask()
923 LocationSize LoadSizeBits = LoadMI->getMemSizeInBits(); in matchCombineLoadWithAndMask()
926 // The mask may not be larger than the in-memory type, as it might cover sign in matchCombineLoadWithAndMask()
936 // Most targets cannot deal with loads of size < 8 and need to re-legalize to in matchCombineLoadWithAndMask()
941 const MachineMemOperand &MMO = LoadMI->getMMO(); in matchCombineLoadWithAndMask()
946 if (LoadMI->isSimple()) in matchCombineLoadWithAndMask()
963 LoadMI->eraseFromParent(); in matchCombineLoadWithAndMask()
989 return MDT->dominates(&DefMI, &UseMI); in dominates()
1013 auto LoadSizeBits = LoadMI->getMemSizeInBits(); in matchSextTruncSextLoad()
1045 uint64_t MemBits = LoadDef->getMemSizeInBits().getValue(); in matchSextInRegOfLoad()
1055 // Don't bother creating a non-power-2 sextload, it will likely be broken up in matchSextInRegOfLoad()
1060 const MachineMemOperand &MMO = LoadDef->getMMO(); in matchSextInRegOfLoad()
1065 if (LoadDef->isSimple()) in matchSextInRegOfLoad()
1072 {MRI.getType(LoadDef->getDstReg()), in matchSextInRegOfLoad()
1073 MRI.getType(LoadDef->getPointerReg())}, in matchSextInRegOfLoad()
1077 MatchInfo = std::make_tuple(LoadDef->getDstReg(), NewSizeBits); in matchSextInRegOfLoad()
1091 // %ext = G_SEXT_INREG %ld, 8 in applySextInRegOfLoad()
1095 auto &MMO = LoadDef->getMMO(); in applySextInRegOfLoad()
1101 LoadDef->getPointerReg(), *NewMMO); in applySextInRegOfLoad()
1110 auto *MF = MI->getMF(); in canFoldInAddressingMode()
1111 auto *Addr = getOpcodeDef<GPtrAdd>(MI->getPointerReg(), MRI); in canFoldInAddressingMode()
1116 if (auto CstOff = getIConstantVRegVal(Addr->getOffsetReg(), MRI)) in canFoldInAddressingMode()
1117 AM.BaseOffs = CstOff->getSExtValue(); // [reg +/- imm] in canFoldInAddressingMode()
1119 AM.Scale = 1; // [reg +/- reg] in canFoldInAddressingMode()
1122 MF->getDataLayout(), AM, in canFoldInAddressingMode()
1123 getTypeForLLT(MI->getMMO().getMemoryType(), in canFoldInAddressingMode()
1124 MF->getFunction().getContext()), in canFoldInAddressingMode()
1125 MI->getMMO().getAddrSpace()); in canFoldInAddressingMode()
1163 "post-index-use-threshold", cl::Hidden, cl::init(32),
1165 "considered for post-indexing."));
1173 // %offset:_(s64) = G_CONSTANT i64 -256 in findPostIndexCandidate()
1199 if (!PtrAdd || MRI.use_nodbg_empty(PtrAdd->getReg(0))) in findPostIndexCandidate()
1207 Offset = PtrAdd->getOffsetReg(); in findPostIndexCandidate()
1209 !TLI.isIndexingLegal(LdSt, PtrAdd->getBaseReg(), Offset, in findPostIndexCandidate()
1219 if (OffsetDef->getOpcode() != TargetOpcode::G_CONSTANT) in findPostIndexCandidate()
1224 for (auto &BasePtrUse : MRI.use_nodbg_instructions(PtrAdd->getBaseReg())) { in findPostIndexCandidate()
1228 // If the user is a later load/store that can be post-indexed, then don't in findPostIndexCandidate()
1239 Register PtrAddDefReg = BasePtrUseDef->getReg(0); in findPostIndexCandidate()
1255 Addr = PtrAdd->getReg(0); in findPostIndexCandidate()
1256 Base = PtrAdd->getBaseReg(); in findPostIndexCandidate()
1265 auto &MF = *LdSt.getParent()->getParent(); in findPreIndexCandidate()
1281 if (BaseDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) in findPreIndexCandidate()
1286 if (Base == St->getValueReg()) in findPreIndexCandidate()
1291 if (St->getValueReg() == Addr) in findPreIndexCandidate()
1295 // Avoid increasing cross-block register pressure. in findPreIndexCandidate()
1338 if (!LoadMI->isSimple()) in matchCombineExtractedVectorLoad()
1348 if (MI.getParent() != LoadMI->getParent()) in matchCombineExtractedVectorLoad()
1352 for (auto II = LoadMI->getIterator(), IE = MI.getIterator(); II != IE; ++II) { in matchCombineExtractedVectorLoad()
1353 if (II->isLoadFoldBarrier()) in matchCombineExtractedVectorLoad()
1360 // if we are in the post-legalization phase. in matchCombineExtractedVectorLoad()
1361 MachineMemOperand MMO = LoadMI->getMMO(); in matchCombineExtractedVectorLoad()
1371 int Elt = CVal->getZExtValue(); in matchCombineExtractedVectorLoad()
1384 Register VecPtr = LoadMI->getPointerReg(); in matchCombineExtractedVectorLoad()
1414 LoadMI->getPointerReg(), MRI.getType(LoadMI->getOperand(0).getReg()), in matchCombineExtractedVectorLoad()
1419 LoadMI->eraseFromParent(); in matchCombineExtractedVectorLoad()
1454 *OldCst->getOperand(1).getCImm()); in applyCombineIndexedLoadStore()
1470 MIB->cloneMemRefs(*MI.getMF(), MI); in applyCombineIndexedLoadStore()
1548 DestRemReg = OtherMI->getOperand(0).getReg(); in applyCombineDivRem()
1550 DestDivReg = OtherMI->getOperand(0).getReg(); in applyCombineDivRem()
1557 // Check which instruction is first in the block so we don't break def-use in applyCombineDivRem()
1559 // instruction is first so we pick it's operands, avoiding use-before-def in applyCombineDivRem()
1567 { FirstInst->getOperand(1), FirstInst->getOperand(2) }); in applyCombineDivRem()
1569 OtherMI->eraseFromParent(); in applyCombineDivRem()
1591 if (BrIt == MBB->begin()) in matchOptBrCondByInvertingCond()
1593 assert(std::next(BrIt) == MBB->end() && "expected G_BR to be a terminator"); in matchOptBrCondByInvertingCond()
1596 if (BrCond->getOpcode() != TargetOpcode::G_BRCOND) in matchOptBrCondByInvertingCond()
1601 MachineBasicBlock *BrCondTarget = BrCond->getOperand(1).getMBB(); in matchOptBrCondByInvertingCond()
1603 MBB->isLayoutSuccessor(BrCondTarget); in matchOptBrCondByInvertingCond()
1610 LLT Ty = MRI.getType(BrCond->getOperand(0).getReg()); in applyOptBrCondByInvertingCond()
1616 auto Xor = Builder.buildXor(Ty, BrCond->getOperand(0), True); in applyOptBrCondByInvertingCond()
1618 auto *FallthroughBB = BrCond->getOperand(1).getMBB(); in applyOptBrCondByInvertingCond()
1626 BrCond->getOperand(0).setReg(Xor.getReg(0)); in applyOptBrCondByInvertingCond()
1627 BrCond->getOperand(1).setMBB(BrTarget); in applyOptBrCondByInvertingCond()
1695 APFloat Folded = constantFoldFpUnary(MI, MRI, Cst->getValue()); in applyCombineConstantFoldFpUnary()
1706 // --> in matchPtrAddImmedChain()
1719 if (!Add2Def || Add2Def->getOpcode() != TargetOpcode::G_PTR_ADD) in matchPtrAddImmedChain()
1722 Register Base = Add2Def->getOperand(1).getReg(); in matchPtrAddImmedChain()
1723 Register Imm2 = Add2Def->getOperand(2).getReg(); in matchPtrAddImmedChain()
1736 AccessTy = getTypeForLLT(MRI.getType(LdSt->getReg(0)), in matchPtrAddImmedChain()
1742 APInt CombinedImm = MaybeImmVal->Value + MaybeImm2Val->Value; in matchPtrAddImmedChain()
1747 AMOld.BaseOffs = MaybeImmVal->Value.getSExtValue(); in matchPtrAddImmedChain()
1782 // --> in matchShiftImmedChain()
1798 if (Shl2Def->getOpcode() != Opcode) in matchShiftImmedChain()
1801 Register Base = Shl2Def->getOperand(1).getReg(); in matchShiftImmedChain()
1802 Register Imm2 = Shl2Def->getOperand(2).getReg(); in matchShiftImmedChain()
1809 (MaybeImmVal->Value.getZExtValue() + MaybeImm2Val->Value).getZExtValue(); in matchShiftImmedChain()
1842 Imm = ScalarSizeInBits - 1; in applyShiftImmedChain()
1861 // --> in matchShiftOfShiftedLogic()
1873 // Match a one-use bitwise logic op. in matchShiftOfShiftedLogic()
1879 unsigned LogicOpcode = LogicMI->getOpcode(); in matchShiftOfShiftedLogic()
1884 // Find a matching one-use shift by constant. in matchShiftOfShiftedLogic()
1887 if (!MaybeImmVal || MaybeImmVal->Value == 0) in matchShiftOfShiftedLogic()
1890 const uint64_t C1Val = MaybeImmVal->Value.getZExtValue(); in matchShiftOfShiftedLogic()
1893 // Shift should match previous one and should be a one-use. in matchShiftOfShiftedLogic()
1894 if (MI->getOpcode() != ShiftOpcode || in matchShiftOfShiftedLogic()
1895 !MRI.hasOneNonDBGUse(MI->getOperand(0).getReg())) in matchShiftOfShiftedLogic()
1900 getIConstantVRegValWithLookThrough(MI->getOperand(2).getReg(), MRI); in matchShiftOfShiftedLogic()
1904 ShiftVal = MaybeImmVal->Value.getSExtValue(); in matchShiftOfShiftedLogic()
1909 Register LogicMIReg1 = LogicMI->getOperand(1).getReg(); in matchShiftOfShiftedLogic()
1911 Register LogicMIReg2 = LogicMI->getOperand(2).getReg(); in matchShiftOfShiftedLogic()
1947 Register Shift1Base = MatchInfo.Shift2->getOperand(1).getReg(); in applyShiftOfShiftedLogic()
1956 MatchInfo.Shift2->eraseFromParent(); in applyShiftOfShiftedLogic()
1965 Builder.buildInstr(MatchInfo.Logic->getOpcode(), {Dest}, {Shift1, Shift2}); in applyShiftOfShiftedLogic()
1968 MatchInfo.Logic->eraseFromParent(); in applyShiftOfShiftedLogic()
1975 // Combine (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2) in matchCommuteShift()
1976 // Combine (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2) in matchCommuteShift()
1997 assert((SrcDef->getOpcode() == TargetOpcode::G_ADD || in matchCommuteShift()
1998 SrcDef->getOpcode() == TargetOpcode::G_OR) && "Unexpected op"); in matchCommuteShift()
2003 B.buildInstr(SrcDef->getOpcode(), {DstReg}, {S1, S2}); in matchCommuteShift()
2016 ShiftVal = MaybeImmVal->Value.exactLogBase2(); in matchCombineMulToShl()
2017 return (static_cast<int32_t>(ShiftVal) != -1); in matchCombineMulToShl()
2032 // shl ([sza]ext x), y => zext (shl x, y), if shift does not overflow source
2064 int64_t ShiftAmt = MaybeShiftAmtVal->getSExtValue(); in matchCombineShlOfExtend()
2068 unsigned MinLeadingZeros = KB->getKnownZeroes(ExtSrc).countl_one(); in matchCombineShlOfExtend()
2094 if (!Unmerge || Unmerge->getNumDefs() != Merge.getNumSources()) in matchCombineMergeUnmerge()
2098 if (MergedValues[I] != Unmerge->getReg(I)) in matchCombineMergeUnmerge()
2101 MatchInfo = Unmerge->getSourceReg(); in matchCombineMergeUnmerge()
2125 LLT SrcMergeTy = MRI.getType(SrcInstr->getSourceReg(0)); in matchCombineUnmergeMergeToPlainValues()
2132 for (unsigned Idx = 0; Idx < SrcInstr->getNumSources(); ++Idx) in matchCombineUnmergeMergeToPlainValues()
2133 Operands.push_back(SrcInstr->getSourceReg(Idx)); in matchCombineUnmergeMergeToPlainValues()
2141 assert((MI.getNumOperands() - 1 == Operands.size()) && in applyCombineUnmergeMergeToPlainValues()
2143 unsigned NumElems = MI.getNumOperands() - 1; in applyCombineUnmergeMergeToPlainValues()
2170 unsigned SrcIdx = MI.getNumOperands() - 1; in matchCombineUnmergeConstant()
2173 if (SrcInstr->getOpcode() != TargetOpcode::G_CONSTANT && in matchCombineUnmergeConstant()
2174 SrcInstr->getOpcode() != TargetOpcode::G_FCONSTANT) in matchCombineUnmergeConstant()
2177 const MachineOperand &CstVal = SrcInstr->getOperand(1); in matchCombineUnmergeConstant()
2178 APInt Val = SrcInstr->getOpcode() == TargetOpcode::G_CONSTANT in matchCombineUnmergeConstant()
2179 ? CstVal.getCImm()->getValue() in matchCombineUnmergeConstant()
2180 : CstVal.getFPImm()->getValueAPF().bitcastToAPInt(); in matchCombineUnmergeConstant()
2197 assert((MI.getNumOperands() - 1 == Csts.size()) && in applyCombineUnmergeConstant()
2199 unsigned NumElems = MI.getNumOperands() - 1; in applyCombineUnmergeConstant()
2210 unsigned SrcIdx = MI.getNumOperands() - 1; in matchCombineUnmergeUndef()
2213 unsigned NumElems = MI.getNumOperands() - 1; in matchCombineUnmergeUndef()
2277 assert(ZExtInstr && ZExtInstr->getOpcode() == TargetOpcode::G_ZEXT && in applyCombineUnmergeZExtToZExt()
2280 Register ZExtSrcReg = ZExtInstr->getOperand(1).getReg(); in applyCombineUnmergeZExtToZExt()
2322 ShiftVal = MaybeImmVal->Value.getSExtValue(); in matchCombineShiftToUnmerge()
2338 unsigned NarrowShiftAmt = ShiftVal - HalfSize; in applyCombineShiftToUnmerge()
2346 // dst = G_MERGE_VALUES (G_LSHR hi, C - 32), 0 in applyCombineShiftToUnmerge()
2360 // dst = G_MERGE_VALUES 0, (G_SHL hi, C - 32) in applyCombineShiftToUnmerge()
2372 Builder.buildConstant(HalfTy, HalfSize - 1)); in applyCombineShiftToUnmerge()
2375 // (G_ASHR i64:x, 32) -> in applyCombineShiftToUnmerge()
2378 } else if (ShiftVal == Size - 1) { in applyCombineShiftToUnmerge()
2380 // (G_ASHR i64:x, 63) -> in applyCombineShiftToUnmerge()
2387 Builder.buildConstant(HalfTy, ShiftVal - HalfSize)); in applyCombineShiftToUnmerge()
2389 // (G_ASHR i64:x, C) ->, for C >= 32 in applyCombineShiftToUnmerge()
2390 // G_MERGE_VALUES (G_ASHR hi_32(x), C - 32), (G_ASHR hi_32(x), 31) in applyCombineShiftToUnmerge()
2486 // G_INTTOPTR uses zero-extension in matchCombineConstPtrAddToI2P()
2488 NewCst += RHSCst->sextOrTrunc(DstTy.getSizeInBits()); in matchCombineConstPtrAddToI2P()
2526 return KB->getKnownBits(Reg).countMinLeadingZeros() >= DstSize - SrcSize; in matchCombineZextTrunc()
2536 "Expected a G_[ASZ]EXT"); in matchCombineExtOfExt()
2542 // Match exts with the same opcode, anyext([sz]ext) and sext(zext). in matchCombineExtOfExt()
2544 unsigned SrcOpc = SrcMI->getOpcode(); in matchCombineExtOfExt()
2549 MatchInfo = std::make_tuple(SrcMI->getOperand(1).getReg(), SrcOpc); in matchCombineExtOfExt()
2560 "Expected a G_[ASZ]EXT"); in applyCombineExtOfExt()
2574 // - anyext([sz]ext x) to [sz]ext x in applyCombineExtOfExt()
2575 // - sext(zext x) to zext x in applyCombineExtOfExt()
2590 unsigned SrcOpc = SrcMI->getOpcode(); in matchCombineTruncOfExt()
2593 MatchInfo = std::make_pair(SrcMI->getOperand(1).getReg(), SrcOpc); in matchCombineTruncOfExt()
2623 // ShiftTy > 32 > TruncTy -> 32 in getMidVTForTruncRightShiftCombine()
2627 // TODO: We could also reduce to 16 bits, but that's more target-dependent. in getMidVTForTruncRightShiftCombine()
2652 switch (SrcMI->getOpcode()) { in matchCombineTruncOfShift()
2659 KnownBits Known = KB->getKnownBits(SrcMI->getOperand(2).getReg()); in matchCombineTruncOfShift()
2680 KnownBits Known = KB->getKnownBits(SrcMI->getOperand(2).getReg()); in matchCombineTruncOfShift()
2681 if (Known.getMaxValue().ugt(NewShiftTy.getScalarSizeInBits() - in matchCombineTruncOfShift()
2689 {SrcMI->getOpcode(), in matchCombineTruncOfShift()
2705 Register ShiftAmt = ShiftMI->getOperand(2).getReg(); in applyCombineTruncOfShift()
2706 Register ShiftSrc = ShiftMI->getOperand(1).getReg(); in applyCombineTruncOfShift()
2711 .buildInstr(ShiftMI->getOpcode(), {NewShiftTy}, {ShiftSrc, ShiftAmt}) in applyCombineTruncOfShift()
2764 return Idx->getZExtValue() >= VecTy.getNumElements(); in matchInsertExtractVecEltOutOfBounds()
2773 OpIdx = Cst->isZero() ? 3 : 2; in matchConstantSelectCmp()
2789 MachineInstr *I1 = InstAndDef1->MI; in matchEqualDefs()
2790 MachineInstr *I2 = InstAndDef2->MI; in matchEqualDefs()
2818 if (I1->mayLoadOrStore() && !I1->isDereferenceableInvariantLoad()) in matchEqualDefs()
2823 if (I1->mayLoadOrStore() && I2->mayLoadOrStore()) { in matchEqualDefs()
2829 if (!I2->isDereferenceableInvariantLoad() || in matchEqualDefs()
2830 (LS1->getMemSizeInBits() != LS2->getMemSizeInBits())) in matchEqualDefs()
2839 // SOMETHING implicit-def $physreg in matchEqualDefs()
2844 if (any_of(I1->uses(), [](const MachineOperand &MO) { in matchEqualDefs()
2855 return I1->isIdenticalTo(*I2); in matchEqualDefs()
2861 // On the off-chance that there's some target instruction feeding into the in matchEqualDefs()
2870 return I1->findRegisterDefOperandIdx(InstAndDef1->Reg, /*TRI=*/nullptr) == in matchEqualDefs()
2871 I2->findRegisterDefOperandIdx(InstAndDef2->Reg, /*TRI=*/nullptr); in matchEqualDefs()
2881 return MaybeCst && MaybeCst->getBitWidth() <= 64 && in matchConstantOp()
2882 MaybeCst->getSExtValue() == C; in matchConstantOp()
2892 return MaybeCst->Value.isExactlyValue(C); in matchConstantFPOp()
2925 return (VRegAndVal->Value.uge(DstTy.getSizeInBits())); in matchConstantLargerBitWidth()
2941 APInt NewConst = VRegAndVal->Value.urem( in applyFunnelShiftConstantModulo()
3005 Builder.buildFConstant(MI.getOperand(0), CFP->getValueAPF()); in replaceInstWithFConstant()
3023 // ((0-A) + B) -> B - A in matchSimplifyAddToSub()
3024 // (A + (0-B)) -> A - B in matchSimplifyAddToSub()
3045 if (MRI.hasOneUse(DstReg) && MRI.use_instr_begin(DstReg)->getOpcode() == in matchCombineInsertVecElts()
3054 CurrInst->getOperand(0).getReg(), MRI, in matchCombineInsertVecElts()
3063 if (CurrInst->getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT) in matchCombineInsertVecElts()
3065 if (TmpInst->getOpcode() == TargetOpcode::G_BUILD_VECTOR) { in matchCombineInsertVecElts()
3066 for (unsigned I = 1; I < TmpInst->getNumOperands(); ++I) { in matchCombineInsertVecElts()
3067 if (!MatchInfo[I - 1].isValid()) in matchCombineInsertVecElts()
3068 MatchInfo[I - 1] = TmpInst->getOperand(I).getReg(); in matchCombineInsertVecElts()
3074 return TmpInst->getOpcode() == TargetOpcode::G_IMPLICIT_DEF || in matchCombineInsertVecElts()
3106 // Matches: logic (hand x, ...), (hand y, ...) -> hand (logic x, y), ... in matchHoistLogicOpWithSameOpcodeHands()
3130 unsigned HandOpcode = LeftHandInst->getOpcode(); in matchHoistLogicOpWithSameOpcodeHands()
3131 if (HandOpcode != RightHandInst->getOpcode()) in matchHoistLogicOpWithSameOpcodeHands()
3133 if (!LeftHandInst->getOperand(1).isReg() || in matchHoistLogicOpWithSameOpcodeHands()
3134 !RightHandInst->getOperand(1).isReg()) in matchHoistLogicOpWithSameOpcodeHands()
3137 // Make sure the types match up, and if we're doing this post-legalization, in matchHoistLogicOpWithSameOpcodeHands()
3139 Register X = LeftHandInst->getOperand(1).getReg(); in matchHoistLogicOpWithSameOpcodeHands()
3140 Register Y = RightHandInst->getOperand(1).getReg(); in matchHoistLogicOpWithSameOpcodeHands()
3154 // Match: logic (ext X), (ext Y) --> ext (logic X, Y) in matchHoistLogicOpWithSameOpcodeHands()
3158 // Match: logic (trunc X), (trunc Y) -> trunc (logic X, Y) in matchHoistLogicOpWithSameOpcodeHands()
3160 const DataLayout &DL = MF->getDataLayout(); in matchHoistLogicOpWithSameOpcodeHands()
3161 LLVMContext &Ctx = MF->getFunction().getContext(); in matchHoistLogicOpWithSameOpcodeHands()
3177 // Match: logic (binop x, z), (binop y, z) -> binop (logic x, y), z in matchHoistLogicOpWithSameOpcodeHands()
3178 MachineOperand &ZOp = LeftHandInst->getOperand(2); in matchHoistLogicOpWithSameOpcodeHands()
3179 if (!matchEqualDefs(ZOp, RightHandInst->getOperand(2))) in matchHoistLogicOpWithSameOpcodeHands()
3251 Builder.buildSExtInReg(MI.getOperand(0).getReg(), Src, Size - ShiftAmt); in applyAshShlToSextInreg()
3255 /// and(and(x, C1), C2) -> C1&C2 ? and(x, C1&C2) : 0
3277 replaceRegWith(MRI, Dst, Zero->getOperand(0).getReg()); in matchOverlappingAnd()
3310 KnownBits RHSBits = KB->getKnownBits(RHS); in matchRedundantAnd()
3314 KnownBits LHSBits = KB->getKnownBits(LHS); in matchRedundantAnd()
3354 KnownBits LHSBits = KB->getKnownBits(LHS); in matchRedundantOr()
3355 KnownBits RHSBits = KB->getKnownBits(RHS); in matchRedundantOr()
3384 return KB->computeNumSignBits(Src) >= (TypeSize - ExtBits + 1); in matchRedundantSExtInReg()
3389 // For i1, Cst will always be -1 regardless of boolean contents. in isConstValidTrue()
3390 return (ScalarSizeBits == 1 && Cst == -1) || in isConstValidTrue()
3421 switch (Def->getOpcode()) { in matchNotCmp()
3430 // When we apply the combine we will invert the predicate. in matchNotCmp()
3436 // When we apply the combine we will invert the predicate. in matchNotCmp()
3441 // ~(x & y) -> ~x | ~y in matchNotCmp()
3442 // ~(x | y) -> ~x & ~y in matchNotCmp()
3445 RegsToNegate.push_back(Def->getOperand(1).getReg()); in matchNotCmp()
3446 RegsToNegate.push_back(Def->getOperand(2).getReg()); in matchNotCmp()
3476 // For each comparison, invert the opcode. For each AND and OR, change the in applyNotCmp()
3478 switch (Def->getOpcode()) { in applyNotCmp()
3483 MachineOperand &PredOp = Def->getOperand(1); in applyNotCmp()
3490 Def->setDesc(Builder.getTII().get(TargetOpcode::G_OR)); in applyNotCmp()
3493 Def->setDesc(Builder.getTII().get(TargetOpcode::G_AND)); in applyNotCmp()
3536 // Fold (xor (and x, y), y) -> (and (not x), y) in applyXorOfAndWithSameReg()
3542 MI.getOperand(1).setReg(Not->getOperand(0).getReg()); in applyXorOfAndWithSameReg()
3579 // Fold (urem x, pow2) -> (and x, pow2-1) in applySimplifyURemByPow2()
3580 auto NegOne = Builder.buildConstant(Ty, -1); in applySimplifyURemByPow2()
3597 if (Select->getOpcode() != TargetOpcode::G_SELECT || in matchFoldBinOpIntoSelect()
3602 if (Select->getOpcode() != TargetOpcode::G_SELECT || in matchFoldBinOpIntoSelect()
3607 MachineInstr *SelectLHS = MRI.getVRegDef(Select->getOperand(2).getReg()); in matchFoldBinOpIntoSelect()
3608 MachineInstr *SelectRHS = MRI.getVRegDef(Select->getOperand(3).getReg()); in matchFoldBinOpIntoSelect()
3647 Register SelectCond = Select->getOperand(1).getReg(); in applyFoldBinOpIntoSelect()
3648 Register SelectTrue = Select->getOperand(2).getReg(); in applyFoldBinOpIntoSelect()
3649 Register SelectFalse = Select->getOperand(3).getReg(); in applyFoldBinOpIntoSelect()
3656 // We have a select-of-constants followed by a binary operator with a in applyFoldBinOpIntoSelect()
3658 // Example: add (select Cond, CT, CF), CBO --> select Cond, CT + CBO, CF + CBO in applyFoldBinOpIntoSelect()
3678 assert(Root->getOpcode() == TargetOpcode::G_OR && "Expected G_OR only!"); in findCandidatesForLoadOrCombine()
3706 // are at most #bytes - 1 ORs. in findCandidatesForLoadOrCombine()
3708 MRI.getType(Root->getOperand(0).getReg()).getSizeInBytes() - 1; in findCandidatesForLoadOrCombine()
3713 Register OrLHS = Curr->getOperand(1).getReg(); in findCandidatesForLoadOrCombine()
3714 Register OrRHS = Curr->getOperand(2).getReg(); in findCandidatesForLoadOrCombine()
3732 // We're going to try and merge each register into a wider power-of-2 type, in findCandidatesForLoadOrCombine()
3751 "Expected Reg to only have one non-debug use?"); in matchLoadAndBytePosition()
3768 if (!Load->isUnordered() || Load->getMemSizeInBits() != MemSizeInBits) in matchLoadAndBytePosition()
3796 // Earliest instruction-order load in the pattern. in findLoadOffsetsForLoadOrCombine()
3799 // Latest instruction-order load in the pattern. in findLoadOffsetsForLoadOrCombine()
3821 MachineBasicBlock *LoadMBB = Load->getParent(); in findLoadOffsetsForLoadOrCombine()
3828 auto &LoadMMO = Load->getMMO(); in findLoadOffsetsForLoadOrCombine()
3831 if (MMO->getAddrSpace() != LoadMMO.getAddrSpace()) in findLoadOffsetsForLoadOrCombine()
3837 if (!mi_match(Load->getOperand(1).getReg(), MRI, in findLoadOffsetsForLoadOrCombine()
3839 LoadPtr = Load->getOperand(1).getReg(); in findLoadOffsetsForLoadOrCombine()
3843 // Don't combine things like a[i], a[i] -> a bigger load. in findLoadOffsetsForLoadOrCombine()
3849 // a[i], b[i + 1] -> a bigger load. in findLoadOffsetsForLoadOrCombine()
3863 // a[i] << 16, a[i + k] << 16 -> a bigger load. in findLoadOffsetsForLoadOrCombine()
3895 for (const auto &MI : instructionsWithoutDebug(EarliestLoad->getIterator(), in findLoadOffsetsForLoadOrCombine()
3896 LatestLoad->getIterator())) { in findLoadOffsetsForLoadOrCombine()
3912 // Assuming a little-endian target, transform: in matchLoadOrCombine()
3928 // possible load is into a byte, we need at least a 16-bit wide type. in matchLoadOrCombine()
3933 // Match a collection of non-OR instructions in the pattern. in matchLoadOrCombine()
3938 // We have a collection of non-OR instructions. Figure out how wide each of in matchLoadOrCombine()
3941 const unsigned NarrowMemSizeInBits = WideMemSizeInBits / RegsToVisit->size(); in matchLoadOrCombine()
3978 // load x[i] -> byte 2 in matchLoadOrCombine()
3979 // load x[i+1] -> byte 0 ---> wide_load x[i] in matchLoadOrCombine()
3980 // load x[i+2] -> byte 1 in matchLoadOrCombine()
3988 ZeroOffsetIdx->second != LowestIdx) in matchLoadOrCombine()
3993 Register Ptr = LowestIdxLoad->getPointerReg(); in matchLoadOrCombine()
3994 const MachineMemOperand &MMO = LowestIdxLoad->getMMO(); in matchLoadOrCombine()
4035 switch (ExtMI->getOpcode()) { in matchExtendThroughPhis()
4055 switch (DefMI->getOpcode()) { in matchExtendThroughPhis()
4078 Register DstReg = ExtMI->getOperand(0).getReg(); in applyExtendThroughPhis()
4093 auto *MBB = SrcMI->getParent(); in applyExtendThroughPhis()
4094 MachineBasicBlock::iterator InsertPt = ++SrcMI->getIterator(); in applyExtendThroughPhis()
4095 if (InsertPt != MBB->end() && InsertPt->isPHI()) in applyExtendThroughPhis()
4096 InsertPt = MBB->getFirstNonPHI(); in applyExtendThroughPhis()
4098 Builder.setInsertPt(*SrcMI->getParent(), InsertPt); in applyExtendThroughPhis()
4100 auto NewExt = Builder.buildExtOrTrunc(ExtMI->getOpcode(), ExtTy, SrcReg); in applyExtendThroughPhis()
4114 NewPhi.addUse(NewSrc->getOperand(0).getReg()); in applyExtendThroughPhis()
4117 ExtMI->eraseFromParent(); in applyExtendThroughPhis()
4129 if (!Cst || Cst->Value.getZExtValue() >= SrcTy.getNumElements()) in matchExtractVecEltBuildVec()
4132 unsigned VecIdx = Cst->Value.getZExtValue(); in matchExtractVecEltBuildVec()
4137 if (SrcVecMI->getOpcode() == TargetOpcode::G_TRUNC) { in matchExtractVecEltBuildVec()
4138 SrcVecMI = MRI.getVRegDef(SrcVecMI->getOperand(1).getReg()); in matchExtractVecEltBuildVec()
4141 if (SrcVecMI->getOpcode() != TargetOpcode::G_BUILD_VECTOR && in matchExtractVecEltBuildVec()
4142 SrcVecMI->getOpcode() != TargetOpcode::G_BUILD_VECTOR_TRUNC) in matchExtractVecEltBuildVec()
4150 Reg = SrcVecMI->getOperand(VecIdx + 1).getReg(); in matchExtractVecEltBuildVec()
4189 // replace ext{1,2,3,4} with %s{1,2,3,4} in matchExtractAllEltsFromBuildVector()
4202 unsigned Idx = Cst->getZExtValue(); in matchExtractAllEltsFromBuildVector()
4219 replaceRegWith(MRI, ExtMI->getOperand(0).getReg(), Pair.first); in applyExtractAllEltsFromBuildVector()
4220 ExtMI->eraseFromParent(); in applyExtractAllEltsFromBuildVector()
4254 // Given constants C0 and C1 such that C0 + C1 is bit-width: in matchOrShiftToFunnelShift()
4255 // (or (shl x, C0), (lshr y, C1)) -> (fshl x, y, C0) or (fshr x, y, C1) in matchOrShiftToFunnelShift()
4266 // (or (shl x, amt), (lshr y, (sub bw, amt))) -> (fshl x, y, amt) in matchOrShiftToFunnelShift()
4272 // (or (shl x, (sub bw, amt)), (lshr y, amt)) -> (fshr x, y, amt) in matchOrShiftToFunnelShift()
4313 // Fold (rot x, c) -> (rot x, c % BitSize)
4323 OutOfRange |= CI->getValue().uge(Bitsize); in matchRotateOutOfRange()
4352 // - The RHS is unknown: Constants are always on RHS. If the RHS is unknown in matchICmpToTrueFalseKnownBits()
4354 // - The RHS is zero: we don't need to know the LHS to do unsigned <0 and in matchICmpToTrueFalseKnownBits()
4356 auto KnownRHS = KB->getKnownBits(MI.getOperand(3).getReg()); in matchICmpToTrueFalseKnownBits()
4362 // ? uge 0 -> always true in matchICmpToTrueFalseKnownBits()
4363 // ? ult 0 -> always false in matchICmpToTrueFalseKnownBits()
4371 auto KnownLHS = KB->getKnownBits(MI.getOperand(2).getReg()); in matchICmpToTrueFalseKnownBits()
4446 auto KnownLHS = KB->getKnownBits(LHS); in matchICmpToLHSKnownBits()
4505 if (!LI || !LI->isLegalOrCustom({TargetOpcode::G_SBFX, {Ty, ExtractTy}})) in matchBitfieldExtractFromSExtInReg()
4530 Register Dst = And->getReg(0); in matchBitfieldExtractFromAnd()
4535 if (LI && !LI->isLegalOrCustom({TargetOpcode::G_UBFX, {Ty, ExtractTy}})) in matchBitfieldExtractFromAnd()
4541 if (!mi_match(And->getReg(0), MRI, in matchBitfieldExtractFromAnd()
4578 if (!LI || !LI->isLegalOrCustom({ExtrOpcode, {Ty, ExtractTy}})) in matchBitfieldExtractFromShr()
4602 const int64_t Pos = ShrAmt - ShlAmt; in matchBitfieldExtractFromShr()
4603 const int64_t Width = Size - ShrAmt; in matchBitfieldExtractFromShr()
4621 if (LI && !LI->isLegalOrCustom({TargetOpcode::G_UBFX, {Ty, ExtractTy}})) in matchBitfieldExtractFromShrAnd()
4655 const int64_t Width = llvm::countr_one(UMask) - ShrAmt; in matchBitfieldExtractFromShrAnd()
4684 auto C1 = getIConstantVRegVal(Src1Def->getOffsetReg(), MRI); in reassociationCanBreakAddressingModePattern()
4699 unsigned ConvUseOpc = ConvUseMI->getOpcode(); in reassociationCanBreakAddressingModePattern()
4702 Register DefReg = ConvUseMI->getOperand(0).getReg(); in reassociationCanBreakAddressingModePattern()
4706 ConvUseOpc = ConvUseMI->getOpcode(); in reassociationCanBreakAddressingModePattern()
4717 unsigned AS = MRI.getType(LdStMI->getPointerReg()).getAddressSpace(); in reassociationCanBreakAddressingModePattern()
4718 Type *AccessTy = getTypeForLLT(LdStMI->getMMO().getMemoryType(), in reassociationCanBreakAddressingModePattern()
4719 PtrAdd.getMF()->getFunction().getContext()); in reassociationCanBreakAddressingModePattern()
4720 const auto &TLI = *PtrAdd.getMF()->getSubtarget().getTargetLowering(); in reassociationCanBreakAddressingModePattern()
4721 if (!TLI.isLegalAddressingMode(PtrAdd.getMF()->getDataLayout(), AM, in reassociationCanBreakAddressingModePattern()
4727 if (!TLI.isLegalAddressingMode(PtrAdd.getMF()->getDataLayout(), AM, in reassociationCanBreakAddressingModePattern()
4738 // G_PTR_ADD(BASE, G_ADD(X, C)) -> G_PTR_ADD(G_PTR_ADD(BASE, X), C) in matchReassocConstantInnerRHS()
4740 if (RHS->getOpcode() != TargetOpcode::G_ADD) in matchReassocConstantInnerRHS()
4742 auto C2 = getIConstantVRegVal(RHS->getOperand(2).getReg(), MRI); in matchReassocConstantInnerRHS()
4750 Builder.buildPtrAdd(PtrTy, Src1Reg, RHS->getOperand(1).getReg()); in matchReassocConstantInnerRHS()
4753 MI.getOperand(2).setReg(RHS->getOperand(2).getReg()); in matchReassocConstantInnerRHS()
4763 // G_PTR_ADD (G_PTR_ADD X, C), Y) -> (G_PTR_ADD (G_PTR_ADD(X, Y), C) in matchReassocConstantInnerLHS()
4776 LHSPtrAdd->moveBefore(&MI); in matchReassocConstantInnerLHS()
4779 auto NewCst = B.buildConstant(MRI.getType(RHSReg), LHSCstOff->Value); in matchReassocConstantInnerLHS()
4784 LHSPtrAdd->getOperand(2).setReg(RHSReg); in matchReassocConstantInnerLHS()
4794 // G_PTR_ADD(G_PTR_ADD(BASE, C1), C2) -> G_PTR_ADD(BASE, C1+C2) in matchReassocFoldConstantsInSubTree()
4800 Register LHSSrc1 = LHSPtrAdd->getBaseReg(); in matchReassocFoldConstantsInSubTree()
4801 Register LHSSrc2 = LHSPtrAdd->getOffsetReg(); in matchReassocFoldConstantsInSubTree()
4823 // re-association opportunities. in matchReassocPtrAdd()
4825 // G_PTR_ADD(BASE, G_ADD(X, C)) -> G_PTR_ADD(G_PTR_ADD(BASE, X), C) in matchReassocPtrAdd()
4827 // 2) Folding two constants in each sub-tree as long as such folding in matchReassocPtrAdd()
4829 // G_PTR_ADD(G_PTR_ADD(BASE, C1), C2) -> G_PTR_ADD(BASE, C1+C2) in matchReassocPtrAdd()
4832 // G_PTR_ADD (G_PTR_ADD X, C), Y) -> G_PTR_ADD (G_PTR_ADD(X, Y), C) in matchReassocPtrAdd()
4857 if (OpLHSDef->getOpcode() != Opc) in tryReassocBinOp()
4861 Register OpLHSLHS = OpLHSDef->getOperand(1).getReg(); in tryReassocBinOp()
4862 Register OpLHSRHS = OpLHSDef->getOperand(2).getReg(); in tryReassocBinOp()
4871 // (Opc (Opc X, C1), C2) -> (Opc X, (Opc C1, C2)) in tryReassocBinOp()
4879 // Reassociate: (op (op x, c1), y) -> (op (op x, y), c1) in tryReassocBinOp()
4937 ConstantFP::get(MI.getMF()->getFunction().getContext(), *MaybeCst); in matchConstantFoldFPBinOp()
4959 APFloat Op1F = Op1Cst->getValueAPF(); in matchConstantFoldFMA()
4960 Op1F.fusedMultiplyAdd(Op2Cst->getValueAPF(), Op3Cst->getValueAPF(), in matchConstantFoldFMA()
4962 MatchInfo = ConstantFP::get(MI.getMF()->getFunction().getContext(), Op1F); in matchConstantFoldFMA()
4998 // e.g. for 64-bit x, y: in matchNarrowBinopFeedingAnd()
5004 unsigned LHSOpc = LHSInst->getOpcode(); in matchNarrowBinopFeedingAnd()
5021 auto Mask = Cst->Value; in matchNarrowBinopFeedingAnd()
5042 Register BinOpLHS = LHSInst->getOperand(1).getReg(); in matchNarrowBinopFeedingAnd()
5043 Register BinOpRHS = LHSInst->getOperand(2).getReg(); in matchNarrowBinopFeedingAnd()
5049 auto Ext = Builder.buildZExt(WideTy, NarrowBinOp); in matchNarrowBinopFeedingAnd() local
5051 MI.getOperand(1).setReg(Ext.getReg(0)); in matchNarrowBinopFeedingAnd()
5076 // (G_*MULO x, 0) -> 0 + no carry out in matchMulOBy0()
5094 // (G_*ADDE x, y, 0) -> (G_*ADDO x, y) in matchAddEToAddO()
5095 // (G_*SUBE x, y, 0) -> (G_*SUBO x, y) in matchAddEToAddO()
5130 // (x + y) - z -> x (if y == z) in matchSubAddSameReg()
5131 // (x + y) - z -> y (if x == z) in matchSubAddSameReg()
5148 // x - (y + z) -> 0 - y (if x == z) in matchSubAddSameReg()
5149 // x - (y + z) -> 0 - z (if x == y) in matchSubAddSameReg()
5198 APInt Divisor = CI->getValue(); in buildUDivUsingMul()
5235 KB ? KB->getKnownBits(LHS).countMinLeadingZeros() : 0; in buildUDivUsingMul()
5241 const APInt &Divisor = CI->getValue(); in buildUDivUsingMul()
5264 assert((!magics.IsAdd || magics.PreShift == 0) && "Unexpected pre-shift"); in buildUDivUsingMul()
5275 SelNPQ ? APInt::getOneBitSet(EltBits, EltBits - 1) in buildUDivUsingMul()
5298 "Non-build_vector operation should have been a scalar"); in buildUDivUsingMul()
5313 // For vectors we might have a mix of non-NPQ/NPQ paths, so use in buildUDivUsingMul()
5314 // G_UMULH to act as a SRL-by-1 for NPQ, else multiply by zero. in buildUDivUsingMul()
5352 MRI, RHS, [](const Constant *C) { return C && !C->isNullValue(); }); in matchUDivByConst()
5373 MRI, RHS, [](const Constant *C) { return C && !C->isNullValue(); }); in matchUDivByConst()
5378 replaceSingleDefInstWithReg(MI, NewMI->getOperand(0).getReg()); in applyUDivByConst()
5403 MRI, RHS, [](const Constant *C) { return C && !C->isNullValue(); }); in matchSDivByConst()
5412 replaceSingleDefInstWithReg(MI, NewMI->getOperand(0).getReg()); in applySDivByConst()
5442 APInt Divisor = CI->getValue(); in buildSDivUsingMul()
5487 return CI && (CI->getValue().isPowerOf2() || in matchDivByPow2()
5488 (IsSigned && CI->getValue().isNegatedPowerOf2())); in matchDivByPow2()
5509 // %sign = %G_ASHR %lhs, $(bitwidth - 1) in applySDivByPow2()
5527 Ty, LHS, Builder.buildConstant(ShiftAmtTy, BitWidth - 1)); in applySDivByPow2()
5529 // Add (LHS < 0) ? abs2 - 1 : 0; in applySDivByPow2()
5534 // Special case: (sdiv X, 1) -> X in applySDivByPow2()
5535 // Special Case: (sdiv X, -1) -> 0-X in applySDivByPow2()
5537 auto MinusOne = Builder.buildConstant(Ty, -1); in applySDivByPow2()
5574 return CI->getValue().isPowerOf2() && !CI->getValue().isOne(); in matchUMulHToLShr()
5610 // fold (fadd x, fneg(y)) -> (fsub x, y) in matchRedundantNegOperands()
5611 // fold (fadd fneg(y), x) -> (fsub x, y) in matchRedundantNegOperands()
5617 /// fold (fsub x, fneg(y)) -> (fadd x, y) in matchRedundantNegOperands()
5622 // fold (fmul fneg(x), fneg(y)) -> (fmul x, y) in matchRedundantNegOperands()
5623 // fold (fdiv fneg(x), fneg(y)) -> (fdiv x, y) in matchRedundantNegOperands()
5624 // fold (fmad fneg(x), fneg(y), z) -> (fmad x, y, z) in matchRedundantNegOperands()
5625 // fold (fma fneg(x), fneg(y), z) -> (fma x, y, z) in matchRedundantNegOperands()
5657 // -0.0 is always allowed in matchFsubToFneg()
5658 if (LHSCst->Value.isNegZero()) in matchFsubToFneg()
5662 if (LHSCst->Value.isPosZero()) in matchFsubToFneg()
5697 const auto &TLI = *MF->getSubtarget().getTargetLowering(); in canCombineFMadOrFMA()
5698 const TargetOptions &Options = MF->getTarget().Options; in canCombineFMadOrFMA()
5705 // Floating-point multiply-add with intermediate rounding. in canCombineFMadOrFMA()
5707 // Floating-point multiply-add without intermediate rounding. in canCombineFMadOrFMA()
5747 // fold (fadd (fmul x, y), z) -> (fma x, y, z) in matchCombineFAddFMulToFMadOrFMA()
5752 {LHS.MI->getOperand(1).getReg(), in matchCombineFAddFMulToFMadOrFMA()
5753 LHS.MI->getOperand(2).getReg(), RHS.Reg}); in matchCombineFAddFMulToFMadOrFMA()
5758 // fold (fadd x, (fmul y, z)) -> (fma y, z, x) in matchCombineFAddFMulToFMadOrFMA()
5763 {RHS.MI->getOperand(1).getReg(), in matchCombineFAddFMulToFMadOrFMA()
5764 RHS.MI->getOperand(2).getReg(), LHS.Reg}); in matchCombineFAddFMulToFMadOrFMA()
5780 const auto &TLI = *MI.getMF()->getSubtarget().getTargetLowering(); in matchCombineFAddFpExtFMulToFMadOrFMA()
5798 // fold (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z) in matchCombineFAddFpExtFMulToFMadOrFMA()
5803 MRI.getType(FpExtSrc->getOperand(1).getReg()))) { in matchCombineFAddFpExtFMulToFMadOrFMA()
5805 auto FpExtX = B.buildFPExt(DstType, FpExtSrc->getOperand(1).getReg()); in matchCombineFAddFpExtFMulToFMadOrFMA()
5806 auto FpExtY = B.buildFPExt(DstType, FpExtSrc->getOperand(2).getReg()); in matchCombineFAddFpExtFMulToFMadOrFMA()
5813 // fold (fadd z, (fpext (fmul x, y))) -> (fma (fpext x), (fpext y), z) in matchCombineFAddFpExtFMulToFMadOrFMA()
5818 MRI.getType(FpExtSrc->getOperand(1).getReg()))) { in matchCombineFAddFpExtFMulToFMadOrFMA()
5820 auto FpExtX = B.buildFPExt(DstType, FpExtSrc->getOperand(1).getReg()); in matchCombineFAddFpExtFMulToFMadOrFMA()
5821 auto FpExtY = B.buildFPExt(DstType, FpExtSrc->getOperand(2).getReg()); in matchCombineFAddFpExtFMulToFMadOrFMA()
5858 // fold (fadd (fma x, y, (fmul u, v)), z) -> (fma x, y, (fma u, v, z)) in matchCombineFAddFMAFMulToFMadOrFMA()
5859 if (LHS.MI->getOpcode() == PreferredFusedOpcode && in matchCombineFAddFMAFMulToFMadOrFMA()
5860 (MRI.getVRegDef(LHS.MI->getOperand(3).getReg())->getOpcode() == in matchCombineFAddFMAFMulToFMadOrFMA()
5862 MRI.hasOneNonDBGUse(LHS.MI->getOperand(0).getReg()) && in matchCombineFAddFMAFMulToFMadOrFMA()
5863 MRI.hasOneNonDBGUse(LHS.MI->getOperand(3).getReg())) { in matchCombineFAddFMAFMulToFMadOrFMA()
5867 // fold (fadd z, (fma x, y, (fmul u, v))) -> (fma x, y, (fma u, v, z)) in matchCombineFAddFMAFMulToFMadOrFMA()
5868 else if (RHS.MI->getOpcode() == PreferredFusedOpcode && in matchCombineFAddFMAFMulToFMadOrFMA()
5869 (MRI.getVRegDef(RHS.MI->getOperand(3).getReg())->getOpcode() == in matchCombineFAddFMAFMulToFMadOrFMA()
5871 MRI.hasOneNonDBGUse(RHS.MI->getOperand(0).getReg()) && in matchCombineFAddFMAFMulToFMadOrFMA()
5872 MRI.hasOneNonDBGUse(RHS.MI->getOperand(3).getReg())) { in matchCombineFAddFMAFMulToFMadOrFMA()
5878 MachineInstr *FMulMI = MRI.getVRegDef(FMA->getOperand(3).getReg()); in matchCombineFAddFMAFMulToFMadOrFMA()
5879 Register X = FMA->getOperand(1).getReg(); in matchCombineFAddFMAFMulToFMadOrFMA()
5880 Register Y = FMA->getOperand(2).getReg(); in matchCombineFAddFMAFMulToFMadOrFMA()
5881 Register U = FMulMI->getOperand(1).getReg(); in matchCombineFAddFMAFMulToFMadOrFMA()
5882 Register V = FMulMI->getOperand(2).getReg(); in matchCombineFAddFMAFMulToFMadOrFMA()
5907 const auto &TLI = *MI.getMF()->getSubtarget().getTargetLowering(); in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5939 // -> (fma x, y, (fma (fpext u), (fpext v), z)) in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5940 if (LHS.MI->getOpcode() == PreferredFusedOpcode && in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5941 mi_match(LHS.MI->getOperand(3).getReg(), MRI, in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5945 MRI.getType(FMulMI->getOperand(0).getReg()))) { in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5947 buildMatchInfo(FMulMI->getOperand(1).getReg(), in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5948 FMulMI->getOperand(2).getReg(), RHS.Reg, in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5949 LHS.MI->getOperand(1).getReg(), in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5950 LHS.MI->getOperand(2).getReg(), B); in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5956 // -> (fma (fpext x), (fpext y), (fma (fpext u), (fpext v), z)) in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5957 // FIXME: This turns two single-precision and one double-precision in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5958 // operation into two double-precision operations, which might not be in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5961 FMAMI->getOpcode() == PreferredFusedOpcode) { in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5962 MachineInstr *FMulMI = MRI.getVRegDef(FMAMI->getOperand(3).getReg()); in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5965 MRI.getType(FMAMI->getOperand(0).getReg()))) { in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5967 Register X = FMAMI->getOperand(1).getReg(); in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5968 Register Y = FMAMI->getOperand(2).getReg(); in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5971 buildMatchInfo(FMulMI->getOperand(1).getReg(), in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5972 FMulMI->getOperand(2).getReg(), RHS.Reg, X, Y, B); in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5980 // -> (fma x, y, (fma (fpext u), (fpext v), z)) in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5981 if (RHS.MI->getOpcode() == PreferredFusedOpcode && in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5982 mi_match(RHS.MI->getOperand(3).getReg(), MRI, in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5986 MRI.getType(FMulMI->getOperand(0).getReg()))) { in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5988 buildMatchInfo(FMulMI->getOperand(1).getReg(), in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5989 FMulMI->getOperand(2).getReg(), LHS.Reg, in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5990 RHS.MI->getOperand(1).getReg(), in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5991 RHS.MI->getOperand(2).getReg(), B); in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5997 // -> (fma (fpext x), (fpext y), (fma (fpext u), (fpext v), z)) in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5998 // FIXME: This turns two single-precision and one double-precision in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
5999 // operation into two double-precision operations, which might not be in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
6002 FMAMI->getOpcode() == PreferredFusedOpcode) { in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
6003 MachineInstr *FMulMI = MRI.getVRegDef(FMAMI->getOperand(3).getReg()); in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
6006 MRI.getType(FMAMI->getOperand(0).getReg()))) { in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
6008 Register X = FMAMI->getOperand(1).getReg(); in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
6009 Register Y = FMAMI->getOperand(2).getReg(); in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
6012 buildMatchInfo(FMulMI->getOperand(1).getReg(), in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
6013 FMulMI->getOperand(2).getReg(), LHS.Reg, X, Y, B); in matchCombineFAddFpExtFMulToFMadOrFMAAggressive()
6047 // fold (fsub (fmul x, y), z) -> (fma x, y, -z) in matchCombineFSubFMulToFMadOrFMA()
6054 {LHS.MI->getOperand(1).getReg(), in matchCombineFSubFMulToFMadOrFMA()
6055 LHS.MI->getOperand(2).getReg(), NegZ}); in matchCombineFSubFMulToFMadOrFMA()
6059 // fold (fsub x, (fmul y, z)) -> (fma -y, z, x) in matchCombineFSubFMulToFMadOrFMA()
6064 B.buildFNeg(DstTy, RHS.MI->getOperand(1).getReg()).getReg(0); in matchCombineFSubFMulToFMadOrFMA()
6066 {NegY, RHS.MI->getOperand(2).getReg(), LHS.Reg}); in matchCombineFSubFMulToFMadOrFMA()
6090 // fold (fsub (fneg (fmul x, y)), z) -> (fma (fneg x), y, (fneg z)) in matchCombineFSubFNegFMulToFMadOrFMA()
6093 MRI.hasOneNonDBGUse(FMulMI->getOperand(0).getReg()))) && in matchCombineFSubFNegFMulToFMadOrFMA()
6097 B.buildFNeg(DstTy, FMulMI->getOperand(1).getReg()).getReg(0); in matchCombineFSubFNegFMulToFMadOrFMA()
6100 {NegX, FMulMI->getOperand(2).getReg(), NegZ}); in matchCombineFSubFNegFMulToFMadOrFMA()
6105 // fold (fsub x, (fneg (fmul, y, z))) -> (fma y, z, x) in matchCombineFSubFNegFMulToFMadOrFMA()
6108 MRI.hasOneNonDBGUse(FMulMI->getOperand(0).getReg()))) && in matchCombineFSubFNegFMulToFMadOrFMA()
6112 {FMulMI->getOperand(1).getReg(), in matchCombineFSubFNegFMulToFMadOrFMA()
6113 FMulMI->getOperand(2).getReg(), LHSReg}); in matchCombineFSubFNegFMulToFMadOrFMA()
6137 // fold (fsub (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), (fneg z)) in matchCombineFSubFpExtFMulToFMadOrFMA()
6143 B.buildFPExt(DstTy, FMulMI->getOperand(1).getReg()).getReg(0); in matchCombineFSubFpExtFMulToFMadOrFMA()
6145 B.buildFPExt(DstTy, FMulMI->getOperand(2).getReg()).getReg(0); in matchCombineFSubFpExtFMulToFMadOrFMA()
6153 // fold (fsub x, (fpext (fmul y, z))) -> (fma (fneg (fpext y)), (fpext z), x) in matchCombineFSubFpExtFMulToFMadOrFMA()
6159 B.buildFPExt(DstTy, FMulMI->getOperand(1).getReg()).getReg(0); in matchCombineFSubFpExtFMulToFMadOrFMA()
6162 B.buildFPExt(DstTy, FMulMI->getOperand(2).getReg()).getReg(0); in matchCombineFSubFpExtFMulToFMadOrFMA()
6180 const auto &TLI = *MI.getMF()->getSubtarget().getTargetLowering(); in matchCombineFSubFpExtFNegFMulToFMadOrFMA()
6196 // fold (fsub (fpext (fneg (fmul x, y))), z) -> in matchCombineFSubFpExtFNegFMulToFMadOrFMA()
6198 // fold (fsub (fneg (fpext (fmul x, y))), z) -> in matchCombineFSubFpExtFNegFMulToFMadOrFMA()
6204 MRI.getType(FMulMI->getOperand(0).getReg()))) { in matchCombineFSubFpExtFNegFMulToFMadOrFMA()
6207 buildMatchInfo(FMAReg, FMulMI->getOperand(1).getReg(), in matchCombineFSubFpExtFNegFMulToFMadOrFMA()
6208 FMulMI->getOperand(2).getReg(), RHSReg, B); in matchCombineFSubFpExtFNegFMulToFMadOrFMA()
6214 // fold (fsub x, (fpext (fneg (fmul y, z)))) -> (fma (fpext y), (fpext z), x) in matchCombineFSubFpExtFNegFMulToFMadOrFMA()
6215 // fold (fsub x, (fneg (fpext (fmul y, z)))) -> (fma (fpext y), (fpext z), x) in matchCombineFSubFpExtFNegFMulToFMadOrFMA()
6220 MRI.getType(FMulMI->getOperand(0).getReg()))) { in matchCombineFSubFpExtFNegFMulToFMadOrFMA()
6222 buildMatchInfo(MI.getOperand(0).getReg(), FMulMI->getOperand(1).getReg(), in matchCombineFSubFpExtFNegFMulToFMadOrFMA()
6223 FMulMI->getOperand(2).getReg(), LHSReg, B); in matchCombineFSubFpExtFNegFMulToFMadOrFMA()
6250 if (!MaybeCst || !MaybeCst->getValueAPF().isNaN()) in matchCombineFMinMaxNaN()
6265 // A + (B - A) -> B in matchAddSubSameReg()
6266 // (B - A) + A -> B in matchAddSubSameReg()
6312 if (Lo == Hi && ShiftAmount->Value == DstEltTy.getSizeInBits()) { in matchBuildVectorIdentityFold()
6343 return ShiftAmt->Value.getZExtValue() == MatchTy.getSizeInBits() && in matchTruncLshrBuildVectorFold()
6417 // Match a floating point compare with a less-than/greater-than predicate. in matchFPSelectToMinMax()
6445 // unless we have fmaximum/fminimum. In that case, we know -0 < 0. in matchFPSelectToMinMax()
6449 // non-zero. in matchFPSelectToMinMax()
6451 if (!KnownNonZeroSide || !KnownNonZeroSide->Value.isNonZero()) { in matchFPSelectToMinMax()
6453 if (!KnownNonZeroSide || !KnownNonZeroSide->Value.isNonZero()) in matchFPSelectToMinMax()
6481 // (X + Y) == X --> Y == 0 in matchRedundantBinOpInEquality()
6482 // (X + Y) != X --> Y != 0 in matchRedundantBinOpInEquality()
6483 // (X - Y) == X --> Y == 0 in matchRedundantBinOpInEquality()
6484 // (X - Y) != X --> Y != 0 in matchRedundantBinOpInEquality()
6485 // (X ^ Y) == X --> Y == 0 in matchRedundantBinOpInEquality()
6486 // (X ^ Y) != X --> Y != 0 in matchRedundantBinOpInEquality()
6515 return CI && CI->uge(ResTy.getScalarSizeInBits()); in matchShiftsTooBig()
6540 if (MRI.getVRegDef(LHS)->getOpcode() != in matchCommuteConstantToRHS()
6545 return MRI.getVRegDef(RHS)->getOpcode() != in matchCommuteConstantToRHS()
6589 return IConstant && IConstant->Value == 1; in isOneOrOneSplat()
6602 return IConstant && IConstant->Value == 0; in isZeroOrZeroSplat()
6614 unsigned NumSources = BuildVector->getNumSources(); in isConstantSplatVector()
6618 getOpcodeDef<GImplicitDef>(BuildVector->getSourceReg(I), MRI); in isConstantSplatVector()
6624 getIConstantVRegValWithLookThrough(BuildVector->getSourceReg(I), MRI); in isConstantSplatVector()
6625 if (IConstant && IConstant->Value == SplatValue) in isConstantSplatVector()
6638 return IConstant->Value; in getConstantOrConstantSplatVector()
6643 unsigned NumSources = BuildVector->getNumSources(); in getConstantOrConstantSplatVector()
6648 getIConstantVRegValWithLookThrough(BuildVector->getSourceReg(I), MRI); in getConstantOrConstantSplatVector()
6652 Value = IConstant->Value; in getConstantOrConstantSplatVector()
6653 else if (*Value != IConstant->Value) in getConstantOrConstantSplatVector()
6669 unsigned NumSources = BuildVector->getNumSources(); in isConstantOrConstantVectorI()
6672 getIConstantVRegValWithLookThrough(BuildVector->getSourceReg(I), MRI); in isConstantOrConstantVectorI()
6682 uint32_t Flags = Select->getFlags(); in tryFoldSelectOfConstants()
6683 Register Dest = Select->getReg(0); in tryFoldSelectOfConstants()
6684 Register Cond = Select->getCondReg(); in tryFoldSelectOfConstants()
6685 Register True = Select->getTrueReg(); in tryFoldSelectOfConstants()
6686 Register False = Select->getFalseReg(); in tryFoldSelectOfConstants()
6687 LLT CondTy = MRI.getType(Select->getCondReg()); in tryFoldSelectOfConstants()
6688 LLT TrueTy = MRI.getType(Select->getTrueReg()); in tryFoldSelectOfConstants()
6706 APInt TrueValue = TrueOpt->Value; in tryFoldSelectOfConstants()
6707 APInt FalseValue = FalseOpt->Value; in tryFoldSelectOfConstants()
6709 // select Cond, 1, 0 --> zext (Cond) in tryFoldSelectOfConstants()
6718 // select Cond, -1, 0 --> sext (Cond) in tryFoldSelectOfConstants()
6727 // select Cond, 0, 1 --> zext (!Cond) in tryFoldSelectOfConstants()
6738 // select Cond, 0, -1 --> sext (!Cond) in tryFoldSelectOfConstants()
6749 // select Cond, C1, C1-1 --> add (zext Cond), C1-1 in tryFoldSelectOfConstants()
6750 if (TrueValue - 1 == FalseValue) { in tryFoldSelectOfConstants()
6760 // select Cond, C1, C1+1 --> add (sext Cond), C1+1 in tryFoldSelectOfConstants()
6771 // select Cond, Pow2, 0 --> (zext Cond) << log2(Pow2) in tryFoldSelectOfConstants()
6784 // select Cond, -1, C --> or (sext Cond), C in tryFoldSelectOfConstants()
6795 // select Cond, C, -1 --> or (sext (not Cond)), C in tryFoldSelectOfConstants()
6814 uint32_t Flags = Select->getFlags(); in tryFoldBoolSelectToLogic()
6815 Register DstReg = Select->getReg(0); in tryFoldBoolSelectToLogic()
6816 Register Cond = Select->getCondReg(); in tryFoldBoolSelectToLogic()
6817 Register True = Select->getTrueReg(); in tryFoldBoolSelectToLogic()
6818 Register False = Select->getFalseReg(); in tryFoldBoolSelectToLogic()
6819 LLT CondTy = MRI.getType(Select->getCondReg()); in tryFoldBoolSelectToLogic()
6820 LLT TrueTy = MRI.getType(Select->getTrueReg()); in tryFoldBoolSelectToLogic()
6832 // select Cond, Cond, F --> or Cond, F in tryFoldBoolSelectToLogic()
6833 // select Cond, 1, F --> or Cond, F in tryFoldBoolSelectToLogic()
6837 Register Ext = MRI.createGenericVirtualRegister(TrueTy); in tryFoldBoolSelectToLogic() local
6838 B.buildZExtOrTrunc(Ext, Cond); in tryFoldBoolSelectToLogic()
6840 B.buildOr(DstReg, Ext, FreezeFalse, Flags); in tryFoldBoolSelectToLogic()
6845 // select Cond, T, Cond --> and Cond, T in tryFoldBoolSelectToLogic()
6846 // select Cond, T, 0 --> and Cond, T in tryFoldBoolSelectToLogic()
6850 Register Ext = MRI.createGenericVirtualRegister(TrueTy); in tryFoldBoolSelectToLogic() local
6851 B.buildZExtOrTrunc(Ext, Cond); in tryFoldBoolSelectToLogic()
6853 B.buildAnd(DstReg, Ext, FreezeTrue); in tryFoldBoolSelectToLogic()
6858 // select Cond, T, 1 --> or (not Cond), T in tryFoldBoolSelectToLogic()
6865 // Then an ext to match the destination register. in tryFoldBoolSelectToLogic()
6866 Register Ext = MRI.createGenericVirtualRegister(TrueTy); in tryFoldBoolSelectToLogic() local
6867 B.buildZExtOrTrunc(Ext, Inner); in tryFoldBoolSelectToLogic()
6869 B.buildOr(DstReg, Ext, FreezeTrue, Flags); in tryFoldBoolSelectToLogic()
6874 // select Cond, 0, F --> and (not Cond), F in tryFoldBoolSelectToLogic()
6881 // Then an ext to match the destination register. in tryFoldBoolSelectToLogic()
6882 Register Ext = MRI.createGenericVirtualRegister(TrueTy); in tryFoldBoolSelectToLogic() local
6883 B.buildZExtOrTrunc(Ext, Inner); in tryFoldBoolSelectToLogic()
6885 B.buildAnd(DstReg, Ext, FreezeFalse); in tryFoldBoolSelectToLogic()
6896 GICmp *Cmp = cast<GICmp>(MRI.getVRegDef(Select->getCondReg())); in matchSelectIMinMax()
6898 Register DstReg = Select->getReg(0); in matchSelectIMinMax()
6899 Register True = Select->getTrueReg(); in matchSelectIMinMax()
6900 Register False = Select->getFalseReg(); in matchSelectIMinMax()
6907 if (!MRI.hasOneNonDBGUse(Cmp->getReg(0))) in matchSelectIMinMax()
6910 CmpInst::Predicate Pred = Cmp->getCond(); in matchSelectIMinMax()
6916 Register CmpLHS = Cmp->getLHSReg(); in matchSelectIMinMax()
6917 Register CmpRHS = Cmp->getRHSReg(); in matchSelectIMinMax()
6925 // (icmp X, Y) ? X : Y -> integer minmax. in matchSelectIMinMax()
6979 /// into a single comparison using range-based reasoning.
6983 assert(Logic->getOpcode() != TargetOpcode::G_XOR && "unexpected xor"); in tryFoldAndOrOrICmpsUsingRanges()
6984 bool IsAnd = Logic->getOpcode() == TargetOpcode::G_AND; in tryFoldAndOrOrICmpsUsingRanges()
6985 Register DstReg = Logic->getReg(0); in tryFoldAndOrOrICmpsUsingRanges()
6986 Register LHS = Logic->getLHSReg(); in tryFoldAndOrOrICmpsUsingRanges()
6987 Register RHS = Logic->getRHSReg(); in tryFoldAndOrOrICmpsUsingRanges()
6988 unsigned Flags = Logic->getFlags(); in tryFoldAndOrOrICmpsUsingRanges()
7001 if (!MRI.hasOneNonDBGUse(Cmp1->getReg(0)) || in tryFoldAndOrOrICmpsUsingRanges()
7002 !MRI.hasOneNonDBGUse(Cmp2->getReg(0))) in tryFoldAndOrOrICmpsUsingRanges()
7008 getIConstantVRegValWithLookThrough(Cmp1->getRHSReg(), MRI); in tryFoldAndOrOrICmpsUsingRanges()
7011 C1 = MaybeC1->Value; in tryFoldAndOrOrICmpsUsingRanges()
7014 getIConstantVRegValWithLookThrough(Cmp2->getRHSReg(), MRI); in tryFoldAndOrOrICmpsUsingRanges()
7017 C2 = MaybeC2->Value; in tryFoldAndOrOrICmpsUsingRanges()
7019 Register R1 = Cmp1->getLHSReg(); in tryFoldAndOrOrICmpsUsingRanges()
7020 Register R2 = Cmp2->getLHSReg(); in tryFoldAndOrOrICmpsUsingRanges()
7021 CmpInst::Predicate Pred1 = Cmp1->getCond(); in tryFoldAndOrOrICmpsUsingRanges()
7022 CmpInst::Predicate Pred2 = Cmp2->getCond(); in tryFoldAndOrOrICmpsUsingRanges()
7023 LLT CmpTy = MRI.getType(Cmp1->getReg(0)); in tryFoldAndOrOrICmpsUsingRanges()
7043 getIConstantVRegValWithLookThrough(Add->getRHSReg(), MRI); in tryFoldAndOrOrICmpsUsingRanges()
7045 R1 = Add->getLHSReg(); in tryFoldAndOrOrICmpsUsingRanges()
7046 Offset1 = MaybeOffset1->Value; in tryFoldAndOrOrICmpsUsingRanges()
7051 getIConstantVRegValWithLookThrough(Add->getRHSReg(), MRI); in tryFoldAndOrOrICmpsUsingRanges()
7053 R2 = Add->getLHSReg(); in tryFoldAndOrOrICmpsUsingRanges()
7054 Offset2 = MaybeOffset2->Value; in tryFoldAndOrOrICmpsUsingRanges()
7077 // We need non-wrapping ranges. in tryFoldAndOrOrICmpsUsingRanges()
7081 // Check whether we have equal-size ranges that only differ by one bit. in tryFoldAndOrOrICmpsUsingRanges()
7084 APInt UpperDiff = (CR1.getUpper() - 1) ^ (CR2.getUpper() - 1); in tryFoldAndOrOrICmpsUsingRanges()
7085 APInt CR1Size = CR1.getUpper() - CR1.getLower(); in tryFoldAndOrOrICmpsUsingRanges()
7087 CR1Size != CR2.getUpper() - CR2.getLower()) in tryFoldAndOrOrICmpsUsingRanges()
7095 CR = CR->inverse(); in tryFoldAndOrOrICmpsUsingRanges()
7099 CR->getEquivalentICmp(NewPred, NewC, Offset); in tryFoldAndOrOrICmpsUsingRanges()
7142 assert(Logic->getOpcode() != TargetOpcode::G_XOR && "unexpecte xor"); in tryFoldLogicOfFCmps()
7143 Register DestReg = Logic->getReg(0); in tryFoldLogicOfFCmps()
7144 Register LHS = Logic->getLHSReg(); in tryFoldLogicOfFCmps()
7145 Register RHS = Logic->getRHSReg(); in tryFoldLogicOfFCmps()
7146 bool IsAnd = Logic->getOpcode() == TargetOpcode::G_AND; in tryFoldLogicOfFCmps()
7158 LLT CmpTy = MRI.getType(Cmp1->getReg(0)); in tryFoldLogicOfFCmps()
7159 LLT CmpOperandTy = MRI.getType(Cmp1->getLHSReg()); in tryFoldLogicOfFCmps()
7165 !MRI.hasOneNonDBGUse(Logic->getReg(0)) || in tryFoldLogicOfFCmps()
7166 !MRI.hasOneNonDBGUse(Cmp1->getReg(0)) || in tryFoldLogicOfFCmps()
7167 !MRI.hasOneNonDBGUse(Cmp2->getReg(0)) || in tryFoldLogicOfFCmps()
7168 MRI.getType(Cmp1->getLHSReg()) != MRI.getType(Cmp2->getLHSReg())) in tryFoldLogicOfFCmps()
7171 CmpInst::Predicate PredL = Cmp1->getCond(); in tryFoldLogicOfFCmps()
7172 CmpInst::Predicate PredR = Cmp2->getCond(); in tryFoldLogicOfFCmps()
7173 Register LHS0 = Cmp1->getLHSReg(); in tryFoldLogicOfFCmps()
7174 Register LHS1 = Cmp1->getRHSReg(); in tryFoldLogicOfFCmps()
7175 Register RHS0 = Cmp2->getLHSReg(); in tryFoldLogicOfFCmps()
7176 Register RHS1 = Cmp2->getRHSReg(); in tryFoldLogicOfFCmps()
7189 unsigned Flags = Cmp1->getFlags() | Cmp2->getFlags(); in tryFoldLogicOfFCmps()
7243 Register Dst = Add->getReg(0); in matchAddOverflow()
7244 Register Carry = Add->getReg(1); in matchAddOverflow()
7245 Register LHS = Add->getLHSReg(); in matchAddOverflow()
7246 Register RHS = Add->getRHSReg(); in matchAddOverflow()
7247 bool IsSigned = Add->isSigned(); in matchAddOverflow()
7251 // Fold addo, if the carry is dead -> add, undef. in matchAddOverflow()
7279 // Fold addo(c1, c2) -> c3, carry. in matchAddOverflow()
7283 APInt Result = IsSigned ? MaybeLHS->sadd_ov(*MaybeRHS, Overflow) in matchAddOverflow()
7284 : MaybeLHS->uadd_ov(*MaybeRHS, Overflow); in matchAddOverflow()
7292 // Fold (addo x, 0) -> x, no carry in matchAddOverflow()
7302 // uaddo (X +nuw C0), C1 -> uaddo X, C0 + C1 in matchAddOverflow()
7303 // saddo (X +nsw C0), C1 -> saddo X, C0 + C1 in matchAddOverflow()
7305 if (MaybeRHS && AddLHS && MRI.hasOneNonDBGUse(Add->getReg(0)) && in matchAddOverflow()
7306 ((IsSigned && AddLHS->getFlag(MachineInstr::MIFlag::NoSWrap)) || in matchAddOverflow()
7307 (!IsSigned && AddLHS->getFlag(MachineInstr::MIFlag::NoUWrap)))) { in matchAddOverflow()
7309 getConstantOrConstantSplatVector(AddLHS->getRHSReg()); in matchAddOverflow()
7312 APInt NewC = IsSigned ? MaybeAddRHS->sadd_ov(*MaybeRHS, Overflow) in matchAddOverflow()
7313 : MaybeAddRHS->uadd_ov(*MaybeRHS, Overflow); in matchAddOverflow()
7318 B.buildSAddo(Dst, Carry, AddLHS->getLHSReg(), ConstRHS); in matchAddOverflow()
7325 B.buildUAddo(Dst, Carry, AddLHS->getLHSReg(), ConstRHS); in matchAddOverflow()
7332 // We try to combine addo to non-overflowing add. in matchAddOverflow()
7337 // We try to combine uaddo to non-overflowing add. in matchAddOverflow()
7340 ConstantRange::fromKnownBits(KB->getKnownBits(LHS), /*IsSigned=*/false); in matchAddOverflow()
7342 ConstantRange::fromKnownBits(KB->getKnownBits(RHS), /*IsSigned=*/false); in matchAddOverflow()
7366 // We try to combine saddo to non-overflowing add. in matchAddOverflow()
7370 if (KB->computeNumSignBits(RHS) > 1 && KB->computeNumSignBits(LHS) > 1) { in matchAddOverflow()
7379 ConstantRange::fromKnownBits(KB->getKnownBits(LHS), /*IsSigned=*/true); in matchAddOverflow()
7381 ConstantRange::fromKnownBits(KB->getKnownBits(RHS), /*IsSigned=*/true); in matchAddOverflow()
7410 Root->eraseFromParent(); in applyBuildFnMO()
7414 bool OptForSize = MI.getMF()->getFunction().hasOptSize(); in matchFPowIExpansion()
7430 ExpVal = -ExpVal; in applyExpandFPowI()
7451 // If the original exponent was negative, invert the result, producing in applyExpandFPowI()
7464 GTrunc *Trunc = cast<GTrunc>(getDefIgnoringCopies(Sext->getSrcReg(), MRI)); in matchSextOfTrunc()
7466 Register Dst = Sext->getReg(0); in matchSextOfTrunc()
7467 Register Src = Trunc->getSrcReg(); in matchSextOfTrunc()
7497 GTrunc *Trunc = cast<GTrunc>(getDefIgnoringCopies(Zext->getSrcReg(), MRI)); in matchZextOfTrunc()
7499 Register Dst = Zext->getReg(0); in matchZextOfTrunc()
7500 Register Src = Trunc->getSrcReg(); in matchZextOfTrunc()
7533 Register Dst = Zext->getReg(0); in matchNonNegZext()
7534 Register Src = Zext->getSrcReg(); in matchNonNegZext()