Lines Matching +full:flip +full:- +full:vertical
1 //===- RegisterCoalescer.cpp - Generic Register Coalescing Interface ------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
13 //===----------------------------------------------------------------------===//
69 STATISTIC(NumReMats , "Number of instructions re-materialized");
75 static cl::opt<bool> EnableJoining("join-liveintervals",
79 static cl::opt<bool> UseTerminalRule("terminal-rule",
85 EnableJoinSplits("join-splitedges",
90 EnableGlobalCopies("join-globalcopies",
95 VerifyCoalescing("verify-coalescing",
100 "late-remat-update-threshold", cl::Hidden,
109 "large-interval-size-threshold", cl::Hidden,
115 "large-interval-freq-threshold", cl::Hidden,
150 /// Debug variable location tracking -- for each VReg, maintain an
151 /// ordered-by-slot-index set of DBG_VALUEs, to help quick
168 /// True if the coalescer should aggressively coalesce fall-thru
272 /// We found a non-trivially-coalescable copy. If the source value number is
283 /// We found a non-trivially-coalescable copy.
288 /// - the first element is true if an interval was modified,
289 /// - the second element is true if the destination interval needs
315 /// %0:sub0<def,read-undef> = ...
316 /// %1 = COPY %0 <-- Coalescing COPY reveals undef
317 /// = use %1:sub1 <-- hidden undef use
342 /// This method does the proper fixing of the live-ranges when the afore
347 if (LIS->shrinkToUses(LI, Dead)) { in shrinkToUses()
351 LIS->splitSeparateComponents(*LI, SplitLIs); in shrinkToUses()
361 LIS->RemoveMachineInstrFromMaps(*MI); in deleteInstr()
362 MI->eraseFromParent(); in deleteInstr()
407 INITIALIZE_PASS_BEGIN(RegisterCoalescer, "register-coalescer",
413 INITIALIZE_PASS_END(RegisterCoalescer, "register-coalescer", in INITIALIZE_PASS_DEPENDENCY()
420 if (MI->isCopy()) { in INITIALIZE_PASS_DEPENDENCY()
421 Dst = MI->getOperand(0).getReg(); in INITIALIZE_PASS_DEPENDENCY()
422 DstSub = MI->getOperand(0).getSubReg(); in INITIALIZE_PASS_DEPENDENCY()
423 Src = MI->getOperand(1).getReg(); in INITIALIZE_PASS_DEPENDENCY()
424 SrcSub = MI->getOperand(1).getSubReg(); in INITIALIZE_PASS_DEPENDENCY()
425 } else if (MI->isSubregToReg()) { in INITIALIZE_PASS_DEPENDENCY()
426 Dst = MI->getOperand(0).getReg(); in INITIALIZE_PASS_DEPENDENCY()
427 DstSub = tri.composeSubRegIndices(MI->getOperand(0).getSubReg(), in INITIALIZE_PASS_DEPENDENCY()
428 MI->getOperand(3).getImm()); in INITIALIZE_PASS_DEPENDENCY()
429 Src = MI->getOperand(2).getReg(); in INITIALIZE_PASS_DEPENDENCY()
430 SrcSub = MI->getOperand(2).getSubReg(); in INITIALIZE_PASS_DEPENDENCY()
439 /// contain non-branches should also be vacated, but this can be handled by an
440 /// earlier pass similar to early if-conversion.
442 if (MBB->pred_size() != 1 || MBB->succ_size() != 1) in isSplitEdge()
473 const MachineRegisterInfo &MRI = MI->getMF()->getRegInfo(); in setRegisters()
487 } else if (!MRI.getRegClass(Src)->contains(Dst)) { in setRegisters()
497 // Copies between different sub-registers are never coalescable. in setRegisters()
506 // SrcReg will be merged with a sub-register of DstReg. in setRegisters()
510 // DstReg will be merged with a sub-register of SrcReg. in setRegisters()
514 // This is a straight copy without sub-registers. in setRegisters()
522 // Prefer SrcReg to be a sub-register of DstReg. in setRegisters()
540 bool CoalescerPair::flip() { in flip() function in CoalescerPair
602 Edit->eliminateDeadDefs(DeadDefs); in eliminateDeadDefs()
621 LIS->getInterval(CP.isFlipped() ? CP.getDstReg() : CP.getSrcReg()); in adjustCopiesBackFrom()
623 LIS->getInterval(CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg()); in adjustCopiesBackFrom()
624 SlotIndex CopyIdx = LIS->getInstructionIndex(*CopyMI).getRegSlot(); in adjustCopiesBackFrom()
626 // We have a non-trivially-coalescable copy with IntA being the source and in adjustCopiesBackFrom()
634 // B1 = A3 <- this copy in adjustCopiesBackFrom()
644 VNInfo *BValNo = BS->valno; in adjustCopiesBackFrom()
649 if (BValNo->def != CopyIdx) return false; in adjustCopiesBackFrom()
656 VNInfo *AValNo = AS->valno; in adjustCopiesBackFrom()
660 MachineInstr *ACopyMI = LIS->getInstructionFromIndex(AValNo->def); in adjustCopiesBackFrom()
662 if (!CP.isCoalescable(ACopyMI) || !ACopyMI->isFullCopy()) in adjustCopiesBackFrom()
667 IntB.FindSegmentContaining(AValNo->def.getPrevSlot()); in adjustCopiesBackFrom()
674 LIS->getInstructionFromIndex(ValS->end.getPrevSlot()); in adjustCopiesBackFrom()
675 if (!ValSEndInst || ValSEndInst->getParent() != CopyMI->getParent()) in adjustCopiesBackFrom()
679 // live-range starts. If there are no intervening live segments between them in adjustCopiesBackFrom()
685 SlotIndex FillerStart = ValS->end, FillerEnd = BS->start; in adjustCopiesBackFrom()
689 BValNo->def = FillerStart; in adjustCopiesBackFrom()
697 if (BValNo != ValS->valno) in adjustCopiesBackFrom()
698 IntB.MergeValueNumberInto(BValNo, ValS->valno); in adjustCopiesBackFrom()
705 if (SS != S.end() && SlotIndex::isSameInstr(SS->start, SS->end)) { in adjustCopiesBackFrom()
712 LIS->getMBBStartIdx(LIS->getMBBFromIndex(FillerStart)); in adjustCopiesBackFrom()
717 VNInfo *SubValSNo = S.getVNInfoAt(AValNo->def.getPrevSlot()); in adjustCopiesBackFrom()
727 ValSEndInst->findRegisterUseOperandIdx(IntB.reg(), /*TRI=*/nullptr, true); in adjustCopiesBackFrom()
728 if (UIdx != -1) { in adjustCopiesBackFrom()
729 ValSEndInst->getOperand(UIdx).setIsKill(false); in adjustCopiesBackFrom()
733 CopyMI->substituteRegister(IntA.reg(), IntB.reg(), 0, *TRI); in adjustCopiesBackFrom()
736 bool RecomputeLiveRange = AS->end == CopyIdx; in adjustCopiesBackFrom()
740 if (SS != S.end() && SS->end == CopyIdx) { in adjustCopiesBackFrom()
759 if (LIS->hasPHIKill(IntA, AValNo)) in hasOtherReachingDefs()
766 --BI; in hasOtherReachingDefs()
767 for (; BI != IntB.end() && ASeg.end >= BI->start; ++BI) { in hasOtherReachingDefs()
768 if (BI->valno == BValNo) in hasOtherReachingDefs()
770 if (BI->start <= ASeg.start && BI->end > ASeg.start) in hasOtherReachingDefs()
772 if (BI->start > ASeg.start && BI->start < ASeg.end) in hasOtherReachingDefs()
790 // to be removed. This segment is going to be merged with a pre-existing in addSegmentsWithValNo()
810 LIS->getInterval(CP.isFlipped() ? CP.getDstReg() : CP.getSrcReg()); in removeCopyByCommutingDef()
812 LIS->getInterval(CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg()); in removeCopyByCommutingDef()
814 // We found a non-trivially-coalescable copy with IntA being the source and in removeCopyByCommutingDef()
822 // B1 = A3 <- this copy in removeCopyByCommutingDef()
824 // = op A3 <- more uses in removeCopyByCommutingDef()
830 // B1 = B2 <- now an identity copy in removeCopyByCommutingDef()
832 // = op B2 <- more uses in removeCopyByCommutingDef()
836 SlotIndex CopyIdx = LIS->getInstructionIndex(*CopyMI).getRegSlot(); in removeCopyByCommutingDef()
838 assert(BValNo != nullptr && BValNo->def == CopyIdx); in removeCopyByCommutingDef()
842 assert(AValNo && !AValNo->isUnused() && "COPY source not live"); in removeCopyByCommutingDef()
843 if (AValNo->isPHIDef()) in removeCopyByCommutingDef()
845 MachineInstr *DefMI = LIS->getInstructionFromIndex(AValNo->def); in removeCopyByCommutingDef()
848 if (!DefMI->isCommutable()) in removeCopyByCommutingDef()
850 // If DefMI is a two-address instruction then commuting it will change the in removeCopyByCommutingDef()
852 int DefIdx = DefMI->findRegisterDefOperandIdx(IntA.reg(), /*TRI=*/nullptr); in removeCopyByCommutingDef()
853 assert(DefIdx != -1); in removeCopyByCommutingDef()
855 if (!DefMI->isRegTiedToUseOperand(DefIdx, &UseOpIdx)) in removeCopyByCommutingDef()
865 // operands then all possible variants (i.e. op#1<->op#2, op#1<->op#3, in removeCopyByCommutingDef()
866 // op#2<->op#3) of commute transformation should be considered/tried here. in removeCopyByCommutingDef()
868 if (!TII->findCommutedOpIndices(*DefMI, UseOpIdx, NewDstIdx)) in removeCopyByCommutingDef()
871 MachineOperand &NewDstMO = DefMI->getOperand(NewDstIdx); in removeCopyByCommutingDef()
873 if (NewReg != IntB.reg() || !IntB.Query(AValNo->def).isKill()) in removeCopyByCommutingDef()
883 for (MachineOperand &MO : MRI->use_nodbg_operands(IntA.reg())) { in removeCopyByCommutingDef()
885 unsigned OpNo = &MO - &UseMI->getOperand(0); in removeCopyByCommutingDef()
886 SlotIndex UseIdx = LIS->getInstructionIndex(*UseMI); in removeCopyByCommutingDef()
888 if (US == IntA.end() || US->valno != AValNo) in removeCopyByCommutingDef()
891 if (UseMI->isRegTiedToDefOperand(OpNo)) in removeCopyByCommutingDef()
895 LLVM_DEBUG(dbgs() << "\tremoveCopyByCommutingDef: " << AValNo->def << '\t' in removeCopyByCommutingDef()
900 MachineBasicBlock *MBB = DefMI->getParent(); in removeCopyByCommutingDef()
902 TII->commuteInstruction(*DefMI, false, UseOpIdx, NewDstIdx); in removeCopyByCommutingDef()
906 !MRI->constrainRegClass(IntB.reg(), MRI->getRegClass(IntA.reg()))) in removeCopyByCommutingDef()
909 LIS->ReplaceMachineInstrInMaps(*DefMI, *NewMI); in removeCopyByCommutingDef()
911 MBB->insert(Pos, NewMI); in removeCopyByCommutingDef()
912 MBB->erase(DefMI); in removeCopyByCommutingDef()
926 llvm::make_early_inc_range(MRI->use_operands(IntA.reg()))) { in removeCopyByCommutingDef()
930 if (UseMI->isDebugInstr()) { in removeCopyByCommutingDef()
936 SlotIndex UseIdx = LIS->getInstructionIndex(*UseMI).getRegSlot(true); in removeCopyByCommutingDef()
939 if (US->valno != AValNo) in removeCopyByCommutingDef()
949 if (!UseMI->isCopy()) in removeCopyByCommutingDef()
951 if (UseMI->getOperand(0).getReg() != IntB.reg() || in removeCopyByCommutingDef()
952 UseMI->getOperand(0).getSubReg()) in removeCopyByCommutingDef()
962 assert(DVNI->def == DefIdx); in removeCopyByCommutingDef()
969 assert(SubBValNo->def == CopyIdx); in removeCopyByCommutingDef()
979 BumpPtrAllocator &Allocator = LIS->getVNInfoAllocator(); in removeCopyByCommutingDef()
982 LaneBitmask Mask = MRI->getMaxLaneMaskForVReg(IntA.reg()); in removeCopyByCommutingDef()
985 LaneBitmask Mask = MRI->getMaxLaneMaskForVReg(IntB.reg()); in removeCopyByCommutingDef()
990 const SlotIndexes &Indexes = *LIS->getSlotIndexes(); in removeCopyByCommutingDef()
1013 BSubValNo->def = ASubValNo->def; in removeCopyByCommutingDef()
1024 if (S->start.getBaseIndex() == CopyIdx.getBaseIndex()) in removeCopyByCommutingDef()
1029 BValNo->def = AValNo->def; in removeCopyByCommutingDef()
1034 LIS->removeVRegDefAt(IntA, AValNo->def); in removeCopyByCommutingDef()
1068 /// BB0/BB2: ----
1072 /// |-------
1104 LIS->getInterval(CP.isFlipped() ? CP.getDstReg() : CP.getSrcReg()); in removePartialRedundancy()
1106 LIS->getInterval(CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg()); in removePartialRedundancy()
1109 SlotIndex CopyIdx = LIS->getInstructionIndex(CopyMI).getRegSlot(true); in removePartialRedundancy()
1111 assert(AValNo && !AValNo->isUnused() && "COPY source not live"); in removePartialRedundancy()
1112 if (!AValNo->isPHIDef()) in removePartialRedundancy()
1116 if (IntB.overlaps(LIS->getMBBStartIdx(&MBB), CopyIdx)) in removePartialRedundancy()
1124 VNInfo *PVal = IntA.getVNInfoBefore(LIS->getMBBEndIdx(Pred)); in removePartialRedundancy()
1125 MachineInstr *DefMI = LIS->getInstructionFromIndex(PVal->def); in removePartialRedundancy()
1126 if (!DefMI || !DefMI->isFullCopy()) { in removePartialRedundancy()
1131 if (DefMI->getOperand(0).getReg() != IntA.reg() || in removePartialRedundancy()
1132 DefMI->getOperand(1).getReg() != IntB.reg() || in removePartialRedundancy()
1133 DefMI->getParent() != Pred) { in removePartialRedundancy()
1142 if (VNI->isUnused()) in removePartialRedundancy()
1144 if (PVal->def < VNI->def && VNI->def < LIS->getMBBEndIdx(Pred)) { in removePartialRedundancy()
1167 if (CopyLeftBB && CopyLeftBB->succ_size() > 1) in removePartialRedundancy()
1173 auto InsPos = CopyLeftBB->getFirstTerminator(); in removePartialRedundancy()
1178 if (InsPos != CopyLeftBB->end()) { in removePartialRedundancy()
1179 SlotIndex InsPosIdx = LIS->getInstructionIndex(*InsPos).getRegSlot(true); in removePartialRedundancy()
1180 if (IntB.overlaps(InsPosIdx, LIS->getMBBEndIdx(CopyLeftBB))) in removePartialRedundancy()
1189 TII->get(TargetOpcode::COPY), IntB.reg()) in removePartialRedundancy()
1192 LIS->InsertMachineInstrInMaps(*NewCopyMI).getRegSlot(); in removePartialRedundancy()
1193 IntB.createDeadDef(NewCopyIdx, LIS->getVNInfoAllocator()); in removePartialRedundancy()
1195 SR.createDeadDef(NewCopyIdx, LIS->getVNInfoAllocator()); in removePartialRedundancy()
1209 // Note: This is fine to remove the copy before updating the live-ranges. in removePartialRedundancy()
1210 // While updating the live-ranges, we only look at slot indices and in removePartialRedundancy()
1218 LIS->pruneValue(*static_cast<LiveRange *>(&IntB), CopyIdx.getRegSlot(), in removePartialRedundancy()
1220 BValNo->markUnused(); in removePartialRedundancy()
1226 for (MachineOperand &MO : MRI->use_nodbg_operands(IntB.reg())) { in removePartialRedundancy()
1228 SlotIndex UseIdx = LIS->getInstructionIndex(MI); in removePartialRedundancy()
1235 LIS->extendToIndices(IntB, EndPoints); in removePartialRedundancy()
1242 LIS->pruneValue(SR, CopyIdx.getRegSlot(), &EndPoints); in removePartialRedundancy()
1243 BValNo->markUnused(); in removePartialRedundancy()
1260 *LIS->getSlotIndexes()); in removePartialRedundancy()
1261 LIS->extendToIndices(SR, EndPoints, Undefs); in removePartialRedundancy()
1266 // Finally, update the live-range of IntA. in removePartialRedundancy()
1298 LiveInterval &SrcInt = LIS->getInterval(SrcReg); in reMaterializeTrivialDef()
1299 SlotIndex CopyIdx = LIS->getInstructionIndex(*CopyMI); in reMaterializeTrivialDef()
1303 if (ValNo->isPHIDef() || ValNo->isUnused()) in reMaterializeTrivialDef()
1305 MachineInstr *DefMI = LIS->getInstructionFromIndex(ValNo->def); in reMaterializeTrivialDef()
1308 if (DefMI->isCopyLike()) { in reMaterializeTrivialDef()
1312 if (!TII->isAsCheapAsAMove(*DefMI)) in reMaterializeTrivialDef()
1323 if (!DefMI->isSafeToMove(AA, SawStore)) in reMaterializeTrivialDef()
1325 const MCInstrDesc &MCID = DefMI->getDesc(); in reMaterializeTrivialDef()
1328 // Only support subregister destinations when the def is read-undef. in reMaterializeTrivialDef()
1329 MachineOperand &DstOperand = CopyMI->getOperand(0); in reMaterializeTrivialDef()
1342 const unsigned DefSubIdx = DefMI->getOperand(0).getSubReg(); in reMaterializeTrivialDef()
1343 const TargetRegisterClass *DefRC = TII->getRegClass(MCID, 0, TRI, *MF); in reMaterializeTrivialDef()
1344 if (!DefMI->isImplicitDef()) { in reMaterializeTrivialDef()
1348 unsigned NewDstIdx = TRI->composeSubRegIndices(CP.getSrcIdx(), DefSubIdx); in reMaterializeTrivialDef()
1350 NewDstReg = TRI->getSubReg(DstReg, NewDstIdx); in reMaterializeTrivialDef()
1354 if (!DefRC->contains(NewDstReg)) in reMaterializeTrivialDef()
1369 DebugLoc DL = CopyMI->getDebugLoc(); in reMaterializeTrivialDef()
1370 MachineBasicBlock *MBB = CopyMI->getParent(); in reMaterializeTrivialDef()
1388 const TargetRegisterClass *DstRC = MRI->getRegClass(DstReg); in reMaterializeTrivialDef()
1390 TRI->getCommonSubClass(DefRC, DstRC); in reMaterializeTrivialDef()
1413 ImplicitOps.reserve(CopyMI->getNumOperands() - in reMaterializeTrivialDef()
1414 CopyMI->getDesc().getNumOperands()); in reMaterializeTrivialDef()
1415 for (unsigned I = CopyMI->getDesc().getNumOperands(), in reMaterializeTrivialDef()
1416 E = CopyMI->getNumOperands(); in reMaterializeTrivialDef()
1418 MachineOperand &MO = CopyMI->getOperand(I); in reMaterializeTrivialDef()
1428 CopyMI->eraseFromParent(); in reMaterializeTrivialDef()
1435 // We also expect to have tied implicit-defs of super registers originating in reMaterializeTrivialDef()
1437 // $edi = MOV32r0 implicit-def dead $eflags, implicit-def $rdi in reMaterializeTrivialDef()
1438 // undef %0.sub_32bit = MOV32r0 implicit-def dead $eflags, implicit-def %0 in reMaterializeTrivialDef()
1440 // The implicit-def of the super register may have been reduced to in reMaterializeTrivialDef()
1459 ((TRI->getSubReg(MO.getReg(), DefSubIdx) == in reMaterializeTrivialDef()
1461 TRI->isSubRegisterEq(NewMI.getOperand(0).getReg(), in reMaterializeTrivialDef()
1472 assert(!MRI->shouldTrackSubRegLiveness(DstReg) && in reMaterializeTrivialDef()
1473 "subrange update for implicit-def of super register may not be " in reMaterializeTrivialDef()
1484 NewRC = TRI->getMatchingSuperRegClass(NewRC, DefRC, NewIdx); in reMaterializeTrivialDef()
1486 NewRC = TRI->getCommonSubClass(NewRC, DefRC); in reMaterializeTrivialDef()
1490 LiveInterval &DstInt = LIS->getInterval(DstReg); in reMaterializeTrivialDef()
1492 SR.LaneMask = TRI->composeSubRegIndexLaneMask(DstIdx, SR.LaneMask); in reMaterializeTrivialDef()
1494 MRI->setRegClass(DstReg, NewRC); in reMaterializeTrivialDef()
1511 // %2:sub_16bit<def, read-undef> = COPY %1:sub_16bit in reMaterializeTrivialDef()
1519 SlotIndex CurrIdx = LIS->getInstructionIndex(NewMI); in reMaterializeTrivialDef()
1522 LaneBitmask MaxMask = MRI->getMaxLaneMaskForVReg(DstReg); in reMaterializeTrivialDef()
1523 VNInfo::Allocator& Alloc = LIS->getVNInfoAllocator(); in reMaterializeTrivialDef()
1531 SR->createDeadDef(DefIndex, Alloc); in reMaterializeTrivialDef()
1537 // %1:sub1<def,read-undef> = LOAD CONSTANT 1 in reMaterializeTrivialDef()
1540 // %2:sub1<def, read-undef> = LOAD CONSTANT 1 in reMaterializeTrivialDef()
1545 SlotIndex CurrIdx = LIS->getInstructionIndex(NewMI); in reMaterializeTrivialDef()
1546 LaneBitmask DstMask = TRI->getSubRegIndexLaneMask(NewIdx); in reMaterializeTrivialDef()
1550 VNInfo::Allocator &Alloc = LIS->getVNInfoAllocator(); in reMaterializeTrivialDef()
1558 // VNI is in ValNo - remove any segments in this SubRange that have in reMaterializeTrivialDef()
1582 // The New instruction may be defining a sub-register of what's actually in reMaterializeTrivialDef()
1593 // Record small dead def live-ranges for all the subregisters in reMaterializeTrivialDef()
1604 // dead ECX = remat ; implicit-def CL in reMaterializeTrivialDef()
1607 // no live-ranges would have been created for ECX. in reMaterializeTrivialDef()
1609 SlotIndex NewMIIdx = LIS->getInstructionIndex(NewMI); in reMaterializeTrivialDef()
1610 for (MCRegUnit Unit : TRI->regunits(NewMI.getOperand(0).getReg())) in reMaterializeTrivialDef()
1611 if (LiveRange *LR = LIS->getCachedRegUnit(Unit)) in reMaterializeTrivialDef()
1612 LR->createDeadDef(NewMIIdx.getRegSlot(), LIS->getVNInfoAllocator()); in reMaterializeTrivialDef()
1621 SlotIndex NewMIIdx = LIS->getInstructionIndex(NewMI); in reMaterializeTrivialDef()
1623 for (MCRegUnit Unit : TRI->regunits(Reg)) in reMaterializeTrivialDef()
1624 if (LiveRange *LR = LIS->getCachedRegUnit(Unit)) in reMaterializeTrivialDef()
1625 LR->createDeadDef(NewMIIdx.getRegSlot(), LIS->getVNInfoAllocator()); in reMaterializeTrivialDef()
1633 if (MRI->use_nodbg_empty(SrcReg)) { in reMaterializeTrivialDef()
1635 llvm::make_early_inc_range(MRI->use_operands(SrcReg))) { in reMaterializeTrivialDef()
1637 if (UseMI->isDebugInstr()) { in reMaterializeTrivialDef()
1644 MBB->splice(std::next(NewMI.getIterator()), UseMI->getParent(), UseMI); in reMaterializeTrivialDef()
1654 for (MachineOperand &UseMO : MRI->use_nodbg_operands(SrcReg)) { in reMaterializeTrivialDef()
1655 if (UseMO.getParent()->isCopyLike()) in reMaterializeTrivialDef()
1686 SlotIndex Idx = LIS->getInstructionIndex(*CopyMI); in eliminateUndefCopy()
1687 const LiveInterval &SrcLI = LIS->getInterval(SrcReg); in eliminateUndefCopy()
1690 LaneBitmask SrcMask = TRI->getSubRegIndexLaneMask(SrcSubIdx); in eliminateUndefCopy()
1700 // If the undef copy defines a live-out value (i.e. an input to a PHI def), in eliminateUndefCopy()
1702 LiveInterval &DstLI = LIS->getInterval(DstReg); in eliminateUndefCopy()
1706 VNInfo *V = DstLI.getVNInfoAt(Seg->end); in eliminateUndefCopy()
1710 if (((V && V->isPHIDef()) || (!V && !DstLI.liveAt(Idx)))) { in eliminateUndefCopy()
1711 for (unsigned i = CopyMI->getNumOperands(); i != 0; --i) { in eliminateUndefCopy()
1712 MachineOperand &MO = CopyMI->getOperand(i-1); in eliminateUndefCopy()
1715 CopyMI->removeOperand(i - 1); in eliminateUndefCopy()
1718 CopyMI->getOpcode() == TargetOpcode::SUBREG_TO_REG); in eliminateUndefCopy()
1719 CopyMI->removeOperand(i-1); in eliminateUndefCopy()
1723 CopyMI->setDesc(TII->get(TargetOpcode::IMPLICIT_DEF)); in eliminateUndefCopy()
1738 LaneBitmask DstMask = TRI->getSubRegIndexLaneMask(DstSubIdx); in eliminateUndefCopy()
1744 assert(SVNI != nullptr && SlotIndex::isSameInstr(SVNI->def, RegIndex)); in eliminateUndefCopy()
1749 LIS->removeVRegDefAt(DstLI, RegIndex); in eliminateUndefCopy()
1752 for (MachineOperand &MO : MRI->reg_nodbg_operands(DstReg)) { in eliminateUndefCopy()
1756 SlotIndex UseIdx = LIS->getInstructionIndex(MI); in eliminateUndefCopy()
1757 LaneBitmask UseMask = TRI->getSubRegIndexLaneMask(MO.getSubReg()); in eliminateUndefCopy()
1782 for (MachineOperand &MO : CopyMI->all_defs()) in eliminateUndefCopy()
1785 LIS->shrinkToUses(&DstLI); in eliminateUndefCopy()
1792 LaneBitmask Mask = TRI->getSubRegIndexLaneMask(SubRegIdx); in addUndefFlag()
1819 LiveInterval *DstInt = DstIsPhys ? nullptr : &LIS->getInterval(DstReg); in updateRegDefsUses()
1821 if (DstInt && DstInt->hasSubRanges() && DstReg != SrcReg) { in updateRegDefsUses()
1822 for (MachineOperand &MO : MRI->reg_operands(DstReg)) { in updateRegDefsUses()
1829 SlotIndex UseIdx = LIS->getInstructionIndex(MI).getRegSlot(true); in updateRegDefsUses()
1836 I = MRI->reg_instr_begin(SrcReg), E = MRI->reg_instr_end(); in updateRegDefsUses()
1840 // Each instruction can only be rewritten once because sub-register in updateRegDefsUses()
1842 // the UseMI operands removes them from the SrcReg use-def chain, but when in updateRegDefsUses()
1850 std::tie(Reads, Writes) = UseMI->readsWritesVirtualRegister(SrcReg, &Ops); in updateRegDefsUses()
1852 // If SrcReg wasn't read, it may still be the case that DstReg is live-in in updateRegDefsUses()
1853 // because SrcReg is a sub-register. in updateRegDefsUses()
1854 if (DstInt && !Reads && SubIdx && !UseMI->isDebugInstr()) in updateRegDefsUses()
1855 Reads = DstInt->liveAt(LIS->getInstructionIndex(*UseMI)); in updateRegDefsUses()
1859 MachineOperand &MO = UseMI->getOperand(Op); in updateRegDefsUses()
1861 // Adjust <undef> flags in case of sub-register joins. We don't want to in updateRegDefsUses()
1862 // turn a full def into a read-modify-write sub-register def and vice in updateRegDefsUses()
1870 unsigned SubUseIdx = TRI->composeSubRegIndices(SubIdx, MO.getSubReg()); in updateRegDefsUses()
1871 if (SubUseIdx != 0 && MRI->shouldTrackSubRegLiveness(DstReg)) { in updateRegDefsUses()
1872 if (!DstInt->hasSubRanges()) { in updateRegDefsUses()
1873 BumpPtrAllocator &Allocator = LIS->getVNInfoAllocator(); in updateRegDefsUses()
1874 LaneBitmask FullMask = MRI->getMaxLaneMaskForVReg(DstInt->reg()); in updateRegDefsUses()
1875 LaneBitmask UsedLanes = TRI->getSubRegIndexLaneMask(SubIdx); in updateRegDefsUses()
1877 DstInt->createSubRangeFrom(Allocator, UsedLanes, *DstInt); in updateRegDefsUses()
1878 // The unused lanes are just empty live-ranges at this point. in updateRegDefsUses()
1882 DstInt->createSubRange(Allocator, UnusedLanes); in updateRegDefsUses()
1884 SlotIndex MIIdx = UseMI->isDebugInstr() in updateRegDefsUses()
1885 ? LIS->getSlotIndexes()->getIndexBefore(*UseMI) in updateRegDefsUses()
1886 : LIS->getInstructionIndex(*UseMI); in updateRegDefsUses()
1900 if (!UseMI->isDebugInstr()) in updateRegDefsUses()
1901 dbgs() << LIS->getInstructionIndex(*UseMI) << "\t"; in updateRegDefsUses()
1911 if (!MRI->isReserved(CP.getDstReg())) { in canJoinPhys()
1916 LiveInterval &JoinVInt = LIS->getInterval(CP.getSrcReg()); in canJoinPhys()
1927 for (const MachineBasicBlock *Pred : MBB->predecessors()) { in copyValueUndefInPredecessors()
1928 SlotIndex PredEnd = LIS->getMBBEndIdx(Pred); in copyValueUndefInPredecessors()
1931 if (V->id != SLRQ.valueOutOrDead()->id) in copyValueUndefInPredecessors()
1944 for (MachineOperand &MO : MRI->use_nodbg_operands(Reg)) { in setUndefOnPrunedSubRegUses()
1949 LaneBitmask SubRegMask = TRI->getSubRegIndexLaneMask(SubRegIdx); in setUndefOnPrunedSubRegUses()
1950 SlotIndex Pos = LIS->getInstructionIndex(*MO.getParent()); in setUndefOnPrunedSubRegUses()
1965 LIS->shrinkToUses(&LI); in setUndefOnPrunedSubRegUses()
1972 LLVM_DEBUG(dbgs() << LIS->getInstructionIndex(*CopyMI) << '\t' << *CopyMI); in joinCopy()
1981 auto SrcRC = MRI->getRegClass(CP.getSrcReg()); in joinCopy()
1982 auto DstRC = MRI->getRegClass(CP.getDstReg()); in joinCopy()
1989 if (!TRI->shouldCoalesce(CopyMI, SrcRC, SrcIdx, DstRC, DstIdx, in joinCopy()
1999 if (!CP.isPhys() && CopyMI->allDefsAreDead()) { in joinCopy()
2010 if (UndefMI->isImplicitDef()) in joinCopy()
2021 LiveInterval &LI = LIS->getInterval(CP.getSrcReg()); in joinCopy()
2023 const SlotIndex CopyIdx = LIS->getInstructionIndex(*CopyMI); in joinCopy()
2032 MachineBasicBlock *MBB = CopyMI->getParent(); in joinCopy()
2081 if (!CP.isPartial() && LIS->getInterval(CP.getSrcReg()).size() > in joinCopy()
2082 LIS->getInterval(CP.getDstReg()).size()) in joinCopy()
2083 CP.flip(); in joinCopy()
2087 << TRI->getRegClassName(CP.getNewRC()) << " with "; in joinCopy()
2090 << TRI->getSubRegIndexName(CP.getDstIdx()) << " and " in joinCopy()
2092 << TRI->getSubRegIndexName(CP.getSrcIdx()) << '\n'; in joinCopy()
2126 LiveInterval &DstLI = LIS->getInterval(DstReg); in joinCopy()
2147 // Coalescing to a virtual register that is of a sub-register class of the in joinCopy()
2151 MRI->setRegClass(CP.getDstReg(), CP.getNewRC()); in joinCopy()
2154 // Removing sub-register copies can ease the register class constraints. in joinCopy()
2174 LiveInterval &LI = LIS->getInterval(CP.getDstReg()); in joinCopy()
2180 LIS->shrinkToUses(S, LI.reg()); in joinCopy()
2188 // is not up-to-date, need to update the merged live interval here. in joinCopy()
2193 LiveInterval &LI = LIS->getInterval(CP.getDstReg()); in joinCopy()
2199 LIS->removeInterval(CP.getSrcReg()); in joinCopy()
2202 TRI->updateRegAllocHint(CP.getSrcReg(), CP.getDstReg(), *MF); in joinCopy()
2206 << " -> " << printReg(CP.getDstReg(), TRI, CP.getDstIdx()) << '\n'; in joinCopy()
2211 dbgs() << LIS->getInterval(CP.getDstReg()); in joinCopy()
2223 assert(MRI->isReserved(DstReg) && "Not a reserved register"); in joinReservedPhysReg()
2224 LiveInterval &RHS = LIS->getInterval(SrcReg); in joinReservedPhysReg()
2232 // - we don't properly track the live range of reserved registers. in joinReservedPhysReg()
2236 if (!MRI->isConstantPhysReg(DstReg)) { in joinReservedPhysReg()
2237 for (MCRegUnit Unit : TRI->regunits(DstReg)) { in joinReservedPhysReg()
2240 if (!MRI->isReserved(*RI)) in joinReservedPhysReg()
2243 if (RHS.overlaps(LIS->getRegUnit(Unit))) { in joinReservedPhysReg()
2252 if (LIS->checkRegMaskInterference(RHS, RegMaskUsable) && in joinReservedPhysReg()
2274 CopyMI = MRI->getVRegDef(SrcReg); in joinReservedPhysReg()
2284 if (!MRI->hasOneNonDBGUse(SrcReg)) { in joinReservedPhysReg()
2289 if (!LIS->intervalIsInOneMBB(RHS)) { in joinReservedPhysReg()
2294 MachineInstr &DestMI = *MRI->getVRegDef(SrcReg); in joinReservedPhysReg()
2295 CopyMI = &*MRI->use_instr_nodbg_begin(SrcReg); in joinReservedPhysReg()
2296 SlotIndex CopyRegIdx = LIS->getInstructionIndex(*CopyMI).getRegSlot(); in joinReservedPhysReg()
2297 SlotIndex DestRegIdx = LIS->getInstructionIndex(DestMI).getRegSlot(); in joinReservedPhysReg()
2299 if (!MRI->isConstantPhysReg(DstReg)) { in joinReservedPhysReg()
2303 SlotIndexes *Indexes = LIS->getSlotIndexes(); in joinReservedPhysReg()
2304 for (SlotIndex SI = Indexes->getNextNonNullIndex(DestRegIdx); in joinReservedPhysReg()
2305 SI != CopyRegIdx; SI = Indexes->getNextNonNullIndex(SI)) { in joinReservedPhysReg()
2306 MachineInstr *MI = LIS->getInstructionFromIndex(SI); in joinReservedPhysReg()
2307 if (MI->readsRegister(DstReg, TRI)) { in joinReservedPhysReg()
2319 LIS->removePhysRegDefAt(DstReg.asMCReg(), CopyRegIdx); in joinReservedPhysReg()
2323 for (MCRegUnit Unit : TRI->regunits(DstReg)) { in joinReservedPhysReg()
2324 LiveRange &LR = LIS->getRegUnit(Unit); in joinReservedPhysReg()
2325 LR.createDeadDef(DestRegIdx, LIS->getVNInfoAllocator()); in joinReservedPhysReg()
2330 MRI->clearKillFlags(CP.getSrcReg()); in joinReservedPhysReg()
2335 //===----------------------------------------------------------------------===//
2337 //===----------------------------------------------------------------------===//
2344 // The live range of a single SSA value forms a sub-tree of the dominator tree.
2370 // %dst = COPY %ext <-- Remove this COPY, trim the live range of %ext.
2375 // %dst:ssub0<def,read-undef> = FOO
2402 /// joined. Objects of this class are always created in pairs - one for each
2470 /// Per-value info for LI. The lane bit masks are all relative to the final
2492 /// is a predecessor to a phi-value. These IMPLICIT_DEF instructions can be
2563 /// 1 %dst = VLOAD <-- Define all lanes in %dst
2564 /// 2 %src = FOO <-- ValNo to be joined with %dst:ssub0
2565 /// 3 %dst:ssub1 = BAR <-- Partial redef doesn't clear taint in ssub0
2566 /// 4 %dst:ssub0 = COPY %src <-- Conflict resolved, ssub0 wasn't read
2584 /// %src = COPY %dst <-- This value to be pruned.
2585 /// %dst = COPY %src <-- This value is a copy of a pruned value.
2595 NewVNInfo(newVNInfo), CP(cp), LIS(lis), Indexes(LIS->getSlotIndexes()), in JoinVals()
2596 TRI(TRI), Assignments(LR.getNumValNums(), -1), in JoinVals()
2652 for (const MachineOperand &MO : DefMI->all_defs()) { in computeWriteLanes()
2655 L |= TRI->getSubRegIndexLaneMask( in computeWriteLanes()
2656 TRI->composeSubRegIndices(SubIdx, MO.getSubReg())); in computeWriteLanes()
2667 while (!VNI->isPHIDef()) { in followCopyChain()
2668 SlotIndex Def = VNI->def; in followCopyChain()
2669 MachineInstr *MI = Indexes->getInstructionFromIndex(Def); in followCopyChain()
2671 if (!MI->isFullCopy()) in followCopyChain()
2673 Register SrcReg = MI->getOperand(1).getReg(); in followCopyChain()
2677 const LiveInterval &LI = LIS->getInterval(SrcReg); in followCopyChain()
2689 LaneBitmask SMask = TRI->composeSubRegIndexLaneMask(SubIdx, S.LaneMask); in followCopyChain()
2737 return Orig0->def == Orig1->def && Reg0 == Reg1; in valuesIdentical()
2745 if (VNI->isUnused()) { in analyzeValue()
2752 if (VNI->isPHIDef()) { in analyzeValue()
2755 : TRI->getSubRegIndexLaneMask(SubIdx); in analyzeValue()
2758 DefMI = Indexes->getInstructionFromIndex(VNI->def); in analyzeValue()
2763 if (DefMI->isImplicitDef()) { in analyzeValue()
2771 // If this is a read-modify-write instruction, there may be more valid in analyzeValue()
2782 // %src:ssub1<def,read-undef> = FOO %src:ssub2 in analyzeValue()
2784 // The <read-undef> flag on the def operand means that old lane values are in analyzeValue()
2787 V.RedefVNI = LR.Query(VNI->def).valueIn(); in analyzeValue()
2791 computeAssignment(V.RedefVNI->id, Other); in analyzeValue()
2792 V.ValidLanes |= Vals[V.RedefVNI->id].ValidLanes; in analyzeValue()
2797 if (DefMI->isImplicitDef()) { in analyzeValue()
2809 // Find the value in Other that overlaps VNI->def, if any. in analyzeValue()
2810 LiveQueryResult OtherLRQ = Other.LR.Query(VNI->def); in analyzeValue()
2817 assert(SlotIndex::isSameInstr(VNI->def, OtherVNI->def) && "Broken LRQ"); in analyzeValue()
2821 if (OtherVNI->def < VNI->def) in analyzeValue()
2822 Other.computeAssignment(OtherVNI->id, *this); in analyzeValue()
2823 else if (VNI->def < OtherVNI->def && OtherLRQ.valueIn()) { in analyzeValue()
2824 // This is an early-clobber def overlapping a live-in value in the other in analyzeValue()
2830 Val &OtherV = Other.Vals[OtherVNI->id]; in analyzeValue()
2832 // revisiting OtherVNI->id in JoinVals::computeAssignment() below before it in analyzeValue()
2834 if (!OtherV.isAnalyzed() || Other.Assignments[OtherVNI->id] == -1) in analyzeValue()
2839 if (VNI->isPHIDef()) in analyzeValue()
2854 assert(!SlotIndex::isSameInstr(VNI->def, V.OtherVNI->def) && "Broken LRQ"); in analyzeValue()
2858 Other.computeAssignment(V.OtherVNI->id, *this); in analyzeValue()
2859 Val &OtherV = Other.Vals[V.OtherVNI->id]; in analyzeValue()
2874 Indexes->getInstructionFromIndex(V.OtherVNI->def); in analyzeValue()
2875 MachineBasicBlock *OtherMBB = OtherImpDef->getParent(); in analyzeValue()
2877 (DefMI->getParent() != OtherMBB || LIS->isLiveInToMBB(LR, OtherMBB))) { in analyzeValue()
2878 LLVM_DEBUG(dbgs() << "IMPLICIT_DEF defined at " << V.OtherVNI->def in analyzeValue()
2880 << printMBBReference(*DefMI->getParent()) in analyzeValue()
2883 } else if (OtherMBB->hasEHPadSuccessor()) { in analyzeValue()
2889 dbgs() << "IMPLICIT_DEF defined at " << V.OtherVNI->def in analyzeValue()
2900 if (VNI->isPHIDef()) in analyzeValue()
2904 if (DefMI->isImplicitDef()) in analyzeValue()
2907 // Include the non-conflict where DefMI is a coalescable copy that kills in analyzeValue()
2918 if (OtherLRQ.isKill() && OtherLRQ.endPoint() <= VNI->def) in analyzeValue()
2924 // %this = COPY %ext <-- Erase this copy in analyzeValue()
2926 if (DefMI->isFullCopy() && !CP.isPartial() && in analyzeValue()
2940 // mapping, though - OtherVNI will map to multiple values: in analyzeValue()
2942 // 1 %dst:ssub0 = FOO <-- OtherVNI in analyzeValue()
2943 // 2 %src = BAR <-- VNI in analyzeValue()
2944 // 3 %dst:ssub1 = COPY killed %src <-- Eliminate this copy. in analyzeValue()
2956 // %dst<def,early-clobber> = ASM killed %src in analyzeValue()
2962 assert(VNI->def.isEarlyClobber() && in analyzeValue()
2971 if ((TRI->getSubRegIndexLaneMask(Other.SubIdx) & ~V.WriteLanes).none()) in analyzeValue()
2975 auto &OtherLI = LIS->getInterval(Other.Reg); in analyzeValue()
2980 LaneBitmask OtherMask = TRI->getSubRegIndexLaneMask(Other.SubIdx); in analyzeValue()
2984 // If we are clobbering some active lanes of OtherVNI at VNI->def, it is in analyzeValue()
2989 TRI->composeSubRegIndexLaneMask(Other.SubIdx, OtherSR.LaneMask); in analyzeValue()
2993 auto OtherSRQ = OtherSR.Query(VNI->def); in analyzeValue()
2994 if (OtherSRQ.valueIn() && OtherSRQ.endPoint() > VNI->def) { in analyzeValue()
3007 MachineBasicBlock *MBB = Indexes->getMBBFromIndex(VNI->def); in analyzeValue()
3008 if (OtherLRQ.endPoint() >= Indexes->getMBBEndIdx(MBB)) in analyzeValue()
3016 // that now - the recursive analyzeValue() calls must go upwards in the in analyzeValue()
3026 assert(Assignments[ValNo] != -1 && "Bad recursion?"); in computeAssignment()
3034 assert(Other.Vals[V.OtherVNI->id].isAnalyzed() && "Missing recursion"); in computeAssignment()
3035 Assignments[ValNo] = Other.Assignments[V.OtherVNI->id]; in computeAssignment()
3037 << LR.getValNumInfo(ValNo)->def << " into " in computeAssignment()
3038 << printReg(Other.Reg) << ':' << V.OtherVNI->id << '@' in computeAssignment()
3039 << V.OtherVNI->def << " --> @" in computeAssignment()
3040 << NewVNInfo[Assignments[ValNo]]->def << '\n'); in computeAssignment()
3046 Val &OtherV = Other.Vals[V.OtherVNI->id]; in computeAssignment()
3063 << '@' << LR.getValNumInfo(i)->def << '\n'); in mapValues()
3074 MachineBasicBlock *MBB = Indexes->getMBBFromIndex(VNI->def); in taintExtent()
3075 SlotIndex MBBEnd = Indexes->getMBBEndIdx(MBB); in taintExtent()
3078 LiveInterval::iterator OtherI = Other.LR.find(VNI->def); in taintExtent()
3083 SlotIndex End = OtherI->end; in taintExtent()
3086 << OtherI->valno->id << '@' << OtherI->start << '\n'); in taintExtent()
3090 << OtherI->valno->id << '@' << OtherI->start << " to " in taintExtent()
3098 if (++OtherI == Other.LR.end() || OtherI->start >= MBBEnd) in taintExtent()
3102 const Val &OV = Other.Vals[OtherI->valno->id]; in taintExtent()
3119 unsigned S = TRI->composeSubRegIndices(SubIdx, MO.getSubReg()); in usesLanes()
3120 if ((Lanes & TRI->getSubRegIndexLaneMask(S)).any()) in usesLanes()
3133 << LR.getValNumInfo(i)->def in resolveConflicts()
3141 const Val &OtherV = Other.Vals[V.OtherVNI->id]; in resolveConflicts()
3154 // Now look at the instructions from VNI->def to TaintExtent (inclusive). in resolveConflicts()
3155 MachineBasicBlock *MBB = Indexes->getMBBFromIndex(VNI->def); in resolveConflicts()
3156 MachineBasicBlock::iterator MI = MBB->begin(); in resolveConflicts()
3157 if (!VNI->isPHIDef()) { in resolveConflicts()
3158 MI = Indexes->getInstructionFromIndex(VNI->def); in resolveConflicts()
3159 if (!VNI->def.isEarlyClobber()) { in resolveConflicts()
3164 assert(!SlotIndex::isSameInstr(VNI->def, TaintExtent.front().first) && in resolveConflicts()
3165 "Interference ends on VNI->def. Should have been handled earlier"); in resolveConflicts()
3167 Indexes->getInstructionFromIndex(TaintExtent.front().first); in resolveConflicts()
3171 assert(MI != MBB->end() && "Bad LastMI"); in resolveConflicts()
3180 LastMI = Indexes->getInstructionFromIndex(TaintExtent[TaintNum].first); in resolveConflicts()
3205 V.Pruned = Other.isPrunedValue(V.OtherVNI->id, *this); in isPrunedValue()
3213 SlotIndex Def = LR.getValNumInfo(i)->def; in pruneValues()
3219 LIS->pruneValue(Other.LR, Def, &EndPoints); in pruneValues()
3221 // instructions are only inserted to provide a live-out value for PHI in pruneValues()
3224 Val &OtherV = Other.Vals[Vals[i].OtherVNI->id]; in pruneValues()
3229 // Remove <def,read-undef> flags. This def is now a partial redef. in pruneValues()
3233 Indexes->getInstructionFromIndex(Def)->operands()) { in pruneValues()
3257 LIS->pruneValue(LR, Def, &EndPoints); in pruneValues()
3269 // Check if the segment consists of a copied live-through value (i.e. the copy
3273 return Q.valueIn() && Q.valueIn()->isPHIDef() && Q.valueIn() == Q.valueOut(); in isLiveThrough()
3277 /// %31 and %45 at 800. (The vertical lines represent live range segments.)
3290 /// | | v0 | v0 <--+
3291 /// 992B ; backedge -> bb.1 | + + |
3294 /// live-out!
3308 /// Since the main-range values %31.v1 and %45.v0 were proved to be
3327 SlotIndex Def = LR.getValNumInfo(i)->def; in pruneSubRegValues()
3330 OtherDef = V.OtherVNI->def; in pruneSubRegValues()
3343 ValueOut->def == Def))) { in pruneSubRegValues()
3347 LIS->pruneValue(S, Def, &EndPoints); in pruneSubRegValues()
3350 ValueOut->markUnused(); in pruneSubRegValues()
3356 LIS->extendToIndices(S, EndPoints); in pruneSubRegValues()
3361 if (ValueOut->isPHIDef()) in pruneSubRegValues()
3388 if (VNI->def == Def) in isDefInSubRange()
3401 if (VNI->isUnused() || VNI->isPHIDef() || isDefInSubRange(LI, VNI->def)) in pruneMainSegments()
3415 VNI->markUnused(); in removeImplicitDefs()
3426 SlotIndex Def = VNI->def; in eraseInstrs()
3450 NewEnd = I->end; in eraseInstrs()
3456 VNI->markUnused(); in eraseInstrs()
3458 if (LI != nullptr && LI->hasSubRanges()) { in eraseInstrs()
3464 for (LiveInterval::SubRange &SR : LI->subranges()) { in eraseInstrs()
3468 if (I->start > Def) in eraseInstrs()
3469 ED = ED.isValid() ? std::min(ED, I->start) : I->start; in eraseInstrs()
3471 LE = LE.isValid() ? std::max(LE, I->end) : I->end; in eraseInstrs()
3483 std::prev(S)->end = NewEnd; in eraseInstrs()
3495 MachineInstr *MI = Indexes->getInstructionFromIndex(Def); in eraseInstrs()
3497 if (MI->isCopy()) { in eraseInstrs()
3498 Register Reg = MI->getOperand(1).getReg(); in eraseInstrs()
3504 LIS->RemoveMachineInstrFromMaps(*MI); in eraseInstrs()
3505 MI->eraseFromParent(); in eraseInstrs()
3569 if (i != n-1) in joinSubRegRanges()
3574 LIS->extendToIndices(LRange, EndPoints); in joinSubRegRanges()
3582 BumpPtrAllocator &Allocator = LIS->getVNInfoAllocator(); in mergeSubRangeInto()
3594 *LIS->getSlotIndexes(), *TRI, ComposeSubRegIdx); in mergeSubRangeInto()
3610 LiveInterval &RHS = LIS->getInterval(CP.getSrcReg()); in joinVirtRegs()
3611 LiveInterval &LHS = LIS->getInterval(CP.getDstReg()); in joinVirtRegs()
3612 bool TrackSubRegLiveness = MRI->shouldTrackSubRegLiveness(*CP.getNewRC()); in joinVirtRegs()
3634 BumpPtrAllocator &Allocator = LIS->getVNInfoAllocator(); in joinVirtRegs()
3640 LaneBitmask Mask = DstIdx == 0 ? CP.getNewRC()->getLaneMask() in joinVirtRegs()
3641 : TRI->getSubRegIndexLaneMask(DstIdx); in joinVirtRegs()
3648 LaneBitmask Mask = TRI->composeSubRegIndexLaneMask(DstIdx, R.LaneMask); in joinVirtRegs()
3658 LaneBitmask Mask = SrcIdx == 0 ? CP.getNewRC()->getLaneMask() in joinVirtRegs()
3659 : TRI->getSubRegIndexLaneMask(SrcIdx); in joinVirtRegs()
3664 LaneBitmask Mask = TRI->composeSubRegIndexLaneMask(SrcIdx, R.LaneMask); in joinVirtRegs()
3677 LHS.createSubRangeFrom(LIS->getVNInfoAllocator(), in joinVirtRegs()
3678 CP.getNewRC()->getLaneMask(), LHS); in joinVirtRegs()
3679 mergeSubRangeInto(LHS, RHS, TRI->getSubRegIndexLaneMask(CP.getSrcIdx()), CP, in joinVirtRegs()
3699 shrinkToUses(&LIS->getInterval(ShrinkRegs.pop_back_val())); in joinVirtRegs()
3704 // If the RHS covers any PHI locations that were tracked for debug-info, we in joinVirtRegs()
3709 for (unsigned InstID : RegIt->second) { in joinVirtRegs()
3712 const SlotIndex &SI = PHIIt->second.SI; in joinVirtRegs()
3716 if (LII == RHS.end() || LII->start > SI) in joinVirtRegs()
3721 // %1:gr16 = some-inst in joinVirtRegs()
3722 // -> in joinVirtRegs()
3723 // %2:gr32.sub_16bit = some-inst in joinVirtRegs()
3726 // %2:gr32.sub_16bit = some-inst in joinVirtRegs()
3728 // -> in joinVirtRegs()
3729 // %3:gr32.sub_16bit = some-inst in joinVirtRegs()
3734 if (PHIIt->second.SubReg && PHIIt->second.SubReg != CP.getSrcIdx()) in joinVirtRegs()
3738 PHIIt->second.Reg = CP.getDstReg(); in joinVirtRegs()
3740 // If we merge into a sub-register of a larger class (test above), in joinVirtRegs()
3743 PHIIt->second.SubReg = CP.getSrcIdx(); in joinVirtRegs()
3749 auto InstrNums = RegIt->second; in joinVirtRegs()
3756 RegIt->second.insert(RegIt->second.end(), InstrNums.begin(), in joinVirtRegs()
3768 MRI->clearKillFlags(LHS.reg()); in joinVirtRegs()
3769 MRI->clearKillFlags(RHS.reg()); in joinVirtRegs()
3778 if (i != n-1) in joinVirtRegs()
3783 LIS->extendToIndices((LiveRange&)LHS, EndPoints); in joinVirtRegs()
3795 const SlotIndexes &Slots = *LIS->getSlotIndexes(); in buildVRegToDbgValueMap()
3802 for (const auto &Op : X->debug_operands()) { in buildVRegToDbgValueMap()
3812 // Once a non-debug instruction is found, record the slot index of the in buildVRegToDbgValueMap()
3865 auto &DbgValueSet = VRegMapIt->second; in checkMergingChangesDbgValuesImpl()
3875 &LastUndefIdx](SlotIndex Idx) -> bool { in checkMergingChangesDbgValuesImpl()
3876 // Our worst-case performance typically happens with asan, causing very in checkMergingChangesDbgValuesImpl()
3878 // result for this edge-case. in checkMergingChangesDbgValuesImpl()
3896 auto Resolution = RegVals.getResolution(OtherIt->valno->id); in checkMergingChangesDbgValuesImpl()
3903 // Iterate over both the live-range of the "Other" register, and the set of in checkMergingChangesDbgValuesImpl()
3907 if (DbgValueSetIt->first < SegmentIt->end) { in checkMergingChangesDbgValuesImpl()
3910 if (DbgValueSetIt->first >= SegmentIt->start) { in checkMergingChangesDbgValuesImpl()
3911 bool HasReg = DbgValueSetIt->second->hasDebugOperandForReg(Reg); in checkMergingChangesDbgValuesImpl()
3912 bool ShouldUndefReg = ShouldUndef(DbgValueSetIt->first); in checkMergingChangesDbgValuesImpl()
3915 DbgValueSetIt->second->setDebugValueUndef(); in checkMergingChangesDbgValuesImpl()
3940 /// C-style comparator that sorts first based on the loop depth of the basic
3947 if (LHS->Depth != RHS->Depth) in compareMBBPriority()
3948 return LHS->Depth > RHS->Depth ? -1 : 1; in compareMBBPriority()
3951 if (LHS->IsSplit != RHS->IsSplit) in compareMBBPriority()
3952 return LHS->IsSplit ? -1 : 1; in compareMBBPriority()
3956 unsigned cl = LHS->MBB->pred_size() + LHS->MBB->succ_size(); in compareMBBPriority()
3957 unsigned cr = RHS->MBB->pred_size() + RHS->MBB->succ_size(); in compareMBBPriority()
3959 return cl > cr ? -1 : 1; in compareMBBPriority()
3962 return LHS->MBB->getNumber() < RHS->MBB->getNumber() ? -1 : 1; in compareMBBPriority()
3967 if (!Copy->isCopy()) in isLocalCopy()
3970 if (Copy->getOperand(1).isUndef()) in isLocalCopy()
3973 Register SrcReg = Copy->getOperand(1).getReg(); in isLocalCopy()
3974 Register DstReg = Copy->getOperand(0).getReg(); in isLocalCopy()
3978 return LIS->intervalIsInOneMBB(LIS->getInterval(SrcReg)) in isLocalCopy()
3979 || LIS->intervalIsInOneMBB(LIS->getInterval(DstReg)); in isLocalCopy()
3984 if (!LIS->hasInterval(reg)) in lateLiveIntervalUpdate()
3986 LiveInterval &LI = LIS->getInterval(reg); in lateLiveIntervalUpdate()
4033 for (const MachineInstr &MI : MRI->reg_nodbg_instructions(DstReg)) in isTerminalReg()
4058 const LiveInterval &DstLI = LIS->getInterval(DstReg); in applyTerminalRule()
4059 for (const MachineInstr &MI : MRI->reg_nodbg_instructions(SrcReg)) { in applyTerminalRule()
4075 // Check if OtherReg is a non-terminal. in applyTerminalRule()
4079 if (LIS->getInterval(OtherReg).overlaps(DstLI)) { in applyTerminalRule()
4090 LLVM_DEBUG(dbgs() << MBB->getName() << ":\n"); in copyCoalesceInMBB()
4092 // Collect all copy-like instructions in MBB. Don't start coalescing anything in copyCoalesceInMBB()
4098 // Coalesce copies bottom-up to coalesce local defs before local uses. They in copyCoalesceInMBB()
4158 MBBs.reserve(MF->size()); in joinAllIntervals()
4160 MBBs.push_back(MBBPriorityInfo(&MBB, Loops->getLoopDepth(&MBB), in joinAllIntervals()
4224 // If there are PHIs tracked by debug-info, they will need updating during in runOnMachineFunction()
4226 SlotIndexes *Slots = LIS->getSlotIndexes(); in runOnMachineFunction()
4227 for (const auto &DebugPHI : MF->DebugPHIPositions) { in runOnMachineFunction()
4231 SlotIndex SI = Slots->getMBBStartIdx(MBB); in runOnMachineFunction()
4243 MF->verify(this, "Before register coalescing"); in runOnMachineFunction()
4255 // Removing sub-register operands may allow GR32_ABCD -> GR32 and DPR_VFP2 -> in runOnMachineFunction()
4262 if (MRI->reg_nodbg_empty(Reg)) in runOnMachineFunction()
4264 if (MRI->recomputeRegClass(Reg)) { in runOnMachineFunction()
4266 << TRI->getRegClassName(MRI->getRegClass(Reg)) << '\n'); in runOnMachineFunction()
4269 LiveInterval &LI = LIS->getInterval(Reg); in runOnMachineFunction()
4273 if (!MRI->shouldTrackSubRegLiveness(Reg)) { in runOnMachineFunction()
4277 LaneBitmask MaxMask = MRI->getMaxLaneMaskForVReg(Reg); in runOnMachineFunction()
4289 // After coalescing, update any PHIs that are being tracked by debug-info in runOnMachineFunction()
4291 for (auto &p : MF->DebugPHIPositions) { in runOnMachineFunction()
4294 p.second.Reg = it->second.Reg; in runOnMachineFunction()
4295 p.second.SubReg = it->second.SubReg; in runOnMachineFunction()
4303 MF->verify(this, "After register coalescing"); in runOnMachineFunction()
4308 LIS->print(O); in print()