Lines Matching +full:latch +full:- +full:bit
1 //===- HexagonHardwareLoops.cpp - Identify and generate hardware loops ----===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
11 // zero-cycle overhead.
20 // - Countable loops (w/ ind. var for a trip count)
21 // - Assumes loops are normalized by IndVarSimplify
22 // - Try inner-most loops first
23 // - No function calls in loops.
25 //===----------------------------------------------------------------------===//
69 static cl::opt<int> HWLoopLimit("hexagon-max-hwloop", cl::Hidden, cl::init(-1));
72 static cl::opt<std::string> PHFn("hexagon-hwloop-phfn", cl::Hidden,
77 static cl::opt<bool> HWCreatePreheader("hexagon-hwloop-preheader",
84 static cl::opt<bool> SpecPreheader("hwloop-spec-preheader", cl::Hidden,
255 /// Check if the given operand has a compile-time known constant
260 /// Check if the operand has a compile-time known constant value.
280 /// The desired flow is: phi ---> bump -+-> comparison-in-latch.
282 /// +-> back to phi
287 /// phi -+-> bump ---> back to phi
289 /// +-> comparison-in-latch (against upper_bound-bump),
296 /// couldn't be identified, or if the value in the latch's comparison
297 /// cannot be adjusted to reflect the post-bump value.
395 if (L->isOutermost()) { in runOnMachineFunction()
409 MachineBasicBlock *Header = L->getHeader(); in findInductionRegister()
410 MachineBasicBlock *Preheader = MLI->findLoopPreheader(L, SpecPreheader); in findInductionRegister()
411 MachineBasicBlock *Latch = L->getLoopLatch(); in findInductionRegister() local
412 MachineBasicBlock *ExitingBlock = L->findLoopControlBlock(); in findInductionRegister()
413 if (!Header || !Preheader || !Latch || !ExitingBlock) in findInductionRegister()
420 // Mapping: R.next -> (R, bump), where R, R.next and bump are derived in findInductionRegister()
430 for (instr_iterator I = Header->instr_begin(), E = Header->instr_end(); in findInductionRegister()
431 I != E && I->isPHI(); ++I) { in findInductionRegister()
435 // latch block, and see if is a result of an addition of form "reg+imm", in findInductionRegister()
437 for (unsigned i = 1, n = Phi->getNumOperands(); i < n; i += 2) { in findInductionRegister()
438 if (Phi->getOperand(i+1).getMBB() != Latch) in findInductionRegister()
441 Register PhiOpReg = Phi->getOperand(i).getReg(); in findInductionRegister()
442 MachineInstr *DI = MRI->getVRegDef(PhiOpReg); in findInductionRegister()
444 if (DI->getDesc().isAdd()) { in findInductionRegister()
447 Register IndReg = DI->getOperand(1).getReg(); in findInductionRegister()
448 MachineOperand &Opnd2 = DI->getOperand(2); in findInductionRegister()
450 if (MRI->getVRegDef(IndReg) == Phi && checkForImmediate(Opnd2, V)) { in findInductionRegister()
451 Register UpdReg = DI->getOperand(0).getReg(); in findInductionRegister()
460 bool NotAnalyzed = TII->analyzeBranch(*ExitingBlock, TB, FB, Cond, false); in findInductionRegister()
466 if (!TII->getPredReg(Cond, PredR, PredPos, PredRegFlags)) in findInductionRegister()
469 MachineInstr *PredI = MRI->getVRegDef(PredR); in findInductionRegister()
470 if (!PredI->isCompare()) in findInductionRegister()
476 TII->analyzeCompare(*PredI, CmpReg1, CmpReg2, CmpMask, CmpImm); in findInductionRegister()
503 Reg = F->second.first; in findInductionRegister()
504 IVBump = F->second.second; in findInductionRegister()
505 IVOp = MRI->getVRegDef(F->first); in findInductionRegister()
571 MachineBasicBlock *TopMBB = L->getTopBlock(); in getLoopTripCount()
572 MachineBasicBlock::pred_iterator PI = TopMBB->pred_begin(); in getLoopTripCount()
573 assert(PI != TopMBB->pred_end() && in getLoopTripCount()
576 if (PI == TopMBB->pred_end()) // dead loop? in getLoopTripCount()
579 if (PI != TopMBB->pred_end()) // multiple backedges? in getLoopTripCount()
584 if (L->contains(Incoming)) { in getLoopTripCount()
585 if (L->contains(Backedge)) in getLoopTripCount()
588 } else if (!L->contains(Backedge)) in getLoopTripCount()
594 MachineBasicBlock *ExitingBlock = L->findLoopControlBlock(); in getLoopTripCount()
605 MachineBasicBlock *Preheader = MLI->findLoopPreheader(L, SpecPreheader); in getLoopTripCount()
608 MachineInstr *IV_Phi = MRI->getVRegDef(IVReg); in getLoopTripCount()
609 MachineBasicBlock *Latch = L->getLoopLatch(); in getLoopTripCount() local
610 for (unsigned i = 1, n = IV_Phi->getNumOperands(); i < n; i += 2) { in getLoopTripCount()
611 MachineBasicBlock *MBB = IV_Phi->getOperand(i+1).getMBB(); in getLoopTripCount()
613 InitialValue = &IV_Phi->getOperand(i); in getLoopTripCount()
614 else if (MBB == Latch) in getLoopTripCount()
615 IVReg = IV_Phi->getOperand(i).getReg(); // Want IV reg after bump. in getLoopTripCount()
622 bool NotAnalyzed = TII->analyzeBranch(*ExitingBlock, TB, FB, Cond, false); in getLoopTripCount()
626 MachineBasicBlock *Header = L->getHeader(); in getLoopTripCount()
627 // TB must be non-null. If FB is also non-null, one of them must be in getLoopTripCount()
631 if (ExitingBlock != Latch && (TB == Latch || FB == Latch)) { in getLoopTripCount()
634 bool NotAnalyzed = TII->analyzeBranch(*Latch, LTB, LFB, LCond, false); in getLoopTripCount()
637 if (TB == Latch) in getLoopTripCount()
648 // If TB is not the header, it means that the "not-taken" path must lead in getLoopTripCount()
650 bool Negated = TII->predOpcodeHasNot(Cond) ^ (TB != Header); in getLoopTripCount()
653 if (!TII->getPredReg(Cond, PredReg, PredPos, PredRegFlags)) in getLoopTripCount()
655 MachineInstr *CondI = MRI->getVRegDef(PredReg); in getLoopTripCount()
656 unsigned CondOpc = CondI->getOpcode(); in getLoopTripCount()
661 TII->analyzeCompare(*CondI, CmpReg1, CmpReg2, Mask, ImmValue); in getLoopTripCount()
673 // between the operands (e.g. is-less-than), rather than to find out in getLoopTripCount()
677 const MachineOperand &Op1 = CondI->getOperand(1); in getLoopTripCount()
678 const MachineOperand &Op2 = CondI->getOperand(2); in getLoopTripCount()
701 if (InitialValue->isReg()) { in getLoopTripCount()
702 Register R = InitialValue->getReg(); in getLoopTripCount()
703 MachineBasicBlock *DefBB = MRI->getVRegDef(R)->getParent(); in getLoopTripCount()
704 if (!MDT->properlyDominates(DefBB, Header)) { in getLoopTripCount()
709 OldInsts.push_back(MRI->getVRegDef(R)); in getLoopTripCount()
711 if (EndValue->isReg()) { in getLoopTripCount()
712 Register R = EndValue->getReg(); in getLoopTripCount()
713 MachineBasicBlock *DefBB = MRI->getVRegDef(R)->getParent(); in getLoopTripCount()
714 if (!MDT->properlyDominates(DefBB, Header)) { in getLoopTripCount()
719 OldInsts.push_back(MRI->getVRegDef(R)); in getLoopTripCount()
741 if (Start->isReg()) { in computeCount()
742 const MachineInstr *StartValInstr = MRI->getVRegDef(Start->getReg()); in computeCount()
743 if (StartValInstr && (StartValInstr->getOpcode() == Hexagon::A2_tfrsi || in computeCount()
744 StartValInstr->getOpcode() == Hexagon::A2_tfrpi)) in computeCount()
745 Start = &StartValInstr->getOperand(1); in computeCount()
747 if (End->isReg()) { in computeCount()
748 const MachineInstr *EndValInstr = MRI->getVRegDef(End->getReg()); in computeCount()
749 if (EndValInstr && (EndValInstr->getOpcode() == Hexagon::A2_tfrsi || in computeCount()
750 EndValInstr->getOpcode() == Hexagon::A2_tfrpi)) in computeCount()
751 End = &EndValInstr->getOperand(1); in computeCount()
754 if (!Start->isReg() && !Start->isImm()) in computeCount()
756 if (!End->isReg() && !End->isImm()) in computeCount()
763 // Avoid certain wrap-arounds. This doesn't detect all wrap-arounds. in computeCount()
778 if (loopCountMayWrapOrUnderFlow(Start, End, Loop->getLoopPreheader(), Loop, in computeCount()
782 if (Start->isImm() && End->isImm()) { in computeCount()
784 int64_t StartV = Start->getImm(); in computeCount()
785 int64_t EndV = End->getImm(); in computeCount()
786 int64_t Dist = EndV - StartV; in computeCount()
802 Dist = Dist > 0 ? Dist+1 : Dist-1; in computeCount()
811 // "Normalized" distance, i.e. with the bump set to +-1. in computeCount()
812 int64_t Dist1 = (IVBump > 0) ? (Dist + (IVBump - 1)) / IVBump in computeCount()
813 : (-Dist + (-IVBump - 1)) / (-IVBump); in computeCount()
833 MachineBasicBlock *PH = MLI->findLoopPreheader(Loop, SpecPreheader); in computeCount()
835 MachineBasicBlock::iterator InsertPos = PH->getFirstTerminator(); in computeCount()
837 if (InsertPos != PH->end()) in computeCount()
838 DL = InsertPos->getDebugLoc(); in computeCount()
841 // will be "reg - imm". Hexagon's "subtract immediate" instruction in computeCount()
842 // is actually "reg + -imm". in computeCount()
845 // then the iteration count (computed as End-Start) will need to be in computeCount()
849 IVBump = -IVBump; in computeCount()
854 bool RegToImm = Start->isReg() && End->isImm(); // for (reg..imm) in computeCount()
855 bool RegToReg = Start->isReg() && End->isReg(); // for (reg..reg) in computeCount()
858 if (Start->isImm()) in computeCount()
859 StartV = Start->getImm(); in computeCount()
860 if (End->isImm()) in computeCount()
861 EndV = End->getImm(); in computeCount()
865 // Count = (End - Start + (IVBump-1)) / IVBump in computeCount()
867 // Count = (End - Start + (IVBump-1)+1) / IVBump in computeCount()
868 // The "IVBump-1" part is the adjustment (AdjV). We can avoid in computeCount()
874 if (Start->isImm()) in computeCount()
875 StartV--; in computeCount()
876 else if (End->isImm()) in computeCount()
883 if (Start->isImm()) in computeCount()
884 StartV -= (IVBump-1); in computeCount()
885 else if (End->isImm()) in computeCount()
886 EndV += (IVBump-1); in computeCount()
888 AdjV += (IVBump-1); in computeCount()
893 if (Start->isReg()) { in computeCount()
894 R = Start->getReg(); in computeCount()
895 SR = Start->getSubReg(); in computeCount()
897 R = End->getReg(); in computeCount()
898 SR = End->getSubReg(); in computeCount()
900 const TargetRegisterClass *RC = MRI->getRegClass(R); in computeCount()
901 // Hardware loops cannot handle 64-bit registers. If it's a double in computeCount()
912 if (Start->isImm() && StartV == 0) { in computeCount()
913 DistR = End->getReg(); in computeCount()
914 DistSR = End->getSubReg(); in computeCount()
916 const MCInstrDesc &SubD = RegToReg ? TII->get(Hexagon::A2_sub) : in computeCount()
917 (RegToImm ? TII->get(Hexagon::A2_subri) : in computeCount()
918 TII->get(Hexagon::A2_addi)); in computeCount()
920 Register SubR = MRI->createVirtualRegister(IntRC); in computeCount()
925 SubIB.addReg(End->getReg(), 0, End->getSubReg()) in computeCount()
926 .addReg(Start->getReg(), 0, Start->getSubReg()); in computeCount()
929 .addReg(Start->getReg(), 0, Start->getSubReg()); in computeCount()
935 const MachineInstr *EndValInstr = MRI->getVRegDef(End->getReg()); in computeCount()
936 if (EndValInstr->getOpcode() == Hexagon::A2_addi && in computeCount()
937 EndValInstr->getOperand(1).getSubReg() == 0 && in computeCount()
938 EndValInstr->getOperand(2).getImm() == StartV) { in computeCount()
939 DistR = EndValInstr->getOperand(1).getReg(); in computeCount()
941 Register SubR = MRI->createVirtualRegister(IntRC); in computeCount()
944 SubIB.addReg(End->getReg(), 0, End->getSubReg()) in computeCount()
945 .addImm(-StartV); in computeCount()
961 Register AddR = MRI->createVirtualRegister(IntRC); in computeCount()
962 MCInstrDesc const &AddD = TII->get(Hexagon::A2_addi); in computeCount()
983 Register LsrR = MRI->createVirtualRegister(IntRC); in computeCount()
984 const MCInstrDesc &LsrD = TII->get(Hexagon::S2_lsr_i_r); in computeCount()
1001 if (MI->getDesc().isCall()) in isInvalidLoopOperation()
1002 return !TII->doesNotReturn(*MI); in isInvalidLoopOperation()
1011 if (MI->modifiesRegister(R, TRI)) in isInvalidLoopOperation()
1022 << printMBBReference(**L->block_begin())); in containsInvalidInstruction()
1023 for (MachineBasicBlock *MBB : L->getBlocks()) { in containsInvalidInstruction()
1042 for (const MachineOperand &MO : MI->operands()) { in isDead()
1047 if (MRI->use_nodbg_empty(Reg)) in isDead()
1055 use_nodbg_iterator I = MRI->use_nodbg_begin(Reg); in isDead()
1056 use_nodbg_iterator End = MRI->use_nodbg_end(); in isDead()
1057 if (std::next(I) != End || !I->getParent()->isPHI()) in isDead()
1060 MachineInstr *OnePhi = I->getParent(); in isDead()
1061 for (const MachineOperand &OPO : OnePhi->operands()) { in isDead()
1067 for (use_nodbg_iterator J = MRI->use_nodbg_begin(OPReg); in isDead()
1095 for (const MachineOperand &MO : MI->operands()) { in removeIfDead()
1102 llvm::make_early_inc_range(MRI->use_operands(Reg))) { in removeIfDead()
1111 MI->eraseFromParent(); in removeIfDead()
1113 DeadPhis[i]->eraseFromParent(); in removeIfDead()
1129 assert(L->getHeader() && "Loop without a header?"); in convertToHardwareLoop()
1151 // 1 - Hardware loop is being generated for the inner most loop. in convertToHardwareLoop()
1152 // 0 - Hardware loop is being generated for the outer loop. in convertToHardwareLoop()
1180 MachineBasicBlock *LastMBB = L->findLoopControlBlock(); in convertToHardwareLoop()
1185 MachineBasicBlock::iterator LastI = LastMBB->getFirstTerminator(); in convertToHardwareLoop()
1186 if (LastI == LastMBB->end()) in convertToHardwareLoop()
1189 // Is the induction variable bump feeding the latch condition? in convertToHardwareLoop()
1195 MachineBasicBlock *Preheader = MLI->findLoopPreheader(L, SpecPreheader); in convertToHardwareLoop()
1202 MachineBasicBlock::iterator InsertPos = Preheader->getFirstTerminator(); in convertToHardwareLoop()
1211 if (TripCount->isReg()) { in convertToHardwareLoop()
1214 MachineInstr *TCDef = MRI->getVRegDef(TripCount->getReg()); in convertToHardwareLoop()
1215 MachineBasicBlock *BBDef = TCDef->getParent(); in convertToHardwareLoop()
1216 if (!MDT->dominates(BBDef, Preheader)) in convertToHardwareLoop()
1221 MachineBasicBlock *TopBlock = L->getTopBlock(); in convertToHardwareLoop()
1222 MachineBasicBlock *ExitingBlock = L->findLoopControlBlock(); in convertToHardwareLoop()
1224 if (ExitingBlock != L->getLoopLatch()) { in convertToHardwareLoop()
1228 if (TII->analyzeBranch(*ExitingBlock, TB, FB, Cond, false)) in convertToHardwareLoop()
1231 if (L->contains(TB)) in convertToHardwareLoop()
1233 else if (L->contains(FB)) in convertToHardwareLoop()
1242 LLVM_DEBUG(dbgs() << "Change to hardware loop at "; L->dump()); in convertToHardwareLoop()
1244 if (InsertPos != Preheader->end()) in convertToHardwareLoop()
1245 DL = InsertPos->getDebugLoc(); in convertToHardwareLoop()
1247 if (TripCount->isReg()) { in convertToHardwareLoop()
1249 Register CountReg = MRI->createVirtualRegister(&Hexagon::IntRegsRegClass); in convertToHardwareLoop()
1250 BuildMI(*Preheader, InsertPos, DL, TII->get(TargetOpcode::COPY), CountReg) in convertToHardwareLoop()
1251 .addReg(TripCount->getReg(), 0, TripCount->getSubReg()); in convertToHardwareLoop()
1253 BuildMI(*Preheader, InsertPos, DL, TII->get(LOOP_r)).addMBB(LoopStart) in convertToHardwareLoop()
1256 assert(TripCount->isImm() && "Expecting immediate value for trip count"); in convertToHardwareLoop()
1260 int64_t CountImm = TripCount->getImm(); in convertToHardwareLoop()
1261 if (!TII->isValidOffset(LOOP_i, CountImm, TRI)) { in convertToHardwareLoop()
1262 Register CountReg = MRI->createVirtualRegister(&Hexagon::IntRegsRegClass); in convertToHardwareLoop()
1263 BuildMI(*Preheader, InsertPos, DL, TII->get(Hexagon::A2_tfrsi), CountReg) in convertToHardwareLoop()
1265 BuildMI(*Preheader, InsertPos, DL, TII->get(LOOP_r)) in convertToHardwareLoop()
1268 BuildMI(*Preheader, InsertPos, DL, TII->get(LOOP_i)) in convertToHardwareLoop()
1273 LoopStart->setMachineBlockAddressTaken(); in convertToHardwareLoop()
1276 DebugLoc LastIDL = LastI->getDebugLoc(); in convertToHardwareLoop()
1277 BuildMI(*LastMBB, LastI, LastIDL, TII->get(ENDLOOP)).addMBB(LoopStart); in convertToHardwareLoop()
1280 // - a conditional branch followed by an unconditional branch, or in convertToHardwareLoop()
1281 // - a conditional branch to the loop start. in convertToHardwareLoop()
1282 if (LastI->getOpcode() == Hexagon::J2_jumpt || in convertToHardwareLoop()
1283 LastI->getOpcode() == Hexagon::J2_jumpf) { in convertToHardwareLoop()
1285 MachineBasicBlock *BranchTarget = LastI->getOperand(1).getMBB(); in convertToHardwareLoop()
1286 LastI = LastMBB->erase(LastI); in convertToHardwareLoop()
1287 if (!L->contains(BranchTarget)) { in convertToHardwareLoop()
1288 if (LastI != LastMBB->end()) in convertToHardwareLoop()
1289 LastI = LastMBB->erase(LastI); in convertToHardwareLoop()
1291 TII->insertBranch(*LastMBB, BranchTarget, nullptr, Cond, LastIDL); in convertToHardwareLoop()
1295 LastMBB->erase(LastI); in convertToHardwareLoop()
1321 MachineBasicBlock *BB = BumpI->getParent(); in orderBumpCompare()
1322 if (CmpI->getParent() != BB) in orderBumpCompare()
1328 for (instr_iterator I(BumpI), E = BB->instr_end(); I != E; ++I) in orderBumpCompare()
1333 Register PredR = CmpI->getOperand(0).getReg(); in orderBumpCompare()
1335 instr_iterator CmpIt = CmpI->getIterator(), NextIt = std::next(CmpIt); in orderBumpCompare()
1336 for (instr_iterator I = NextIt, E = BB->instr_end(); I != E; ++I) { in orderBumpCompare()
1338 for (unsigned i = 0, n = In->getNumOperands(); i < n; ++i) { in orderBumpCompare()
1339 MachineOperand &MO = In->getOperand(i); in orderBumpCompare()
1347 BB->splice(++BumpI->getIterator(), BB, CmpI->getIterator()); in orderBumpCompare()
1365 if (LoopFeederPhi.find(MO->getReg()) == LoopFeederPhi.end()) { in isLoopFeeder()
1367 << printMBBReference(**L->block_begin())); in isLoopFeeder()
1369 if (llvm::is_contained(L->getBlocks(), A)) in isLoopFeeder()
1371 MachineInstr *Def = MRI->getVRegDef(MO->getReg()); in isLoopFeeder()
1372 LoopFeederPhi.insert(std::make_pair(MO->getReg(), Def)); in isLoopFeeder()
1384 assert(Phi->isPHI() && "Expecting a Phi."); in phiMayWrapOrUnderflow()
1387 for (int i = 1, n = Phi->getNumOperands(); i < n; i += 2) in phiMayWrapOrUnderflow()
1388 if (isLoopFeeder(L, MBB, Phi, &(Phi->getOperand(i)), LoopFeederPhi)) in phiMayWrapOrUnderflow()
1389 if (loopCountMayWrapOrUnderFlow(&(Phi->getOperand(i)), EndVal, in phiMayWrapOrUnderflow()
1390 Phi->getParent(), L, LoopFeederPhi)) in phiMayWrapOrUnderflow()
1397 /// first itertion of a do-while loop. In this case, we cannot generate a
1414 if (!InitVal->isReg()) in loopCountMayWrapOrUnderFlow()
1417 if (!EndVal->isImm()) in loopCountMayWrapOrUnderFlow()
1424 return (EndVal->getImm() == Imm); in loopCountMayWrapOrUnderFlow()
1426 Register Reg = InitVal->getReg(); in loopCountMayWrapOrUnderFlow()
1432 MachineInstr *Def = MRI->getVRegDef(Reg); in loopCountMayWrapOrUnderFlow()
1438 if (Def->isPHI() && !phiMayWrapOrUnderflow(Def, EndVal, Def->getParent(), in loopCountMayWrapOrUnderFlow()
1441 if (Def->isCopy() && !loopCountMayWrapOrUnderFlow(&(Def->getOperand(1)), in loopCountMayWrapOrUnderFlow()
1442 EndVal, Def->getParent(), in loopCountMayWrapOrUnderFlow()
1449 for (MachineRegisterInfo::use_instr_nodbg_iterator I = MRI->use_instr_nodbg_begin(Reg), in loopCountMayWrapOrUnderFlow()
1450 E = MRI->use_instr_nodbg_end(); I != E; ++I) { in loopCountMayWrapOrUnderFlow()
1455 if (!TII->analyzeCompare(*MI, CmpReg1, CmpReg2, CmpMask, CmpValue)) in loopCountMayWrapOrUnderFlow()
1460 if (TII->analyzeBranch(*MI->getParent(), TBB, FBB, Cond, false)) in loopCountMayWrapOrUnderFlow()
1464 getComparisonKind(MI->getOpcode(), nullptr, nullptr, 0); in loopCountMayWrapOrUnderFlow()
1467 if (TII->predOpcodeHasNot(Cond) ^ (TBB != MBB)) in loopCountMayWrapOrUnderFlow()
1483 // OK - this is a hack that needs to be improved. We really need to analyze in loopCountMayWrapOrUnderFlow()
1486 if (!Def->isCopy() && !Def->isPHI()) in loopCountMayWrapOrUnderFlow()
1509 MachineInstr *DI = MRI->getVRegDef(R); in checkForImmediate()
1510 unsigned DOpc = DI->getOpcode(); in checkForImmediate()
1520 if (!checkForImmediate(DI->getOperand(1), TV)) in checkForImmediate()
1528 const MachineOperand &S1 = DI->getOperand(1); in checkForImmediate()
1529 const MachineOperand &S2 = DI->getOperand(2); in checkForImmediate()
1537 const MachineOperand &S1 = DI->getOperand(1); in checkForImmediate()
1538 const MachineOperand &S3 = DI->getOperand(3); in checkForImmediate()
1542 unsigned Sub2 = DI->getOperand(2).getImm(); in checkForImmediate()
1543 unsigned Sub4 = DI->getOperand(4).getImm(); in checkForImmediate()
1581 MachineInstr *DI = MRI->getVRegDef(R); in setImmediate()
1583 const TargetRegisterClass *RC = MRI->getRegClass(R); in setImmediate()
1584 Register NewR = MRI->createVirtualRegister(RC); in setImmediate()
1585 MachineBasicBlock &B = *DI->getParent(); in setImmediate()
1586 DebugLoc DL = DI->getDebugLoc(); in setImmediate()
1587 BuildMI(B, DI, DL, TII->get(DI->getOpcode()), NewR).addImm(Val); in setImmediate()
1592 MachineBasicBlock *Header = L->getHeader(); in fixupInductionVariable()
1593 MachineBasicBlock *Latch = L->getLoopLatch(); in fixupInductionVariable() local
1594 MachineBasicBlock *ExitingBlock = L->findLoopControlBlock(); in fixupInductionVariable()
1596 if (!(Header && Latch && ExitingBlock)) in fixupInductionVariable()
1609 // %1 = PHI ..., [ latch, %2 ] in fixupInductionVariable()
1613 for (instr_iterator I = Header->instr_begin(), E = Header->instr_end(); in fixupInductionVariable()
1614 I != E && I->isPHI(); ++I) { in fixupInductionVariable()
1618 for (unsigned i = 1, n = Phi->getNumOperands(); i < n; i += 2) { in fixupInductionVariable()
1619 if (Phi->getOperand(i+1).getMBB() != Latch) in fixupInductionVariable()
1622 Register PhiReg = Phi->getOperand(i).getReg(); in fixupInductionVariable()
1623 MachineInstr *DI = MRI->getVRegDef(PhiReg); in fixupInductionVariable()
1625 if (DI->getDesc().isAdd()) { in fixupInductionVariable()
1628 Register IndReg = DI->getOperand(1).getReg(); in fixupInductionVariable()
1629 MachineOperand &Opnd2 = DI->getOperand(2); in fixupInductionVariable()
1631 if (MRI->getVRegDef(IndReg) == Phi && checkForImmediate(Opnd2, V)) { in fixupInductionVariable()
1632 Register UpdReg = DI->getOperand(0).getReg(); in fixupInductionVariable()
1645 bool NotAnalyzed = TII->analyzeBranch(*ExitingBlock, TB, FB, Cond, false); in fixupInductionVariable()
1649 if (ExitingBlock != Latch && (TB == Latch || FB == Latch)) { in fixupInductionVariable()
1652 bool NotAnalyzed = TII->analyzeBranch(*Latch, LTB, LFB, LCond, false); in fixupInductionVariable()
1656 // Since latch is not the exiting block, the latch branch should be an in fixupInductionVariable()
1658 if (TB == Latch) in fixupInductionVariable()
1665 // The latch/exit block does not go back to the header. in fixupInductionVariable()
1669 // In this case, the LoopBody -> TB should not be a back edge otherwise in fixupInductionVariable()
1671 // This case can happen when the Latch has two jumps like this: in fixupInductionVariable()
1672 // Jmp_c OuterLoopHeader <-- TB in fixupInductionVariable()
1673 // Jmp InnerLoopHeader <-- FB in fixupInductionVariable()
1674 if (MDT->dominates(TB, FB)) in fixupInductionVariable()
1681 // into Cond, followed by the predicate register. For non-negated branches in fixupInductionVariable()
1687 if (!Cond[CSz-1].isReg()) in fixupInductionVariable()
1690 Register P = Cond[CSz - 1].getReg(); in fixupInductionVariable()
1691 MachineInstr *PredDef = MRI->getVRegDef(P); in fixupInductionVariable()
1693 if (!PredDef->isCompare()) in fixupInductionVariable()
1703 for (MachineOperand &MO : PredDef->operands()) { in fixupInductionVariable()
1735 if (CmpRegs.count(I->first)) in fixupInductionVariable()
1741 const RegisterBump &RB = I->second; in fixupInductionVariable()
1751 for (unsigned i = 1, n = PredDef->getNumOperands(); i < n; ++i) { in fixupInductionVariable()
1752 MachineOperand &MO = PredDef->getOperand(i); in fixupInductionVariable()
1755 << ") = " << *(MRI->getVRegDef(I->first))); in fixupInductionVariable()
1759 IndI = MRI->getVRegDef(I->first); in fixupInductionVariable()
1763 << ") = " << *(MRI->getVRegDef(MO.getReg()))); in fixupInductionVariable()
1767 nonIndI = MRI->getVRegDef(MO.getReg()); in fixupInductionVariable()
1772 nonIndI->getOpcode() == Hexagon::A2_addi && in fixupInductionVariable()
1773 nonIndI->getOperand(2).isImm() && in fixupInductionVariable()
1774 nonIndI->getOperand(2).getImm() == - RB.second) { in fixupInductionVariable()
1777 IndMO->setReg(I->first); in fixupInductionVariable()
1778 nonIndMO->setReg(nonIndI->getOperand(1).getReg()); in fixupInductionVariable()
1788 getComparisonKind(PredDef->getOpcode(), nullptr, nullptr, 0); in fixupInductionVariable()
1797 // Handle Overflow (64-bit). in fixupInductionVariable()
1798 if (((V > 0) && (CmpImm > INT64_MAX - V)) || in fixupInductionVariable()
1799 ((V < 0) && (CmpImm < INT64_MIN - V))) in fixupInductionVariable()
1803 // the immediate to be constant-extended. There are some exceptions in fixupInductionVariable()
1805 if (CmpImmOp->isImm() && !TII->isExtendable(*PredDef) && in fixupInductionVariable()
1806 !TII->isValidOffset(PredDef->getOpcode(), CmpImm, TRI, false)) in fixupInductionVariable()
1810 // after the fixup, the compare would use a yet-undefined register. in fixupInductionVariable()
1811 MachineInstr *BumpI = MRI->getVRegDef(I->first); in fixupInductionVariable()
1818 for (MachineOperand &MO : PredDef->operands()) { in fixupInductionVariable()
1820 MO.setReg(I->first); in fixupInductionVariable()
1830 /// createPreheaderForLoop - Create a preheader for a given loop.
1833 if (MachineBasicBlock *TmpPH = MLI->findLoopPreheader(L, SpecPreheader)) in createPreheaderForLoop()
1838 MachineBasicBlock *Header = L->getHeader(); in createPreheaderForLoop()
1839 MachineBasicBlock *Latch = L->getLoopLatch(); in createPreheaderForLoop() local
1840 MachineBasicBlock *ExitingBlock = L->findLoopControlBlock(); in createPreheaderForLoop()
1841 MachineFunction *MF = Header->getParent(); in createPreheaderForLoop()
1845 if ((!PHFn.empty()) && (PHFn != MF->getName())) in createPreheaderForLoop()
1849 if (!Latch || !ExitingBlock || Header->hasAddressTaken()) in createPreheaderForLoop()
1858 MBBVector Preds(Header->pred_begin(), Header->pred_end()); in createPreheaderForLoop()
1862 if (TII->analyzeBranch(*ExitingBlock, TB, FB, Tmp1, false)) in createPreheaderForLoop()
1866 bool NotAnalyzed = TII->analyzeBranch(*PB, TB, FB, Tmp1, false); in createPreheaderForLoop()
1871 MachineBasicBlock *NewPH = MF->CreateMachineBasicBlock(); in createPreheaderForLoop()
1872 MF->insert(Header->getIterator(), NewPH); in createPreheaderForLoop()
1874 if (Header->pred_size() > 2) { in createPreheaderForLoop()
1876 // the loop latch. Any additional predecessors of the header should in createPreheaderForLoop()
1880 for (instr_iterator I = Header->instr_begin(), E = Header->instr_end(); in createPreheaderForLoop()
1881 I != E && I->isPHI(); ++I) { in createPreheaderForLoop()
1884 const MCInstrDesc &PD = TII->get(TargetOpcode::PHI); in createPreheaderForLoop()
1885 MachineInstr *NewPN = MF->CreateMachineInstr(PD, DL); in createPreheaderForLoop()
1886 NewPH->insert(NewPH->end(), NewPN); in createPreheaderForLoop()
1888 Register PR = PN->getOperand(0).getReg(); in createPreheaderForLoop()
1889 const TargetRegisterClass *RC = MRI->getRegClass(PR); in createPreheaderForLoop()
1890 Register NewPR = MRI->createVirtualRegister(RC); in createPreheaderForLoop()
1891 NewPN->addOperand(MachineOperand::CreateReg(NewPR, true)); in createPreheaderForLoop()
1893 // Copy all non-latch operands of a header's PHI node to the newly in createPreheaderForLoop()
1895 for (unsigned i = 1, n = PN->getNumOperands(); i < n; i += 2) { in createPreheaderForLoop()
1896 Register PredR = PN->getOperand(i).getReg(); in createPreheaderForLoop()
1897 unsigned PredRSub = PN->getOperand(i).getSubReg(); in createPreheaderForLoop()
1898 MachineBasicBlock *PredB = PN->getOperand(i+1).getMBB(); in createPreheaderForLoop()
1899 if (PredB == Latch) in createPreheaderForLoop()
1904 NewPN->addOperand(MO); in createPreheaderForLoop()
1905 NewPN->addOperand(MachineOperand::CreateMBB(PredB)); in createPreheaderForLoop()
1910 for (int i = PN->getNumOperands()-2; i > 0; i -= 2) { in createPreheaderForLoop()
1911 MachineBasicBlock *PredB = PN->getOperand(i+1).getMBB(); in createPreheaderForLoop()
1912 if (PredB != Latch) { in createPreheaderForLoop()
1913 PN->removeOperand(i+1); in createPreheaderForLoop()
1914 PN->removeOperand(i); in createPreheaderForLoop()
1917 PN->addOperand(MachineOperand::CreateReg(NewPR, false)); in createPreheaderForLoop()
1918 PN->addOperand(MachineOperand::CreateMBB(NewPH)); in createPreheaderForLoop()
1921 assert(Header->pred_size() == 2); in createPreheaderForLoop()
1923 // The header has only two predecessors, but the non-latch predecessor in createPreheaderForLoop()
1928 for (instr_iterator I = Header->instr_begin(), E = Header->instr_end(); in createPreheaderForLoop()
1929 I != E && I->isPHI(); ++I) { in createPreheaderForLoop()
1931 for (unsigned i = 1, n = PN->getNumOperands(); i < n; i += 2) { in createPreheaderForLoop()
1932 MachineOperand &MO = PN->getOperand(i+1); in createPreheaderForLoop()
1933 if (MO.getMBB() != Latch) in createPreheaderForLoop()
1948 if (PB != Latch) { in createPreheaderForLoop()
1950 bool NotAnalyzed = TII->analyzeBranch(*PB, TB, FB, Tmp2, false); in createPreheaderForLoop()
1954 TII->insertBranch(*PB, NewPH, nullptr, EmptyCond, DL); in createPreheaderForLoop()
1955 PB->ReplaceUsesOfBlockWith(Header, NewPH); in createPreheaderForLoop()
1959 // It can happen that the latch block will fall through into the header. in createPreheaderForLoop()
1962 bool LatchNotAnalyzed = TII->analyzeBranch(*Latch, TB, FB, Tmp2, false); in createPreheaderForLoop()
1966 TII->insertBranch(*Latch, Header, nullptr, EmptyCond, DL); in createPreheaderForLoop()
1969 TII->insertBranch(*NewPH, Header, nullptr, EmptyCond, DL); in createPreheaderForLoop()
1970 NewPH->addSuccessor(Header); in createPreheaderForLoop()
1972 MachineLoop *ParentLoop = L->getParentLoop(); in createPreheaderForLoop()
1974 ParentLoop->addBasicBlockToLoop(NewPH, *MLI); in createPreheaderForLoop()
1978 if (MachineDomTreeNode *HN = MDT->getNode(Header)) { in createPreheaderForLoop()
1979 if (MachineDomTreeNode *DHN = HN->getIDom()) { in createPreheaderForLoop()
1980 MDT->addNewBlock(NewPH, DHN->getBlock()); in createPreheaderForLoop()
1981 MDT->changeImmediateDominator(Header, NewPH); in createPreheaderForLoop()