Lines Matching refs:MI
86 static bool isVectorPredicated(MachineInstr *MI) { in isVectorPredicated() argument
87 int PIdx = llvm::findFirstVPTPredOperandIdx(*MI); in isVectorPredicated()
88 return PIdx != -1 && MI->getOperand(PIdx + 1).getReg() == ARM::VPR; in isVectorPredicated()
91 static bool isVectorPredicate(MachineInstr *MI) { in isVectorPredicate() argument
92 return MI->findRegisterDefOperandIdx(ARM::VPR, /*TRI=*/nullptr) != -1; in isVectorPredicate()
95 static bool hasVPRUse(MachineInstr &MI) { in hasVPRUse() argument
96 return MI.findRegisterUseOperandIdx(ARM::VPR, /*TRI=*/nullptr) != -1; in hasVPRUse()
99 static bool isDomainMVE(MachineInstr *MI) { in isDomainMVE() argument
100 uint64_t Domain = MI->getDesc().TSFlags & ARMII::DomainMask; in isDomainMVE()
104 static int getVecSize(const MachineInstr &MI) { in getVecSize() argument
105 const MCInstrDesc &MCID = MI.getDesc(); in getVecSize()
110 static bool shouldInspect(MachineInstr &MI) { in shouldInspect() argument
111 if (MI.isDebugInstr()) in shouldInspect()
113 return isDomainMVE(&MI) || isVectorPredicate(&MI) || hasVPRUse(MI); in shouldInspect()
116 static bool isHorizontalReduction(const MachineInstr &MI) { in isHorizontalReduction() argument
117 const MCInstrDesc &MCID = MI.getDesc(); in isHorizontalReduction()
183 VPTBlock(MachineInstr *MI) { Insts.push_back(MI); } in VPTBlock() argument
201 void insert(MachineInstr *MI) { in insert() argument
202 Insts.push_back(MI); in insert()
225 void CreateVPTBlock(MachineInstr *MI) { in CreateVPTBlock() argument
226 assert((CurrentPredicates.size() || MI->getParent()->isLiveIn(ARM::VPR)) in CreateVPTBlock()
228 Blocks.emplace_back(MI); in CreateVPTBlock()
232 PredicatedInsts[MI] = CurrentPredicates; in CreateVPTBlock()
235 void addInst(MachineInstr *MI) { in addInst() argument
236 Blocks.back().insert(MI); in addInst()
237 PredicatedInsts[MI] = CurrentPredicates; in addInst()
240 void addPredicate(MachineInstr *MI) { in addPredicate() argument
241 LLVM_DEBUG(dbgs() << "ARM Loops: Adding VPT Predicate: " << *MI); in addPredicate()
242 CurrentPredicates.insert(MI); in addPredicate()
245 void resetPredicate(MachineInstr *MI) { in resetPredicate() argument
246 LLVM_DEBUG(dbgs() << "ARM Loops: Resetting VPT Predicate: " << *MI); in resetPredicate()
248 CurrentPredicates.insert(MI); in resetPredicate()
253 bool isPredicatedOnVCTP(MachineInstr *MI, bool Exclusive = false) { in isPredicatedOnVCTP() argument
254 SetVector<MachineInstr *> &Predicates = PredicatedInsts[MI]; in isPredicatedOnVCTP()
258 if (getVPTInstrPredicate(*MI) == ARMVCC::Else) in isPredicatedOnVCTP()
285 all_of(Block.getInsts(), [](const MachineInstr *MI) { in hasImplicitlyValidVPT() argument
286 return !MI->mayStore() && !MI->mayLoad() && in hasImplicitlyValidVPT()
287 !isHorizontalReduction(*MI) && !isVCTP(MI); in hasImplicitlyValidVPT()
291 auto IsOperandPredicated = [&](MachineInstr *MI, unsigned Idx) { in hasImplicitlyValidVPT() argument
292 MachineInstr *Op = RDA.getMIOperand(MI, MI->getOperand(Idx)); in hasImplicitlyValidVPT()
296 auto IsOperandInvariant = [&](MachineInstr *MI, unsigned Idx) { in hasImplicitlyValidVPT() argument
297 MachineOperand &MO = MI->getOperand(Idx); in hasImplicitlyValidVPT()
302 RDA.getGlobalReachingDefs(MI, MO.getReg(), Defs); in hasImplicitlyValidVPT()
325 !any_of(drop_begin(Block.getInsts()), [](const MachineInstr *MI) { in isValid() argument
326 return getVPTInstrPredicate(*MI) == ARMVCC::Else; in isValid()
341 for (auto *MI : Insts) { in isValid() local
343 if (isVCTP(MI) && getVPTInstrPredicate(*MI) != ARMVCC::Then) in isValid()
346 if (MI->getOpcode() == ARM::MVE_VPST || isVectorPredicate(MI)) in isValid()
351 if (!isPredicatedOnVCTP(MI)) { in isValid()
352 LLVM_DEBUG(dbgs() << "ARM Loops: Can't convert: " << *MI); in isValid()
400 bool ValidateMVEInst(MachineInstr *MI);
402 void AnalyseMVEInst(MachineInstr *MI) { in AnalyseMVEInst()
403 CannotTailPredicate = !ValidateMVEInst(MI); in AnalyseMVEInst()
415 bool AddVCTP(MachineInstr *MI);
458 for (auto *MI : VCTPs) in dump() local
459 dbgs() << " - " << *MI; in dump()
504 void RevertWhile(MachineInstr *MI) const;
505 void RevertDo(MachineInstr *MI) const;
507 bool RevertLoopDec(MachineInstr *MI) const;
509 void RevertLoopEnd(MachineInstr *MI, bool SkipCmp = false) const;
511 void RevertLoopEndDec(MachineInstr *MI) const;
528 static bool TryRemove(MachineInstr *MI, ReachingDefAnalysis &RDA, in INITIALIZE_PASS()
574 if (!RDA.isSafeToRemove(MI, Uses, Ignore)) in INITIALIZE_PASS()
579 LLVM_DEBUG(dbgs() << "ARM Loops: Able to remove: " << *MI in INITIALIZE_PASS()
585 RDA.collectKilledOperands(MI, Killed); in INITIALIZE_PASS()
732 for (MachineInstr *MI : DoubleWidthResultInstrs) { in ValidateTailPredicate()
733 unsigned InstrVecSize = getVecSize(*MI); in ValidateTailPredicate()
736 << "VecSize:\n" << *MI); in ValidateTailPredicate()
745 auto IsValidSub = [](MachineInstr *MI, int ExpectedVecWidth) { in ValidateTailPredicate() argument
746 return -getAddSubImmediate(*MI) == ExpectedVecWidth; in ValidateTailPredicate()
765 for (auto *MI : ElementChain) { in ValidateTailPredicate() local
766 if (isMovRegOpcode(MI->getOpcode())) in ValidateTailPredicate()
769 if (isSubImmOpcode(MI->getOpcode())) { in ValidateTailPredicate()
770 if (FoundSub || !IsValidSub(MI, ExpectedVectorWidth)) { in ValidateTailPredicate()
772 " count: " << *MI); in ValidateTailPredicate()
778 " count: " << *MI); in ValidateTailPredicate()
811 static bool retainsPreviousHalfElement(const MachineInstr &MI) { in retainsPreviousHalfElement() argument
812 const MCInstrDesc &MCID = MI.getDesc(); in retainsPreviousHalfElement()
820 static bool producesDoubleWidthResult(const MachineInstr &MI) { in producesDoubleWidthResult() argument
821 const MCInstrDesc &MCID = MI.getDesc(); in producesDoubleWidthResult()
830 static bool canGenerateNonZeros(const MachineInstr &MI) { in canGenerateNonZeros() argument
834 if (producesDoubleWidthResult(MI)) in canGenerateNonZeros()
837 switch (MI.getOpcode()) { in canGenerateNonZeros()
859 static bool producesFalseLanesZero(MachineInstr &MI, in producesFalseLanesZero() argument
863 if (canGenerateNonZeros(MI)) in producesFalseLanesZero()
866 bool isPredicated = isVectorPredicated(&MI); in producesFalseLanesZero()
869 if (MI.mayLoad()) in producesFalseLanesZero()
878 bool AllowScalars = isHorizontalReduction(MI); in producesFalseLanesZero()
879 for (auto &MO : MI.operands()) { in producesFalseLanesZero()
885 int PIdx = llvm::findFirstVPTPredOperandIdx(MI); in producesFalseLanesZero()
894 RDA.getGlobalReachingDefs(&MI, MO.getReg(), Defs); in producesFalseLanesZero()
898 if (Def == &MI || FalseLanesZero.count(Def) || IsZeroInit(Def)) in producesFalseLanesZero()
905 LLVM_DEBUG(dbgs() << "ARM Loops: Always False Zeros: " << MI); in producesFalseLanesZero()
939 for (auto &MI : *Header) { in ValidateLiveOuts()
940 if (!shouldInspect(MI)) in ValidateLiveOuts()
943 if (isVCTP(&MI) || isVPTOpcode(MI.getOpcode())) in ValidateLiveOuts()
946 bool isPredicated = isVectorPredicated(&MI); in ValidateLiveOuts()
948 retainsPreviousHalfElement(MI) || isHorizontalReduction(MI); in ValidateLiveOuts()
951 Predicated.insert(&MI); in ValidateLiveOuts()
952 if (producesFalseLanesZero(MI, QPRs, RDA, FalseLanesZero)) in ValidateLiveOuts()
953 FalseLanesZero.insert(&MI); in ValidateLiveOuts()
954 else if (MI.getNumDefs() == 0) in ValidateLiveOuts()
957 LLVM_DEBUG(dbgs() << " Unpredicated instruction that retainsOrReduces: " << MI); in ValidateLiveOuts()
959 } else if (!isPredicated && MI.getOpcode() != ARM::MQPRCopy) in ValidateLiveOuts()
960 FalseLanesUnknown.insert(&MI); in ValidateLiveOuts()
975 auto HasPredicatedUsers = [this](MachineInstr *MI, const MachineOperand &MO, in ValidateLiveOuts()
978 RDA.getGlobalUses(MI, MO.getReg().asMCReg(), Uses); in ValidateLiveOuts()
980 if (Use != MI && !Predicated.count(Use)) in ValidateLiveOuts()
993 for (auto *MI : reverse(FalseLanesUnknown)) { in ValidateLiveOuts() local
994 for (auto &MO : MI->operands()) { in ValidateLiveOuts()
997 if (!HasPredicatedUsers(MI, MO, Predicated)) { in ValidateLiveOuts()
999 << TRI.getRegAsmName(MO.getReg()) << " at " << *MI); in ValidateLiveOuts()
1000 NonPredicated.insert(MI); in ValidateLiveOuts()
1005 if (!NonPredicated.contains(MI)) in ValidateLiveOuts()
1006 Predicated.insert(MI); in ValidateLiveOuts()
1025 if (auto *MI = RDA.getLocalLiveOutMIDef(Header, RegMask.PhysReg)) in ValidateLiveOuts() local
1026 LiveOutMIs.insert(MI); in ValidateLiveOuts()
1038 MachineInstr *MI = Worklist.pop_back_val(); in ValidateLiveOuts() local
1039 if (MI->getOpcode() == ARM::MQPRCopy) { in ValidateLiveOuts()
1040 VMOVCopies.insert(MI); in ValidateLiveOuts()
1042 RDA.getUniqueReachingMIDef(MI, MI->getOperand(1).getReg()); in ValidateLiveOuts()
1045 } else if (NonPredicated.count(MI) && FalseLanesUnknown.contains(MI)) { in ValidateLiveOuts()
1046 LLVM_DEBUG(dbgs() << " Unable to handle live out: " << *MI); in ValidateLiveOuts()
1101 bool LowOverheadLoop::AddVCTP(MachineInstr *MI) { in AddVCTP() argument
1102 LLVM_DEBUG(dbgs() << "ARM Loops: Adding VCTP: " << *MI); in AddVCTP()
1104 VCTPs.push_back(MI); in AddVCTP()
1111 if (!Prev->getOperand(1).isIdenticalTo(MI->getOperand(1)) || in AddVCTP()
1112 !RDA.hasSameReachingDef(Prev, MI, MI->getOperand(1).getReg().asMCReg())) { in AddVCTP()
1117 VCTPs.push_back(MI); in AddVCTP()
1121 static bool ValidateMVEStore(MachineInstr *MI, MachineLoop *ML) { in ValidateMVEStore() argument
1148 if (MI->getOpcode() != ARM::MVE_VSTRWU32 || !IsStackOp(MI)) in ValidateMVEStore()
1154 if (MI->memoperands().size() == 0) in ValidateMVEStore()
1156 int FI = GetFrameIndex(MI->memoperands().front()); in ValidateMVEStore()
1158 auto &FrameInfo = MI->getParent()->getParent()->getFrameInfo(); in ValidateMVEStore()
1164 SmallPtrSet<MachineBasicBlock *, 4> Visited{MI->getParent()}; in ValidateMVEStore()
1199 bool LowOverheadLoop::ValidateMVEInst(MachineInstr *MI) { in ValidateMVEInst() argument
1203 if (!shouldInspect(*MI)) in ValidateMVEInst()
1206 if (MI->getOpcode() == ARM::MVE_VPSEL || in ValidateMVEInst()
1207 MI->getOpcode() == ARM::MVE_VPNOT) { in ValidateMVEInst()
1220 if (isVCTP(MI) && !AddVCTP(MI)) in ValidateMVEInst()
1225 const MCInstrDesc &MCID = MI->getDesc(); in ValidateMVEInst()
1227 unsigned LastOpIdx = MI->getNumOperands() - 1; in ValidateMVEInst()
1229 const MachineOperand &MO = MI->getOperand(LastOpIdx - Op.index()); in ValidateMVEInst()
1234 VPTstate.addInst(MI); in ValidateMVEInst()
1236 } else if (MI->getOpcode() != ARM::MVE_VPST) { in ValidateMVEInst()
1237 LLVM_DEBUG(dbgs() << "ARM Loops: Found instruction using vpr: " << *MI); in ValidateMVEInst()
1247 if (isDomainMVE(MI) && RequiresExplicitPredication) { in ValidateMVEInst()
1248 if (MI->getOpcode() == ARM::MQPRCopy) in ValidateMVEInst()
1250 if (!IsUse && producesDoubleWidthResult(*MI)) { in ValidateMVEInst()
1251 DoubleWidthResultInstrs.insert(MI); in ValidateMVEInst()
1256 << "ARM Loops: Can't tail predicate: " << *MI); in ValidateMVEInst()
1262 if (MI->mayStore() && !ValidateMVEStore(MI, &ML)) in ValidateMVEInst()
1267 if (isVectorPredicate(MI)) { in ValidateMVEInst()
1270 if (!isVectorPredicated(MI)) in ValidateMVEInst()
1271 VPTstate.resetPredicate(MI); in ValidateMVEInst()
1273 VPTstate.addPredicate(MI); in ValidateMVEInst()
1278 if (isVPTOpcode(MI->getOpcode())) in ValidateMVEInst()
1279 VPTstate.CreateVPTBlock(MI); in ValidateMVEInst()
1332 for (auto &MI : *MBB) { in ProcessLoop()
1333 if (isLoopStart(MI)) in ProcessLoop()
1334 return &MI; in ProcessLoop()
1354 for (auto &MI : *MBB) { in ProcessLoop()
1355 if (MI.isDebugValue()) in ProcessLoop()
1357 else if (MI.getOpcode() == ARM::t2LoopDec) in ProcessLoop()
1358 LoLoop.Dec = &MI; in ProcessLoop()
1359 else if (MI.getOpcode() == ARM::t2LoopEnd) in ProcessLoop()
1360 LoLoop.End = &MI; in ProcessLoop()
1361 else if (MI.getOpcode() == ARM::t2LoopEndDec) in ProcessLoop()
1362 LoLoop.End = LoLoop.Dec = &MI; in ProcessLoop()
1363 else if (isLoopStart(MI)) in ProcessLoop()
1364 LoLoop.Start = &MI; in ProcessLoop()
1365 else if (MI.getDesc().isCall()) { in ProcessLoop()
1374 LoLoop.AnalyseMVEInst(&MI); in ProcessLoop()
1408 void ARMLowOverheadLoops::RevertWhile(MachineInstr *MI) const { in RevertWhile()
1409 LLVM_DEBUG(dbgs() << "ARM Loops: Reverting to cmp: " << *MI); in RevertWhile()
1410 MachineBasicBlock *DestBB = getWhileLoopStartTargetBB(*MI); in RevertWhile()
1411 unsigned BrOpc = BBUtils->isBBInRange(MI, DestBB, 254) ? in RevertWhile()
1414 RevertWhileLoopStartLR(MI, TII, BrOpc); in RevertWhile()
1417 void ARMLowOverheadLoops::RevertDo(MachineInstr *MI) const { in RevertDo()
1418 LLVM_DEBUG(dbgs() << "ARM Loops: Reverting to mov: " << *MI); in RevertDo()
1419 RevertDoLoopStart(MI, TII); in RevertDo()
1422 bool ARMLowOverheadLoops::RevertLoopDec(MachineInstr *MI) const { in RevertLoopDec()
1423 LLVM_DEBUG(dbgs() << "ARM Loops: Reverting to sub: " << *MI); in RevertLoopDec()
1424 MachineBasicBlock *MBB = MI->getParent(); in RevertLoopDec()
1426 for (auto I = MachineBasicBlock::iterator(MI), E = MBB->end(); I != E; ++I) { in RevertLoopDec()
1435 RDA->isSafeToDefRegAt(MI, MCRegister::from(ARM::CPSR), Ignore); in RevertLoopDec()
1437 llvm::RevertLoopDec(MI, TII, SetFlags); in RevertLoopDec()
1442 void ARMLowOverheadLoops::RevertLoopEnd(MachineInstr *MI, bool SkipCmp) const { in RevertLoopEnd() argument
1443 LLVM_DEBUG(dbgs() << "ARM Loops: Reverting to cmp, br: " << *MI); in RevertLoopEnd()
1445 MachineBasicBlock *DestBB = MI->getOperand(1).getMBB(); in RevertLoopEnd()
1446 unsigned BrOpc = BBUtils->isBBInRange(MI, DestBB, 254) ? in RevertLoopEnd()
1449 llvm::RevertLoopEnd(MI, TII, BrOpc, SkipCmp); in RevertLoopEnd()
1453 void ARMLowOverheadLoops::RevertLoopEndDec(MachineInstr *MI) const { in RevertLoopEndDec()
1454 LLVM_DEBUG(dbgs() << "ARM Loops: Reverting to subs, br: " << *MI); in RevertLoopEndDec()
1455 assert(MI->getOpcode() == ARM::t2LoopEndDec && "Expected a t2LoopEndDec!"); in RevertLoopEndDec()
1456 MachineBasicBlock *MBB = MI->getParent(); in RevertLoopEndDec()
1459 BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(ARM::t2SUBri)); in RevertLoopEndDec()
1461 MIB.add(MI->getOperand(1)); in RevertLoopEndDec()
1468 MachineBasicBlock *DestBB = MI->getOperand(2).getMBB(); in RevertLoopEndDec()
1470 BBUtils->isBBInRange(MI, DestBB, 254) ? ARM::tBcc : ARM::t2Bcc; in RevertLoopEndDec()
1473 MIB = BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(BrOpc)); in RevertLoopEndDec()
1474 MIB.add(MI->getOperand(2)); // branch target in RevertLoopEndDec()
1478 MI->eraseFromParent(); in RevertLoopEndDec()
1560 auto RemovePredicate = [](MachineInstr *MI) { in ConvertVPTBlocks() argument
1561 if (MI->isDebugInstr()) in ConvertVPTBlocks()
1563 LLVM_DEBUG(dbgs() << "ARM Loops: Removing predicate from: " << *MI); in ConvertVPTBlocks()
1564 int PIdx = llvm::findFirstVPTPredOperandIdx(*MI); in ConvertVPTBlocks()
1566 assert(MI->getOperand(PIdx).getImm() == ARMVCC::Then && in ConvertVPTBlocks()
1568 MI->getOperand(PIdx).setImm(ARMVCC::None); in ConvertVPTBlocks()
1569 MI->getOperand(PIdx + 1).setReg(0); in ConvertVPTBlocks()
1736 for (auto *MI : VMOVCopies) { in Expand() local
1737 LLVM_DEBUG(dbgs() << "Converting copy to VMOVD: " << *MI); in Expand()
1738 assert(MI->getOpcode() == ARM::MQPRCopy && "Only expected MQPRCOPY!"); in Expand()
1739 MachineBasicBlock *MBB = MI->getParent(); in Expand()
1740 Register Dst = MI->getOperand(0).getReg(); in Expand()
1741 Register Src = MI->getOperand(1).getReg(); in Expand()
1742 auto MIB1 = BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(ARM::VMOVD), in Expand()
1748 auto MIB2 = BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(ARM::VMOVD), in Expand()
1754 MI->eraseFromParent(); in Expand()
1842 for (auto *MI : MQPRCopies) { in RevertNonLoops() local
1843 LLVM_DEBUG(dbgs() << "Converting copy to VORR: " << *MI); in RevertNonLoops()
1844 assert(MI->getOpcode() == ARM::MQPRCopy && "Only expected MQPRCOPY!"); in RevertNonLoops()
1845 MachineBasicBlock *MBB = MI->getParent(); in RevertNonLoops()
1846 auto MIB = BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(ARM::MVE_VORR), in RevertNonLoops()
1847 MI->getOperand(0).getReg()) in RevertNonLoops()
1848 .add(MI->getOperand(1)) in RevertNonLoops()
1849 .add(MI->getOperand(1)); in RevertNonLoops()
1850 addUnpredicatedMveVpredROp(MIB, MI->getOperand(0).getReg()); in RevertNonLoops()
1851 MI->eraseFromParent(); in RevertNonLoops()