Lines Matching +full:un +full:- +full:masked
1 //===- MachineVerifier.cpp - Machine Code Verifier ------------------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
19 // The machine code verifier is enabled with the command-line option
20 // -verify-machineinstrs.
21 //===----------------------------------------------------------------------===//
144 // Add Reg and any sub-registers to RV
148 append_range(RV, TRI->subregs(Reg.asMCReg())); in addRegWithSubRegs()
207 // Live-out registers are either in regsLiveOut or vregsPassed.
221 return Reg.id() < TRI->getNumRegs() && TRI->isInAllocatableClass(Reg) && in isAllocatable()
407 for (SlotIndexes::MBBIndexIterator I = Indexes->MBBIndexBegin(), in verifySlotIndexes()
408 E = Indexes->MBBIndexEnd(); I != E; ++I) { in verifySlotIndexes()
409 assert(!Last.isValid() || I->first > Last); in verifySlotIndexes()
410 Last = I->first; in verifySlotIndexes()
420 MRI->getNumVirtRegs()) in verifyProperties()
427 this->MF = &MF; in verify()
437 // If we're mid-GlobalISel and we already triggered the fallback path then in verify()
451 auto *LISWrapper = PASS->getAnalysisIfAvailable<LiveIntervalsWrapperPass>(); in verify()
452 LiveInts = LISWrapper ? &LISWrapper->getLIS() : nullptr; in verify()
454 auto *LVWrapper = PASS->getAnalysisIfAvailable<LiveVariablesWrapperPass>(); in verify()
456 LiveVars = LVWrapper ? &LVWrapper->getLV() : nullptr; in verify()
457 LiveStks = PASS->getAnalysisIfAvailable<LiveStacks>(); in verify()
458 auto *SIWrapper = PASS->getAnalysisIfAvailable<SlotIndexesWrapperPass>(); in verify()
459 Indexes = SIWrapper ? &SIWrapper->getSI() : nullptr; in verify()
463 LiveInts = MFAM->getCachedResult<LiveIntervalsAnalysis>(Func); in verify()
465 LiveVars = MFAM->getCachedResult<LiveVariablesAnalysis>(Func); in verify()
466 // TODO: LiveStks = MFAM->getCachedResult<LiveStacksAnalysis>(Func); in verify()
467 Indexes = MFAM->getCachedResult<SlotIndexesAnalysis>(Func); in verify()
548 LiveInts->print(errs()); in report()
550 MF->print(errs(), Indexes); in report()
553 << "- function: " << MF->getName() << "\n"; in report()
558 report(msg, MBB->getParent()); in report()
559 errs() << "- basic block: " << printMBBReference(*MBB) << ' ' in report()
560 << MBB->getName() << " (" << (const void *)MBB << ')'; in report()
562 errs() << " [" << Indexes->getMBBStartIdx(MBB) in report()
563 << ';' << Indexes->getMBBEndIdx(MBB) << ')'; in report()
569 report(msg, MI->getParent()); in report()
570 errs() << "- instruction: "; in report()
571 if (Indexes && Indexes->hasIndex(*MI)) in report()
572 errs() << Indexes->getInstructionIndex(*MI) << '\t'; in report()
573 MI->print(errs(), /*IsStandalone=*/true); in report()
579 report(msg, MO->getParent()); in report()
580 errs() << "- operand " << MONum << ": "; in report()
581 MO->print(errs(), MOVRegType, TRI); in report()
590 errs() << "- at: " << Pos << '\n'; in report_context()
594 errs() << "- interval: " << LI << '\n'; in report_context()
606 errs() << "- segment: " << S << '\n'; in report_context()
610 errs() << "- ValNo: " << VNI.id << " (def " << VNI.def << ")\n"; in report_context()
614 errs() << "- liverange: " << LR << '\n'; in report_context_liverange()
618 errs() << "- p. register: " << printReg(PReg, TRI) << '\n'; in report_context()
622 errs() << "- v. register: " << printReg(VReg, TRI) << '\n'; in report_context_vreg()
629 errs() << "- regunit: " << printRegUnit(VRegOrUnit, TRI) << '\n'; in report_context_vreg_regunit()
634 errs() << "- lanemask: " << PrintLaneMask(LaneMask) << '\n'; in report_context_lanemask()
641 for (const MachineBasicBlock *Succ : MBB->successors()) in markReachable()
648 regsReserved = MRI->reservedRegsFrozen() ? MRI->getReservedRegs() in visitMachineFunctionBefore()
649 : TRI->getReservedRegs(*MF); in visitMachineFunctionBefore()
651 if (!MF->empty()) in visitMachineFunctionBefore()
652 markReachable(&MF->front()); in visitMachineFunctionBefore()
670 MRI->verifyUseLists(); in visitMachineFunctionBefore()
672 if (!MF->empty()) in visitMachineFunctionBefore()
681 if (!MF->getProperties().hasProperty( in visitMachineBasicBlockBefore()
682 MachineFunctionProperties::Property::NoPHIs) && MRI->tracksLiveness()) { in visitMachineBasicBlockBefore()
683 // If this block has allocatable physical registers live-in, check that in visitMachineBasicBlockBefore()
685 for (const auto &LI : MBB->liveins()) { in visitMachineBasicBlockBefore()
686 if (isAllocatable(LI.PhysReg) && !MBB->isEHPad() && in visitMachineBasicBlockBefore()
687 MBB->getIterator() != MBB->getParent()->begin() && in visitMachineBasicBlockBefore()
688 !MBB->isInlineAsmBrIndirectTarget()) { in visitMachineBasicBlockBefore()
689 report("MBB has allocatable live-in, but isn't entry, landing-pad, or " in visitMachineBasicBlockBefore()
690 "inlineasm-br-indirect-target.", in visitMachineBasicBlockBefore()
697 if (MBB->isIRBlockAddressTaken()) { in visitMachineBasicBlockBefore()
698 if (!MBB->getAddressTakenIRBlock()->hasAddressTaken()) in visitMachineBasicBlockBefore()
699 report("ir-block-address-taken is associated with basic block not used by " in visitMachineBasicBlockBefore()
706 for (const auto *succ : MBB->successors()) { in visitMachineBasicBlockBefore()
707 if (succ->isEHPad()) in visitMachineBasicBlockBefore()
719 for (const MachineBasicBlock *Pred : MBB->predecessors()) { in visitMachineBasicBlockBefore()
729 const MCAsmInfo *AsmInfo = TM->getMCAsmInfo(); in visitMachineBasicBlockBefore()
730 const BasicBlock *BB = MBB->getBasicBlock(); in visitMachineBasicBlockBefore()
731 const Function &F = MF->getFunction(); in visitMachineBasicBlockBefore()
734 AsmInfo->getExceptionHandlingType() == ExceptionHandling::SjLj && in visitMachineBasicBlockBefore()
735 BB && isa<SwitchInst>(BB->getTerminator())) && in visitMachineBasicBlockBefore()
742 if (!TII->analyzeBranch(*const_cast<MachineBasicBlock *>(MBB), TBB, FBB, in visitMachineBasicBlockBefore()
748 if (!MBB->empty() && MBB->back().isBarrier() && in visitMachineBasicBlockBefore()
749 !TII->isPredicated(MBB->back())) { in visitMachineBasicBlockBefore()
750 report("MBB exits via unconditional fall-through but ends with a " in visitMachineBasicBlockBefore()
754 report("MBB exits via unconditional fall-through but has a condition!", in visitMachineBasicBlockBefore()
759 if (MBB->empty()) { in visitMachineBasicBlockBefore()
762 } else if (!MBB->back().isBarrier()) { in visitMachineBasicBlockBefore()
765 } else if (!MBB->back().isTerminator()) { in visitMachineBasicBlockBefore()
771 if (MBB->empty()) { in visitMachineBasicBlockBefore()
772 report("MBB exits via conditional branch/fall-through but doesn't " in visitMachineBasicBlockBefore()
774 } else if (MBB->back().isBarrier()) { in visitMachineBasicBlockBefore()
775 report("MBB exits via conditional branch/fall-through but ends with a " in visitMachineBasicBlockBefore()
777 } else if (!MBB->back().isTerminator()) { in visitMachineBasicBlockBefore()
778 report("MBB exits via conditional branch/fall-through but the branch " in visitMachineBasicBlockBefore()
784 if (MBB->empty()) { in visitMachineBasicBlockBefore()
787 } else if (!MBB->back().isBarrier()) { in visitMachineBasicBlockBefore()
790 } else if (!MBB->back().isTerminator()) { in visitMachineBasicBlockBefore()
804 if (TBB && !MBB->isSuccessor(TBB)) in visitMachineBasicBlockBefore()
808 if (FBB && !MBB->isSuccessor(FBB)) in visitMachineBasicBlockBefore()
822 MachineFunction::const_iterator MBBI = std::next(MBB->getIterator()); in visitMachineBasicBlockBefore()
823 if (MBBI == MF->end()) { in visitMachineBasicBlockBefore()
825 } else if (!MBB->isSuccessor(&*MBBI)) in visitMachineBasicBlockBefore()
826 report("MBB exits via conditional branch/fall-through but the CFG " in visitMachineBasicBlockBefore()
831 // Verify that there aren't any extra un-accounted-for successors. in visitMachineBasicBlockBefore()
832 for (const MachineBasicBlock *SuccMBB : MBB->successors()) { in visitMachineBasicBlockBefore()
838 if (Fallthrough && SuccMBB == MBB->getNextNode()) in visitMachineBasicBlockBefore()
840 // Also accept successors which are for exception-handling or might be in visitMachineBasicBlockBefore()
842 if (SuccMBB->isEHPad() || SuccMBB->isInlineAsmBrIndirectTarget()) in visitMachineBasicBlockBefore()
851 if (MRI->tracksLiveness()) { in visitMachineBasicBlockBefore()
852 for (const auto &LI : MBB->liveins()) { in visitMachineBasicBlockBefore()
854 report("MBB live-in list contains non-physical register", MBB); in visitMachineBasicBlockBefore()
857 for (const MCPhysReg &SubReg : TRI->subregs_inclusive(LI.PhysReg)) in visitMachineBasicBlockBefore()
862 const MachineFrameInfo &MFI = MF->getFrameInfo(); in visitMachineBasicBlockBefore()
865 for (const MCPhysReg &SubReg : TRI->subregs_inclusive(I)) in visitMachineBasicBlockBefore()
873 lastIndex = Indexes->getMBBStartIdx(MBB); in visitMachineBasicBlockBefore()
877 // stand-alone unbundled instructions.
879 if (Indexes && Indexes->hasIndex(*MI)) { in visitMachineBundleBefore()
880 SlotIndex idx = Indexes->getInstructionIndex(*MI); in visitMachineBundleBefore()
888 // Ensure non-terminators don't follow terminators. in visitMachineBundleBefore()
889 if (MI->isTerminator()) { in visitMachineBundleBefore()
894 // precede non-terminators. in visitMachineBundleBefore()
895 if (FirstTerminator->getOpcode() != TargetOpcode::G_INVOKE_REGION_START) { in visitMachineBundleBefore()
896 report("Non-terminator instruction after the first terminator", MI); in visitMachineBundleBefore()
906 if (MI->getNumOperands() < 2) { in verifyInlineAsm()
910 if (!MI->getOperand(0).isSymbol()) in verifyInlineAsm()
912 if (!MI->getOperand(1).isImm()) in verifyInlineAsm()
917 if (!isUInt<6>(MI->getOperand(1).getImm())) in verifyInlineAsm()
918 report("Unknown asm flags", &MI->getOperand(1), 1); in verifyInlineAsm()
924 for (unsigned e = MI->getNumOperands(); OpNo < e; OpNo += NumOps) { in verifyInlineAsm()
925 const MachineOperand &MO = MI->getOperand(OpNo); in verifyInlineAsm()
933 if (OpNo > MI->getNumOperands()) in verifyInlineAsm()
937 if (OpNo < MI->getNumOperands() && MI->getOperand(OpNo).isMetadata()) in verifyInlineAsm()
941 for (unsigned e = MI->getNumOperands(); OpNo < e; ++OpNo) { in verifyInlineAsm()
942 const MachineOperand &MO = MI->getOperand(OpNo); in verifyInlineAsm()
947 if (MI->getOpcode() == TargetOpcode::INLINEASM_BR) { in verifyInlineAsm()
948 const MachineBasicBlock *MBB = MI->getParent(); in verifyInlineAsm()
950 for (unsigned i = InlineAsm::MIOp_FirstOperand, e = MI->getNumOperands(); in verifyInlineAsm()
952 const MachineOperand &MO = MI->getOperand(i); in verifyInlineAsm()
965 if (!MBB->isSuccessor(IndirectTargetMBB)) in verifyInlineAsm()
969 if (!IndirectTargetMBB->isPredecessor(MBB)) in verifyInlineAsm()
997 report("operand types must be all-vector or all-scalar", MI); in verifyVectorElementMatch()
1015 auto Opcode = MI->getOpcode(); in verifyGIntrinsicSideEffects()
1018 unsigned IntrID = cast<GIntrinsic>(MI)->getIntrinsicID(); in verifyGIntrinsicSideEffects()
1021 MF->getFunction().getContext(), static_cast<Intrinsic::ID>(IntrID)); in verifyGIntrinsicSideEffects()
1024 report(Twine(TII->getName(Opcode), in verifyGIntrinsicSideEffects()
1030 report(Twine(TII->getName(Opcode), " used with readnone intrinsic"), MI); in verifyGIntrinsicSideEffects()
1039 auto Opcode = MI->getOpcode(); in verifyGIntrinsicConvergence()
1042 unsigned IntrID = cast<GIntrinsic>(MI)->getIntrinsicID(); in verifyGIntrinsicConvergence()
1045 MF->getFunction().getContext(), static_cast<Intrinsic::ID>(IntrID)); in verifyGIntrinsicConvergence()
1048 report(Twine(TII->getName(Opcode), " used with a convergent intrinsic"), in verifyGIntrinsicConvergence()
1054 Twine(TII->getName(Opcode), " used with a non-convergent intrinsic"), in verifyGIntrinsicConvergence()
1067 const MCInstrDesc &MCID = MI->getDesc(); in verifyPreISelGenericInstruction()
1068 unsigned NumOps = MI->getNumOperands(); in verifyPreISelGenericInstruction()
1071 if (MI->isBranch() && !MI->isIndirectBranch()) { in verifyPreISelGenericInstruction()
1073 for (const MachineOperand &Op : MI->operands()) { in verifyPreISelGenericInstruction()
1098 const MachineOperand *MO = &MI->getOperand(I); in verifyPreISelGenericInstruction()
1099 if (!MO->isReg()) { in verifyPreISelGenericInstruction()
1104 LLT OpTy = MRI->getType(MO->getReg()); in verifyPreISelGenericInstruction()
1121 for (unsigned I = 0; I < MI->getNumOperands(); ++I) { in verifyPreISelGenericInstruction()
1122 const MachineOperand *MO = &MI->getOperand(I); in verifyPreISelGenericInstruction()
1123 if (MO->isReg() && MO->getReg().isPhysical()) in verifyPreISelGenericInstruction()
1128 if (MI->getNumOperands() < MCID.getNumOperands()) in verifyPreISelGenericInstruction()
1132 if (!TII->verifyInstruction(*MI, ErrorInfo)) in verifyPreISelGenericInstruction()
1136 unsigned Opc = MI->getOpcode(); in verifyPreISelGenericInstruction()
1142 if (!MI->getOperand(2).isImm()) { in verifyPreISelGenericInstruction()
1147 Register Dst = MI->getOperand(0).getReg(); in verifyPreISelGenericInstruction()
1148 Register Src = MI->getOperand(1).getReg(); in verifyPreISelGenericInstruction()
1149 LLT SrcTy = MRI->getType(Src); in verifyPreISelGenericInstruction()
1150 int64_t Imm = MI->getOperand(2).getImm(); in verifyPreISelGenericInstruction()
1161 const RegisterBank *SrcRB = RBI->getRegBank(Src, *MRI, *TRI); in verifyPreISelGenericInstruction()
1162 const RegisterBank *DstRB = RBI->getRegBank(Dst, *MRI, *TRI); in verifyPreISelGenericInstruction()
1170 // Don't allow a class change. Do allow member class->regbank. in verifyPreISelGenericInstruction()
1171 const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(Dst); in verifyPreISelGenericInstruction()
1172 if (DstRC && DstRC != MRI->getRegClassOrNull(Src)) { in verifyPreISelGenericInstruction()
1184 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1188 if (MI->getOpcode() == TargetOpcode::G_CONSTANT) { in verifyPreISelGenericInstruction()
1189 if (!MI->getOperand(1).isCImm()) { in verifyPreISelGenericInstruction()
1194 const ConstantInt *CI = MI->getOperand(1).getCImm(); in verifyPreISelGenericInstruction()
1195 if (CI->getBitWidth() != DstTy.getSizeInBits()) in verifyPreISelGenericInstruction()
1198 if (!MI->getOperand(1).isFPImm()) { in verifyPreISelGenericInstruction()
1202 const ConstantFP *CF = MI->getOperand(1).getFPImm(); in verifyPreISelGenericInstruction()
1204 if (APFloat::getSizeInBits(CF->getValueAPF().getSemantics()) != in verifyPreISelGenericInstruction()
1216 LLT ValTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1217 LLT PtrTy = MRI->getType(MI->getOperand(1).getReg()); in verifyPreISelGenericInstruction()
1223 if (!MI->hasOneMemOperand()) { in verifyPreISelGenericInstruction()
1227 const MachineMemOperand &MMO = **MI->memoperands_begin(); in verifyPreISelGenericInstruction()
1228 if (MI->getOpcode() == TargetOpcode::G_ZEXTLOAD || in verifyPreISelGenericInstruction()
1229 MI->getOpcode() == TargetOpcode::G_SEXTLOAD) { in verifyPreISelGenericInstruction()
1233 } else if (MI->getOpcode() == TargetOpcode::G_LOAD) { in verifyPreISelGenericInstruction()
1237 } else if (MI->getOpcode() == TargetOpcode::G_STORE) { in verifyPreISelGenericInstruction()
1259 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1260 if (!DstTy.isValid() || !all_of(drop_begin(MI->operands()), in verifyPreISelGenericInstruction()
1264 LLT Ty = MRI->getType(MO.getReg()); in verifyPreISelGenericInstruction()
1275 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1276 LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); in verifyPreISelGenericInstruction()
1294 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1295 LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); in verifyPreISelGenericInstruction()
1304 if (MI->getOpcode() == TargetOpcode::G_INTTOPTR) { in verifyPreISelGenericInstruction()
1309 } else if (MI->getOpcode() == TargetOpcode::G_PTRTOINT) { in verifyPreISelGenericInstruction()
1315 assert(MI->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST); in verifyPreISelGenericInstruction()
1327 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1328 LLT PtrTy = MRI->getType(MI->getOperand(1).getReg()); in verifyPreISelGenericInstruction()
1329 LLT OffsetTy = MRI->getType(MI->getOperand(2).getReg()); in verifyPreISelGenericInstruction()
1340 const DataLayout &DL = MF->getDataLayout(); in verifyPreISelGenericInstruction()
1353 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1354 LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); in verifyPreISelGenericInstruction()
1355 LLT MaskTy = MRI->getType(MI->getOperand(2).getReg()); in verifyPreISelGenericInstruction()
1380 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1381 LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); in verifyPreISelGenericInstruction()
1392 switch (MI->getOpcode()) { in verifyPreISelGenericInstruction()
1407 LLT SelTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1408 LLT CondTy = MRI->getType(MI->getOperand(1).getReg()); in verifyPreISelGenericInstruction()
1422 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1423 LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); in verifyPreISelGenericInstruction()
1427 const unsigned NumOps = MI->getNumOperands(); in verifyPreISelGenericInstruction()
1428 if (DstTy.getSizeInBits() != SrcTy.getSizeInBits() * (NumOps - 1)) in verifyPreISelGenericInstruction()
1432 if (MRI->getType(MI->getOperand(I).getReg()) != SrcTy) in verifyPreISelGenericInstruction()
1439 unsigned NumDsts = MI->getNumOperands() - 1; in verifyPreISelGenericInstruction()
1440 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1442 if (MRI->getType(MI->getOperand(i).getReg()) != DstTy) { in verifyPreISelGenericInstruction()
1448 LLT SrcTy = MRI->getType(MI->getOperand(NumDsts).getReg()); in verifyPreISelGenericInstruction()
1478 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1479 LLT SrcEltTy = MRI->getType(MI->getOperand(1).getReg()); in verifyPreISelGenericInstruction()
1488 if (DstTy.getNumElements() != MI->getNumOperands() - 1) in verifyPreISelGenericInstruction()
1491 for (const MachineOperand &MO : llvm::drop_begin(MI->operands(), 2)) in verifyPreISelGenericInstruction()
1492 if (MRI->getType(MI->getOperand(1).getReg()) != MRI->getType(MO.getReg())) in verifyPreISelGenericInstruction()
1500 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1501 LLT SrcEltTy = MRI->getType(MI->getOperand(1).getReg()); in verifyPreISelGenericInstruction()
1505 for (const MachineOperand &MO : llvm::drop_begin(MI->operands(), 2)) in verifyPreISelGenericInstruction()
1506 if (MRI->getType(MI->getOperand(1).getReg()) != MRI->getType(MO.getReg())) in verifyPreISelGenericInstruction()
1518 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1519 LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); in verifyPreISelGenericInstruction()
1524 if (MI->getNumOperands() < 3) in verifyPreISelGenericInstruction()
1527 for (const MachineOperand &MO : llvm::drop_begin(MI->operands(), 2)) in verifyPreISelGenericInstruction()
1528 if (MRI->getType(MI->getOperand(1).getReg()) != MRI->getType(MO.getReg())) in verifyPreISelGenericInstruction()
1531 SrcTy.getElementCount() * (MI->getNumOperands() - 1)) in verifyPreISelGenericInstruction()
1537 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1538 LLT SrcTy = MRI->getType(MI->getOperand(2).getReg()); in verifyPreISelGenericInstruction()
1549 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1550 LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); in verifyPreISelGenericInstruction()
1551 LLT SrcTy2 = MRI->getType(MI->getOperand(2).getReg()); in verifyPreISelGenericInstruction()
1578 const MachineOperand &SrcOp = MI->getOperand(1); in verifyPreISelGenericInstruction()
1584 const MachineOperand &OffsetOp = MI->getOperand(2); in verifyPreISelGenericInstruction()
1590 unsigned DstSize = MRI->getType(MI->getOperand(0).getReg()).getSizeInBits(); in verifyPreISelGenericInstruction()
1591 unsigned SrcSize = MRI->getType(SrcOp.getReg()).getSizeInBits(); in verifyPreISelGenericInstruction()
1600 const MachineOperand &SrcOp = MI->getOperand(2); in verifyPreISelGenericInstruction()
1606 const MachineOperand &OffsetOp = MI->getOperand(3); in verifyPreISelGenericInstruction()
1612 unsigned DstSize = MRI->getType(MI->getOperand(0).getReg()).getSizeInBits(); in verifyPreISelGenericInstruction()
1613 unsigned SrcSize = MRI->getType(SrcOp.getReg()).getSizeInBits(); in verifyPreISelGenericInstruction()
1624 if (!MI->getOperand(1).isJTI()) in verifyPreISelGenericInstruction()
1626 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1632 if (!MRI->getType(MI->getOperand(0).getReg()).isPointer()) in verifyPreISelGenericInstruction()
1635 if (!MI->getOperand(1).isJTI()) in verifyPreISelGenericInstruction()
1638 const auto &IdxOp = MI->getOperand(2); in verifyPreISelGenericInstruction()
1639 if (!IdxOp.isReg() || MRI->getType(IdxOp.getReg()).isPointer()) in verifyPreISelGenericInstruction()
1649 const MachineOperand &IntrIDOp = MI->getOperand(MI->getNumExplicitDefs()); in verifyPreISelGenericInstruction()
1663 if (!MI->getOperand(2).isImm()) { in verifyPreISelGenericInstruction()
1668 LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); in verifyPreISelGenericInstruction()
1669 int64_t Imm = MI->getOperand(2).getImm(); in verifyPreISelGenericInstruction()
1677 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1683 if (!MI->getOperand(1).isCImm()) { in verifyPreISelGenericInstruction()
1687 if (MI->getOperand(1).getCImm()->isZero()) { in verifyPreISelGenericInstruction()
1694 const MachineOperand &Src0Op = MI->getOperand(1); in verifyPreISelGenericInstruction()
1700 const MachineOperand &Src1Op = MI->getOperand(2); in verifyPreISelGenericInstruction()
1706 const MachineOperand &IndexOp = MI->getOperand(3); in verifyPreISelGenericInstruction()
1712 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1713 LLT Src0Ty = MRI->getType(Src0Op.getReg()); in verifyPreISelGenericInstruction()
1714 LLT Src1Ty = MRI->getType(Src1Op.getReg()); in verifyPreISelGenericInstruction()
1751 const MachineOperand &SrcOp = MI->getOperand(1); in verifyPreISelGenericInstruction()
1757 const MachineOperand &IndexOp = MI->getOperand(2); in verifyPreISelGenericInstruction()
1763 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1764 LLT SrcTy = MRI->getType(SrcOp.getReg()); in verifyPreISelGenericInstruction()
1792 const MachineOperand &MaskOp = MI->getOperand(3); in verifyPreISelGenericInstruction()
1798 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1799 LLT Src0Ty = MRI->getType(MI->getOperand(1).getReg()); in verifyPreISelGenericInstruction()
1800 LLT Src1Ty = MRI->getType(MI->getOperand(2).getReg()); in verifyPreISelGenericInstruction()
1830 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1831 LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); in verifyPreISelGenericInstruction()
1854 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1855 LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); in verifyPreISelGenericInstruction()
1856 LLT IdxTy = MRI->getType(MI->getOperand(2).getReg()); in verifyPreISelGenericInstruction()
1868 auto TLI = MF->getSubtarget().getTargetLowering(); in verifyPreISelGenericInstruction()
1870 TLI->getVectorIdxTy(MF->getDataLayout()).getFixedSizeInBits()) { in verifyPreISelGenericInstruction()
1878 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1879 LLT VecTy = MRI->getType(MI->getOperand(1).getReg()); in verifyPreISelGenericInstruction()
1880 LLT ScaTy = MRI->getType(MI->getOperand(2).getReg()); in verifyPreISelGenericInstruction()
1881 LLT IdxTy = MRI->getType(MI->getOperand(3).getReg()); in verifyPreISelGenericInstruction()
1898 auto TLI = MF->getSubtarget().getTargetLowering(); in verifyPreISelGenericInstruction()
1900 TLI->getVectorIdxTy(MF->getDataLayout()).getFixedSizeInBits()) { in verifyPreISelGenericInstruction()
1908 const MachineOperand &DstOp = MI->getOperand(0); in verifyPreISelGenericInstruction()
1909 const MachineOperand &AllocOp = MI->getOperand(1); in verifyPreISelGenericInstruction()
1910 const MachineOperand &AlignOp = MI->getOperand(2); in verifyPreISelGenericInstruction()
1912 if (!DstOp.isReg() || !MRI->getType(DstOp.getReg()).isPointer()) { in verifyPreISelGenericInstruction()
1917 if (!AllocOp.isReg() || !MRI->getType(AllocOp.getReg()).isScalar()) { in verifyPreISelGenericInstruction()
1931 ArrayRef<MachineMemOperand *> MMOs = MI->memoperands(); in verifyPreISelGenericInstruction()
1937 if ((!MMOs[0]->isStore() || MMOs[0]->isLoad()) || in verifyPreISelGenericInstruction()
1938 (MMOs[1]->isStore() || !MMOs[1]->isLoad())) { in verifyPreISelGenericInstruction()
1943 if (MMOs[0]->getSize() != MMOs[1]->getSize()) in verifyPreISelGenericInstruction()
1946 LLT DstPtrTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1947 LLT SrcPtrTy = MRI->getType(MI->getOperand(1).getReg()); in verifyPreISelGenericInstruction()
1954 if (DstPtrTy.getAddressSpace() != MMOs[0]->getAddrSpace()) in verifyPreISelGenericInstruction()
1956 if (SrcPtrTy.getAddressSpace() != MMOs[1]->getAddrSpace()) in verifyPreISelGenericInstruction()
1960 if (!MI->getOperand(3).isImm() || (MI->getOperand(3).getImm() & ~1LL)) in verifyPreISelGenericInstruction()
1967 ArrayRef<MachineMemOperand *> MMOs = MI->memoperands(); in verifyPreISelGenericInstruction()
1974 if ((!MMOs[0]->isStore() || MMOs[0]->isLoad())) { in verifyPreISelGenericInstruction()
1979 LLT DstPtrTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
1985 if (DstPtrTy.getAddressSpace() != MMOs[0]->getAddrSpace()) in verifyPreISelGenericInstruction()
1988 if (!MI->getOperand(MI->getNumOperands() - 1).isImm() || in verifyPreISelGenericInstruction()
1989 (MI->getOperand(MI->getNumOperands() - 1).getImm() & ~1LL)) in verifyPreISelGenericInstruction()
1995 const MachineOperand &KindOp = MI->getOperand(0); in verifyPreISelGenericInstruction()
1996 if (!MI->getOperand(0).isImm()) { in verifyPreISelGenericInstruction()
2000 int64_t Kind = MI->getOperand(0).getImm(); in verifyPreISelGenericInstruction()
2007 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
2008 LLT Src1Ty = MRI->getType(MI->getOperand(1).getReg()); in verifyPreISelGenericInstruction()
2009 LLT Src2Ty = MRI->getType(MI->getOperand(2).getReg()); in verifyPreISelGenericInstruction()
2033 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
2041 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
2053 LLT Src1Ty = MRI->getType(MI->getOperand(1).getReg()); in verifyPreISelGenericInstruction()
2054 LLT Src2Ty = MRI->getType(MI->getOperand(2).getReg()); in verifyPreISelGenericInstruction()
2069 LLT DestTy = MRI->getType(MI->getOperand(0).getReg()); in verifyPreISelGenericInstruction()
2075 LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); in verifyPreISelGenericInstruction()
2083 const MachineOperand &TestMO = MI->getOperand(2); in verifyPreISelGenericInstruction()
2085 report("floating-point class set (operand 2) must be an immediate", MI); in verifyPreISelGenericInstruction()
2090 report("Incorrect floating-point class set (operand 2)", MI); in verifyPreISelGenericInstruction()
2096 const MachineOperand &AddrOp = MI->getOperand(0); in verifyPreISelGenericInstruction()
2097 if (!AddrOp.isReg() || !MRI->getType(AddrOp.getReg()).isPointer()) { in verifyPreISelGenericInstruction()
2101 const MachineOperand &RWOp = MI->getOperand(1); in verifyPreISelGenericInstruction()
2103 report("rw operand must be an immediate 0-1", &RWOp, 1); in verifyPreISelGenericInstruction()
2106 const MachineOperand &LocalityOp = MI->getOperand(2); in verifyPreISelGenericInstruction()
2108 report("locality operand must be an immediate 0-3", &LocalityOp, 2); in verifyPreISelGenericInstruction()
2111 const MachineOperand &CacheTypeOp = MI->getOperand(3); in verifyPreISelGenericInstruction()
2113 report("cache type operand must be an immediate 0-1", &CacheTypeOp, 3); in verifyPreISelGenericInstruction()
2119 if (MI->getOperand(2).getImm() < 1) in verifyPreISelGenericInstruction()
2124 if (!MI->getOperand(1).isCPI()) in verifyPreISelGenericInstruction()
2126 if (!MRI->getType(MI->getOperand(0).getReg()).isPointer()) in verifyPreISelGenericInstruction()
2131 const MachineOperand &AddrOp = MI->getOperand(1); in verifyPreISelGenericInstruction()
2132 if (!AddrOp.isReg() || !MRI->getType(AddrOp.getReg()).isPointer()) in verifyPreISelGenericInstruction()
2142 const MCInstrDesc &MCID = MI->getDesc(); in visitMachineInstrBefore()
2143 if (MI->getNumOperands() < MCID.getNumOperands()) { in visitMachineInstrBefore()
2146 << MI->getNumOperands() << " given.\n"; in visitMachineInstrBefore()
2149 if (MI->getFlag(MachineInstr::NoConvergent) && !MCID.isConvergent()) in visitMachineInstrBefore()
2152 if (MI->isPHI()) { in visitMachineInstrBefore()
2153 if (MF->getProperties().hasProperty( in visitMachineInstrBefore()
2158 report("Found PHI instruction after non-PHI", MI); in visitMachineInstrBefore()
2163 if (MI->isInlineAsm()) in visitMachineInstrBefore()
2167 if (TII->isUnspillableTerminator(MI)) { in visitMachineInstrBefore()
2168 if (!MI->getOperand(0).isReg() || !MI->getOperand(0).isDef()) in visitMachineInstrBefore()
2170 Register Def = MI->getOperand(0).getReg(); in visitMachineInstrBefore()
2172 !MF->getProperties().hasProperty( in visitMachineInstrBefore()
2174 std::distance(MRI->use_nodbg_begin(Def), MRI->use_nodbg_end()) > 1) in visitMachineInstrBefore()
2178 // A fully-formed DBG_VALUE must have a location. Ignore partially formed in visitMachineInstrBefore()
2181 if (MI->isDebugValue() && MI->getNumOperands() == 4) in visitMachineInstrBefore()
2182 if (!MI->getDebugLoc()) in visitMachineInstrBefore()
2187 if (MI->isMetaInstruction() && MI->peekDebugInstrNum()) in visitMachineInstrBefore()
2191 for (MachineMemOperand *Op : MI->memoperands()) { in visitMachineInstrBefore()
2192 if (Op->isLoad() && !MI->mayLoad()) in visitMachineInstrBefore()
2194 if (Op->isStore() && !MI->mayStore()) in visitMachineInstrBefore()
2201 bool mapped = !LiveInts->isNotInMIMap(*MI); in visitMachineInstrBefore()
2202 if (MI->isDebugOrPseudoInstr()) { in visitMachineInstrBefore()
2205 } else if (MI->isInsideBundle()) { in visitMachineInstrBefore()
2221 if (!TII->verifyInstruction(*MI, ErrorInfo)) in visitMachineInstrBefore()
2225 switch (MI->getOpcode()) { in visitMachineInstrBefore()
2227 const MachineOperand &DstOp = MI->getOperand(0); in visitMachineInstrBefore()
2228 const MachineOperand &SrcOp = MI->getOperand(1); in visitMachineInstrBefore()
2232 LLT DstTy = MRI->getType(DstReg); in visitMachineInstrBefore()
2233 LLT SrcTy = MRI->getType(SrcReg); in visitMachineInstrBefore()
2249 TypeSize SrcSize = TRI->getRegSizeInBits(SrcReg, *MRI); in visitMachineInstrBefore()
2250 TypeSize DstSize = TRI->getRegSizeInBits(DstReg, *MRI); in visitMachineInstrBefore()
2253 TRI->getMinimalPhysRegClassLLT(SrcReg, DstTy); in visitMachineInstrBefore()
2255 SrcSize = TRI->getRegSizeInBits(*SrcRC); in visitMachineInstrBefore()
2260 TRI->getMinimalPhysRegClassLLT(DstReg, SrcTy); in visitMachineInstrBefore()
2262 DstSize = TRI->getRegSizeInBits(*DstRC); in visitMachineInstrBefore()
2290 if (!MI->getOperand(SO.getIDPos()).isImm() || in visitMachineInstrBefore()
2291 !MI->getOperand(SO.getNBytesPos()).isImm() || in visitMachineInstrBefore()
2292 !MI->getOperand(SO.getNCallArgsPos()).isImm()) { in visitMachineInstrBefore()
2298 if (Offset >= MI->getNumOperands()) { in visitMachineInstrBefore()
2302 if (!MI->getOperand(Offset - 1).isImm() || in visitMachineInstrBefore()
2303 MI->getOperand(Offset - 1).getImm() != StackMaps::ConstantOp || in visitMachineInstrBefore()
2304 !MI->getOperand(Offset).isImm()) in visitMachineInstrBefore()
2317 unsigned LastGCPtrIdx = SO.getNumAllocaIdx() - 2; in visitMachineInstrBefore()
2318 for (unsigned Idx = 0; Idx < MI->getNumDefs(); Idx++) { in visitMachineInstrBefore()
2320 if (!MI->isRegTiedToUseOperand(Idx, &UseOpIdx)) { in visitMachineInstrBefore()
2325 report("STATEPOINT def tied to non-gc operand", MI); in visitMachineInstrBefore()
2334 if (unsigned SubIdx = MI->getOperand(2).getSubReg()) in visitMachineInstrBefore()
2335 InsertedSize = TRI->getSubRegIdxSize(SubIdx); in visitMachineInstrBefore()
2337 InsertedSize = TRI->getRegSizeInBits(MI->getOperand(2).getReg(), *MRI); in visitMachineInstrBefore()
2338 unsigned SubRegSize = TRI->getSubRegIdxSize(MI->getOperand(3).getImm()); in visitMachineInstrBefore()
2346 unsigned NumOps = MI->getNumOperands(); in visitMachineInstrBefore()
2353 const MachineOperand &RegOp = MI->getOperand(I); in visitMachineInstrBefore()
2354 const MachineOperand &SubRegOp = MI->getOperand(I + 1); in visitMachineInstrBefore()
2360 SubRegOp.getImm() >= TRI->getNumSubRegIndices()) { in visitMachineInstrBefore()
2366 Register DstReg = MI->getOperand(0).getReg(); in visitMachineInstrBefore()
2370 if (MI->getOperand(0).getSubReg()) in visitMachineInstrBefore()
2380 const MachineInstr *MI = MO->getParent(); in visitMachineOperand()
2381 const MCInstrDesc &MCID = MI->getDesc(); in visitMachineOperand()
2384 NumDefs = (MONum == 0 && MO->isReg()) ? NumDefs : 0; in visitMachineOperand()
2389 if (!MO->isReg()) in visitMachineOperand()
2391 else if (!MO->isDef() && !MCOI.isOptionalDef()) in visitMachineOperand()
2393 else if (MO->isImplicit()) in visitMachineOperand()
2398 // e.g., LDM_RET in the arm back end. Check non-variadic operands only. in visitMachineOperand()
2399 bool IsOptional = MI->isVariadic() && MONum == MCID.getNumOperands() - 1; in visitMachineOperand()
2401 if (MO->isReg()) { in visitMachineOperand()
2402 if (MO->isDef() && !MCOI.isOptionalDef() && !MCID.variadicOpsAreDefs()) in visitMachineOperand()
2404 if (MO->isImplicit()) in visitMachineOperand()
2410 !MO->isReg() && !MO->isFI()) in visitMachineOperand()
2412 if (MO->isReg()) { in visitMachineOperand()
2415 !TII->isPCRelRegisterOperandLegal(*MO))) in visitMachineOperand()
2416 report("Expected a non-register operand.", MO, MONum); in visitMachineOperand()
2421 if (TiedTo != -1) { in visitMachineOperand()
2422 if (!MO->isReg()) in visitMachineOperand()
2424 else if (!MO->isTied()) in visitMachineOperand()
2426 else if (unsigned(TiedTo) != MI->findTiedOperandIdx(MONum)) in visitMachineOperand()
2428 else if (MO->getReg().isPhysical()) { in visitMachineOperand()
2429 const MachineOperand &MOTied = MI->getOperand(TiedTo); in visitMachineOperand()
2433 MO->getReg() != MOTied.getReg()) in visitMachineOperand()
2436 } else if (MO->isReg() && MO->isTied()) in visitMachineOperand()
2438 } else if (!MI->isVariadic()) { in visitMachineOperand()
2440 if (!MO->isValidExcessOperand()) in visitMachineOperand()
2441 report("Extra explicit operand on non-variadic instruction", MO, MONum); in visitMachineOperand()
2444 switch (MO->getType()) { in visitMachineOperand()
2448 if (MI->isDebugInstr() && MO->isUse()) { in visitMachineOperand()
2449 if (!MO->isDebug()) in visitMachineOperand()
2451 } else if (MO->isDebug()) { in visitMachineOperand()
2455 const Register Reg = MO->getReg(); in visitMachineOperand()
2458 if (MRI->tracksLiveness() && !MI->isDebugInstr()) in visitMachineOperand()
2461 if (MO->isDef() && MO->isUndef() && !MO->getSubReg() && in visitMachineOperand()
2462 MO->getReg().isVirtual()) // TODO: Apply to physregs too in visitMachineOperand()
2466 if (MO->isTied()) { in visitMachineOperand()
2467 unsigned OtherIdx = MI->findTiedOperandIdx(MONum); in visitMachineOperand()
2468 const MachineOperand &OtherMO = MI->getOperand(OtherIdx); in visitMachineOperand()
2473 if (MI->findTiedOperandIdx(OtherIdx) != MONum) in visitMachineOperand()
2477 if (-1 == MCID.getOperandConstraint(OtherIdx, MCOI::TIED_TO)) in visitMachineOperand()
2487 // Verify two-address constraints after the twoaddressinstruction pass. in visitMachineOperand()
2488 // Both twoaddressinstruction pass and phi-node-elimination pass call in visitMachineOperand()
2489 // MRI->leaveSSA() to set MF as not IsSSA, we should do the verification in visitMachineOperand()
2490 // after twoaddressinstruction pass not after phi-node-elimination pass. So in visitMachineOperand()
2492 // TiedOpsRewritten property to verify two-address constraints, this in visitMachineOperand()
2495 if (MF->getProperties().hasProperty( in visitMachineOperand()
2497 MO->isUse() && MI->isRegTiedToDefOperand(MONum, &DefIdx) && in visitMachineOperand()
2498 Reg != MI->getOperand(DefIdx).getReg()) in visitMachineOperand()
2499 report("Two-address instruction operands must be identical", MO, MONum); in visitMachineOperand()
2502 unsigned SubIdx = MO->getSubReg(); in visitMachineOperand()
2511 TII->getRegClass(MCID, MONum, TRI, *MF)) { in visitMachineOperand()
2512 if (!DRC->contains(Reg)) { in visitMachineOperand()
2515 << TRI->getRegClassName(DRC) << " register.\n"; in visitMachineOperand()
2519 if (MO->isRenamable()) { in visitMachineOperand()
2520 if (MRI->isReserved(Reg)) { in visitMachineOperand()
2527 const TargetRegisterClass *RC = MRI->getRegClassOrNull(Reg); in visitMachineOperand()
2537 if (MO->isUndef()) in visitMachineOperand()
2545 if (isFunctionTracksDebugUserValues || !MO->isUse() || in visitMachineOperand()
2546 !MI->isDebugValue() || !MRI->def_empty(Reg)) { in visitMachineOperand()
2547 // If we're post-Select, we can't have gvregs anymore. in visitMachineOperand()
2555 LLT Ty = MRI->getType(Reg); in visitMachineOperand()
2562 const RegisterBank *RegBank = MRI->getRegBankOrNull(Reg); in visitMachineOperand()
2563 const RegisterBankInfo *RBI = MF->getSubtarget().getRegBankInfo(); in visitMachineOperand()
2565 // If we're post-RegBankSelect, the gvreg must have a bank. in visitMachineOperand()
2575 RBI->getMaximumSize(RegBank->getID()) < Ty.getSizeInBits()) { in visitMachineOperand()
2578 errs() << "Register bank " << RegBank->getName() << " too small(" in visitMachineOperand()
2579 << RBI->getMaximumSize(RegBank->getID()) << ") to fit " in visitMachineOperand()
2580 << Ty.getSizeInBits() << "-bits\n"; in visitMachineOperand()
2596 TII->getRegClass(MCID, MONum, TRI, *MF)) { in visitMachineOperand()
2600 << TRI->getRegClassName( in visitMachineOperand()
2601 TII->getRegClass(MCID, MONum, TRI, *MF)) in visitMachineOperand()
2610 TRI->getSubClassWithSubReg(RC, SubIdx); in visitMachineOperand()
2613 errs() << "Register class " << TRI->getRegClassName(RC) in visitMachineOperand()
2619 errs() << "Register class " << TRI->getRegClassName(RC) in visitMachineOperand()
2626 TII->getRegClass(MCID, MONum, TRI, *MF)) { in visitMachineOperand()
2629 TRI->getLargestLegalSuperClass(RC, *MF); in visitMachineOperand()
2634 DRC = TRI->getMatchingSuperRegClass(SuperRC, DRC, SubIdx); in visitMachineOperand()
2636 report("No matching super-reg register class.", MO, MONum); in visitMachineOperand()
2640 if (!RC->hasSuperClassEq(DRC)) { in visitMachineOperand()
2642 errs() << "Expected a " << TRI->getRegClassName(DRC) in visitMachineOperand()
2643 << " register, but got a " << TRI->getRegClassName(RC) in visitMachineOperand()
2653 regMasks.push_back(MO->getRegMask()); in visitMachineOperand()
2657 if (MI->isPHI() && !MO->getMBB()->isSuccessor(MI->getParent())) in visitMachineOperand()
2662 if (LiveStks && LiveStks->hasInterval(MO->getIndex()) && in visitMachineOperand()
2663 LiveInts && !LiveInts->isNotInMIMap(*MI)) { in visitMachineOperand()
2664 int FI = MO->getIndex(); in visitMachineOperand()
2665 LiveInterval &LI = LiveStks->getInterval(FI); in visitMachineOperand()
2666 SlotIndex Idx = LiveInts->getInstructionIndex(*MI); in visitMachineOperand()
2668 bool stores = MI->mayStore(); in visitMachineOperand()
2669 bool loads = MI->mayLoad(); in visitMachineOperand()
2670 // For a memory-to-memory move, we need to check if the frame in visitMachineOperand()
2674 for (auto *MMO : MI->memoperands()) { in visitMachineOperand()
2675 const PseudoSourceValue *PSV = MMO->getPseudoValue(); in visitMachineOperand()
2680 if (Value->getFrameIndex() != FI) continue; in visitMachineOperand()
2682 if (MMO->isStore()) in visitMachineOperand()
2703 if (MO->getCFIIndex() >= MF->getFrameInstructions().size()) in visitMachineOperand()
2717 const MachineInstr *MI = MO->getParent(); in checkLivenessAtUse()
2719 bool HasValue = LRQ.valueIn() || (MI->isPHI() && LRQ.valueOut()); in checkLivenessAtUse()
2728 if (MO->isKill() && !LRQ.isKill()) { in checkLivenessAtUse()
2751 // Check that there is an early-clobber def of the same superregister in checkLivenessAtDef()
2753 if (((SubRangeCheck || MO->getSubReg() == 0) && VNI->def != DefIdx) || in checkLivenessAtDef()
2754 !SlotIndex::isSameInstr(VNI->def, DefIdx) || in checkLivenessAtDef()
2755 (VNI->def != DefIdx && in checkLivenessAtDef()
2756 (!VNI->def.isEarlyClobber() || !DefIdx.isRegister()))) { in checkLivenessAtDef()
2757 report("Inconsistent valno->def", MO, MONum); in checkLivenessAtDef()
2774 if (MO->isDead()) { in checkLivenessAtDef()
2779 // could be other non-dead defs of other subregs, or we could have other in checkLivenessAtDef()
2783 if (SubRangeCheck || MO->getSubReg() == 0) { in checkLivenessAtDef()
2795 const MachineInstr *MI = MO->getParent(); in checkLiveness()
2796 const Register Reg = MO->getReg(); in checkLiveness()
2797 const unsigned SubRegIdx = MO->getSubReg(); in checkLiveness()
2801 if (LiveInts->hasInterval(Reg)) { in checkLiveness()
2802 LI = &LiveInts->getInterval(Reg); in checkLiveness()
2803 if (SubRegIdx != 0 && (MO->isDef() || !MO->isUndef()) && !LI->empty() && in checkLiveness()
2804 !LI->hasSubRanges() && MRI->shouldTrackSubRegLiveness(Reg)) in checkLiveness()
2812 if (MO->readsReg()) { in checkLiveness()
2813 if (MO->isKill()) in checkLiveness()
2819 if (LiveVars && Reg.isVirtual() && MO->isKill() && in checkLiveness()
2820 !MI->isBundledWithPred()) { in checkLiveness()
2821 LiveVariables::VarInfo &VI = LiveVars->getVarInfo(Reg); in checkLiveness()
2827 if (LiveInts && !LiveInts->isNotInMIMap(*MI)) { in checkLiveness()
2829 if (MI->isPHI()) { in checkLiveness()
2831 UseIdx = LiveInts->getMBBEndIdx( in checkLiveness()
2832 MI->getOperand(MONum + 1).getMBB()).getPrevSlot(); in checkLiveness()
2834 UseIdx = LiveInts->getInstructionIndex(*MI); in checkLiveness()
2838 for (MCRegUnit Unit : TRI->regunits(Reg.asMCReg())) { in checkLiveness()
2839 if (MRI->isReservedRegUnit(Unit)) in checkLiveness()
2841 if (const LiveRange *LR = LiveInts->getCachedRegUnit(Unit)) in checkLiveness()
2850 if (LI->hasSubRanges() && !MO->isDef()) { in checkLiveness()
2852 ? TRI->getSubRegIndexLaneMask(SubRegIdx) in checkLiveness()
2853 : MRI->getMaxLaneMaskForVReg(Reg); in checkLiveness()
2855 for (const LiveInterval::SubRange &SR : LI->subranges()) { in checkLiveness()
2860 if (LRQ.valueIn() || (MI->isPHI() && LRQ.valueOut())) in checkLiveness()
2870 if (MI->isPHI() && LiveInMask != MOMask) { in checkLiveness()
2887 for (const MCPhysReg &SubReg : TRI->subregs(Reg)) { in checkLiveness()
2894 // If there is an additional implicit-use of a super register we stop in checkLiveness()
2899 for (const MachineOperand &MOP : MI->uses()) { in checkLiveness()
2906 if (llvm::is_contained(TRI->subregs(MOP.getReg()), Reg)) in checkLiveness()
2912 } else if (MRI->def_empty(Reg)) { in checkLiveness()
2915 BBInfo &MInfo = MBBInfoMap[MI->getParent()]; in checkLiveness()
2921 else if (!MI->isPHI()) in checkLiveness()
2927 if (MO->isDef()) { in checkLiveness()
2930 if (MO->isDead()) in checkLiveness()
2936 if (MRI->isSSA() && Reg.isVirtual() && in checkLiveness()
2937 std::next(MRI->def_begin(Reg)) != MRI->def_end()) in checkLiveness()
2941 if (LiveInts && !LiveInts->isNotInMIMap(*MI)) { in checkLiveness()
2942 SlotIndex DefIdx = LiveInts->getInstructionIndex(*MI); in checkLiveness()
2943 DefIdx = DefIdx.getRegSlot(MO->isEarlyClobber()); in checkLiveness()
2948 if (LI->hasSubRanges()) { in checkLiveness()
2950 ? TRI->getSubRegIndexLaneMask(SubRegIdx) in checkLiveness()
2951 : MRI->getMaxLaneMaskForVReg(Reg); in checkLiveness()
2952 for (const LiveInterval::SubRange &SR : LI->subranges()) { in checkLiveness()
2965 // Normal stand-alone instructions are also considered 'bundles', and this
2968 BBInfo &MInfo = MBBInfoMap[MI->getParent()]; in visitMachineBundleAfter()
2971 // Kill any masked registers. in visitMachineBundleAfter()
2989 SlotIndex stop = Indexes->getMBBEndIdx(MBB); in visitMachineBasicBlockAfter()
3041 // and pays easily for double look-ups (even in Dense with SparseUniverseMax in filterAndAdd()
3063 // practice even in the worst-by-execution time cases having all elements
3066 // worst-case memory usage within 2x of figures determined empirically for
3067 // "all Dense" scenario in such worst-by-execution-time cases.
3072 // Implements both a transfer function and a (binary, in-place) join operator
3074 // (out_b = in_b \ filter_b). filter_b is expected to be set-up ahead of time.
3083 // Set-up the filter_b. \pre Input register set \p RS must have no duplicates.
3091 // Double-duty the Filter: to maintain VRegs a set (and the join operation in add()
3106 if (MF->empty()) in calcRegsPassed()
3118 for (const MachineBasicBlock *Pred : MB->predecessors()) { in calcRegsPassed()
3135 // First push live-in regs to predecessors' vregsRequired. in calcRegsRequired()
3169 for (const MachineBasicBlock *Pred : MBB->predecessors()) { in calcRegsRequired()
3229 report("PHI operand is not live-out from predecessor", &MO0, I); in checkPHIOps()
3289 if (!MF->empty()) { in visitMachineFunctionAfter()
3290 BBInfo &MInfo = MBBInfoMap[&MF->front()]; in visitMachineFunctionAfter()
3302 // Check live-in list of each MBB. If a register is live into MBB, check in visitMachineFunctionAfter()
3304 // this must come from a definition in the predecesssor or its live-in in visitMachineFunctionAfter()
3305 // list, this will catch a live-through case where the predecessor does not in visitMachineFunctionAfter()
3306 // have the register in its live-in list. This currently only checks in visitMachineFunctionAfter()
3309 if (MRI->tracksLiveness()) in visitMachineFunctionAfter()
3321 errs() << TRI->getName(LiveInReg) in visitMachineFunctionAfter()
3328 for (auto CSInfo : MF->getCallSitesInfo()) in visitMachineFunctionAfter()
3329 if (!CSInfo.first->isCall()) in visitMachineFunctionAfter()
3332 // If there's debug-info, check that we don't have any duplicate value in visitMachineFunctionAfter()
3334 if (MF->getFunction().getSubprogram()) { in visitMachineFunctionAfter()
3350 for (unsigned I = 0, E = MRI->getNumVirtRegs(); I != E; ++I) { in verifyLiveVariables()
3352 LiveVariables::VarInfo &VI = LiveVars->getVarInfo(Reg); in verifyLiveVariables()
3376 for (unsigned I = 0, E = MRI->getNumVirtRegs(); I != E; ++I) { in verifyLiveIntervals()
3380 if (MRI->reg_nodbg_empty(Reg)) in verifyLiveIntervals()
3383 if (!LiveInts->hasInterval(Reg)) { in verifyLiveIntervals()
3389 const LiveInterval &LI = LiveInts->getInterval(Reg); in verifyLiveIntervals()
3395 for (unsigned i = 0, e = TRI->getNumRegUnits(); i != e; ++i) in verifyLiveIntervals()
3396 if (const LiveRange *LR = LiveInts->getCachedRegUnit(i)) in verifyLiveIntervals()
3403 if (VNI->isUnused()) in verifyLiveRangeValue()
3406 const VNInfo *DefVNI = LR.getVNInfoAt(VNI->def); in verifyLiveRangeValue()
3422 const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(VNI->def); in verifyLiveRangeValue()
3430 if (VNI->isPHIDef()) { in verifyLiveRangeValue()
3431 if (VNI->def != LiveInts->getMBBStartIdx(MBB)) { in verifyLiveRangeValue()
3439 // Non-PHI def. in verifyLiveRangeValue()
3440 const MachineInstr *MI = LiveInts->getInstructionFromIndex(VNI->def); in verifyLiveRangeValue()
3452 if (!MOI->isReg() || !MOI->isDef()) in verifyLiveRangeValue()
3455 if (MOI->getReg() != Reg) in verifyLiveRangeValue()
3458 if (!MOI->getReg().isPhysical() || !TRI->hasRegUnit(MOI->getReg(), Reg)) in verifyLiveRangeValue()
3462 (TRI->getSubRegIndexLaneMask(MOI->getSubReg()) & LaneMask).none()) in verifyLiveRangeValue()
3465 if (MOI->isEarlyClobber()) in verifyLiveRangeValue()
3478 if (!VNI->def.isEarlyClobber()) { in verifyLiveRangeValue()
3479 report("Early clobber def must be at an early-clobber slot", MBB); in verifyLiveRangeValue()
3483 } else if (!VNI->def.isRegister()) { in verifyLiveRangeValue()
3484 report("Non-PHI, non-early clobber def must be at a register slot", MBB); in verifyLiveRangeValue()
3499 if (VNI->id >= LR.getNumValNums() || VNI != LR.getValNumInfo(VNI->id)) { in verifyLiveRangeSegment()
3506 if (VNI->isUnused()) { in verifyLiveRangeSegment()
3512 const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(S.start); in verifyLiveRangeSegment()
3519 SlotIndex MBBStartIdx = LiveInts->getMBBStartIdx(MBB); in verifyLiveRangeSegment()
3520 if (S.start != MBBStartIdx && S.start != VNI->def) { in verifyLiveRangeSegment()
3527 LiveInts->getMBBFromIndex(S.end.getPrevSlot()); in verifyLiveRangeSegment()
3535 // Checks for non-live-out segments. in verifyLiveRangeSegment()
3536 if (S.end != LiveInts->getMBBEndIdx(EndMBB)) { in verifyLiveRangeSegment()
3538 if (!Reg.isVirtual() && VNI->isPHIDef() && S.start == VNI->def && in verifyLiveRangeSegment()
3539 S.end == VNI->def.getDeadSlot()) in verifyLiveRangeSegment()
3544 LiveInts->getInstructionFromIndex(S.end.getPrevSlot()); in verifyLiveRangeSegment()
3570 // early-clobber slot if it is being redefined by an early-clobber def. in verifyLiveRangeSegment()
3572 // an early-clobber slot if the last use is tied to an early-clobber def. in verifyLiveRangeSegment()
3573 if (MF->getProperties().hasProperty( in verifyLiveRangeSegment()
3576 if (I + 1 == LR.end() || (I + 1)->start != S.end) { in verifyLiveRangeSegment()
3594 if (!MOI->isReg() || MOI->getReg() != Reg) in verifyLiveRangeSegment()
3596 unsigned Sub = MOI->getSubReg(); in verifyLiveRangeSegment()
3598 Sub != 0 ? TRI->getSubRegIndexLaneMask(Sub) : LaneBitmask::getAll(); in verifyLiveRangeSegment()
3599 if (MOI->isDef()) { in verifyLiveRangeSegment()
3603 // mask for subregister defs. Read-undef defs will be handled by in verifyLiveRangeSegment()
3607 if (MOI->isDead()) in verifyLiveRangeSegment()
3612 if (MOI->readsReg()) in verifyLiveRangeSegment()
3630 if (!MRI->shouldTrackSubRegLiveness(Reg) || LaneMask.any() || in verifyLiveRangeSegment()
3643 MachineFunction::const_iterator MFI = MBB->getIterator(); in verifyLiveRangeSegment()
3644 // Is this live segment the beginning of a non-PHIDef VN? in verifyLiveRangeSegment()
3645 if (S.start == VNI->def && !VNI->isPHIDef()) { in verifyLiveRangeSegment()
3646 // Not live-in to any blocks. in verifyLiveRangeSegment()
3655 LiveInterval &OwnerLI = LiveInts->getInterval(Reg); in verifyLiveRangeSegment()
3660 assert(LiveInts->isLiveInToMBB(LR, &*MFI)); in verifyLiveRangeSegment()
3662 if (!Reg.isVirtual() && MFI->isEHPad()) { in verifyLiveRangeSegment()
3669 // Is VNI a PHI-def in the current block? in verifyLiveRangeSegment()
3670 bool IsPHI = VNI->isPHIDef() && in verifyLiveRangeSegment()
3671 VNI->def == LiveInts->getMBBStartIdx(&*MFI); in verifyLiveRangeSegment()
3673 // Check that VNI is live-out of all predecessors. in verifyLiveRangeSegment()
3674 for (const MachineBasicBlock *Pred : MFI->predecessors()) { in verifyLiveRangeSegment()
3675 SlotIndex PEnd = LiveInts->getMBBEndIdx(Pred); in verifyLiveRangeSegment()
3676 // Predecessor of landing pad live-out on last call. in verifyLiveRangeSegment()
3677 if (MFI->isEHPad()) { in verifyLiveRangeSegment()
3680 PEnd = Indexes->getInstructionIndex(MI).getBoundaryIndex(); in verifyLiveRangeSegment()
3687 // All predecessors must have a live-out value. However for a phi in verifyLiveRangeSegment()
3698 << LiveInts->getMBBStartIdx(&*MFI) << ", not live before " in verifyLiveRangeSegment()
3703 // Only PHI-defs can take different predecessor values. in verifyLiveRangeSegment()
3707 errs() << "Valno #" << PVNI->id << " live out of " in verifyLiveRangeSegment()
3709 << VNI->id << " live into " << printMBBReference(*MFI) << '@' in verifyLiveRangeSegment()
3710 << LiveInts->getMBBStartIdx(&*MFI) << '\n'; in verifyLiveRangeSegment()
3735 LaneBitmask MaxMask = MRI->getMaxLaneMaskForVReg(Reg); in verifyLiveInterval()
3768 errs() << ' ' << I->id; in verifyLiveInterval()
3799 unsigned FrameSetupOpcode = TII->getCallFrameSetupOpcode(); in verifyStackFrame()
3800 unsigned FrameDestroyOpcode = TII->getCallFrameDestroyOpcode(); in verifyStackFrame()
3805 SPState.resize(MF->getNumBlockIDs()); in verifyStackFrame()
3818 const MachineBasicBlock *StackPred = DFI.getPath(DFI.getPathLength() - 2); in verifyStackFrame()
3821 BBState.EntryValue = SPState[StackPred->getNumber()].ExitValue; in verifyStackFrame()
3822 BBState.EntryIsSetup = SPState[StackPred->getNumber()].ExitIsSetup; in verifyStackFrame()
3827 if ((int)MBB->getCallFrameSize() != -BBState.EntryValue) { in verifyStackFrame()
3831 errs() << "Call frame size on entry " << MBB->getCallFrameSize() in verifyStackFrame()
3833 << -BBState.EntryValue << '\n'; in verifyStackFrame()
3841 if (!MRI->isSSA() && !MF->getFrameInfo().adjustsStack()) in verifyStackFrame()
3844 BBState.ExitValue -= TII->getFrameTotalSize(I); in verifyStackFrame()
3849 int Size = TII->getFrameTotalSize(I); in verifyStackFrame()
3852 int AbsSPAdj = BBState.ExitValue < 0 ? -BBState.ExitValue : in verifyStackFrame()
3859 if (!MRI->isSSA() && !MF->getFrameInfo().adjustsStack()) in verifyStackFrame()
3866 SPState[MBB->getNumber()] = BBState; in verifyStackFrame()
3870 for (const MachineBasicBlock *Pred : MBB->predecessors()) { in verifyStackFrame()
3872 (SPState[Pred->getNumber()].ExitValue != BBState.EntryValue || in verifyStackFrame()
3873 SPState[Pred->getNumber()].ExitIsSetup != BBState.EntryIsSetup)) { in verifyStackFrame()
3876 << " has exit state (" << SPState[Pred->getNumber()].ExitValue in verifyStackFrame()
3877 << ", " << SPState[Pred->getNumber()].ExitIsSetup << "), while " in verifyStackFrame()
3885 for (const MachineBasicBlock *Succ : MBB->successors()) { in verifyStackFrame()
3887 (SPState[Succ->getNumber()].EntryValue != BBState.ExitValue || in verifyStackFrame()
3888 SPState[Succ->getNumber()].EntryIsSetup != BBState.ExitIsSetup)) { in verifyStackFrame()
3891 << " has entry state (" << SPState[Succ->getNumber()].EntryValue in verifyStackFrame()
3892 << ", " << SPState[Succ->getNumber()].EntryIsSetup << "), while " in verifyStackFrame()
3899 if (!MBB->empty() && MBB->back().isReturn()) { in verifyStackFrame()