Lines Matching +full:disp +full:- +full:sysreg
1 //===- AArch64InstrInfo.cpp - AArch64 Instruction Information -------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
11 //===----------------------------------------------------------------------===//
67 "aarch64-tbz-offset-bits", cl::Hidden, cl::init(14),
71 "aarch64-cbz-offset-bits", cl::Hidden, cl::init(19),
75 BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19),
79 BDisplacementBits("aarch64-b-offset-bits", cl::Hidden, cl::init(26),
87 /// GetInstSize - Return the number of bytes of code the specified
92 const Function &F = MF->getFunction(); in getInstSizeInBytes()
93 const MCAsmInfo *MAI = MF->getTarget().getMCAsmInfo(); in getInstSizeInBytes()
101 // Meta-instructions emit no code. in getInstSizeInBytes()
118 // Anything not explicitly designated otherwise (i.e. pseudo-instructions in getInstSizeInBytes()
120 // 4-byte insn. in getInstSizeInBytes()
141 // If `patchable-function-entry` is set, PATCHABLE_FUNCTION_ENTER in getInstSizeInBytes()
143 // they are expanded to 36-byte XRay sleds. in getInstSizeInBytes()
145 F.getFnAttributeAsParsedInteger("patchable-function-entry", 9) * 4; in getInstSizeInBytes()
149 // An XRay sled can be 4 bytes of alignment plus a 32-byte block. in getInstSizeInBytes()
171 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end(); in getInstBundleLength()
172 while (++I != E && I->isInsideBundle()) { in getInstBundleLength()
173 assert(!I->isBundle() && "No nested bundle!"); in getInstBundleLength()
181 // Block ends with fall-through condbranch. in parseCondBranch()
182 switch (LastInst->getOpcode()) { in parseCondBranch()
186 Target = LastInst->getOperand(1).getMBB(); in parseCondBranch()
187 Cond.push_back(LastInst->getOperand(0)); in parseCondBranch()
193 Target = LastInst->getOperand(1).getMBB(); in parseCondBranch()
194 Cond.push_back(MachineOperand::CreateImm(-1)); in parseCondBranch()
195 Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode())); in parseCondBranch()
196 Cond.push_back(LastInst->getOperand(0)); in parseCondBranch()
202 Target = LastInst->getOperand(2).getMBB(); in parseCondBranch()
203 Cond.push_back(MachineOperand::CreateImm(-1)); in parseCondBranch()
204 Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode())); in parseCondBranch()
205 Cond.push_back(LastInst->getOperand(0)); in parseCondBranch()
206 Cond.push_back(LastInst->getOperand(1)); in parseCondBranch()
274 // Offsets outside of the signed 33-bit range are not supported for ADRP + in insertIndirectBranch()
278 "Branch offsets outside of the signed 33-bit range not supported"); in insertIndirectBranch()
289 RS->enterBasicBlockEnd(MBB); in insertIndirectBranch()
293 if (!RS->isRegUsed(Reg)) { in insertIndirectBranch()
295 RS->setRegUsed(Reg); in insertIndirectBranch()
301 Register Scavenged = RS->FindUnusedReg(&AArch64::GPR64RegClass); in insertIndirectBranch()
305 RS->setRegUsed(Scavenged); in insertIndirectBranch()
311 AArch64FunctionInfo *AFI = MBB.getParent()->getInfo<AArch64FunctionInfo>(); in insertIndirectBranch()
312 if (!AFI || AFI->hasRedZone().value_or(true)) in insertIndirectBranch()
321 .addImm(-16); in insertIndirectBranch()
344 if (I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB || in analyzeBranch()
345 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) { in analyzeBranch()
346 --I; in analyzeBranch()
356 unsigned LastOpc = LastInst->getOpcode(); in analyzeBranch()
357 if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) { in analyzeBranch()
359 TBB = LastInst->getOperand(0).getMBB(); in analyzeBranch()
363 // Block ends with fall-through condbranch. in analyzeBranch()
372 unsigned SecondLastOpc = SecondLastInst->getOpcode(); in analyzeBranch()
378 LastInst->eraseFromParent(); in analyzeBranch()
380 LastOpc = LastInst->getOpcode(); in analyzeBranch()
381 if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) { in analyzeBranch()
383 TBB = LastInst->getOperand(0).getMBB(); in analyzeBranch()
387 SecondLastOpc = SecondLastInst->getOpcode(); in analyzeBranch()
397 LastInst->eraseFromParent(); in analyzeBranch()
399 LastOpc = LastInst->getOpcode(); in analyzeBranch()
400 if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) { in analyzeBranch()
405 // Block ends with fall-through condbranch. in analyzeBranch()
412 SecondLastOpc = SecondLastInst->getOpcode(); in analyzeBranch()
416 if (SecondLastInst && I != MBB.begin() && isUnpredicatedTerminator(*--I)) in analyzeBranch()
422 FBB = LastInst->getOperand(0).getMBB(); in analyzeBranch()
429 TBB = SecondLastInst->getOperand(0).getMBB(); in analyzeBranch()
432 I->eraseFromParent(); in analyzeBranch()
441 I->eraseFromParent(); in analyzeBranch()
461 if (I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB || in analyzeBranchPredicate()
462 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) { in analyzeBranchPredicate()
463 --I; in analyzeBranchPredicate()
471 unsigned LastOpc = LastInst->getOpcode(); in analyzeBranchPredicate()
485 MBP.TrueDest = LastInst->getOperand(1).getMBB(); in analyzeBranchPredicate()
492 MBP.LHS = LastInst->getOperand(0); in analyzeBranchPredicate()
501 if (Cond[0].getImm() != -1) { in reverseBranchCondition()
506 // Folded compare-and-branch in reverseBranchCondition()
546 if (!isUncondBranchOpcode(I->getOpcode()) && in removeBranch()
547 !isCondBranchOpcode(I->getOpcode())) in removeBranch()
551 I->eraseFromParent(); in removeBranch()
560 --I; in removeBranch()
561 if (!isCondBranchOpcode(I->getOpcode())) { in removeBranch()
568 I->eraseFromParent(); in removeBranch()
578 if (Cond[0].getImm() != -1) { in instantiateCondBranch()
582 // Folded compare-and-branch in instantiateCondBranch()
610 // Two-way conditional branch. in insertBranch()
624 if (!DefMI->isFullCopy()) in removeCopies()
626 VReg = DefMI->getOperand(1).getReg(); in removeCopies()
644 switch (DefMI->getOpcode()) { in canFoldIntoCSel()
648 if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, /*TRI=*/nullptr, in canFoldIntoCSel()
649 true) == -1) in canFoldIntoCSel()
651 // fall-through to ADDXri and ADDWri. in canFoldIntoCSel()
655 // add x, 1 -> csinc. in canFoldIntoCSel()
656 if (!DefMI->getOperand(2).isImm() || DefMI->getOperand(2).getImm() != 1 || in canFoldIntoCSel()
657 DefMI->getOperand(3).getImm() != 0) in canFoldIntoCSel()
665 // not x -> csinv, represented as orn dst, xzr, src. in canFoldIntoCSel()
666 unsigned ZReg = removeCopies(MRI, DefMI->getOperand(1).getReg()); in canFoldIntoCSel()
677 if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, /*TRI=*/nullptr, in canFoldIntoCSel()
678 true) == -1) in canFoldIntoCSel()
680 // fall-through to SUBXrr and SUBWrr. in canFoldIntoCSel()
684 // neg x -> csneg, represented as sub dst, xzr, src. in canFoldIntoCSel()
685 unsigned ZReg = removeCopies(MRI, DefMI->getOperand(1).getReg()); in canFoldIntoCSel()
698 *NewVReg = DefMI->getOperand(SrcOpNum).getReg(); in canFoldIntoCSel()
709 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); in canInsertSelect()
725 // FIXME: Fold in x+1, -x, and ~x when applicable. in canInsertSelect()
728 // Single-cycle csel, csinc, csinv, and csneg. in canInsertSelect()
756 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); in insertSelect()
901 // FIXME: this implementation should be micro-architecture dependent, so a
902 // micro-architecture target hook should be introduced here in future.
1089 // Check for the 32 -> 64 bit extension case, these instructions can do in isCoalescableExtInstr()
1093 // This is a signed or unsigned 32 -> 64 bit extension. in isCoalescableExtInstr()
1127 if (BaseOpA->isIdenticalTo(*BaseOpB) && in areMemAccessesTriviallyDisjoint()
1168 return Next != MBB->end() && Next->isCFIInstruction(); in isSchedulingBoundary()
1171 /// analyzeCompare - For a comparison instruction, return the source registers
1240 MachineFunction *MF = MBB->getParent(); in UpdateOperandRegClass()
1242 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo(); in UpdateOperandRegClass()
1243 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo(); in UpdateOperandRegClass()
1244 MachineRegisterInfo *MRI = &MF->getRegInfo(); in UpdateOperandRegClass()
1265 if (!OpRegCstraints->contains(Reg)) in UpdateOperandRegClass()
1267 } else if (!OpRegCstraints->hasSubClassEq(MRI->getRegClass(Reg)) && in UpdateOperandRegClass()
1268 !MRI->constrainRegClass(Reg, OpRegCstraints)) in UpdateOperandRegClass()
1275 /// Return the opcode that does not set flags when possible - otherwise
1335 if (To == To->getParent()->begin()) in areCFlagsAccessedBetweenInstrs()
1340 if (To->getParent() != From->getParent()) in areCFlagsAccessedBetweenInstrs()
1345 ++To.getReverse(), To->getParent()->rend(), in areCFlagsAccessedBetweenInstrs()
1363 unsigned MaskOpcode = Mask->getOpcode(); in canRemovePTestInstr()
1364 unsigned PredOpcode = Pred->getOpcode(); in canRemovePTestInstr()
1372 if ((Mask == Pred) && PTest->getOpcode() == AArch64::PTEST_PP_ANY) in canRemovePTestInstr()
1378 if (isPTrueOpcode(MaskOpcode) && Mask->getOperand(1).getImm() == 31 && in canRemovePTestInstr()
1390 // ptest-like instruction. in canRemovePTestInstr()
1391 if ((Mask == Pred) && PTest->getOpcode() == AArch64::PTEST_PP_ANY) in canRemovePTestInstr()
1397 if (isPTrueOpcode(MaskOpcode) && Mask->getOperand(1).getImm() == 31 && in canRemovePTestInstr()
1400 auto PTestLikeMask = MRI->getUniqueVRegDef(Pred->getOperand(1).getReg()); in canRemovePTestInstr()
1401 if (Mask == PTestLikeMask || PTest->getOpcode() == AArch64::PTEST_PP_ANY) in canRemovePTestInstr()
1407 // on 8-bit predicates like the PTEST. Otherwise, for instructions like in canRemovePTestInstr()
1408 // compare that also support 16/32/64-bit predicates, the implicit PTEST in canRemovePTestInstr()
1414 // ptrue p0.b ; P0=1111-1111-1111-1111 in canRemovePTestInstr()
1417 // cmphi p1.s, p0/z, z1.s, z0.s ; P1=0001-0001-0001-0001 in canRemovePTestInstr()
1419 // ptest p0, p1.b ; P1=0001-0001-0001-0001 in canRemovePTestInstr()
1422 // where the compare generates a canonical all active 32-bit predicate in canRemovePTestInstr()
1427 auto PTestLikeMask = MRI->getUniqueVRegDef(Pred->getOperand(1).getReg()); in canRemovePTestInstr()
1430 PTest->getOpcode() == AArch64::PTEST_PP_ANY)) in canRemovePTestInstr()
1436 // If OP in PTEST(PG, OP(PG, ...)) has a flag-setting variant change the in canRemovePTestInstr()
1453 auto *PredMask = MRI->getUniqueVRegDef(Pred->getOperand(1).getReg()); in canRemovePTestInstr()
1460 // flag-setting instructions. in canRemovePTestInstr()
1461 // PTEST(PTRUE_B(31), BRKN(PG, A, B)) -> BRKNS(PG, A, B). in canRemovePTestInstr()
1463 (Mask->getOperand(1).getImm() != 31)) in canRemovePTestInstr()
1468 // PTEST(OP=PTRUE_B(A), OP) -> PTRUES_B(A) in canRemovePTestInstr()
1478 /// optimizePTestInstr - Attempt to remove a ptest of a predicate-generating
1483 auto *Mask = MRI->getUniqueVRegDef(MaskReg); in optimizePTestInstr()
1484 auto *Pred = MRI->getUniqueVRegDef(PredReg); in optimizePTestInstr()
1485 unsigned PredOpcode = Pred->getOpcode(); in optimizePTestInstr()
1501 PTest->eraseFromParent(); in optimizePTestInstr()
1503 Pred->setDesc(get(*NewOp)); in optimizePTestInstr()
1507 Pred->addRegisterDefined(AArch64::NZCV, TRI); in optimizePTestInstr()
1511 if (Pred->registerDefIsDead(AArch64::NZCV, TRI)) { in optimizePTestInstr()
1512 unsigned i = 0, e = Pred->getNumOperands(); in optimizePTestInstr()
1514 MachineOperand &MO = Pred->getOperand(i); in optimizePTestInstr()
1544 if (DeadNZCVIdx != -1) { in optimizeCompareInstr()
1571 if (!MRI->use_nodbg_empty(CmpInstr.getOperand(0).getReg())) in optimizeCompareInstr()
1632 for (auto *BB : MBB->successors()) in areCFlagsAliveInSuccessors()
1633 if (BB->isLiveIn(AArch64::NZCV)) in areCFlagsAliveInSuccessors()
1639 /// or select and -1 otherwise.
1644 return -1; in findCondCodeUseOperandIdxForBranchOrSelect()
1649 return Idx - 2; in findCondCodeUseOperandIdxForBranchOrSelect()
1664 return Idx - 1; in findCondCodeUseOperandIdxForBranchOrSelect()
1741 std::next(CmpInstr.getIterator()), CmpParent->instr_end())) { in examineCFlagsUse()
1748 CCUseInstrs->push_back(&Instr); in examineCFlagsUse()
1767 /// - CmpInstr is either 'ADDS %vreg, 0' or 'SUBS %vreg, 0'
1768 /// - and, MI and CmpInstr are from the same MachineBB
1769 /// - and, condition flags are not alive in successors of the CmpInstr parent
1770 /// - and, if MI opcode is the S form there must be no defs of flags between
1774 /// - and, if C/V flags are not used after CmpInstr
1792 if (!NZVCUsed || NZVCUsed->C) in canInstrSubstituteCmpInstr()
1799 // 2) If MI is add/sub with no-signed-wrap, it produces a poison value when in canInstrSubstituteCmpInstr()
1801 if (NZVCUsed->V && !MI.getFlag(MachineInstr::NoSWrap)) in canInstrSubstituteCmpInstr()
1832 MI->setDesc(get(NewOpc)); in substituteCmpToZero()
1837 MI->addRegisterDefined(AArch64::NZCV, &TRI); in substituteCmpToZero()
1870 if (MI.findRegisterDefOperandIdx(AArch64::NZCV, /*TRI=*/nullptr, true) != -1) in canCmpInstrBeRemoved()
1890 if (!NZCVUsedAfterCmp || NZCVUsedAfterCmp->C || NZCVUsedAfterCmp->V) in canCmpInstrBeRemoved()
1893 if ((MIUsedNZCV.Z && NZCVUsedAfterCmp->N) || in canCmpInstrBeRemoved()
1894 (MIUsedNZCV.N && NZCVUsedAfterCmp->Z)) in canCmpInstrBeRemoved()
1912 /// Remove comparison in csinc-cmp sequence
1958 MachineOperand &CCOperand = CCUseInstr->getOperand(Idx); in removeCmpToZeroOrOne()
1973 auto &Subtarget = MBB.getParent()->getSubtarget<AArch64Subtarget>(); in expandPostRAPseudo()
1980 MBB.getParent()->getSubtarget().getInstrInfo(); in expandPostRAPseudo()
1984 while (FirstEpilogSEH->getFlag(MachineInstr::FrameDestroy) && in expandPostRAPseudo()
1989 BuildMI(MBB, FirstEpilogSEH, DL, TII->get(AArch64::ADRP)) in expandPostRAPseudo()
1992 BuildMI(MBB, FirstEpilogSEH, DL, TII->get(AArch64::ADDXri)) in expandPostRAPseudo()
2001 Module &M = *MBB.getParent()->getFunction().getParent(); in expandPostRAPseudo()
2002 if (M.getStackProtectorGuard() == "sysreg") { in expandPostRAPseudo()
2003 const AArch64SysReg::SysReg *SrcReg = in expandPostRAPseudo()
2006 report_fatal_error("Unknown SysReg for Stack Protector Guard Register"); in expandPostRAPseudo()
2008 // mrs xN, sysreg in expandPostRAPseudo()
2011 .addImm(SrcReg->Encoding); in expandPostRAPseudo()
2019 } else if (Offset >= -256 && Offset <= 255) { in expandPostRAPseudo()
2025 } else if (Offset >= -4095 && Offset <= 4095) { in expandPostRAPseudo()
2038 .addImm(-Offset) in expandPostRAPseudo()
2047 // Cases that are larger than +/- 4095 and not a multiple of 8, or larger in expandPostRAPseudo()
2064 cast<GlobalValue>((*MI.memoperands_begin())->getValue()); in expandPostRAPseudo()
2065 const TargetMachine &TM = MBB.getParent()->getTarget(); in expandPostRAPseudo()
2073 unsigned Reg32 = TRI->getSubReg(Reg, AArch64::sub_32); in expandPostRAPseudo()
2115 unsigned Reg32 = TRI->getSubReg(Reg, AArch64::sub_32); in expandPostRAPseudo()
2136 // to zero. This is equivalent to a register rename of the zero-register.
2260 return MMO->getFlags() & MOSuppressPair; in isLdStPairSuppressed()
2268 (*MI.memoperands_begin())->setFlags(MOSuppressPair); in suppressLdStPair()
2274 return MMO->getFlags() & MOStridedAccess; in isStridedAccess()
2523 "Unexpected instruction - was a new tail call opcode introduced?"); in isTailCallReturnInst()
2541 // 32-bit cases: in convertToFlagSettingOpc()
2568 // 64-bit cases: in convertToFlagSettingOpc()
2638 // For Pre-inc LD/ST, the operand is shifted by one. in isCandidateToMergeOrPair()
2643 // For Pre-indexed addressing quadword instructions, the third operand is the in isCandidateToMergeOrPair()
2672 // Do not pair any callee-save store/reload instructions in the in isCandidateToMergeOrPair()
2676 const MCAsmInfo *MAI = MI.getMF()->getTarget().getMCAsmInfo(); in isCandidateToMergeOrPair()
2677 bool NeedsWinCFI = MAI->usesWindowsCFI() && in isCandidateToMergeOrPair()
2678 MI.getMF()->getFunction().needsUnwindTableEntry(); in isCandidateToMergeOrPair()
2727 if (!Base->isReg()) in getAddrModeFromMemoryOp()
2730 AM.BaseReg = Base->getReg(); in getAddrModeFromMemoryOp()
2872 // -> in canFoldIntoAddrMode()
2890 // -> in canFoldIntoAddrMode()
2893 // Zero-extension looks like an ORRWrs followed by a SUBREG_TO_REG. in canFoldIntoAddrMode()
2898 const MachineRegisterInfo &MRI = AddrI.getMF()->getRegInfo(); in canFoldIntoAddrMode()
2925 int64_t NewOffset) -> bool { in canFoldIntoAddrMode()
2931 MinOffset = -256; in canFoldIntoAddrMode()
2935 MinOffset = -512; in canFoldIntoAddrMode()
2939 MinOffset = -1024; in canFoldIntoAddrMode()
2946 auto canFoldAddSubImmIntoAddrMode = [&](int64_t Disp) -> bool { in canFoldIntoAddrMode() argument
2948 int64_t NewOffset = OldOffset + Disp; in canFoldIntoAddrMode()
2965 ExtAddrMode::Formula Form = ExtAddrMode::Formula::Basic) -> bool { in canFoldIntoAddrMode()
2984 int64_t Disp = 0; in canFoldIntoAddrMode() local
2985 const bool OptSize = MemI.getMF()->getFunction().hasOptSize(); in canFoldIntoAddrMode()
2993 // -> in canFoldIntoAddrMode()
2995 Disp = AddrI.getOperand(2).getImm() << AddrI.getOperand(3).getImm(); in canFoldIntoAddrMode()
2996 return canFoldAddSubImmIntoAddrMode(Disp); in canFoldIntoAddrMode()
3001 // -> in canFoldIntoAddrMode()
3003 Disp = AddrI.getOperand(2).getImm() << AddrI.getOperand(3).getImm(); in canFoldIntoAddrMode()
3004 return canFoldAddSubImmIntoAddrMode(-Disp); in canFoldIntoAddrMode()
3009 // -> in canFoldIntoAddrMode()
3030 // -> in canFoldIntoAddrMode()
3042 // -> in canFoldIntoAddrMode()
3050 // Can fold only sign-/zero-extend of a word. in canFoldIntoAddrMode()
3319 // the [Reg, Reg, {s,u}xtw #N] addressing mode with sign-/zero-extend of the
3418 MachineRegisterInfo &MRI = MemI.getMF()->getRegInfo(); in emitLdStWithAddr()
3468 if (RC->hasSuperClassEq(&AArch64::GPR64RegClass)) { in emitLdStWithAddr()
3497 // Non-paired instruction (e.g., ldr x1, [x0, #8]). in getMemOperandWithOffsetWidth()
3532 if (!BaseOp->isReg() && !BaseOp->isFI()) in getMemOperandWithOffsetWidth()
3541 MachineOperand &OfsOp = LdSt.getOperand(LdSt.getNumExplicitOperands() - 1); in getMemOpBaseRegImmOfsOffsetOperand()
3611 MinOffset = -256; in getMemOpInfo()
3620 MinOffset = -256; in getMemOpInfo()
3627 MinOffset = -256; in getMemOpInfo()
3635 MinOffset = -256; in getMemOpInfo()
3647 MinOffset = -256; in getMemOpInfo()
3660 MinOffset = -256; in getMemOpInfo()
3675 MinOffset = -256; in getMemOpInfo()
3690 MinOffset = -256; in getMemOpInfo()
3700 MinOffset = -64; in getMemOpInfo()
3713 MinOffset = -64; in getMemOpInfo()
3726 MinOffset = -64; in getMemOpInfo()
3734 MinOffset = -1024; in getMemOpInfo()
3743 MinOffset = -512; in getMemOpInfo()
3764 MinOffset = -63; in getMemOpInfo()
3772 MinOffset = -256; in getMemOpInfo()
3780 MinOffset = -256; in getMemOpInfo()
3787 MinOffset = -256; in getMemOpInfo()
3794 MinOffset = -256; in getMemOpInfo()
3801 MinOffset = -256; in getMemOpInfo()
3808 MinOffset = -256; in getMemOpInfo()
3815 MinOffset = -256; in getMemOpInfo()
3842 MinOffset = -8; in getMemOpInfo()
3855 MinOffset = -8; in getMemOpInfo()
3868 MinOffset = -8; in getMemOpInfo()
3881 MinOffset = -8; in getMemOpInfo()
3903 MinOffset = -8; in getMemOpInfo()
3920 MinOffset = -8; in getMemOpInfo()
3932 MinOffset = -8; in getMemOpInfo()
3939 MinOffset = -256; in getMemOpInfo()
3945 MinOffset = -64; in getMemOpInfo()
4133 const MachineFunction *MF = MI.getParent()->getParent(); in getRegClass()
4134 return MF ? MF->getRegInfo().getRegClassOrNull(Reg) : nullptr; in getRegClass()
4226 // If the byte-offset isn't a multiple of the stride, we can't scale this in scaleOffset()
4231 // Convert the byte-offset used by unscaled into an "element" offset used in scaleOffset()
4240 // We can also pair sign-ext and zero-ext instructions. in canPairLdStOpc()
4359 // Pairwise instructions have a 7-bit signed offset field. in shouldClusterMemOps()
4360 if (Offset1 > 63 || Offset1 < -64) in shouldClusterMemOps()
4364 // Note: except for non-equal frame index bases in shouldClusterMemOps()
4370 FirstLdSt.getParent()->getParent()->getFrameInfo(); in shouldClusterMemOps()
4388 return MIB.addReg(TRI->getSubReg(Reg, SubIdx), State); in AddSubReg()
4396 return ((DestReg - SrcReg) & 0x1f) < NumRegs; in forwardCopyWillClobberTuple()
4407 uint16_t DestEncoding = TRI->getEncodingValue(DestReg); in copyPhysRegTuple()
4408 uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg); in copyPhysRegTuple()
4413 SubReg = NumRegs - 1; in copyPhysRegTuple()
4414 End = -1; in copyPhysRegTuple()
4415 Incr = -1; in copyPhysRegTuple()
4436 uint16_t DestEncoding = TRI->getEncodingValue(DestReg); in copyGPRRegTuple()
4437 uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg); in copyGPRRegTuple()
4462 // Cyclone recognizes "ADD Xd, Xn, #0" as a zero-cycle register move. in copyPhysReg()
4463 MCRegister DestRegX = TRI->getMatchingSuperReg( in copyPhysReg()
4465 MCRegister SrcRegX = TRI->getMatchingSuperReg( in copyPhysReg()
4488 // Cyclone recognizes "ORR Xd, XZR, Xm" as a zero-cycle register move. in copyPhysReg()
4489 MCRegister DestRegX = TRI->getMatchingSuperReg( in copyPhysReg()
4491 MCRegister SrcRegX = TRI->getMatchingSuperReg( in copyPhysReg()
4523 // Copy a predicate-as-counter register by ORRing with itself as if it in copyPhysReg()
4528 auto ToPPR = [](MCRegister R) -> MCRegister { in copyPhysReg()
4529 return (R - AArch64::PN0) + AArch64::P0; in copyPhysReg()
4556 // Copy a Z register pair by copying the individual sub-registers. in copyPhysReg()
4569 // Copy a Z register triple by copying the individual sub-registers. in copyPhysReg()
4581 // Copy a Z register quad by copying the individual sub-registers. in copyPhysReg()
4616 // Copy a DDDD register quad by copying the individual sub-registers. in copyPhysReg()
4626 // Copy a DDD register triple by copying the individual sub-registers. in copyPhysReg()
4636 // Copy a DD register pair by copying the individual sub-registers. in copyPhysReg()
4645 // Copy a QQQQ register quad by copying the individual sub-registers. in copyPhysReg()
4655 // Copy a QQQ register triple by copying the individual sub-registers. in copyPhysReg()
4665 // Copy a QQ register pair by copying the individual sub-registers. in copyPhysReg()
4695 .addReg(AArch64::Z0 + (DestReg - AArch64::Q0), RegState::Define) in copyPhysReg()
4696 .addReg(AArch64::Z0 + (SrcReg - AArch64::Q0)) in copyPhysReg()
4697 .addReg(AArch64::Z0 + (SrcReg - AArch64::Q0)); in copyPhysReg()
4707 .addImm(-16); in copyPhysReg()
4802 llvm_unreachable("unimplemented reg-to-reg copy"); in copyPhysReg()
4845 switch (TRI->getSpillSize(*RC)) { in storeRegToStackSlot()
5017 switch (TRI->getSpillSize(*RC)) { in loadRegFromStackSlot()
5178 /// For non-scalable offsets this is simply its byte size.
5194 if (NumPredicateVectors % 8 == 0 || NumPredicateVectors < -64 || in decomposeStackOffsetForFrameOffsets()
5197 NumPredicateVectors -= NumDataVectors * 8; in decomposeStackOffsetForFrameOffsets()
5212 Comment << (NumBytes < 0 ? " - " : " + ") << std::abs(NumBytes); in appendVGScaledOffsetExpr()
5226 Comment << (NumVGScaledBytes < 0 ? " - " : " + ") in appendVGScaledOffsetExpr()
5290 // Non-scalable offsets can use DW_CFA_offset directly. in createCFAOffset()
5344 Sign = -1; in emitFrameOffsetAdj()
5345 Offset = -Offset; in emitFrameOffsetAdj()
5359 // FIXME: If the offset won't fit in 24-bits, compute the offset into a in emitFrameOffsetAdj()
5363 // slightly if DestReg is SP which is always 16-byte aligned, so the scratch in emitFrameOffsetAdj()
5373 TmpReg = MBB.getParent()->getRegInfo().createVirtualRegister( in emitFrameOffsetAdj()
5385 Offset -= ThisVal << LocalShiftSize; in emitFrameOffsetAdj()
5388 auto MBI = BuildMI(MBB, MBBI, DL, TII->get(Opc), TmpReg) in emitFrameOffsetAdj()
5400 if (Sign == -1 || Opc == AArch64::SUBXri || Opc == AArch64::SUBSXri) in emitFrameOffsetAdj()
5403 CFAOffset -= Change; in emitFrameOffsetAdj()
5411 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION)) in emitFrameOffsetAdj()
5424 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_SetFP)).setMIFlag(Flag); in emitFrameOffsetAdj()
5426 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_AddFP)) in emitFrameOffsetAdj()
5435 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc)) in emitFrameOffsetAdj()
5456 // we can use `addsvl` to allocate any scalable stack-slots, which under in emitFrameOffset()
5457 // most circumstances will be only locals, not callee-save slots. in emitFrameOffset()
5458 const Function &F = MBB.getParent()->getFunction(); in emitFrameOffset()
5465 // First emit non-scalable frame offsets, or a simple 'mov'. in emitFrameOffset()
5468 "SP increment/decrement not 8-byte aligned"); in emitFrameOffset()
5471 Bytes = -Bytes; in emitFrameOffset()
5478 ? StackOffset::getFixed(-Bytes) in emitFrameOffset()
5494 CFAOffset += StackOffset::getScalable(-NumDataVectors * 16); in emitFrameOffset()
5590 return &*--InsertPt; in foldMemoryOperandImpl()
5595 // %0:sub_32<def,read-undef> = COPY %wzr; GPR64common:%0 in foldMemoryOperandImpl()
5609 return &*--InsertPt; in foldMemoryOperandImpl()
5614 // %0:sub_32<def,read-undef> = COPY %1; GPR64:%0, GPR32:%1 in foldMemoryOperandImpl()
5619 // LDRWui %0:sub_32<def,read-undef>, %stack.0 in foldMemoryOperandImpl()
5644 MachineInstr &LoadMI = *--InsertPt; in foldMemoryOperandImpl()
5748 Offset = Offset - (NewOffset * Scale); in isAArch64FrameOffsetLegal()
5792 MI.setDesc(TII->get(UnscaledOp)); in rewriteAArch64FrameIndex()
5823 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi. in isCombineInstrSettingFlag()
5842 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi. in isCombineInstrCandidate32()
5861 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi. in isCombineInstrCandidate64()
5904 TargetOptions Options = Inst.getParent()->getParent()->getTarget().Options; in isCombineInstrCandidateFP()
5906 // the target options or if FADD/FSUB has the contract fast-math flag. in isCombineInstrCandidateFP()
5926 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); in canCombine()
5932 if (!MI || MI->getParent() != &MBB || (unsigned)MI->getOpcode() != CombineOpc) in canCombine()
5935 if (!MRI.hasOneNonDBGUse(MI->getOperand(0).getReg())) in canCombine()
5939 assert(MI->getNumOperands() >= 4 && MI->getOperand(0).isReg() && in canCombine()
5940 MI->getOperand(1).isReg() && MI->getOperand(2).isReg() && in canCombine()
5941 MI->getOperand(3).isReg() && "MAdd/MSub must have a least 4 regs"); in canCombine()
5943 if (MI->getOperand(3).getReg() != ZeroReg) in canCombine()
5948 MI->findRegisterDefOperandIdx(AArch64::NZCV, /*TRI=*/nullptr, true) == -1) in canCombine()
5962 // Is \param MO defined by a floating-point multiply and can be combined?
5977 // == Floating-point types == in isAssociativeAndCommutative()
5978 // -- Floating-point instructions -- in isAssociativeAndCommutative()
5988 // -- Advanced SIMD instructions -- in isAssociativeAndCommutative()
6004 // -- SVE instructions -- in isAssociativeAndCommutative()
6013 return Inst.getParent()->getParent()->getTarget().Options.UnsafeFPMath || in isAssociativeAndCommutative()
6018 // -- Base instructions -- in isAssociativeAndCommutative()
6022 // The machine-combiner does not support three-source-operands machine in isAssociativeAndCommutative()
6034 // -- Advanced SIMD instructions -- in isAssociativeAndCommutative()
6035 // Opcodes MULv1i64 and MULv2i64 don't exist because there is no 64-bit MUL in isAssociativeAndCommutative()
6057 // -- SVE instructions -- in isAssociativeAndCommutative()
6089 if (Cmp_NZCV == -1) in getMaddPatterns()
6216 /// Floating-Point Support
6228 auto Match = [&](int Opcode, int Operand, unsigned Pattern) -> bool { in getFMAPatterns()
6366 auto Match = [&](unsigned Opcode, int Operand, unsigned Pattern) -> bool { in getFMULPatterns()
6367 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); in getFMULPatterns()
6372 // Ignore No-op COPYs in FMUL(COPY(DUP(..))) in getFMULPatterns()
6373 if (MI && MI->getOpcode() == TargetOpcode::COPY && in getFMULPatterns()
6374 MI->getOperand(1).getReg().isVirtual()) in getFMULPatterns()
6375 MI = MRI.getUniqueVRegDef(MI->getOperand(1).getReg()); in getFMULPatterns()
6376 if (MI && MI->getOpcode() == Opcode) { in getFMULPatterns()
6417 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); in getFNEGPatterns()
6419 auto Match = [&](unsigned Opcode, unsigned Pattern) -> bool { in getFNEGPatterns()
6422 if (MI != nullptr && (MI->getOpcode() == Opcode) && in getFNEGPatterns()
6423 MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()) && in getFNEGPatterns()
6426 MI->getFlag(MachineInstr::MIFlag::FmContract) && in getFNEGPatterns()
6427 MI->getFlag(MachineInstr::MIFlag::FmNsz)) { in getFNEGPatterns()
6448 /// \param Pattern - combiner pattern
6566 // A - (B + C) ==> (A - B) - C or (A - C) - B in getMiscPatterns()
6583 -1) in getMiscPatterns()
6637 /// genFusedMultiply - Generate fused multiply instructions.
6655 /// replacing the non-combined operand, if any.
6668 Register SrcReg0 = MUL->getOperand(1).getReg(); in genFusedMultiply()
6669 bool Src0IsKill = MUL->getOperand(1).isKill(); in genFusedMultiply()
6670 Register SrcReg1 = MUL->getOperand(2).getReg(); in genFusedMultiply()
6671 bool Src1IsKill = MUL->getOperand(2).isKill(); in genFusedMultiply()
6695 MIB = BuildMI(MF, MIMetadata(Root), TII->get(MaddOpc), ResultReg) in genFusedMultiply()
6700 MIB = BuildMI(MF, MIMetadata(Root), TII->get(MaddOpc), ResultReg) in genFusedMultiply()
6704 .addImm(MUL->getOperand(3).getImm()); in genFusedMultiply()
6706 MIB = BuildMI(MF, MIMetadata(Root), TII->get(MaddOpc), ResultReg) in genFusedMultiply()
6724 const TargetRegisterClass *RC = MRI.getRegClass(MAD->getOperand(0).getReg()); in genFNegatedMAD()
6733 Register SrcReg0 = MAD->getOperand(1).getReg(); in genFNegatedMAD()
6734 Register SrcReg1 = MAD->getOperand(2).getReg(); in genFNegatedMAD()
6735 Register SrcReg2 = MAD->getOperand(3).getReg(); in genFNegatedMAD()
6736 bool Src0IsKill = MAD->getOperand(1).isKill(); in genFNegatedMAD()
6737 bool Src1IsKill = MAD->getOperand(2).isKill(); in genFNegatedMAD()
6738 bool Src2IsKill = MAD->getOperand(3).isKill(); in genFNegatedMAD()
6749 BuildMI(MF, MIMetadata(Root), TII->get(Opc), ResultReg) in genFNegatedMAD()
6773 if (Dup->getOpcode() == TargetOpcode::COPY) in genIndexedMultiply()
6774 Dup = MRI.getUniqueVRegDef(Dup->getOperand(1).getReg()); in genIndexedMultiply()
6776 Register DupSrcReg = Dup->getOperand(1).getReg(); in genIndexedMultiply()
6780 unsigned DupSrcLane = Dup->getOperand(2).getImm(); in genIndexedMultiply()
6788 MIB = BuildMI(MF, MIMetadata(Root), TII->get(MulOpc), ResultReg) in genIndexedMultiply()
6797 /// genFusedMultiplyAcc - Helper to generate fused multiply accumulate
6809 /// genNeg - Helper to generate an intermediate negation of the second operand
6818 BuildMI(MF, MIMetadata(Root), TII->get(MnegOpc), NewVR) in genNeg()
6828 /// genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate
6843 /// genFusedMultiplyIdx - Helper to generate fused multiply accumulate
6855 /// genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate
6871 /// genMaddR - Generate madd instruction and combine mul and add using
6873 /// Example - an ADD intermediate needs to be stored in a register:
6899 Register SrcReg0 = MUL->getOperand(1).getReg(); in genMaddR()
6900 bool Src0IsKill = MUL->getOperand(1).isKill(); in genMaddR()
6901 Register SrcReg1 = MUL->getOperand(2).getReg(); in genMaddR()
6902 bool Src1IsKill = MUL->getOperand(2).isKill(); in genMaddR()
6914 BuildMI(MF, MIMetadata(Root), TII->get(MaddOpc), ResultReg) in genMaddR()
6924 /// A - (B + C) ==> (A - B) - C
6925 /// A - (B + C) ==> (A - C) - B
6940 Register RegB = AddMI->getOperand(IdxOpd1).getReg(); in genSubAdd2SubSub()
6941 bool RegBIsKill = AddMI->getOperand(IdxOpd1).isKill(); in genSubAdd2SubSub()
6942 Register RegC = AddMI->getOperand(IdxOtherOpd).getReg(); in genSubAdd2SubSub()
6943 bool RegCIsKill = AddMI->getOperand(IdxOtherOpd).isKill(); in genSubAdd2SubSub()
6960 BuildMI(MF, MIMetadata(Root), TII->get(Opcode), NewVR) in genSubAdd2SubSub()
6965 BuildMI(MF, MIMetadata(Root), TII->get(Opcode), ResultReg) in genSubAdd2SubSub()
6986 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); in genAlternativeCodeSequence()
7000 // A - (B + C) in genAlternativeCodeSequence()
7001 // ==> (A - B) - C in genAlternativeCodeSequence()
7006 // A - (B + C) in genAlternativeCodeSequence()
7007 // ==> (A - C) - B in genAlternativeCodeSequence()
7016 // --- Create(MADD); in genAlternativeCodeSequence()
7031 // --- Create(MADD); in genAlternativeCodeSequence()
7047 // --- Create(MADD); in genAlternativeCodeSequence()
7081 if (MovI->Opcode == OrrOpc) in genAlternativeCodeSequence()
7082 MIB1 = BuildMI(MF, MIMetadata(Root), TII->get(OrrOpc), NewVR) in genAlternativeCodeSequence()
7084 .addImm(MovI->Op2); in genAlternativeCodeSequence()
7087 assert((MovI->Opcode == AArch64::MOVNWi || in genAlternativeCodeSequence()
7088 MovI->Opcode == AArch64::MOVZWi) && in genAlternativeCodeSequence()
7091 assert((MovI->Opcode == AArch64::MOVNXi || in genAlternativeCodeSequence()
7092 MovI->Opcode == AArch64::MOVZXi) && in genAlternativeCodeSequence()
7094 MIB1 = BuildMI(MF, MIMetadata(Root), TII->get(MovI->Opcode), NewVR) in genAlternativeCodeSequence()
7095 .addImm(MovI->Op1) in genAlternativeCodeSequence()
7096 .addImm(MovI->Op2); in genAlternativeCodeSequence()
7108 // ==> MADD R,A,B,V // = -C + A*B in genAlternativeCodeSequence()
7109 // --- Create(MADD); in genAlternativeCodeSequence()
7128 BuildMI(MF, MIMetadata(Root), TII->get(SubOpc), NewVR) in genAlternativeCodeSequence()
7140 // ==> MSUB R,A,B,C (computes C - A*B) in genAlternativeCodeSequence()
7141 // --- Create(MSUB); in genAlternativeCodeSequence()
7155 // ==> MOV V, -Imm in genAlternativeCodeSequence()
7156 // ==> MADD R,A,B,V // = -Imm + A*B in genAlternativeCodeSequence()
7157 // --- Create(MADD); in genAlternativeCodeSequence()
7181 uint64_t UImm = SignExtend64(-Imm, BitSize); in genAlternativeCodeSequence()
7190 if (MovI->Opcode == OrrOpc) in genAlternativeCodeSequence()
7191 MIB1 = BuildMI(MF, MIMetadata(Root), TII->get(OrrOpc), NewVR) in genAlternativeCodeSequence()
7193 .addImm(MovI->Op2); in genAlternativeCodeSequence()
7196 assert((MovI->Opcode == AArch64::MOVNWi || in genAlternativeCodeSequence()
7197 MovI->Opcode == AArch64::MOVZWi) && in genAlternativeCodeSequence()
7200 assert((MovI->Opcode == AArch64::MOVNXi || in genAlternativeCodeSequence()
7201 MovI->Opcode == AArch64::MOVZXi) && in genAlternativeCodeSequence()
7203 MIB1 = BuildMI(MF, MIMetadata(Root), TII->get(MovI->Opcode), NewVR) in genAlternativeCodeSequence()
7204 .addImm(MovI->Op1) in genAlternativeCodeSequence()
7205 .addImm(MovI->Op2); in genAlternativeCodeSequence()
7695 BuildMI(MF, MIMetadata(Root), TII->get(AArch64::FNEGv4f16), NewVR) in genAlternativeCodeSequence()
7742 BuildMI(MF, MIMetadata(Root), TII->get(AArch64::FNEGv8f16), NewVR) in genAlternativeCodeSequence()
7802 BuildMI(MF, MIMetadata(Root), TII->get(AArch64::FNEGv2f32), NewVR) in genAlternativeCodeSequence()
7822 BuildMI(MF, MIMetadata(Root), TII->get(AArch64::FNEGv4f32), NewVR) in genAlternativeCodeSequence()
7842 BuildMI(MF, MIMetadata(Root), TII->get(AArch64::FNEGv2f64), NewVR) in genAlternativeCodeSequence()
7919 MI->setFlags(Flags); in genAlternativeCodeSequence()
7922 /// Replace csincr-branch sequence by simple conditional branch
7998 MachineFunction *MF = MBB->getParent(); in optimizeCondBranch()
7999 MachineRegisterInfo *MRI = &MF->getRegInfo(); in optimizeCondBranch()
8004 MachineInstr *DefMI = MRI->getVRegDef(VReg); in optimizeCondBranch()
8007 while (DefMI->isCopy()) { in optimizeCondBranch()
8008 Register CopyVReg = DefMI->getOperand(1).getReg(); in optimizeCondBranch()
8009 if (!MRI->hasOneNonDBGUse(CopyVReg)) in optimizeCondBranch()
8011 if (!MRI->hasOneDef(CopyVReg)) in optimizeCondBranch()
8013 DefMI = MRI->getVRegDef(CopyVReg); in optimizeCondBranch()
8016 switch (DefMI->getOpcode()) { in optimizeCondBranch()
8024 if (DefMI->getParent() != MBB) in optimizeCondBranch()
8026 if (!MRI->hasOneNonDBGUse(VReg)) in optimizeCondBranch()
8029 bool Is32Bit = (DefMI->getOpcode() == AArch64::ANDWri); in optimizeCondBranch()
8031 DefMI->getOperand(2).getImm(), Is32Bit ? 32 : 64); in optimizeCondBranch()
8035 MachineOperand &MO = DefMI->getOperand(1); in optimizeCondBranch()
8040 assert(!MRI->def_empty(NewReg) && "Register must be defined."); in optimizeCondBranch()
8056 // For immediate smaller than 32, we need to use the 32-bit in optimizeCondBranch()
8057 // variant (W) in all cases. Indeed the 64-bit variant does not in optimizeCondBranch()
8059 // Therefore, if the input register is 64-bit, we need to take the in optimizeCondBranch()
8060 // 32-bit sub-part. in optimizeCondBranch()
8062 NewMI->getOperand(0).setSubReg(AArch64::sub_32); in optimizeCondBranch()
8069 if (!(DefMI->getOperand(1).getReg() == AArch64::WZR && in optimizeCondBranch()
8070 DefMI->getOperand(2).getReg() == AArch64::WZR) && in optimizeCondBranch()
8071 !(DefMI->getOperand(1).getReg() == AArch64::XZR && in optimizeCondBranch()
8072 DefMI->getOperand(2).getReg() == AArch64::XZR)) in optimizeCondBranch()
8075 if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, /*TRI=*/nullptr, in optimizeCondBranch()
8076 true) != -1) in optimizeCondBranch()
8079 AArch64CC::CondCode CC = (AArch64CC::CondCode)DefMI->getOperand(3).getImm(); in optimizeCondBranch()
8108 {MO_PAGE, "aarch64-page"}, {MO_PAGEOFF, "aarch64-pageoff"}, in getSerializableDirectMachineOperandTargetFlags()
8109 {MO_G3, "aarch64-g3"}, {MO_G2, "aarch64-g2"}, in getSerializableDirectMachineOperandTargetFlags()
8110 {MO_G1, "aarch64-g1"}, {MO_G0, "aarch64-g0"}, in getSerializableDirectMachineOperandTargetFlags()
8111 {MO_HI12, "aarch64-hi12"}}; in getSerializableDirectMachineOperandTargetFlags()
8120 {MO_COFFSTUB, "aarch64-coffstub"}, in getSerializableBitmaskMachineOperandTargetFlags()
8121 {MO_GOT, "aarch64-got"}, in getSerializableBitmaskMachineOperandTargetFlags()
8122 {MO_NC, "aarch64-nc"}, in getSerializableBitmaskMachineOperandTargetFlags()
8123 {MO_S, "aarch64-s"}, in getSerializableBitmaskMachineOperandTargetFlags()
8124 {MO_TLS, "aarch64-tls"}, in getSerializableBitmaskMachineOperandTargetFlags()
8125 {MO_DLLIMPORT, "aarch64-dllimport"}, in getSerializableBitmaskMachineOperandTargetFlags()
8126 {MO_PREL, "aarch64-prel"}, in getSerializableBitmaskMachineOperandTargetFlags()
8127 {MO_TAGGED, "aarch64-tagged"}, in getSerializableBitmaskMachineOperandTargetFlags()
8128 {MO_ARM64EC_CALLMANGLE, "aarch64-arm64ec-callmangle"}, in getSerializableBitmaskMachineOperandTargetFlags()
8136 {{MOSuppressPair, "aarch64-suppress-pair"}, in getSerializableMachineMemOperandTargetFlags()
8137 {MOStridedAccess, "aarch64-strided-access"}}; in getSerializableMachineMemOperandTargetFlags()
8151 /// I2 --> BL OUTLINED_FUNCTION I1
8166 /// I2 --> B OUTLINED_FUNCTION I1
8181 /// I2 --> BL OUTLINED_FUNCTION I1
8192 /// called with a BL instruction, and the outlined function tail-calls the
8198 /// I2 --> BL OUTLINED_FUNCTION I1
8213 /// I2 --> BL OUTLINED_FUNCTION I1
8225 MachineOutlinerThunk, /// Emit a call and tail-call.
8238 const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo(); in findRegisterToSaveLRTo()
8244 if (!ARI->isReservedReg(*MF, Reg) && in findRegisterToSaveLRTo()
8258 const auto &MFIa = a.getMF()->getInfo<AArch64FunctionInfo>(); in outliningCandidatesSigningScopeConsensus()
8259 const auto &MFIb = b.getMF()->getInfo<AArch64FunctionInfo>(); in outliningCandidatesSigningScopeConsensus()
8261 return MFIa->shouldSignReturnAddress(false) == MFIb->shouldSignReturnAddress(false) && in outliningCandidatesSigningScopeConsensus()
8262 MFIa->shouldSignReturnAddress(true) == MFIb->shouldSignReturnAddress(true); in outliningCandidatesSigningScopeConsensus()
8268 const auto &MFIa = a.getMF()->getInfo<AArch64FunctionInfo>(); in outliningCandidatesSigningKeyConsensus()
8269 const auto &MFIb = b.getMF()->getInfo<AArch64FunctionInfo>(); in outliningCandidatesSigningKeyConsensus()
8271 return MFIa->shouldSignWithBKey() == MFIb->shouldSignWithBKey(); in outliningCandidatesSigningKeyConsensus()
8277 a.getMF()->getSubtarget<AArch64Subtarget>(); in outliningCandidatesV8_3OpsConsensus()
8279 b.getMF()->getSubtarget<AArch64Subtarget>(); in outliningCandidatesV8_3OpsConsensus()
8294 // attribute "sign-return-address" and all share the same type of key they in getOutliningCandidateInfo()
8303 // Return true if a and b are non-equal w.r.t. return address in getOutliningCandidateInfo()
8318 // the case of "sign-return-address"="non-leaf" this is an assumption: It is in getOutliningCandidateInfo()
8332 ->getInfo<AArch64FunctionInfo>() in getOutliningCandidateInfo()
8333 ->shouldSignReturnAddress(true)) { in getOutliningCandidateInfo()
8337 // PAuth is enabled - set extra tail call cost, if any. in getOutliningCandidateInfo()
8382 SPValue -= MI.getOperand(2).getImm(); in getOutliningCandidateInfo()
8423 return C.getMF()->getInfo<AArch64FunctionInfo>()->branchTargetEnforcement(); in getOutliningCandidateInfo()
8441 C.getMF()->getFrameInstructions(); in getOutliningCandidateInfo()
8472 !Base->isReg() || Base->getReg() != AArch64::SP) in getOutliningCandidateInfo()
8475 // Fixe-up code below assumes bytes. in getOutliningCandidateInfo()
8546 C.getMF()->getFunction().hasFnAttribute(Attribute::NoReturn); in getOutliningCandidateInfo()
8606 // save LR to SP <- Requires stack instr fixups in OUTLINED_FUNCTION_N in getOutliningCandidateInfo()
8612 // save LR to SP <- Requires stack instr fixups in OUTLINED_FUNCTION_N in getOutliningCandidateInfo()
8629 // To avoid asserting (or generating non-legal code on noassert builds) in getOutliningCandidateInfo()
8698 const auto &CFn = Candidates.front().getMF()->getFunction(); in mergeOutliningCandidateAttributes()
8700 if (CFn.hasFnAttribute("ptrauth-returns")) in mergeOutliningCandidateAttributes()
8701 F.addFnAttr(CFn.getFnAttribute("ptrauth-returns")); in mergeOutliningCandidateAttributes()
8702 if (CFn.hasFnAttribute("ptrauth-auth-traps")) in mergeOutliningCandidateAttributes()
8703 F.addFnAttr(CFn.getFnAttribute("ptrauth-auth-traps")); in mergeOutliningCandidateAttributes()
8705 // function-level attributes of an arbitrary function. in mergeOutliningCandidateAttributes()
8706 if (CFn.hasFnAttribute("sign-return-address")) in mergeOutliningCandidateAttributes()
8707 F.addFnAttr(CFn.getFnAttribute("sign-return-address")); in mergeOutliningCandidateAttributes()
8708 if (CFn.hasFnAttribute("sign-return-address-key")) in mergeOutliningCandidateAttributes()
8709 F.addFnAttr(CFn.getFnAttribute("sign-return-address-key")); in mergeOutliningCandidateAttributes()
8733 if (!AFI || AFI->hasRedZone().value_or(true)) in isFunctionSafeToOutlineFrom()
8737 // streaming-mode changes. We may need to ensure any smstart/smstop pairs are in isFunctionSafeToOutlineFrom()
8740 if (AFI->hasStreamingModeChanges()) in isFunctionSafeToOutlineFrom()
8744 if (MF.getTarget().getMCAsmInfo()->usesWindowsCFI()) in isFunctionSafeToOutlineFrom()
8754 assert(MBB.getParent()->getRegInfo().tracksLiveness() && in getOutlinableRanges()
8790 // Compute liveness bottom-up. in getOutlinableRanges()
8815 // FIND: <safe instr> <-- end of first potential range in getOutlinableRanges()
8832 CreateNewRangeStartingAt(FirstPossibleEndPt->getIterator()); in getOutlinableRanges()
8854 // We found the ranges bottom-up. Mapping expects the top-down. Reverse in getOutlinableRanges()
8869 MachineFunction *MF = MBB->getParent(); in getOutliningTypeImpl()
8870 AArch64FunctionInfo *FuncInfo = MF->getInfo<AArch64FunctionInfo>(); in getOutliningTypeImpl()
8899 if (FuncInfo->getLOHRelated().count(&MI)) in getOutliningTypeImpl()
8917 // Make sure none of the operands are un-outlinable. in getOutliningTypeImpl()
8930 // the later tests. e.g, ADRPs, which are PC-relative use LR, but can always in getOutliningTypeImpl()
8961 if (Callee && Callee->getName() == "\01_mcount") in getOutliningTypeImpl()
8966 // as a tail-call. Explicitly list the call instructions we know about so we in getOutliningTypeImpl()
8967 // don't get unexpected results with call pseudo-instructions. in getOutliningTypeImpl()
8978 MachineFunction *CalleeMF = MF->getMMI().getMachineFunction(*Callee); in getOutliningTypeImpl()
8987 MachineFrameInfo &MFI = CalleeMF->getFrameInfo(); in getOutliningTypeImpl()
9021 (Base->isReg() && Base->getReg() != AArch64::SP)) in fixupPostOutline()
9048 BuildMI(MBB, MBB.begin(), DebugLoc(), TII->get(AArch64::PAUTH_PROLOGUE)) in signOutlinedFunction()
9051 TII->get(AArch64::PAUTH_EPILOGUE)) in signOutlinedFunction()
9062 FI->setOutliningStyle("Tail Call"); in buildOutlinedFrame()
9065 // tail-call. in buildOutlinedFrame()
9066 MachineInstr *Call = &*--MBB.instr_end(); in buildOutlinedFrame()
9068 if (Call->getOpcode() == AArch64::BL) { in buildOutlinedFrame()
9071 assert(Call->getOpcode() == AArch64::BLR || in buildOutlinedFrame()
9072 Call->getOpcode() == AArch64::BLRNoIP); in buildOutlinedFrame()
9076 .add(Call->getOperand(0)) in buildOutlinedFrame()
9079 Call->eraseFromParent(); in buildOutlinedFrame()
9081 FI->setOutliningStyle("Thunk"); in buildOutlinedFrame()
9119 .addImm(-16); in buildOutlinedFrame()
9122 if (MF.getInfo<AArch64FunctionInfo>()->needsDwarfUnwindInfo(MF)) { in buildOutlinedFrame()
9125 unsigned DwarfReg = MRI->getDwarfRegNum(AArch64::LR, true); in buildOutlinedFrame()
9137 MCCFIInstruction::createOffset(nullptr, DwarfReg, -16)); in buildOutlinedFrame()
9152 bool ShouldSignReturnAddr = FI->shouldSignReturnAddress(!IsLeafFunction); in buildOutlinedFrame()
9173 FI->setOutliningStyle("Function"); in buildOutlinedFrame()
9215 // FIXME: This logic should be sunk into a target-specific interface so that in insertOutlinedCall()
9218 assert(Reg && "No callee-saved register available?"); in insertOutlinedCall()
9239 .addImm(-16); in insertOutlinedCall()
9293 // Check that the w->w move is not a zero-extending w->x mov. in isCopyInstrImpl()
9297 MI.findRegisterDefOperandIdx(MI.getOperand(0).getReg() - AArch64::W0 + in isCopyInstrImpl()
9299 /*TRI=*/nullptr) == -1)) in isCopyInstrImpl()
9324 // TODO: Handle cases where Reg is a super- or sub-register of the in isAddImmediate()
9337 Sign *= -1; in isAddImmediate()
9362 auto DestSrc = TII->isCopyLikeInstr(MI); in describeORRLoadedValue()
9366 Register DestReg = DestSrc->Destination->getReg(); in describeORRLoadedValue()
9367 Register SrcReg = DestSrc->Source->getReg(); in describeORRLoadedValue()
9369 auto Expr = DIExpression::get(MI.getMF()->getFunction().getContext(), {}); in describeORRLoadedValue()
9375 // ORRWrs zero-extends to 64-bits, so we need to consider such cases. in describeORRLoadedValue()
9377 TRI->isSuperRegister(DestReg, DescribedReg)) in describeORRLoadedValue()
9382 TRI->isSubRegister(DestReg, DescribedReg)) { in describeORRLoadedValue()
9383 Register SrcSubReg = TRI->getSubReg(SrcReg, AArch64::sub_32); in describeORRLoadedValue()
9387 assert(!TRI->isSuperOrSubRegisterEq(DestReg, DescribedReg) && in describeORRLoadedValue()
9395 // a red zone. This is because relaxing a cross-section branch may require in isFunctionSafeToSplit()
9398 if (MF.getInfo<AArch64FunctionInfo>()->hasRedZone().value_or(true)) in isFunctionSafeToSplit()
9414 // Because jump tables are label-relative instead of table-relative, they all in isMBBSafeToSplitToCold()
9418 const MachineJumpTableInfo *MJTI = MBB.getParent()->getJumpTableInfo(); in isMBBSafeToSplitToCold()
9422 if (MJTI != nullptr && llvm::any_of(MJTI->getJumpTables(), containsMBB)) in isMBBSafeToSplitToCold()
9446 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo(); in describeLoadedValue()
9450 // MOVZWi may be used for producing zero-extended 32-bit immediates in in describeLoadedValue()
9451 // 64-bit parameters, so we need to consider super-registers. in describeLoadedValue()
9452 if (!TRI->isSuperRegisterEq(MI.getOperand(0).getReg(), Reg)) in describeLoadedValue()
9487 return UserMI->getOpcode() == TargetOpcode::G_PTR_ADD; in isExtendLikelyToBeFolded()
9514 // 9-bit signed offset in isLegalAddressingMode()
9518 // 12-bit unsigned offset in isLegalAddressingMode()
9520 if (NumBytes && Offset > 0 && (Offset / NumBytes) <= (1LL << 12) - 1 && in isLegalAddressingMode()
9543 MachineBasicBlock &MBB = *MBBI->getParent(); in probedStackAlloc()
9547 int64_t ProbeSize = MF.getInfo<AArch64FunctionInfo>()->getStackProbeSize(); in probedStackAlloc()
9564 emitFrameOffset(*LoopTestMBB, LoopTestMBB->end(), DL, AArch64::SP, in probedStackAlloc()
9565 AArch64::SP, StackOffset::getFixed(-ProbeSize), TII, Flags); in probedStackAlloc()
9568 BuildMI(*LoopTestMBB, LoopTestMBB->end(), DL, TII->get(AArch64::SUBSXrx64), in probedStackAlloc()
9576 BuildMI(*LoopTestMBB, LoopTestMBB->end(), DL, TII->get(AArch64::Bcc)) in probedStackAlloc()
9582 BuildMI(*LoopBodyMBB, LoopBodyMBB->end(), DL, TII->get(AArch64::STRXui)) in probedStackAlloc()
9589 BuildMI(*LoopBodyMBB, LoopBodyMBB->end(), DL, TII->get(AArch64::B)) in probedStackAlloc()
9595 BuildMI(*ExitMBB, ExitMBB->end(), DL, TII->get(AArch64::ADDXri), AArch64::SP) in probedStackAlloc()
9602 BuildMI(*ExitMBB, ExitMBB->end(), DL, TII->get(AArch64::LDRXui)) in probedStackAlloc()
9608 ExitMBB->splice(ExitMBB->end(), &MBB, std::next(MBBI), MBB.end()); in probedStackAlloc()
9609 ExitMBB->transferSuccessorsAndUpdatePHIs(&MBB); in probedStackAlloc()
9611 LoopTestMBB->addSuccessor(ExitMBB); in probedStackAlloc()
9612 LoopTestMBB->addSuccessor(LoopBodyMBB); in probedStackAlloc()
9613 LoopBodyMBB->addSuccessor(LoopTestMBB); in probedStackAlloc()
9620 return ExitMBB->begin(); in probedStackAlloc()
9656 : MF(Comp->getParent()->getParent()), in AArch64PipelinerLoopInfo()
9657 TII(MF->getSubtarget().getInstrInfo()), in AArch64PipelinerLoopInfo()
9658 TRI(MF->getSubtarget().getRegisterInfo()), MRI(MF->getRegInfo()), in AArch64PipelinerLoopInfo()
9693 /// Clone an instruction from MI. The register of ReplaceOprNum-th operand
9699 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); in cloneInstr()
9700 const TargetInstrInfo *TII = MBB.getParent()->getSubtarget().getInstrInfo(); in cloneInstr()
9702 MBB.getParent()->getSubtarget().getRegisterInfo(); in cloneInstr()
9703 MachineInstr *NewMI = MBB.getParent()->CloneMachineInstr(MI); in cloneInstr()
9705 for (unsigned I = 0; I < NewMI->getNumOperands(); ++I) { in cloneInstr()
9706 if (I == 0 && NewMI->getOperand(0).getReg().isVirtual()) { in cloneInstr()
9708 MRI.getRegClass(NewMI->getOperand(0).getReg())); in cloneInstr()
9709 NewMI->getOperand(I).setReg(Result); in cloneInstr()
9713 TII->getRegClass(NewMI->getDesc(), I, TRI, *MBB.getParent())); in cloneInstr()
9714 NewMI->getOperand(I).setReg(ReplaceReg); in cloneInstr()
9726 // SUBSXrr N, counter, implicit-def $nzcv # compare instruction for the last in createRemainingIterationsGreaterCondition()
9731 // counter = ADDXri counter, 1 # clone from this->Update in createRemainingIterationsGreaterCondition()
9732 // SUBSXrr n, counter, implicit-def $nzcv # clone from this->Comp in createRemainingIterationsGreaterCondition()
9735 // SUBSXri cond, 0, implicit-def $nzcv in createRemainingIterationsGreaterCondition()
9737 assert(CondBranch->getOpcode() == AArch64::Bcc); in createRemainingIterationsGreaterCondition()
9740 (AArch64CC::CondCode)CondBranch->getOperand(0).getImm(); in createRemainingIterationsGreaterCondition()
9741 if (CondBranch->getOperand(1).getMBB() == LoopBB) in createRemainingIterationsGreaterCondition()
9749 AArch64CC::CondCode CC) -> Register { in createRemainingIterationsGreaterCondition()
9751 BuildMI(MBB, MBB.end(), Comp->getDebugLoc(), TII->get(AArch64::CSINCXr)) in createRemainingIterationsGreaterCondition()
9759 if (!LastStage0Insts.empty() && LastStage0Insts[Comp]->getParent() == &MBB) { in createRemainingIterationsGreaterCondition()
9775 LastStage0Insts[Comp]->getOperand(CompCounterOprNum).getReg(); in createRemainingIterationsGreaterCondition()
9780 NextCounter = LastStage0Insts[Update]->getOperand(0).getReg(); in createRemainingIterationsGreaterCondition()
9800 Counter = LastStage0Insts[Comp]->getOperand(CompCounterOprNum).getReg(); in createRemainingIterationsGreaterCondition()
9816 BuildMI(MBB, MBB.end(), Comp->getDebugLoc(), TII->get(AArch64::SUBSXri)) in createRemainingIterationsGreaterCondition()
9841 const MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); in isDefinedOutside()
9842 return MRI.getVRegDef(Reg)->getParent() != BB; in isDefinedOutside()
9861 if (LoopBB->pred_size() != 2) in getIndVarInfo()
9865 const MachineRegisterInfo &MRI = LoopBB->getParent()->getRegInfo(); in getIndVarInfo()
9873 if (Def->getParent() != LoopBB) in getIndVarInfo()
9875 if (Def->isCopy()) { in getIndVarInfo()
9877 if (Def->getOperand(0).getSubReg() || Def->getOperand(1).getSubReg()) in getIndVarInfo()
9879 CurReg = Def->getOperand(1).getReg(); in getIndVarInfo()
9880 } else if (Def->isPHI()) { in getIndVarInfo()
9889 switch (Def->getOpcode()) { in getIndVarInfo()
9910 if (isDefinedOutside(Def->getOperand(2).getReg(), LoopBB)) in getIndVarInfo()
9912 else if (isDefinedOutside(Def->getOperand(1).getReg(), LoopBB)) in getIndVarInfo()
9920 CurReg = Def->getOperand(UpdateCounterOprNum).getReg(); in getIndVarInfo()
9965 "The Loop must be a single-basic-block loop"); in analyzeLoopForPipelining()
9967 MachineInstr *CondBranch = &*LoopBB->getFirstTerminator(); in analyzeLoopForPipelining()
9970 if (CondBranch->getOpcode() != AArch64::Bcc) in analyzeLoopForPipelining()
10007 if (isDefinedOutside(Comp->getOperand(1).getReg(), LoopBB)) in analyzeLoopForPipelining()
10009 else if (isDefinedOutside(Comp->getOperand(2).getReg(), LoopBB)) in analyzeLoopForPipelining()
10024 if (!getIndVarInfo(Comp->getOperand(CompCounterOprNum).getReg(), LoopBB, in analyzeLoopForPipelining()