xref: /freebsd/contrib/llvm-project/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp (revision 9f23cbd6cae82fd77edfad7173432fa8dccd0a95)
1 //===-- ARMBaseInstrInfo.cpp - ARM Instruction Information ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the Base ARM implementation of the TargetInstrInfo class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "ARMBaseInstrInfo.h"
14 #include "ARMBaseRegisterInfo.h"
15 #include "ARMConstantPoolValue.h"
16 #include "ARMFeatures.h"
17 #include "ARMHazardRecognizer.h"
18 #include "ARMMachineFunctionInfo.h"
19 #include "ARMSubtarget.h"
20 #include "MCTargetDesc/ARMAddressingModes.h"
21 #include "MCTargetDesc/ARMBaseInfo.h"
22 #include "MVETailPredUtils.h"
23 #include "llvm/ADT/DenseMap.h"
24 #include "llvm/ADT/STLExtras.h"
25 #include "llvm/ADT/SmallSet.h"
26 #include "llvm/ADT/SmallVector.h"
27 #include "llvm/ADT/Triple.h"
28 #include "llvm/CodeGen/DFAPacketizer.h"
29 #include "llvm/CodeGen/LiveVariables.h"
30 #include "llvm/CodeGen/MachineBasicBlock.h"
31 #include "llvm/CodeGen/MachineConstantPool.h"
32 #include "llvm/CodeGen/MachineFrameInfo.h"
33 #include "llvm/CodeGen/MachineFunction.h"
34 #include "llvm/CodeGen/MachineInstr.h"
35 #include "llvm/CodeGen/MachineInstrBuilder.h"
36 #include "llvm/CodeGen/MachineMemOperand.h"
37 #include "llvm/CodeGen/MachineModuleInfo.h"
38 #include "llvm/CodeGen/MachineOperand.h"
39 #include "llvm/CodeGen/MachinePipeliner.h"
40 #include "llvm/CodeGen/MachineRegisterInfo.h"
41 #include "llvm/CodeGen/MachineScheduler.h"
42 #include "llvm/CodeGen/MultiHazardRecognizer.h"
43 #include "llvm/CodeGen/ScoreboardHazardRecognizer.h"
44 #include "llvm/CodeGen/SelectionDAGNodes.h"
45 #include "llvm/CodeGen/TargetInstrInfo.h"
46 #include "llvm/CodeGen/TargetRegisterInfo.h"
47 #include "llvm/CodeGen/TargetSchedule.h"
48 #include "llvm/IR/Attributes.h"
49 #include "llvm/IR/Constants.h"
50 #include "llvm/IR/DebugLoc.h"
51 #include "llvm/IR/Function.h"
52 #include "llvm/IR/GlobalValue.h"
53 #include "llvm/MC/MCAsmInfo.h"
54 #include "llvm/MC/MCInstrDesc.h"
55 #include "llvm/MC/MCInstrItineraries.h"
56 #include "llvm/Support/BranchProbability.h"
57 #include "llvm/Support/Casting.h"
58 #include "llvm/Support/CommandLine.h"
59 #include "llvm/Support/Compiler.h"
60 #include "llvm/Support/Debug.h"
61 #include "llvm/Support/ErrorHandling.h"
62 #include "llvm/Support/raw_ostream.h"
63 #include "llvm/Target/TargetMachine.h"
64 #include <algorithm>
65 #include <cassert>
66 #include <cstdint>
67 #include <iterator>
68 #include <new>
69 #include <utility>
70 #include <vector>
71 
72 using namespace llvm;
73 
74 #define DEBUG_TYPE "arm-instrinfo"
75 
76 #define GET_INSTRINFO_CTOR_DTOR
77 #include "ARMGenInstrInfo.inc"
78 
79 static cl::opt<bool>
80 EnableARM3Addr("enable-arm-3-addr-conv", cl::Hidden,
81                cl::desc("Enable ARM 2-addr to 3-addr conv"));
82 
83 /// ARM_MLxEntry - Record information about MLA / MLS instructions.
84 struct ARM_MLxEntry {
85   uint16_t MLxOpc;     // MLA / MLS opcode
86   uint16_t MulOpc;     // Expanded multiplication opcode
87   uint16_t AddSubOpc;  // Expanded add / sub opcode
88   bool NegAcc;         // True if the acc is negated before the add / sub.
89   bool HasLane;        // True if instruction has an extra "lane" operand.
90 };
91 
92 static const ARM_MLxEntry ARM_MLxTable[] = {
93   // MLxOpc,          MulOpc,           AddSubOpc,       NegAcc, HasLane
94   // fp scalar ops
95   { ARM::VMLAS,       ARM::VMULS,       ARM::VADDS,      false,  false },
96   { ARM::VMLSS,       ARM::VMULS,       ARM::VSUBS,      false,  false },
97   { ARM::VMLAD,       ARM::VMULD,       ARM::VADDD,      false,  false },
98   { ARM::VMLSD,       ARM::VMULD,       ARM::VSUBD,      false,  false },
99   { ARM::VNMLAS,      ARM::VNMULS,      ARM::VSUBS,      true,   false },
100   { ARM::VNMLSS,      ARM::VMULS,       ARM::VSUBS,      true,   false },
101   { ARM::VNMLAD,      ARM::VNMULD,      ARM::VSUBD,      true,   false },
102   { ARM::VNMLSD,      ARM::VMULD,       ARM::VSUBD,      true,   false },
103 
104   // fp SIMD ops
105   { ARM::VMLAfd,      ARM::VMULfd,      ARM::VADDfd,     false,  false },
106   { ARM::VMLSfd,      ARM::VMULfd,      ARM::VSUBfd,     false,  false },
107   { ARM::VMLAfq,      ARM::VMULfq,      ARM::VADDfq,     false,  false },
108   { ARM::VMLSfq,      ARM::VMULfq,      ARM::VSUBfq,     false,  false },
109   { ARM::VMLAslfd,    ARM::VMULslfd,    ARM::VADDfd,     false,  true  },
110   { ARM::VMLSslfd,    ARM::VMULslfd,    ARM::VSUBfd,     false,  true  },
111   { ARM::VMLAslfq,    ARM::VMULslfq,    ARM::VADDfq,     false,  true  },
112   { ARM::VMLSslfq,    ARM::VMULslfq,    ARM::VSUBfq,     false,  true  },
113 };
114 
115 ARMBaseInstrInfo::ARMBaseInstrInfo(const ARMSubtarget& STI)
116   : ARMGenInstrInfo(ARM::ADJCALLSTACKDOWN, ARM::ADJCALLSTACKUP),
117     Subtarget(STI) {
118   for (unsigned i = 0, e = std::size(ARM_MLxTable); i != e; ++i) {
119     if (!MLxEntryMap.insert(std::make_pair(ARM_MLxTable[i].MLxOpc, i)).second)
120       llvm_unreachable("Duplicated entries?");
121     MLxHazardOpcodes.insert(ARM_MLxTable[i].AddSubOpc);
122     MLxHazardOpcodes.insert(ARM_MLxTable[i].MulOpc);
123   }
124 }
125 
126 // Use a ScoreboardHazardRecognizer for prepass ARM scheduling. TargetInstrImpl
127 // currently defaults to no prepass hazard recognizer.
128 ScheduleHazardRecognizer *
129 ARMBaseInstrInfo::CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI,
130                                                const ScheduleDAG *DAG) const {
131   if (usePreRAHazardRecognizer()) {
132     const InstrItineraryData *II =
133         static_cast<const ARMSubtarget *>(STI)->getInstrItineraryData();
134     return new ScoreboardHazardRecognizer(II, DAG, "pre-RA-sched");
135   }
136   return TargetInstrInfo::CreateTargetHazardRecognizer(STI, DAG);
137 }
138 
139 // Called during:
140 // - pre-RA scheduling
141 // - post-RA scheduling when FeatureUseMISched is set
142 ScheduleHazardRecognizer *ARMBaseInstrInfo::CreateTargetMIHazardRecognizer(
143     const InstrItineraryData *II, const ScheduleDAGMI *DAG) const {
144   MultiHazardRecognizer *MHR = new MultiHazardRecognizer();
145 
146   // We would like to restrict this hazard recognizer to only
147   // post-RA scheduling; we can tell that we're post-RA because we don't
148   // track VRegLiveness.
149   // Cortex-M7: TRM indicates that there is a single ITCM bank and two DTCM
150   //            banks banked on bit 2.  Assume that TCMs are in use.
151   if (Subtarget.isCortexM7() && !DAG->hasVRegLiveness())
152     MHR->AddHazardRecognizer(
153         std::make_unique<ARMBankConflictHazardRecognizer>(DAG, 0x4, true));
154 
155   // Not inserting ARMHazardRecognizerFPMLx because that would change
156   // legacy behavior
157 
158   auto BHR = TargetInstrInfo::CreateTargetMIHazardRecognizer(II, DAG);
159   MHR->AddHazardRecognizer(std::unique_ptr<ScheduleHazardRecognizer>(BHR));
160   return MHR;
161 }
162 
163 // Called during post-RA scheduling when FeatureUseMISched is not set
164 ScheduleHazardRecognizer *ARMBaseInstrInfo::
165 CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
166                                    const ScheduleDAG *DAG) const {
167   MultiHazardRecognizer *MHR = new MultiHazardRecognizer();
168 
169   if (Subtarget.isThumb2() || Subtarget.hasVFP2Base())
170     MHR->AddHazardRecognizer(std::make_unique<ARMHazardRecognizerFPMLx>());
171 
172   auto BHR = TargetInstrInfo::CreateTargetPostRAHazardRecognizer(II, DAG);
173   if (BHR)
174     MHR->AddHazardRecognizer(std::unique_ptr<ScheduleHazardRecognizer>(BHR));
175   return MHR;
176 }
177 
178 MachineInstr *
179 ARMBaseInstrInfo::convertToThreeAddress(MachineInstr &MI, LiveVariables *LV,
180                                         LiveIntervals *LIS) const {
181   // FIXME: Thumb2 support.
182 
183   if (!EnableARM3Addr)
184     return nullptr;
185 
186   MachineFunction &MF = *MI.getParent()->getParent();
187   uint64_t TSFlags = MI.getDesc().TSFlags;
188   bool isPre = false;
189   switch ((TSFlags & ARMII::IndexModeMask) >> ARMII::IndexModeShift) {
190   default: return nullptr;
191   case ARMII::IndexModePre:
192     isPre = true;
193     break;
194   case ARMII::IndexModePost:
195     break;
196   }
197 
198   // Try splitting an indexed load/store to an un-indexed one plus an add/sub
199   // operation.
200   unsigned MemOpc = getUnindexedOpcode(MI.getOpcode());
201   if (MemOpc == 0)
202     return nullptr;
203 
204   MachineInstr *UpdateMI = nullptr;
205   MachineInstr *MemMI = nullptr;
206   unsigned AddrMode = (TSFlags & ARMII::AddrModeMask);
207   const MCInstrDesc &MCID = MI.getDesc();
208   unsigned NumOps = MCID.getNumOperands();
209   bool isLoad = !MI.mayStore();
210   const MachineOperand &WB = isLoad ? MI.getOperand(1) : MI.getOperand(0);
211   const MachineOperand &Base = MI.getOperand(2);
212   const MachineOperand &Offset = MI.getOperand(NumOps - 3);
213   Register WBReg = WB.getReg();
214   Register BaseReg = Base.getReg();
215   Register OffReg = Offset.getReg();
216   unsigned OffImm = MI.getOperand(NumOps - 2).getImm();
217   ARMCC::CondCodes Pred = (ARMCC::CondCodes)MI.getOperand(NumOps - 1).getImm();
218   switch (AddrMode) {
219   default: llvm_unreachable("Unknown indexed op!");
220   case ARMII::AddrMode2: {
221     bool isSub = ARM_AM::getAM2Op(OffImm) == ARM_AM::sub;
222     unsigned Amt = ARM_AM::getAM2Offset(OffImm);
223     if (OffReg == 0) {
224       if (ARM_AM::getSOImmVal(Amt) == -1)
225         // Can't encode it in a so_imm operand. This transformation will
226         // add more than 1 instruction. Abandon!
227         return nullptr;
228       UpdateMI = BuildMI(MF, MI.getDebugLoc(),
229                          get(isSub ? ARM::SUBri : ARM::ADDri), WBReg)
230                      .addReg(BaseReg)
231                      .addImm(Amt)
232                      .add(predOps(Pred))
233                      .add(condCodeOp());
234     } else if (Amt != 0) {
235       ARM_AM::ShiftOpc ShOpc = ARM_AM::getAM2ShiftOpc(OffImm);
236       unsigned SOOpc = ARM_AM::getSORegOpc(ShOpc, Amt);
237       UpdateMI = BuildMI(MF, MI.getDebugLoc(),
238                          get(isSub ? ARM::SUBrsi : ARM::ADDrsi), WBReg)
239                      .addReg(BaseReg)
240                      .addReg(OffReg)
241                      .addReg(0)
242                      .addImm(SOOpc)
243                      .add(predOps(Pred))
244                      .add(condCodeOp());
245     } else
246       UpdateMI = BuildMI(MF, MI.getDebugLoc(),
247                          get(isSub ? ARM::SUBrr : ARM::ADDrr), WBReg)
248                      .addReg(BaseReg)
249                      .addReg(OffReg)
250                      .add(predOps(Pred))
251                      .add(condCodeOp());
252     break;
253   }
254   case ARMII::AddrMode3 : {
255     bool isSub = ARM_AM::getAM3Op(OffImm) == ARM_AM::sub;
256     unsigned Amt = ARM_AM::getAM3Offset(OffImm);
257     if (OffReg == 0)
258       // Immediate is 8-bits. It's guaranteed to fit in a so_imm operand.
259       UpdateMI = BuildMI(MF, MI.getDebugLoc(),
260                          get(isSub ? ARM::SUBri : ARM::ADDri), WBReg)
261                      .addReg(BaseReg)
262                      .addImm(Amt)
263                      .add(predOps(Pred))
264                      .add(condCodeOp());
265     else
266       UpdateMI = BuildMI(MF, MI.getDebugLoc(),
267                          get(isSub ? ARM::SUBrr : ARM::ADDrr), WBReg)
268                      .addReg(BaseReg)
269                      .addReg(OffReg)
270                      .add(predOps(Pred))
271                      .add(condCodeOp());
272     break;
273   }
274   }
275 
276   std::vector<MachineInstr*> NewMIs;
277   if (isPre) {
278     if (isLoad)
279       MemMI =
280           BuildMI(MF, MI.getDebugLoc(), get(MemOpc), MI.getOperand(0).getReg())
281               .addReg(WBReg)
282               .addImm(0)
283               .addImm(Pred);
284     else
285       MemMI = BuildMI(MF, MI.getDebugLoc(), get(MemOpc))
286                   .addReg(MI.getOperand(1).getReg())
287                   .addReg(WBReg)
288                   .addReg(0)
289                   .addImm(0)
290                   .addImm(Pred);
291     NewMIs.push_back(MemMI);
292     NewMIs.push_back(UpdateMI);
293   } else {
294     if (isLoad)
295       MemMI =
296           BuildMI(MF, MI.getDebugLoc(), get(MemOpc), MI.getOperand(0).getReg())
297               .addReg(BaseReg)
298               .addImm(0)
299               .addImm(Pred);
300     else
301       MemMI = BuildMI(MF, MI.getDebugLoc(), get(MemOpc))
302                   .addReg(MI.getOperand(1).getReg())
303                   .addReg(BaseReg)
304                   .addReg(0)
305                   .addImm(0)
306                   .addImm(Pred);
307     if (WB.isDead())
308       UpdateMI->getOperand(0).setIsDead();
309     NewMIs.push_back(UpdateMI);
310     NewMIs.push_back(MemMI);
311   }
312 
313   // Transfer LiveVariables states, kill / dead info.
314   if (LV) {
315     for (const MachineOperand &MO : MI.operands()) {
316       if (MO.isReg() && MO.getReg().isVirtual()) {
317         Register Reg = MO.getReg();
318 
319         LiveVariables::VarInfo &VI = LV->getVarInfo(Reg);
320         if (MO.isDef()) {
321           MachineInstr *NewMI = (Reg == WBReg) ? UpdateMI : MemMI;
322           if (MO.isDead())
323             LV->addVirtualRegisterDead(Reg, *NewMI);
324         }
325         if (MO.isUse() && MO.isKill()) {
326           for (unsigned j = 0; j < 2; ++j) {
327             // Look at the two new MI's in reverse order.
328             MachineInstr *NewMI = NewMIs[j];
329             if (!NewMI->readsRegister(Reg))
330               continue;
331             LV->addVirtualRegisterKilled(Reg, *NewMI);
332             if (VI.removeKill(MI))
333               VI.Kills.push_back(NewMI);
334             break;
335           }
336         }
337       }
338     }
339   }
340 
341   MachineBasicBlock &MBB = *MI.getParent();
342   MBB.insert(MI, NewMIs[1]);
343   MBB.insert(MI, NewMIs[0]);
344   return NewMIs[0];
345 }
346 
347 // Branch analysis.
348 // Cond vector output format:
349 //   0 elements indicates an unconditional branch
350 //   2 elements indicates a conditional branch; the elements are
351 //     the condition to check and the CPSR.
352 //   3 elements indicates a hardware loop end; the elements
353 //     are the opcode, the operand value to test, and a dummy
354 //     operand used to pad out to 3 operands.
355 bool ARMBaseInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
356                                      MachineBasicBlock *&TBB,
357                                      MachineBasicBlock *&FBB,
358                                      SmallVectorImpl<MachineOperand> &Cond,
359                                      bool AllowModify) const {
360   TBB = nullptr;
361   FBB = nullptr;
362 
363   MachineBasicBlock::instr_iterator I = MBB.instr_end();
364   if (I == MBB.instr_begin())
365     return false; // Empty blocks are easy.
366   --I;
367 
368   // Walk backwards from the end of the basic block until the branch is
369   // analyzed or we give up.
370   while (isPredicated(*I) || I->isTerminator() || I->isDebugValue()) {
371     // Flag to be raised on unanalyzeable instructions. This is useful in cases
372     // where we want to clean up on the end of the basic block before we bail
373     // out.
374     bool CantAnalyze = false;
375 
376     // Skip over DEBUG values, predicated nonterminators and speculation
377     // barrier terminators.
378     while (I->isDebugInstr() || !I->isTerminator() ||
379            isSpeculationBarrierEndBBOpcode(I->getOpcode()) ||
380            I->getOpcode() == ARM::t2DoLoopStartTP){
381       if (I == MBB.instr_begin())
382         return false;
383       --I;
384     }
385 
386     if (isIndirectBranchOpcode(I->getOpcode()) ||
387         isJumpTableBranchOpcode(I->getOpcode())) {
388       // Indirect branches and jump tables can't be analyzed, but we still want
389       // to clean up any instructions at the tail of the basic block.
390       CantAnalyze = true;
391     } else if (isUncondBranchOpcode(I->getOpcode())) {
392       TBB = I->getOperand(0).getMBB();
393     } else if (isCondBranchOpcode(I->getOpcode())) {
394       // Bail out if we encounter multiple conditional branches.
395       if (!Cond.empty())
396         return true;
397 
398       assert(!FBB && "FBB should have been null.");
399       FBB = TBB;
400       TBB = I->getOperand(0).getMBB();
401       Cond.push_back(I->getOperand(1));
402       Cond.push_back(I->getOperand(2));
403     } else if (I->isReturn()) {
404       // Returns can't be analyzed, but we should run cleanup.
405       CantAnalyze = true;
406     } else if (I->getOpcode() == ARM::t2LoopEnd &&
407                MBB.getParent()
408                    ->getSubtarget<ARMSubtarget>()
409                    .enableMachinePipeliner()) {
410       if (!Cond.empty())
411         return true;
412       FBB = TBB;
413       TBB = I->getOperand(1).getMBB();
414       Cond.push_back(MachineOperand::CreateImm(I->getOpcode()));
415       Cond.push_back(I->getOperand(0));
416       Cond.push_back(MachineOperand::CreateImm(0));
417     } else {
418       // We encountered other unrecognized terminator. Bail out immediately.
419       return true;
420     }
421 
422     // Cleanup code - to be run for unpredicated unconditional branches and
423     //                returns.
424     if (!isPredicated(*I) &&
425           (isUncondBranchOpcode(I->getOpcode()) ||
426            isIndirectBranchOpcode(I->getOpcode()) ||
427            isJumpTableBranchOpcode(I->getOpcode()) ||
428            I->isReturn())) {
429       // Forget any previous condition branch information - it no longer applies.
430       Cond.clear();
431       FBB = nullptr;
432 
433       // If we can modify the function, delete everything below this
434       // unconditional branch.
435       if (AllowModify) {
436         MachineBasicBlock::iterator DI = std::next(I);
437         while (DI != MBB.instr_end()) {
438           MachineInstr &InstToDelete = *DI;
439           ++DI;
440           // Speculation barriers must not be deleted.
441           if (isSpeculationBarrierEndBBOpcode(InstToDelete.getOpcode()))
442             continue;
443           InstToDelete.eraseFromParent();
444         }
445       }
446     }
447 
448     if (CantAnalyze) {
449       // We may not be able to analyze the block, but we could still have
450       // an unconditional branch as the last instruction in the block, which
451       // just branches to layout successor. If this is the case, then just
452       // remove it if we're allowed to make modifications.
453       if (AllowModify && !isPredicated(MBB.back()) &&
454           isUncondBranchOpcode(MBB.back().getOpcode()) &&
455           TBB && MBB.isLayoutSuccessor(TBB))
456         removeBranch(MBB);
457       return true;
458     }
459 
460     if (I == MBB.instr_begin())
461       return false;
462 
463     --I;
464   }
465 
466   // We made it past the terminators without bailing out - we must have
467   // analyzed this branch successfully.
468   return false;
469 }
470 
471 unsigned ARMBaseInstrInfo::removeBranch(MachineBasicBlock &MBB,
472                                         int *BytesRemoved) const {
473   assert(!BytesRemoved && "code size not handled");
474 
475   MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
476   if (I == MBB.end())
477     return 0;
478 
479   if (!isUncondBranchOpcode(I->getOpcode()) &&
480       !isCondBranchOpcode(I->getOpcode()) && I->getOpcode() != ARM::t2LoopEnd)
481     return 0;
482 
483   // Remove the branch.
484   I->eraseFromParent();
485 
486   I = MBB.end();
487 
488   if (I == MBB.begin()) return 1;
489   --I;
490   if (!isCondBranchOpcode(I->getOpcode()) && I->getOpcode() != ARM::t2LoopEnd)
491     return 1;
492 
493   // Remove the branch.
494   I->eraseFromParent();
495   return 2;
496 }
497 
498 unsigned ARMBaseInstrInfo::insertBranch(MachineBasicBlock &MBB,
499                                         MachineBasicBlock *TBB,
500                                         MachineBasicBlock *FBB,
501                                         ArrayRef<MachineOperand> Cond,
502                                         const DebugLoc &DL,
503                                         int *BytesAdded) const {
504   assert(!BytesAdded && "code size not handled");
505   ARMFunctionInfo *AFI = MBB.getParent()->getInfo<ARMFunctionInfo>();
506   int BOpc   = !AFI->isThumbFunction()
507     ? ARM::B : (AFI->isThumb2Function() ? ARM::t2B : ARM::tB);
508   int BccOpc = !AFI->isThumbFunction()
509     ? ARM::Bcc : (AFI->isThumb2Function() ? ARM::t2Bcc : ARM::tBcc);
510   bool isThumb = AFI->isThumbFunction() || AFI->isThumb2Function();
511 
512   // Shouldn't be a fall through.
513   assert(TBB && "insertBranch must not be told to insert a fallthrough");
514   assert((Cond.size() == 2 || Cond.size() == 0 || Cond.size() == 3) &&
515          "ARM branch conditions have two or three components!");
516 
517   // For conditional branches, we use addOperand to preserve CPSR flags.
518 
519   if (!FBB) {
520     if (Cond.empty()) { // Unconditional branch?
521       if (isThumb)
522         BuildMI(&MBB, DL, get(BOpc)).addMBB(TBB).add(predOps(ARMCC::AL));
523       else
524         BuildMI(&MBB, DL, get(BOpc)).addMBB(TBB);
525     } else if (Cond.size() == 2) {
526       BuildMI(&MBB, DL, get(BccOpc))
527           .addMBB(TBB)
528           .addImm(Cond[0].getImm())
529           .add(Cond[1]);
530     } else
531       BuildMI(&MBB, DL, get(Cond[0].getImm())).add(Cond[1]).addMBB(TBB);
532     return 1;
533   }
534 
535   // Two-way conditional branch.
536   if (Cond.size() == 2)
537     BuildMI(&MBB, DL, get(BccOpc))
538         .addMBB(TBB)
539         .addImm(Cond[0].getImm())
540         .add(Cond[1]);
541   else if (Cond.size() == 3)
542     BuildMI(&MBB, DL, get(Cond[0].getImm())).add(Cond[1]).addMBB(TBB);
543   if (isThumb)
544     BuildMI(&MBB, DL, get(BOpc)).addMBB(FBB).add(predOps(ARMCC::AL));
545   else
546     BuildMI(&MBB, DL, get(BOpc)).addMBB(FBB);
547   return 2;
548 }
549 
550 bool ARMBaseInstrInfo::
551 reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
552   if (Cond.size() == 2) {
553     ARMCC::CondCodes CC = (ARMCC::CondCodes)(int)Cond[0].getImm();
554     Cond[0].setImm(ARMCC::getOppositeCondition(CC));
555     return false;
556   }
557   return true;
558 }
559 
560 bool ARMBaseInstrInfo::isPredicated(const MachineInstr &MI) const {
561   if (MI.isBundle()) {
562     MachineBasicBlock::const_instr_iterator I = MI.getIterator();
563     MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
564     while (++I != E && I->isInsideBundle()) {
565       int PIdx = I->findFirstPredOperandIdx();
566       if (PIdx != -1 && I->getOperand(PIdx).getImm() != ARMCC::AL)
567         return true;
568     }
569     return false;
570   }
571 
572   int PIdx = MI.findFirstPredOperandIdx();
573   return PIdx != -1 && MI.getOperand(PIdx).getImm() != ARMCC::AL;
574 }
575 
576 std::string ARMBaseInstrInfo::createMIROperandComment(
577     const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx,
578     const TargetRegisterInfo *TRI) const {
579 
580   // First, let's see if there is a generic comment for this operand
581   std::string GenericComment =
582       TargetInstrInfo::createMIROperandComment(MI, Op, OpIdx, TRI);
583   if (!GenericComment.empty())
584     return GenericComment;
585 
586   // If not, check if we have an immediate operand.
587   if (!Op.isImm())
588     return std::string();
589 
590   // And print its corresponding condition code if the immediate is a
591   // predicate.
592   int FirstPredOp = MI.findFirstPredOperandIdx();
593   if (FirstPredOp != (int) OpIdx)
594     return std::string();
595 
596   std::string CC = "CC::";
597   CC += ARMCondCodeToString((ARMCC::CondCodes)Op.getImm());
598   return CC;
599 }
600 
601 bool ARMBaseInstrInfo::PredicateInstruction(
602     MachineInstr &MI, ArrayRef<MachineOperand> Pred) const {
603   unsigned Opc = MI.getOpcode();
604   if (isUncondBranchOpcode(Opc)) {
605     MI.setDesc(get(getMatchingCondBranchOpcode(Opc)));
606     MachineInstrBuilder(*MI.getParent()->getParent(), MI)
607       .addImm(Pred[0].getImm())
608       .addReg(Pred[1].getReg());
609     return true;
610   }
611 
612   int PIdx = MI.findFirstPredOperandIdx();
613   if (PIdx != -1) {
614     MachineOperand &PMO = MI.getOperand(PIdx);
615     PMO.setImm(Pred[0].getImm());
616     MI.getOperand(PIdx+1).setReg(Pred[1].getReg());
617 
618     // Thumb 1 arithmetic instructions do not set CPSR when executed inside an
619     // IT block. This affects how they are printed.
620     const MCInstrDesc &MCID = MI.getDesc();
621     if (MCID.TSFlags & ARMII::ThumbArithFlagSetting) {
622       assert(MCID.operands()[1].isOptionalDef() &&
623              "CPSR def isn't expected operand");
624       assert((MI.getOperand(1).isDead() ||
625               MI.getOperand(1).getReg() != ARM::CPSR) &&
626              "if conversion tried to stop defining used CPSR");
627       MI.getOperand(1).setReg(ARM::NoRegister);
628     }
629 
630     return true;
631   }
632   return false;
633 }
634 
635 bool ARMBaseInstrInfo::SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
636                                          ArrayRef<MachineOperand> Pred2) const {
637   if (Pred1.size() > 2 || Pred2.size() > 2)
638     return false;
639 
640   ARMCC::CondCodes CC1 = (ARMCC::CondCodes)Pred1[0].getImm();
641   ARMCC::CondCodes CC2 = (ARMCC::CondCodes)Pred2[0].getImm();
642   if (CC1 == CC2)
643     return true;
644 
645   switch (CC1) {
646   default:
647     return false;
648   case ARMCC::AL:
649     return true;
650   case ARMCC::HS:
651     return CC2 == ARMCC::HI;
652   case ARMCC::LS:
653     return CC2 == ARMCC::LO || CC2 == ARMCC::EQ;
654   case ARMCC::GE:
655     return CC2 == ARMCC::GT;
656   case ARMCC::LE:
657     return CC2 == ARMCC::LT;
658   }
659 }
660 
661 bool ARMBaseInstrInfo::ClobbersPredicate(MachineInstr &MI,
662                                          std::vector<MachineOperand> &Pred,
663                                          bool SkipDead) const {
664   bool Found = false;
665   for (const MachineOperand &MO : MI.operands()) {
666     bool ClobbersCPSR = MO.isRegMask() && MO.clobbersPhysReg(ARM::CPSR);
667     bool IsCPSR = MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR;
668     if (ClobbersCPSR || IsCPSR) {
669 
670       // Filter out T1 instructions that have a dead CPSR,
671       // allowing IT blocks to be generated containing T1 instructions
672       const MCInstrDesc &MCID = MI.getDesc();
673       if (MCID.TSFlags & ARMII::ThumbArithFlagSetting && MO.isDead() &&
674           SkipDead)
675         continue;
676 
677       Pred.push_back(MO);
678       Found = true;
679     }
680   }
681 
682   return Found;
683 }
684 
685 bool ARMBaseInstrInfo::isCPSRDefined(const MachineInstr &MI) {
686   for (const auto &MO : MI.operands())
687     if (MO.isReg() && MO.getReg() == ARM::CPSR && MO.isDef() && !MO.isDead())
688       return true;
689   return false;
690 }
691 
692 static bool isEligibleForITBlock(const MachineInstr *MI) {
693   switch (MI->getOpcode()) {
694   default: return true;
695   case ARM::tADC:   // ADC (register) T1
696   case ARM::tADDi3: // ADD (immediate) T1
697   case ARM::tADDi8: // ADD (immediate) T2
698   case ARM::tADDrr: // ADD (register) T1
699   case ARM::tAND:   // AND (register) T1
700   case ARM::tASRri: // ASR (immediate) T1
701   case ARM::tASRrr: // ASR (register) T1
702   case ARM::tBIC:   // BIC (register) T1
703   case ARM::tEOR:   // EOR (register) T1
704   case ARM::tLSLri: // LSL (immediate) T1
705   case ARM::tLSLrr: // LSL (register) T1
706   case ARM::tLSRri: // LSR (immediate) T1
707   case ARM::tLSRrr: // LSR (register) T1
708   case ARM::tMUL:   // MUL T1
709   case ARM::tMVN:   // MVN (register) T1
710   case ARM::tORR:   // ORR (register) T1
711   case ARM::tROR:   // ROR (register) T1
712   case ARM::tRSB:   // RSB (immediate) T1
713   case ARM::tSBC:   // SBC (register) T1
714   case ARM::tSUBi3: // SUB (immediate) T1
715   case ARM::tSUBi8: // SUB (immediate) T2
716   case ARM::tSUBrr: // SUB (register) T1
717     return !ARMBaseInstrInfo::isCPSRDefined(*MI);
718   }
719 }
720 
721 /// isPredicable - Return true if the specified instruction can be predicated.
722 /// By default, this returns true for every instruction with a
723 /// PredicateOperand.
724 bool ARMBaseInstrInfo::isPredicable(const MachineInstr &MI) const {
725   if (!MI.isPredicable())
726     return false;
727 
728   if (MI.isBundle())
729     return false;
730 
731   if (!isEligibleForITBlock(&MI))
732     return false;
733 
734   const MachineFunction *MF = MI.getParent()->getParent();
735   const ARMFunctionInfo *AFI =
736       MF->getInfo<ARMFunctionInfo>();
737 
738   // Neon instructions in Thumb2 IT blocks are deprecated, see ARMARM.
739   // In their ARM encoding, they can't be encoded in a conditional form.
740   if ((MI.getDesc().TSFlags & ARMII::DomainMask) == ARMII::DomainNEON)
741     return false;
742 
743   // Make indirect control flow changes unpredicable when SLS mitigation is
744   // enabled.
745   const ARMSubtarget &ST = MF->getSubtarget<ARMSubtarget>();
746   if (ST.hardenSlsRetBr() && isIndirectControlFlowNotComingBack(MI))
747     return false;
748   if (ST.hardenSlsBlr() && isIndirectCall(MI))
749     return false;
750 
751   if (AFI->isThumb2Function()) {
752     if (getSubtarget().restrictIT())
753       return isV8EligibleForIT(&MI);
754   }
755 
756   return true;
757 }
758 
759 namespace llvm {
760 
761 template <> bool IsCPSRDead<MachineInstr>(const MachineInstr *MI) {
762   for (const MachineOperand &MO : MI->operands()) {
763     if (!MO.isReg() || MO.isUndef() || MO.isUse())
764       continue;
765     if (MO.getReg() != ARM::CPSR)
766       continue;
767     if (!MO.isDead())
768       return false;
769   }
770   // all definitions of CPSR are dead
771   return true;
772 }
773 
774 } // end namespace llvm
775 
776 /// GetInstSize - Return the size of the specified MachineInstr.
777 ///
778 unsigned ARMBaseInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
779   const MachineBasicBlock &MBB = *MI.getParent();
780   const MachineFunction *MF = MBB.getParent();
781   const MCAsmInfo *MAI = MF->getTarget().getMCAsmInfo();
782 
783   const MCInstrDesc &MCID = MI.getDesc();
784 
785   switch (MI.getOpcode()) {
786   default:
787     // Return the size specified in .td file. If there's none, return 0, as we
788     // can't define a default size (Thumb1 instructions are 2 bytes, Thumb2
789     // instructions are 2-4 bytes, and ARM instructions are 4 bytes), in
790     // contrast to AArch64 instructions which have a default size of 4 bytes for
791     // example.
792     return MCID.getSize();
793   case TargetOpcode::BUNDLE:
794     return getInstBundleLength(MI);
795   case ARM::CONSTPOOL_ENTRY:
796   case ARM::JUMPTABLE_INSTS:
797   case ARM::JUMPTABLE_ADDRS:
798   case ARM::JUMPTABLE_TBB:
799   case ARM::JUMPTABLE_TBH:
800     // If this machine instr is a constant pool entry, its size is recorded as
801     // operand #2.
802     return MI.getOperand(2).getImm();
803   case ARM::SPACE:
804     return MI.getOperand(1).getImm();
805   case ARM::INLINEASM:
806   case ARM::INLINEASM_BR: {
807     // If this machine instr is an inline asm, measure it.
808     unsigned Size = getInlineAsmLength(MI.getOperand(0).getSymbolName(), *MAI);
809     if (!MF->getInfo<ARMFunctionInfo>()->isThumbFunction())
810       Size = alignTo(Size, 4);
811     return Size;
812   }
813   }
814 }
815 
816 unsigned ARMBaseInstrInfo::getInstBundleLength(const MachineInstr &MI) const {
817   unsigned Size = 0;
818   MachineBasicBlock::const_instr_iterator I = MI.getIterator();
819   MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
820   while (++I != E && I->isInsideBundle()) {
821     assert(!I->isBundle() && "No nested bundle!");
822     Size += getInstSizeInBytes(*I);
823   }
824   return Size;
825 }
826 
827 void ARMBaseInstrInfo::copyFromCPSR(MachineBasicBlock &MBB,
828                                     MachineBasicBlock::iterator I,
829                                     unsigned DestReg, bool KillSrc,
830                                     const ARMSubtarget &Subtarget) const {
831   unsigned Opc = Subtarget.isThumb()
832                      ? (Subtarget.isMClass() ? ARM::t2MRS_M : ARM::t2MRS_AR)
833                      : ARM::MRS;
834 
835   MachineInstrBuilder MIB =
836       BuildMI(MBB, I, I->getDebugLoc(), get(Opc), DestReg);
837 
838   // There is only 1 A/R class MRS instruction, and it always refers to
839   // APSR. However, there are lots of other possibilities on M-class cores.
840   if (Subtarget.isMClass())
841     MIB.addImm(0x800);
842 
843   MIB.add(predOps(ARMCC::AL))
844      .addReg(ARM::CPSR, RegState::Implicit | getKillRegState(KillSrc));
845 }
846 
847 void ARMBaseInstrInfo::copyToCPSR(MachineBasicBlock &MBB,
848                                   MachineBasicBlock::iterator I,
849                                   unsigned SrcReg, bool KillSrc,
850                                   const ARMSubtarget &Subtarget) const {
851   unsigned Opc = Subtarget.isThumb()
852                      ? (Subtarget.isMClass() ? ARM::t2MSR_M : ARM::t2MSR_AR)
853                      : ARM::MSR;
854 
855   MachineInstrBuilder MIB = BuildMI(MBB, I, I->getDebugLoc(), get(Opc));
856 
857   if (Subtarget.isMClass())
858     MIB.addImm(0x800);
859   else
860     MIB.addImm(8);
861 
862   MIB.addReg(SrcReg, getKillRegState(KillSrc))
863      .add(predOps(ARMCC::AL))
864      .addReg(ARM::CPSR, RegState::Implicit | RegState::Define);
865 }
866 
867 void llvm::addUnpredicatedMveVpredNOp(MachineInstrBuilder &MIB) {
868   MIB.addImm(ARMVCC::None);
869   MIB.addReg(0);
870   MIB.addReg(0); // tp_reg
871 }
872 
873 void llvm::addUnpredicatedMveVpredROp(MachineInstrBuilder &MIB,
874                                       Register DestReg) {
875   addUnpredicatedMveVpredNOp(MIB);
876   MIB.addReg(DestReg, RegState::Undef);
877 }
878 
879 void llvm::addPredicatedMveVpredNOp(MachineInstrBuilder &MIB, unsigned Cond) {
880   MIB.addImm(Cond);
881   MIB.addReg(ARM::VPR, RegState::Implicit);
882   MIB.addReg(0); // tp_reg
883 }
884 
885 void llvm::addPredicatedMveVpredROp(MachineInstrBuilder &MIB,
886                                     unsigned Cond, unsigned Inactive) {
887   addPredicatedMveVpredNOp(MIB, Cond);
888   MIB.addReg(Inactive);
889 }
890 
891 void ARMBaseInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
892                                    MachineBasicBlock::iterator I,
893                                    const DebugLoc &DL, MCRegister DestReg,
894                                    MCRegister SrcReg, bool KillSrc) const {
895   bool GPRDest = ARM::GPRRegClass.contains(DestReg);
896   bool GPRSrc = ARM::GPRRegClass.contains(SrcReg);
897 
898   if (GPRDest && GPRSrc) {
899     BuildMI(MBB, I, DL, get(ARM::MOVr), DestReg)
900         .addReg(SrcReg, getKillRegState(KillSrc))
901         .add(predOps(ARMCC::AL))
902         .add(condCodeOp());
903     return;
904   }
905 
906   bool SPRDest = ARM::SPRRegClass.contains(DestReg);
907   bool SPRSrc = ARM::SPRRegClass.contains(SrcReg);
908 
909   unsigned Opc = 0;
910   if (SPRDest && SPRSrc)
911     Opc = ARM::VMOVS;
912   else if (GPRDest && SPRSrc)
913     Opc = ARM::VMOVRS;
914   else if (SPRDest && GPRSrc)
915     Opc = ARM::VMOVSR;
916   else if (ARM::DPRRegClass.contains(DestReg, SrcReg) && Subtarget.hasFP64())
917     Opc = ARM::VMOVD;
918   else if (ARM::QPRRegClass.contains(DestReg, SrcReg))
919     Opc = Subtarget.hasNEON() ? ARM::VORRq : ARM::MQPRCopy;
920 
921   if (Opc) {
922     MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opc), DestReg);
923     MIB.addReg(SrcReg, getKillRegState(KillSrc));
924     if (Opc == ARM::VORRq || Opc == ARM::MVE_VORR)
925       MIB.addReg(SrcReg, getKillRegState(KillSrc));
926     if (Opc == ARM::MVE_VORR)
927       addUnpredicatedMveVpredROp(MIB, DestReg);
928     else if (Opc != ARM::MQPRCopy)
929       MIB.add(predOps(ARMCC::AL));
930     return;
931   }
932 
933   // Handle register classes that require multiple instructions.
934   unsigned BeginIdx = 0;
935   unsigned SubRegs = 0;
936   int Spacing = 1;
937 
938   // Use VORRq when possible.
939   if (ARM::QQPRRegClass.contains(DestReg, SrcReg)) {
940     Opc = Subtarget.hasNEON() ? ARM::VORRq : ARM::MVE_VORR;
941     BeginIdx = ARM::qsub_0;
942     SubRegs = 2;
943   } else if (ARM::QQQQPRRegClass.contains(DestReg, SrcReg)) {
944     Opc = Subtarget.hasNEON() ? ARM::VORRq : ARM::MVE_VORR;
945     BeginIdx = ARM::qsub_0;
946     SubRegs = 4;
947   // Fall back to VMOVD.
948   } else if (ARM::DPairRegClass.contains(DestReg, SrcReg)) {
949     Opc = ARM::VMOVD;
950     BeginIdx = ARM::dsub_0;
951     SubRegs = 2;
952   } else if (ARM::DTripleRegClass.contains(DestReg, SrcReg)) {
953     Opc = ARM::VMOVD;
954     BeginIdx = ARM::dsub_0;
955     SubRegs = 3;
956   } else if (ARM::DQuadRegClass.contains(DestReg, SrcReg)) {
957     Opc = ARM::VMOVD;
958     BeginIdx = ARM::dsub_0;
959     SubRegs = 4;
960   } else if (ARM::GPRPairRegClass.contains(DestReg, SrcReg)) {
961     Opc = Subtarget.isThumb2() ? ARM::tMOVr : ARM::MOVr;
962     BeginIdx = ARM::gsub_0;
963     SubRegs = 2;
964   } else if (ARM::DPairSpcRegClass.contains(DestReg, SrcReg)) {
965     Opc = ARM::VMOVD;
966     BeginIdx = ARM::dsub_0;
967     SubRegs = 2;
968     Spacing = 2;
969   } else if (ARM::DTripleSpcRegClass.contains(DestReg, SrcReg)) {
970     Opc = ARM::VMOVD;
971     BeginIdx = ARM::dsub_0;
972     SubRegs = 3;
973     Spacing = 2;
974   } else if (ARM::DQuadSpcRegClass.contains(DestReg, SrcReg)) {
975     Opc = ARM::VMOVD;
976     BeginIdx = ARM::dsub_0;
977     SubRegs = 4;
978     Spacing = 2;
979   } else if (ARM::DPRRegClass.contains(DestReg, SrcReg) &&
980              !Subtarget.hasFP64()) {
981     Opc = ARM::VMOVS;
982     BeginIdx = ARM::ssub_0;
983     SubRegs = 2;
984   } else if (SrcReg == ARM::CPSR) {
985     copyFromCPSR(MBB, I, DestReg, KillSrc, Subtarget);
986     return;
987   } else if (DestReg == ARM::CPSR) {
988     copyToCPSR(MBB, I, SrcReg, KillSrc, Subtarget);
989     return;
990   } else if (DestReg == ARM::VPR) {
991     assert(ARM::GPRRegClass.contains(SrcReg));
992     BuildMI(MBB, I, I->getDebugLoc(), get(ARM::VMSR_P0), DestReg)
993         .addReg(SrcReg, getKillRegState(KillSrc))
994         .add(predOps(ARMCC::AL));
995     return;
996   } else if (SrcReg == ARM::VPR) {
997     assert(ARM::GPRRegClass.contains(DestReg));
998     BuildMI(MBB, I, I->getDebugLoc(), get(ARM::VMRS_P0), DestReg)
999         .addReg(SrcReg, getKillRegState(KillSrc))
1000         .add(predOps(ARMCC::AL));
1001     return;
1002   } else if (DestReg == ARM::FPSCR_NZCV) {
1003     assert(ARM::GPRRegClass.contains(SrcReg));
1004     BuildMI(MBB, I, I->getDebugLoc(), get(ARM::VMSR_FPSCR_NZCVQC), DestReg)
1005         .addReg(SrcReg, getKillRegState(KillSrc))
1006         .add(predOps(ARMCC::AL));
1007     return;
1008   } else if (SrcReg == ARM::FPSCR_NZCV) {
1009     assert(ARM::GPRRegClass.contains(DestReg));
1010     BuildMI(MBB, I, I->getDebugLoc(), get(ARM::VMRS_FPSCR_NZCVQC), DestReg)
1011         .addReg(SrcReg, getKillRegState(KillSrc))
1012         .add(predOps(ARMCC::AL));
1013     return;
1014   }
1015 
1016   assert(Opc && "Impossible reg-to-reg copy");
1017 
1018   const TargetRegisterInfo *TRI = &getRegisterInfo();
1019   MachineInstrBuilder Mov;
1020 
1021   // Copy register tuples backward when the first Dest reg overlaps with SrcReg.
1022   if (TRI->regsOverlap(SrcReg, TRI->getSubReg(DestReg, BeginIdx))) {
1023     BeginIdx = BeginIdx + ((SubRegs - 1) * Spacing);
1024     Spacing = -Spacing;
1025   }
1026 #ifndef NDEBUG
1027   SmallSet<unsigned, 4> DstRegs;
1028 #endif
1029   for (unsigned i = 0; i != SubRegs; ++i) {
1030     Register Dst = TRI->getSubReg(DestReg, BeginIdx + i * Spacing);
1031     Register Src = TRI->getSubReg(SrcReg, BeginIdx + i * Spacing);
1032     assert(Dst && Src && "Bad sub-register");
1033 #ifndef NDEBUG
1034     assert(!DstRegs.count(Src) && "destructive vector copy");
1035     DstRegs.insert(Dst);
1036 #endif
1037     Mov = BuildMI(MBB, I, I->getDebugLoc(), get(Opc), Dst).addReg(Src);
1038     // VORR (NEON or MVE) takes two source operands.
1039     if (Opc == ARM::VORRq || Opc == ARM::MVE_VORR) {
1040       Mov.addReg(Src);
1041     }
1042     // MVE VORR takes predicate operands in place of an ordinary condition.
1043     if (Opc == ARM::MVE_VORR)
1044       addUnpredicatedMveVpredROp(Mov, Dst);
1045     else
1046       Mov = Mov.add(predOps(ARMCC::AL));
1047     // MOVr can set CC.
1048     if (Opc == ARM::MOVr)
1049       Mov = Mov.add(condCodeOp());
1050   }
1051   // Add implicit super-register defs and kills to the last instruction.
1052   Mov->addRegisterDefined(DestReg, TRI);
1053   if (KillSrc)
1054     Mov->addRegisterKilled(SrcReg, TRI);
1055 }
1056 
1057 std::optional<DestSourcePair>
1058 ARMBaseInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
1059   // VMOVRRD is also a copy instruction but it requires
1060   // special way of handling. It is more complex copy version
1061   // and since that we are not considering it. For recognition
1062   // of such instruction isExtractSubregLike MI interface fuction
1063   // could be used.
1064   // VORRq is considered as a move only if two inputs are
1065   // the same register.
1066   if (!MI.isMoveReg() ||
1067       (MI.getOpcode() == ARM::VORRq &&
1068        MI.getOperand(1).getReg() != MI.getOperand(2).getReg()))
1069     return std::nullopt;
1070   return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
1071 }
1072 
1073 std::optional<ParamLoadedValue>
1074 ARMBaseInstrInfo::describeLoadedValue(const MachineInstr &MI,
1075                                       Register Reg) const {
1076   if (auto DstSrcPair = isCopyInstrImpl(MI)) {
1077     Register DstReg = DstSrcPair->Destination->getReg();
1078 
1079     // TODO: We don't handle cases where the forwarding reg is narrower/wider
1080     // than the copy registers. Consider for example:
1081     //
1082     //   s16 = VMOVS s0
1083     //   s17 = VMOVS s1
1084     //   call @callee(d0)
1085     //
1086     // We'd like to describe the call site value of d0 as d8, but this requires
1087     // gathering and merging the descriptions for the two VMOVS instructions.
1088     //
1089     // We also don't handle the reverse situation, where the forwarding reg is
1090     // narrower than the copy destination:
1091     //
1092     //   d8 = VMOVD d0
1093     //   call @callee(s1)
1094     //
1095     // We need to produce a fragment description (the call site value of s1 is
1096     // /not/ just d8).
1097     if (DstReg != Reg)
1098       return std::nullopt;
1099   }
1100   return TargetInstrInfo::describeLoadedValue(MI, Reg);
1101 }
1102 
1103 const MachineInstrBuilder &
1104 ARMBaseInstrInfo::AddDReg(MachineInstrBuilder &MIB, unsigned Reg,
1105                           unsigned SubIdx, unsigned State,
1106                           const TargetRegisterInfo *TRI) const {
1107   if (!SubIdx)
1108     return MIB.addReg(Reg, State);
1109 
1110   if (Register::isPhysicalRegister(Reg))
1111     return MIB.addReg(TRI->getSubReg(Reg, SubIdx), State);
1112   return MIB.addReg(Reg, State, SubIdx);
1113 }
1114 
1115 void ARMBaseInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
1116                                            MachineBasicBlock::iterator I,
1117                                            Register SrcReg, bool isKill, int FI,
1118                                            const TargetRegisterClass *RC,
1119                                            const TargetRegisterInfo *TRI,
1120                                            Register VReg) const {
1121   MachineFunction &MF = *MBB.getParent();
1122   MachineFrameInfo &MFI = MF.getFrameInfo();
1123   Align Alignment = MFI.getObjectAlign(FI);
1124 
1125   MachineMemOperand *MMO = MF.getMachineMemOperand(
1126       MachinePointerInfo::getFixedStack(MF, FI), MachineMemOperand::MOStore,
1127       MFI.getObjectSize(FI), Alignment);
1128 
1129   switch (TRI->getSpillSize(*RC)) {
1130     case 2:
1131       if (ARM::HPRRegClass.hasSubClassEq(RC)) {
1132         BuildMI(MBB, I, DebugLoc(), get(ARM::VSTRH))
1133             .addReg(SrcReg, getKillRegState(isKill))
1134             .addFrameIndex(FI)
1135             .addImm(0)
1136             .addMemOperand(MMO)
1137             .add(predOps(ARMCC::AL));
1138       } else
1139         llvm_unreachable("Unknown reg class!");
1140       break;
1141     case 4:
1142       if (ARM::GPRRegClass.hasSubClassEq(RC)) {
1143         BuildMI(MBB, I, DebugLoc(), get(ARM::STRi12))
1144             .addReg(SrcReg, getKillRegState(isKill))
1145             .addFrameIndex(FI)
1146             .addImm(0)
1147             .addMemOperand(MMO)
1148             .add(predOps(ARMCC::AL));
1149       } else if (ARM::SPRRegClass.hasSubClassEq(RC)) {
1150         BuildMI(MBB, I, DebugLoc(), get(ARM::VSTRS))
1151             .addReg(SrcReg, getKillRegState(isKill))
1152             .addFrameIndex(FI)
1153             .addImm(0)
1154             .addMemOperand(MMO)
1155             .add(predOps(ARMCC::AL));
1156       } else if (ARM::VCCRRegClass.hasSubClassEq(RC)) {
1157         BuildMI(MBB, I, DebugLoc(), get(ARM::VSTR_P0_off))
1158             .addReg(SrcReg, getKillRegState(isKill))
1159             .addFrameIndex(FI)
1160             .addImm(0)
1161             .addMemOperand(MMO)
1162             .add(predOps(ARMCC::AL));
1163       } else
1164         llvm_unreachable("Unknown reg class!");
1165       break;
1166     case 8:
1167       if (ARM::DPRRegClass.hasSubClassEq(RC)) {
1168         BuildMI(MBB, I, DebugLoc(), get(ARM::VSTRD))
1169             .addReg(SrcReg, getKillRegState(isKill))
1170             .addFrameIndex(FI)
1171             .addImm(0)
1172             .addMemOperand(MMO)
1173             .add(predOps(ARMCC::AL));
1174       } else if (ARM::GPRPairRegClass.hasSubClassEq(RC)) {
1175         if (Subtarget.hasV5TEOps()) {
1176           MachineInstrBuilder MIB = BuildMI(MBB, I, DebugLoc(), get(ARM::STRD));
1177           AddDReg(MIB, SrcReg, ARM::gsub_0, getKillRegState(isKill), TRI);
1178           AddDReg(MIB, SrcReg, ARM::gsub_1, 0, TRI);
1179           MIB.addFrameIndex(FI).addReg(0).addImm(0).addMemOperand(MMO)
1180              .add(predOps(ARMCC::AL));
1181         } else {
1182           // Fallback to STM instruction, which has existed since the dawn of
1183           // time.
1184           MachineInstrBuilder MIB = BuildMI(MBB, I, DebugLoc(), get(ARM::STMIA))
1185                                         .addFrameIndex(FI)
1186                                         .addMemOperand(MMO)
1187                                         .add(predOps(ARMCC::AL));
1188           AddDReg(MIB, SrcReg, ARM::gsub_0, getKillRegState(isKill), TRI);
1189           AddDReg(MIB, SrcReg, ARM::gsub_1, 0, TRI);
1190         }
1191       } else
1192         llvm_unreachable("Unknown reg class!");
1193       break;
1194     case 16:
1195       if (ARM::DPairRegClass.hasSubClassEq(RC) && Subtarget.hasNEON()) {
1196         // Use aligned spills if the stack can be realigned.
1197         if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF)) {
1198           BuildMI(MBB, I, DebugLoc(), get(ARM::VST1q64))
1199               .addFrameIndex(FI)
1200               .addImm(16)
1201               .addReg(SrcReg, getKillRegState(isKill))
1202               .addMemOperand(MMO)
1203               .add(predOps(ARMCC::AL));
1204         } else {
1205           BuildMI(MBB, I, DebugLoc(), get(ARM::VSTMQIA))
1206               .addReg(SrcReg, getKillRegState(isKill))
1207               .addFrameIndex(FI)
1208               .addMemOperand(MMO)
1209               .add(predOps(ARMCC::AL));
1210         }
1211       } else if (ARM::QPRRegClass.hasSubClassEq(RC) &&
1212                  Subtarget.hasMVEIntegerOps()) {
1213         auto MIB = BuildMI(MBB, I, DebugLoc(), get(ARM::MVE_VSTRWU32));
1214         MIB.addReg(SrcReg, getKillRegState(isKill))
1215           .addFrameIndex(FI)
1216           .addImm(0)
1217           .addMemOperand(MMO);
1218         addUnpredicatedMveVpredNOp(MIB);
1219       } else
1220         llvm_unreachable("Unknown reg class!");
1221       break;
1222     case 24:
1223       if (ARM::DTripleRegClass.hasSubClassEq(RC)) {
1224         // Use aligned spills if the stack can be realigned.
1225         if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
1226             Subtarget.hasNEON()) {
1227           BuildMI(MBB, I, DebugLoc(), get(ARM::VST1d64TPseudo))
1228               .addFrameIndex(FI)
1229               .addImm(16)
1230               .addReg(SrcReg, getKillRegState(isKill))
1231               .addMemOperand(MMO)
1232               .add(predOps(ARMCC::AL));
1233         } else {
1234           MachineInstrBuilder MIB = BuildMI(MBB, I, DebugLoc(),
1235                                             get(ARM::VSTMDIA))
1236                                         .addFrameIndex(FI)
1237                                         .add(predOps(ARMCC::AL))
1238                                         .addMemOperand(MMO);
1239           MIB = AddDReg(MIB, SrcReg, ARM::dsub_0, getKillRegState(isKill), TRI);
1240           MIB = AddDReg(MIB, SrcReg, ARM::dsub_1, 0, TRI);
1241           AddDReg(MIB, SrcReg, ARM::dsub_2, 0, TRI);
1242         }
1243       } else
1244         llvm_unreachable("Unknown reg class!");
1245       break;
1246     case 32:
1247       if (ARM::QQPRRegClass.hasSubClassEq(RC) ||
1248           ARM::MQQPRRegClass.hasSubClassEq(RC) ||
1249           ARM::DQuadRegClass.hasSubClassEq(RC)) {
1250         if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
1251             Subtarget.hasNEON()) {
1252           // FIXME: It's possible to only store part of the QQ register if the
1253           // spilled def has a sub-register index.
1254           BuildMI(MBB, I, DebugLoc(), get(ARM::VST1d64QPseudo))
1255               .addFrameIndex(FI)
1256               .addImm(16)
1257               .addReg(SrcReg, getKillRegState(isKill))
1258               .addMemOperand(MMO)
1259               .add(predOps(ARMCC::AL));
1260         } else if (Subtarget.hasMVEIntegerOps()) {
1261           BuildMI(MBB, I, DebugLoc(), get(ARM::MQQPRStore))
1262               .addReg(SrcReg, getKillRegState(isKill))
1263               .addFrameIndex(FI)
1264               .addMemOperand(MMO);
1265         } else {
1266           MachineInstrBuilder MIB = BuildMI(MBB, I, DebugLoc(),
1267                                             get(ARM::VSTMDIA))
1268                                         .addFrameIndex(FI)
1269                                         .add(predOps(ARMCC::AL))
1270                                         .addMemOperand(MMO);
1271           MIB = AddDReg(MIB, SrcReg, ARM::dsub_0, getKillRegState(isKill), TRI);
1272           MIB = AddDReg(MIB, SrcReg, ARM::dsub_1, 0, TRI);
1273           MIB = AddDReg(MIB, SrcReg, ARM::dsub_2, 0, TRI);
1274                 AddDReg(MIB, SrcReg, ARM::dsub_3, 0, TRI);
1275         }
1276       } else
1277         llvm_unreachable("Unknown reg class!");
1278       break;
1279     case 64:
1280       if (ARM::MQQQQPRRegClass.hasSubClassEq(RC) &&
1281           Subtarget.hasMVEIntegerOps()) {
1282         BuildMI(MBB, I, DebugLoc(), get(ARM::MQQQQPRStore))
1283             .addReg(SrcReg, getKillRegState(isKill))
1284             .addFrameIndex(FI)
1285             .addMemOperand(MMO);
1286       } else if (ARM::QQQQPRRegClass.hasSubClassEq(RC)) {
1287         MachineInstrBuilder MIB = BuildMI(MBB, I, DebugLoc(), get(ARM::VSTMDIA))
1288                                       .addFrameIndex(FI)
1289                                       .add(predOps(ARMCC::AL))
1290                                       .addMemOperand(MMO);
1291         MIB = AddDReg(MIB, SrcReg, ARM::dsub_0, getKillRegState(isKill), TRI);
1292         MIB = AddDReg(MIB, SrcReg, ARM::dsub_1, 0, TRI);
1293         MIB = AddDReg(MIB, SrcReg, ARM::dsub_2, 0, TRI);
1294         MIB = AddDReg(MIB, SrcReg, ARM::dsub_3, 0, TRI);
1295         MIB = AddDReg(MIB, SrcReg, ARM::dsub_4, 0, TRI);
1296         MIB = AddDReg(MIB, SrcReg, ARM::dsub_5, 0, TRI);
1297         MIB = AddDReg(MIB, SrcReg, ARM::dsub_6, 0, TRI);
1298               AddDReg(MIB, SrcReg, ARM::dsub_7, 0, TRI);
1299       } else
1300         llvm_unreachable("Unknown reg class!");
1301       break;
1302     default:
1303       llvm_unreachable("Unknown reg class!");
1304   }
1305 }
1306 
1307 unsigned ARMBaseInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
1308                                               int &FrameIndex) const {
1309   switch (MI.getOpcode()) {
1310   default: break;
1311   case ARM::STRrs:
1312   case ARM::t2STRs: // FIXME: don't use t2STRs to access frame.
1313     if (MI.getOperand(1).isFI() && MI.getOperand(2).isReg() &&
1314         MI.getOperand(3).isImm() && MI.getOperand(2).getReg() == 0 &&
1315         MI.getOperand(3).getImm() == 0) {
1316       FrameIndex = MI.getOperand(1).getIndex();
1317       return MI.getOperand(0).getReg();
1318     }
1319     break;
1320   case ARM::STRi12:
1321   case ARM::t2STRi12:
1322   case ARM::tSTRspi:
1323   case ARM::VSTRD:
1324   case ARM::VSTRS:
1325   case ARM::VSTR_P0_off:
1326   case ARM::MVE_VSTRWU32:
1327     if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
1328         MI.getOperand(2).getImm() == 0) {
1329       FrameIndex = MI.getOperand(1).getIndex();
1330       return MI.getOperand(0).getReg();
1331     }
1332     break;
1333   case ARM::VST1q64:
1334   case ARM::VST1d64TPseudo:
1335   case ARM::VST1d64QPseudo:
1336     if (MI.getOperand(0).isFI() && MI.getOperand(2).getSubReg() == 0) {
1337       FrameIndex = MI.getOperand(0).getIndex();
1338       return MI.getOperand(2).getReg();
1339     }
1340     break;
1341   case ARM::VSTMQIA:
1342     if (MI.getOperand(1).isFI() && MI.getOperand(0).getSubReg() == 0) {
1343       FrameIndex = MI.getOperand(1).getIndex();
1344       return MI.getOperand(0).getReg();
1345     }
1346     break;
1347   case ARM::MQQPRStore:
1348   case ARM::MQQQQPRStore:
1349     if (MI.getOperand(1).isFI()) {
1350       FrameIndex = MI.getOperand(1).getIndex();
1351       return MI.getOperand(0).getReg();
1352     }
1353     break;
1354   }
1355 
1356   return 0;
1357 }
1358 
1359 unsigned ARMBaseInstrInfo::isStoreToStackSlotPostFE(const MachineInstr &MI,
1360                                                     int &FrameIndex) const {
1361   SmallVector<const MachineMemOperand *, 1> Accesses;
1362   if (MI.mayStore() && hasStoreToStackSlot(MI, Accesses) &&
1363       Accesses.size() == 1) {
1364     FrameIndex =
1365         cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue())
1366             ->getFrameIndex();
1367     return true;
1368   }
1369   return false;
1370 }
1371 
1372 void ARMBaseInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
1373                                             MachineBasicBlock::iterator I,
1374                                             Register DestReg, int FI,
1375                                             const TargetRegisterClass *RC,
1376                                             const TargetRegisterInfo *TRI,
1377                                             Register VReg) const {
1378   DebugLoc DL;
1379   if (I != MBB.end()) DL = I->getDebugLoc();
1380   MachineFunction &MF = *MBB.getParent();
1381   MachineFrameInfo &MFI = MF.getFrameInfo();
1382   const Align Alignment = MFI.getObjectAlign(FI);
1383   MachineMemOperand *MMO = MF.getMachineMemOperand(
1384       MachinePointerInfo::getFixedStack(MF, FI), MachineMemOperand::MOLoad,
1385       MFI.getObjectSize(FI), Alignment);
1386 
1387   switch (TRI->getSpillSize(*RC)) {
1388   case 2:
1389     if (ARM::HPRRegClass.hasSubClassEq(RC)) {
1390       BuildMI(MBB, I, DL, get(ARM::VLDRH), DestReg)
1391           .addFrameIndex(FI)
1392           .addImm(0)
1393           .addMemOperand(MMO)
1394           .add(predOps(ARMCC::AL));
1395     } else
1396       llvm_unreachable("Unknown reg class!");
1397     break;
1398   case 4:
1399     if (ARM::GPRRegClass.hasSubClassEq(RC)) {
1400       BuildMI(MBB, I, DL, get(ARM::LDRi12), DestReg)
1401           .addFrameIndex(FI)
1402           .addImm(0)
1403           .addMemOperand(MMO)
1404           .add(predOps(ARMCC::AL));
1405     } else if (ARM::SPRRegClass.hasSubClassEq(RC)) {
1406       BuildMI(MBB, I, DL, get(ARM::VLDRS), DestReg)
1407           .addFrameIndex(FI)
1408           .addImm(0)
1409           .addMemOperand(MMO)
1410           .add(predOps(ARMCC::AL));
1411     } else if (ARM::VCCRRegClass.hasSubClassEq(RC)) {
1412       BuildMI(MBB, I, DL, get(ARM::VLDR_P0_off), DestReg)
1413           .addFrameIndex(FI)
1414           .addImm(0)
1415           .addMemOperand(MMO)
1416           .add(predOps(ARMCC::AL));
1417     } else
1418       llvm_unreachable("Unknown reg class!");
1419     break;
1420   case 8:
1421     if (ARM::DPRRegClass.hasSubClassEq(RC)) {
1422       BuildMI(MBB, I, DL, get(ARM::VLDRD), DestReg)
1423           .addFrameIndex(FI)
1424           .addImm(0)
1425           .addMemOperand(MMO)
1426           .add(predOps(ARMCC::AL));
1427     } else if (ARM::GPRPairRegClass.hasSubClassEq(RC)) {
1428       MachineInstrBuilder MIB;
1429 
1430       if (Subtarget.hasV5TEOps()) {
1431         MIB = BuildMI(MBB, I, DL, get(ARM::LDRD));
1432         AddDReg(MIB, DestReg, ARM::gsub_0, RegState::DefineNoRead, TRI);
1433         AddDReg(MIB, DestReg, ARM::gsub_1, RegState::DefineNoRead, TRI);
1434         MIB.addFrameIndex(FI).addReg(0).addImm(0).addMemOperand(MMO)
1435            .add(predOps(ARMCC::AL));
1436       } else {
1437         // Fallback to LDM instruction, which has existed since the dawn of
1438         // time.
1439         MIB = BuildMI(MBB, I, DL, get(ARM::LDMIA))
1440                   .addFrameIndex(FI)
1441                   .addMemOperand(MMO)
1442                   .add(predOps(ARMCC::AL));
1443         MIB = AddDReg(MIB, DestReg, ARM::gsub_0, RegState::DefineNoRead, TRI);
1444         MIB = AddDReg(MIB, DestReg, ARM::gsub_1, RegState::DefineNoRead, TRI);
1445       }
1446 
1447       if (DestReg.isPhysical())
1448         MIB.addReg(DestReg, RegState::ImplicitDefine);
1449     } else
1450       llvm_unreachable("Unknown reg class!");
1451     break;
1452   case 16:
1453     if (ARM::DPairRegClass.hasSubClassEq(RC) && Subtarget.hasNEON()) {
1454       if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF)) {
1455         BuildMI(MBB, I, DL, get(ARM::VLD1q64), DestReg)
1456             .addFrameIndex(FI)
1457             .addImm(16)
1458             .addMemOperand(MMO)
1459             .add(predOps(ARMCC::AL));
1460       } else {
1461         BuildMI(MBB, I, DL, get(ARM::VLDMQIA), DestReg)
1462             .addFrameIndex(FI)
1463             .addMemOperand(MMO)
1464             .add(predOps(ARMCC::AL));
1465       }
1466     } else if (ARM::QPRRegClass.hasSubClassEq(RC) &&
1467                Subtarget.hasMVEIntegerOps()) {
1468       auto MIB = BuildMI(MBB, I, DL, get(ARM::MVE_VLDRWU32), DestReg);
1469       MIB.addFrameIndex(FI)
1470         .addImm(0)
1471         .addMemOperand(MMO);
1472       addUnpredicatedMveVpredNOp(MIB);
1473     } else
1474       llvm_unreachable("Unknown reg class!");
1475     break;
1476   case 24:
1477     if (ARM::DTripleRegClass.hasSubClassEq(RC)) {
1478       if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
1479           Subtarget.hasNEON()) {
1480         BuildMI(MBB, I, DL, get(ARM::VLD1d64TPseudo), DestReg)
1481             .addFrameIndex(FI)
1482             .addImm(16)
1483             .addMemOperand(MMO)
1484             .add(predOps(ARMCC::AL));
1485       } else {
1486         MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(ARM::VLDMDIA))
1487                                       .addFrameIndex(FI)
1488                                       .addMemOperand(MMO)
1489                                       .add(predOps(ARMCC::AL));
1490         MIB = AddDReg(MIB, DestReg, ARM::dsub_0, RegState::DefineNoRead, TRI);
1491         MIB = AddDReg(MIB, DestReg, ARM::dsub_1, RegState::DefineNoRead, TRI);
1492         MIB = AddDReg(MIB, DestReg, ARM::dsub_2, RegState::DefineNoRead, TRI);
1493         if (DestReg.isPhysical())
1494           MIB.addReg(DestReg, RegState::ImplicitDefine);
1495       }
1496     } else
1497       llvm_unreachable("Unknown reg class!");
1498     break;
1499    case 32:
1500      if (ARM::QQPRRegClass.hasSubClassEq(RC) ||
1501          ARM::MQQPRRegClass.hasSubClassEq(RC) ||
1502          ARM::DQuadRegClass.hasSubClassEq(RC)) {
1503        if (Alignment >= 16 && getRegisterInfo().canRealignStack(MF) &&
1504            Subtarget.hasNEON()) {
1505          BuildMI(MBB, I, DL, get(ARM::VLD1d64QPseudo), DestReg)
1506              .addFrameIndex(FI)
1507              .addImm(16)
1508              .addMemOperand(MMO)
1509              .add(predOps(ARMCC::AL));
1510        } else if (Subtarget.hasMVEIntegerOps()) {
1511          BuildMI(MBB, I, DL, get(ARM::MQQPRLoad), DestReg)
1512              .addFrameIndex(FI)
1513              .addMemOperand(MMO);
1514        } else {
1515          MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(ARM::VLDMDIA))
1516                                        .addFrameIndex(FI)
1517                                        .add(predOps(ARMCC::AL))
1518                                        .addMemOperand(MMO);
1519          MIB = AddDReg(MIB, DestReg, ARM::dsub_0, RegState::DefineNoRead, TRI);
1520          MIB = AddDReg(MIB, DestReg, ARM::dsub_1, RegState::DefineNoRead, TRI);
1521          MIB = AddDReg(MIB, DestReg, ARM::dsub_2, RegState::DefineNoRead, TRI);
1522          MIB = AddDReg(MIB, DestReg, ARM::dsub_3, RegState::DefineNoRead, TRI);
1523          if (DestReg.isPhysical())
1524            MIB.addReg(DestReg, RegState::ImplicitDefine);
1525        }
1526      } else
1527        llvm_unreachable("Unknown reg class!");
1528      break;
1529   case 64:
1530     if (ARM::MQQQQPRRegClass.hasSubClassEq(RC) &&
1531         Subtarget.hasMVEIntegerOps()) {
1532       BuildMI(MBB, I, DL, get(ARM::MQQQQPRLoad), DestReg)
1533           .addFrameIndex(FI)
1534           .addMemOperand(MMO);
1535     } else if (ARM::QQQQPRRegClass.hasSubClassEq(RC)) {
1536       MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(ARM::VLDMDIA))
1537                                     .addFrameIndex(FI)
1538                                     .add(predOps(ARMCC::AL))
1539                                     .addMemOperand(MMO);
1540       MIB = AddDReg(MIB, DestReg, ARM::dsub_0, RegState::DefineNoRead, TRI);
1541       MIB = AddDReg(MIB, DestReg, ARM::dsub_1, RegState::DefineNoRead, TRI);
1542       MIB = AddDReg(MIB, DestReg, ARM::dsub_2, RegState::DefineNoRead, TRI);
1543       MIB = AddDReg(MIB, DestReg, ARM::dsub_3, RegState::DefineNoRead, TRI);
1544       MIB = AddDReg(MIB, DestReg, ARM::dsub_4, RegState::DefineNoRead, TRI);
1545       MIB = AddDReg(MIB, DestReg, ARM::dsub_5, RegState::DefineNoRead, TRI);
1546       MIB = AddDReg(MIB, DestReg, ARM::dsub_6, RegState::DefineNoRead, TRI);
1547       MIB = AddDReg(MIB, DestReg, ARM::dsub_7, RegState::DefineNoRead, TRI);
1548       if (DestReg.isPhysical())
1549         MIB.addReg(DestReg, RegState::ImplicitDefine);
1550     } else
1551       llvm_unreachable("Unknown reg class!");
1552     break;
1553   default:
1554     llvm_unreachable("Unknown regclass!");
1555   }
1556 }
1557 
1558 unsigned ARMBaseInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
1559                                                int &FrameIndex) const {
1560   switch (MI.getOpcode()) {
1561   default: break;
1562   case ARM::LDRrs:
1563   case ARM::t2LDRs:  // FIXME: don't use t2LDRs to access frame.
1564     if (MI.getOperand(1).isFI() && MI.getOperand(2).isReg() &&
1565         MI.getOperand(3).isImm() && MI.getOperand(2).getReg() == 0 &&
1566         MI.getOperand(3).getImm() == 0) {
1567       FrameIndex = MI.getOperand(1).getIndex();
1568       return MI.getOperand(0).getReg();
1569     }
1570     break;
1571   case ARM::LDRi12:
1572   case ARM::t2LDRi12:
1573   case ARM::tLDRspi:
1574   case ARM::VLDRD:
1575   case ARM::VLDRS:
1576   case ARM::VLDR_P0_off:
1577   case ARM::MVE_VLDRWU32:
1578     if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
1579         MI.getOperand(2).getImm() == 0) {
1580       FrameIndex = MI.getOperand(1).getIndex();
1581       return MI.getOperand(0).getReg();
1582     }
1583     break;
1584   case ARM::VLD1q64:
1585   case ARM::VLD1d8TPseudo:
1586   case ARM::VLD1d16TPseudo:
1587   case ARM::VLD1d32TPseudo:
1588   case ARM::VLD1d64TPseudo:
1589   case ARM::VLD1d8QPseudo:
1590   case ARM::VLD1d16QPseudo:
1591   case ARM::VLD1d32QPseudo:
1592   case ARM::VLD1d64QPseudo:
1593     if (MI.getOperand(1).isFI() && MI.getOperand(0).getSubReg() == 0) {
1594       FrameIndex = MI.getOperand(1).getIndex();
1595       return MI.getOperand(0).getReg();
1596     }
1597     break;
1598   case ARM::VLDMQIA:
1599     if (MI.getOperand(1).isFI() && MI.getOperand(0).getSubReg() == 0) {
1600       FrameIndex = MI.getOperand(1).getIndex();
1601       return MI.getOperand(0).getReg();
1602     }
1603     break;
1604   case ARM::MQQPRLoad:
1605   case ARM::MQQQQPRLoad:
1606     if (MI.getOperand(1).isFI()) {
1607       FrameIndex = MI.getOperand(1).getIndex();
1608       return MI.getOperand(0).getReg();
1609     }
1610     break;
1611   }
1612 
1613   return 0;
1614 }
1615 
1616 unsigned ARMBaseInstrInfo::isLoadFromStackSlotPostFE(const MachineInstr &MI,
1617                                                      int &FrameIndex) const {
1618   SmallVector<const MachineMemOperand *, 1> Accesses;
1619   if (MI.mayLoad() && hasLoadFromStackSlot(MI, Accesses) &&
1620       Accesses.size() == 1) {
1621     FrameIndex =
1622         cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue())
1623             ->getFrameIndex();
1624     return true;
1625   }
1626   return false;
1627 }
1628 
1629 /// Expands MEMCPY to either LDMIA/STMIA or LDMIA_UPD/STMID_UPD
1630 /// depending on whether the result is used.
1631 void ARMBaseInstrInfo::expandMEMCPY(MachineBasicBlock::iterator MI) const {
1632   bool isThumb1 = Subtarget.isThumb1Only();
1633   bool isThumb2 = Subtarget.isThumb2();
1634   const ARMBaseInstrInfo *TII = Subtarget.getInstrInfo();
1635 
1636   DebugLoc dl = MI->getDebugLoc();
1637   MachineBasicBlock *BB = MI->getParent();
1638 
1639   MachineInstrBuilder LDM, STM;
1640   if (isThumb1 || !MI->getOperand(1).isDead()) {
1641     MachineOperand LDWb(MI->getOperand(1));
1642     LDM = BuildMI(*BB, MI, dl, TII->get(isThumb2 ? ARM::t2LDMIA_UPD
1643                                                  : isThumb1 ? ARM::tLDMIA_UPD
1644                                                             : ARM::LDMIA_UPD))
1645               .add(LDWb);
1646   } else {
1647     LDM = BuildMI(*BB, MI, dl, TII->get(isThumb2 ? ARM::t2LDMIA : ARM::LDMIA));
1648   }
1649 
1650   if (isThumb1 || !MI->getOperand(0).isDead()) {
1651     MachineOperand STWb(MI->getOperand(0));
1652     STM = BuildMI(*BB, MI, dl, TII->get(isThumb2 ? ARM::t2STMIA_UPD
1653                                                  : isThumb1 ? ARM::tSTMIA_UPD
1654                                                             : ARM::STMIA_UPD))
1655               .add(STWb);
1656   } else {
1657     STM = BuildMI(*BB, MI, dl, TII->get(isThumb2 ? ARM::t2STMIA : ARM::STMIA));
1658   }
1659 
1660   MachineOperand LDBase(MI->getOperand(3));
1661   LDM.add(LDBase).add(predOps(ARMCC::AL));
1662 
1663   MachineOperand STBase(MI->getOperand(2));
1664   STM.add(STBase).add(predOps(ARMCC::AL));
1665 
1666   // Sort the scratch registers into ascending order.
1667   const TargetRegisterInfo &TRI = getRegisterInfo();
1668   SmallVector<unsigned, 6> ScratchRegs;
1669   for(unsigned I = 5; I < MI->getNumOperands(); ++I)
1670     ScratchRegs.push_back(MI->getOperand(I).getReg());
1671   llvm::sort(ScratchRegs,
1672              [&TRI](const unsigned &Reg1, const unsigned &Reg2) -> bool {
1673                return TRI.getEncodingValue(Reg1) <
1674                       TRI.getEncodingValue(Reg2);
1675              });
1676 
1677   for (const auto &Reg : ScratchRegs) {
1678     LDM.addReg(Reg, RegState::Define);
1679     STM.addReg(Reg, RegState::Kill);
1680   }
1681 
1682   BB->erase(MI);
1683 }
1684 
1685 bool ARMBaseInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
1686   if (MI.getOpcode() == TargetOpcode::LOAD_STACK_GUARD) {
1687     expandLoadStackGuard(MI);
1688     MI.getParent()->erase(MI);
1689     return true;
1690   }
1691 
1692   if (MI.getOpcode() == ARM::MEMCPY) {
1693     expandMEMCPY(MI);
1694     return true;
1695   }
1696 
1697   // This hook gets to expand COPY instructions before they become
1698   // copyPhysReg() calls.  Look for VMOVS instructions that can legally be
1699   // widened to VMOVD.  We prefer the VMOVD when possible because it may be
1700   // changed into a VORR that can go down the NEON pipeline.
1701   if (!MI.isCopy() || Subtarget.dontWidenVMOVS() || !Subtarget.hasFP64())
1702     return false;
1703 
1704   // Look for a copy between even S-registers.  That is where we keep floats
1705   // when using NEON v2f32 instructions for f32 arithmetic.
1706   Register DstRegS = MI.getOperand(0).getReg();
1707   Register SrcRegS = MI.getOperand(1).getReg();
1708   if (!ARM::SPRRegClass.contains(DstRegS, SrcRegS))
1709     return false;
1710 
1711   const TargetRegisterInfo *TRI = &getRegisterInfo();
1712   unsigned DstRegD = TRI->getMatchingSuperReg(DstRegS, ARM::ssub_0,
1713                                               &ARM::DPRRegClass);
1714   unsigned SrcRegD = TRI->getMatchingSuperReg(SrcRegS, ARM::ssub_0,
1715                                               &ARM::DPRRegClass);
1716   if (!DstRegD || !SrcRegD)
1717     return false;
1718 
1719   // We want to widen this into a DstRegD = VMOVD SrcRegD copy.  This is only
1720   // legal if the COPY already defines the full DstRegD, and it isn't a
1721   // sub-register insertion.
1722   if (!MI.definesRegister(DstRegD, TRI) || MI.readsRegister(DstRegD, TRI))
1723     return false;
1724 
1725   // A dead copy shouldn't show up here, but reject it just in case.
1726   if (MI.getOperand(0).isDead())
1727     return false;
1728 
1729   // All clear, widen the COPY.
1730   LLVM_DEBUG(dbgs() << "widening:    " << MI);
1731   MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI);
1732 
1733   // Get rid of the old implicit-def of DstRegD.  Leave it if it defines a Q-reg
1734   // or some other super-register.
1735   int ImpDefIdx = MI.findRegisterDefOperandIdx(DstRegD);
1736   if (ImpDefIdx != -1)
1737     MI.removeOperand(ImpDefIdx);
1738 
1739   // Change the opcode and operands.
1740   MI.setDesc(get(ARM::VMOVD));
1741   MI.getOperand(0).setReg(DstRegD);
1742   MI.getOperand(1).setReg(SrcRegD);
1743   MIB.add(predOps(ARMCC::AL));
1744 
1745   // We are now reading SrcRegD instead of SrcRegS.  This may upset the
1746   // register scavenger and machine verifier, so we need to indicate that we
1747   // are reading an undefined value from SrcRegD, but a proper value from
1748   // SrcRegS.
1749   MI.getOperand(1).setIsUndef();
1750   MIB.addReg(SrcRegS, RegState::Implicit);
1751 
1752   // SrcRegD may actually contain an unrelated value in the ssub_1
1753   // sub-register.  Don't kill it.  Only kill the ssub_0 sub-register.
1754   if (MI.getOperand(1).isKill()) {
1755     MI.getOperand(1).setIsKill(false);
1756     MI.addRegisterKilled(SrcRegS, TRI, true);
1757   }
1758 
1759   LLVM_DEBUG(dbgs() << "replaced by: " << MI);
1760   return true;
1761 }
1762 
1763 /// Create a copy of a const pool value. Update CPI to the new index and return
1764 /// the label UID.
1765 static unsigned duplicateCPV(MachineFunction &MF, unsigned &CPI) {
1766   MachineConstantPool *MCP = MF.getConstantPool();
1767   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
1768 
1769   const MachineConstantPoolEntry &MCPE = MCP->getConstants()[CPI];
1770   assert(MCPE.isMachineConstantPoolEntry() &&
1771          "Expecting a machine constantpool entry!");
1772   ARMConstantPoolValue *ACPV =
1773     static_cast<ARMConstantPoolValue*>(MCPE.Val.MachineCPVal);
1774 
1775   unsigned PCLabelId = AFI->createPICLabelUId();
1776   ARMConstantPoolValue *NewCPV = nullptr;
1777 
1778   // FIXME: The below assumes PIC relocation model and that the function
1779   // is Thumb mode (t1 or t2). PCAdjustment would be 8 for ARM mode PIC, and
1780   // zero for non-PIC in ARM or Thumb. The callers are all of thumb LDR
1781   // instructions, so that's probably OK, but is PIC always correct when
1782   // we get here?
1783   if (ACPV->isGlobalValue())
1784     NewCPV = ARMConstantPoolConstant::Create(
1785         cast<ARMConstantPoolConstant>(ACPV)->getGV(), PCLabelId, ARMCP::CPValue,
1786         4, ACPV->getModifier(), ACPV->mustAddCurrentAddress());
1787   else if (ACPV->isExtSymbol())
1788     NewCPV = ARMConstantPoolSymbol::
1789       Create(MF.getFunction().getContext(),
1790              cast<ARMConstantPoolSymbol>(ACPV)->getSymbol(), PCLabelId, 4);
1791   else if (ACPV->isBlockAddress())
1792     NewCPV = ARMConstantPoolConstant::
1793       Create(cast<ARMConstantPoolConstant>(ACPV)->getBlockAddress(), PCLabelId,
1794              ARMCP::CPBlockAddress, 4);
1795   else if (ACPV->isLSDA())
1796     NewCPV = ARMConstantPoolConstant::Create(&MF.getFunction(), PCLabelId,
1797                                              ARMCP::CPLSDA, 4);
1798   else if (ACPV->isMachineBasicBlock())
1799     NewCPV = ARMConstantPoolMBB::
1800       Create(MF.getFunction().getContext(),
1801              cast<ARMConstantPoolMBB>(ACPV)->getMBB(), PCLabelId, 4);
1802   else
1803     llvm_unreachable("Unexpected ARM constantpool value type!!");
1804   CPI = MCP->getConstantPoolIndex(NewCPV, MCPE.getAlign());
1805   return PCLabelId;
1806 }
1807 
1808 void ARMBaseInstrInfo::reMaterialize(MachineBasicBlock &MBB,
1809                                      MachineBasicBlock::iterator I,
1810                                      Register DestReg, unsigned SubIdx,
1811                                      const MachineInstr &Orig,
1812                                      const TargetRegisterInfo &TRI) const {
1813   unsigned Opcode = Orig.getOpcode();
1814   switch (Opcode) {
1815   default: {
1816     MachineInstr *MI = MBB.getParent()->CloneMachineInstr(&Orig);
1817     MI->substituteRegister(Orig.getOperand(0).getReg(), DestReg, SubIdx, TRI);
1818     MBB.insert(I, MI);
1819     break;
1820   }
1821   case ARM::tLDRpci_pic:
1822   case ARM::t2LDRpci_pic: {
1823     MachineFunction &MF = *MBB.getParent();
1824     unsigned CPI = Orig.getOperand(1).getIndex();
1825     unsigned PCLabelId = duplicateCPV(MF, CPI);
1826     BuildMI(MBB, I, Orig.getDebugLoc(), get(Opcode), DestReg)
1827         .addConstantPoolIndex(CPI)
1828         .addImm(PCLabelId)
1829         .cloneMemRefs(Orig);
1830     break;
1831   }
1832   }
1833 }
1834 
1835 MachineInstr &
1836 ARMBaseInstrInfo::duplicate(MachineBasicBlock &MBB,
1837     MachineBasicBlock::iterator InsertBefore,
1838     const MachineInstr &Orig) const {
1839   MachineInstr &Cloned = TargetInstrInfo::duplicate(MBB, InsertBefore, Orig);
1840   MachineBasicBlock::instr_iterator I = Cloned.getIterator();
1841   for (;;) {
1842     switch (I->getOpcode()) {
1843     case ARM::tLDRpci_pic:
1844     case ARM::t2LDRpci_pic: {
1845       MachineFunction &MF = *MBB.getParent();
1846       unsigned CPI = I->getOperand(1).getIndex();
1847       unsigned PCLabelId = duplicateCPV(MF, CPI);
1848       I->getOperand(1).setIndex(CPI);
1849       I->getOperand(2).setImm(PCLabelId);
1850       break;
1851     }
1852     }
1853     if (!I->isBundledWithSucc())
1854       break;
1855     ++I;
1856   }
1857   return Cloned;
1858 }
1859 
1860 bool ARMBaseInstrInfo::produceSameValue(const MachineInstr &MI0,
1861                                         const MachineInstr &MI1,
1862                                         const MachineRegisterInfo *MRI) const {
1863   unsigned Opcode = MI0.getOpcode();
1864   if (Opcode == ARM::t2LDRpci || Opcode == ARM::t2LDRpci_pic ||
1865       Opcode == ARM::tLDRpci || Opcode == ARM::tLDRpci_pic ||
1866       Opcode == ARM::LDRLIT_ga_pcrel || Opcode == ARM::LDRLIT_ga_pcrel_ldr ||
1867       Opcode == ARM::tLDRLIT_ga_pcrel || Opcode == ARM::t2LDRLIT_ga_pcrel ||
1868       Opcode == ARM::MOV_ga_pcrel || Opcode == ARM::MOV_ga_pcrel_ldr ||
1869       Opcode == ARM::t2MOV_ga_pcrel) {
1870     if (MI1.getOpcode() != Opcode)
1871       return false;
1872     if (MI0.getNumOperands() != MI1.getNumOperands())
1873       return false;
1874 
1875     const MachineOperand &MO0 = MI0.getOperand(1);
1876     const MachineOperand &MO1 = MI1.getOperand(1);
1877     if (MO0.getOffset() != MO1.getOffset())
1878       return false;
1879 
1880     if (Opcode == ARM::LDRLIT_ga_pcrel || Opcode == ARM::LDRLIT_ga_pcrel_ldr ||
1881         Opcode == ARM::tLDRLIT_ga_pcrel || Opcode == ARM::t2LDRLIT_ga_pcrel ||
1882         Opcode == ARM::MOV_ga_pcrel || Opcode == ARM::MOV_ga_pcrel_ldr ||
1883         Opcode == ARM::t2MOV_ga_pcrel)
1884       // Ignore the PC labels.
1885       return MO0.getGlobal() == MO1.getGlobal();
1886 
1887     const MachineFunction *MF = MI0.getParent()->getParent();
1888     const MachineConstantPool *MCP = MF->getConstantPool();
1889     int CPI0 = MO0.getIndex();
1890     int CPI1 = MO1.getIndex();
1891     const MachineConstantPoolEntry &MCPE0 = MCP->getConstants()[CPI0];
1892     const MachineConstantPoolEntry &MCPE1 = MCP->getConstants()[CPI1];
1893     bool isARMCP0 = MCPE0.isMachineConstantPoolEntry();
1894     bool isARMCP1 = MCPE1.isMachineConstantPoolEntry();
1895     if (isARMCP0 && isARMCP1) {
1896       ARMConstantPoolValue *ACPV0 =
1897         static_cast<ARMConstantPoolValue*>(MCPE0.Val.MachineCPVal);
1898       ARMConstantPoolValue *ACPV1 =
1899         static_cast<ARMConstantPoolValue*>(MCPE1.Val.MachineCPVal);
1900       return ACPV0->hasSameValue(ACPV1);
1901     } else if (!isARMCP0 && !isARMCP1) {
1902       return MCPE0.Val.ConstVal == MCPE1.Val.ConstVal;
1903     }
1904     return false;
1905   } else if (Opcode == ARM::PICLDR) {
1906     if (MI1.getOpcode() != Opcode)
1907       return false;
1908     if (MI0.getNumOperands() != MI1.getNumOperands())
1909       return false;
1910 
1911     Register Addr0 = MI0.getOperand(1).getReg();
1912     Register Addr1 = MI1.getOperand(1).getReg();
1913     if (Addr0 != Addr1) {
1914       if (!MRI || !Addr0.isVirtual() || !Addr1.isVirtual())
1915         return false;
1916 
1917       // This assumes SSA form.
1918       MachineInstr *Def0 = MRI->getVRegDef(Addr0);
1919       MachineInstr *Def1 = MRI->getVRegDef(Addr1);
1920       // Check if the loaded value, e.g. a constantpool of a global address, are
1921       // the same.
1922       if (!produceSameValue(*Def0, *Def1, MRI))
1923         return false;
1924     }
1925 
1926     for (unsigned i = 3, e = MI0.getNumOperands(); i != e; ++i) {
1927       // %12 = PICLDR %11, 0, 14, %noreg
1928       const MachineOperand &MO0 = MI0.getOperand(i);
1929       const MachineOperand &MO1 = MI1.getOperand(i);
1930       if (!MO0.isIdenticalTo(MO1))
1931         return false;
1932     }
1933     return true;
1934   }
1935 
1936   return MI0.isIdenticalTo(MI1, MachineInstr::IgnoreVRegDefs);
1937 }
1938 
1939 /// areLoadsFromSameBasePtr - This is used by the pre-regalloc scheduler to
1940 /// determine if two loads are loading from the same base address. It should
1941 /// only return true if the base pointers are the same and the only differences
1942 /// between the two addresses is the offset. It also returns the offsets by
1943 /// reference.
1944 ///
1945 /// FIXME: remove this in favor of the MachineInstr interface once pre-RA-sched
1946 /// is permanently disabled.
1947 bool ARMBaseInstrInfo::areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2,
1948                                                int64_t &Offset1,
1949                                                int64_t &Offset2) const {
1950   // Don't worry about Thumb: just ARM and Thumb2.
1951   if (Subtarget.isThumb1Only()) return false;
1952 
1953   if (!Load1->isMachineOpcode() || !Load2->isMachineOpcode())
1954     return false;
1955 
1956   switch (Load1->getMachineOpcode()) {
1957   default:
1958     return false;
1959   case ARM::LDRi12:
1960   case ARM::LDRBi12:
1961   case ARM::LDRD:
1962   case ARM::LDRH:
1963   case ARM::LDRSB:
1964   case ARM::LDRSH:
1965   case ARM::VLDRD:
1966   case ARM::VLDRS:
1967   case ARM::t2LDRi8:
1968   case ARM::t2LDRBi8:
1969   case ARM::t2LDRDi8:
1970   case ARM::t2LDRSHi8:
1971   case ARM::t2LDRi12:
1972   case ARM::t2LDRBi12:
1973   case ARM::t2LDRSHi12:
1974     break;
1975   }
1976 
1977   switch (Load2->getMachineOpcode()) {
1978   default:
1979     return false;
1980   case ARM::LDRi12:
1981   case ARM::LDRBi12:
1982   case ARM::LDRD:
1983   case ARM::LDRH:
1984   case ARM::LDRSB:
1985   case ARM::LDRSH:
1986   case ARM::VLDRD:
1987   case ARM::VLDRS:
1988   case ARM::t2LDRi8:
1989   case ARM::t2LDRBi8:
1990   case ARM::t2LDRSHi8:
1991   case ARM::t2LDRi12:
1992   case ARM::t2LDRBi12:
1993   case ARM::t2LDRSHi12:
1994     break;
1995   }
1996 
1997   // Check if base addresses and chain operands match.
1998   if (Load1->getOperand(0) != Load2->getOperand(0) ||
1999       Load1->getOperand(4) != Load2->getOperand(4))
2000     return false;
2001 
2002   // Index should be Reg0.
2003   if (Load1->getOperand(3) != Load2->getOperand(3))
2004     return false;
2005 
2006   // Determine the offsets.
2007   if (isa<ConstantSDNode>(Load1->getOperand(1)) &&
2008       isa<ConstantSDNode>(Load2->getOperand(1))) {
2009     Offset1 = cast<ConstantSDNode>(Load1->getOperand(1))->getSExtValue();
2010     Offset2 = cast<ConstantSDNode>(Load2->getOperand(1))->getSExtValue();
2011     return true;
2012   }
2013 
2014   return false;
2015 }
2016 
2017 /// shouldScheduleLoadsNear - This is a used by the pre-regalloc scheduler to
2018 /// determine (in conjunction with areLoadsFromSameBasePtr) if two loads should
2019 /// be scheduled togther. On some targets if two loads are loading from
2020 /// addresses in the same cache line, it's better if they are scheduled
2021 /// together. This function takes two integers that represent the load offsets
2022 /// from the common base address. It returns true if it decides it's desirable
2023 /// to schedule the two loads together. "NumLoads" is the number of loads that
2024 /// have already been scheduled after Load1.
2025 ///
2026 /// FIXME: remove this in favor of the MachineInstr interface once pre-RA-sched
2027 /// is permanently disabled.
2028 bool ARMBaseInstrInfo::shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2,
2029                                                int64_t Offset1, int64_t Offset2,
2030                                                unsigned NumLoads) const {
2031   // Don't worry about Thumb: just ARM and Thumb2.
2032   if (Subtarget.isThumb1Only()) return false;
2033 
2034   assert(Offset2 > Offset1);
2035 
2036   if ((Offset2 - Offset1) / 8 > 64)
2037     return false;
2038 
2039   // Check if the machine opcodes are different. If they are different
2040   // then we consider them to not be of the same base address,
2041   // EXCEPT in the case of Thumb2 byte loads where one is LDRBi8 and the other LDRBi12.
2042   // In this case, they are considered to be the same because they are different
2043   // encoding forms of the same basic instruction.
2044   if ((Load1->getMachineOpcode() != Load2->getMachineOpcode()) &&
2045       !((Load1->getMachineOpcode() == ARM::t2LDRBi8 &&
2046          Load2->getMachineOpcode() == ARM::t2LDRBi12) ||
2047         (Load1->getMachineOpcode() == ARM::t2LDRBi12 &&
2048          Load2->getMachineOpcode() == ARM::t2LDRBi8)))
2049     return false;  // FIXME: overly conservative?
2050 
2051   // Four loads in a row should be sufficient.
2052   if (NumLoads >= 3)
2053     return false;
2054 
2055   return true;
2056 }
2057 
2058 bool ARMBaseInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
2059                                             const MachineBasicBlock *MBB,
2060                                             const MachineFunction &MF) const {
2061   // Debug info is never a scheduling boundary. It's necessary to be explicit
2062   // due to the special treatment of IT instructions below, otherwise a
2063   // dbg_value followed by an IT will result in the IT instruction being
2064   // considered a scheduling hazard, which is wrong. It should be the actual
2065   // instruction preceding the dbg_value instruction(s), just like it is
2066   // when debug info is not present.
2067   if (MI.isDebugInstr())
2068     return false;
2069 
2070   // Terminators and labels can't be scheduled around.
2071   if (MI.isTerminator() || MI.isPosition())
2072     return true;
2073 
2074   // INLINEASM_BR can jump to another block
2075   if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
2076     return true;
2077 
2078   if (isSEHInstruction(MI))
2079     return true;
2080 
2081   // Treat the start of the IT block as a scheduling boundary, but schedule
2082   // t2IT along with all instructions following it.
2083   // FIXME: This is a big hammer. But the alternative is to add all potential
2084   // true and anti dependencies to IT block instructions as implicit operands
2085   // to the t2IT instruction. The added compile time and complexity does not
2086   // seem worth it.
2087   MachineBasicBlock::const_iterator I = MI;
2088   // Make sure to skip any debug instructions
2089   while (++I != MBB->end() && I->isDebugInstr())
2090     ;
2091   if (I != MBB->end() && I->getOpcode() == ARM::t2IT)
2092     return true;
2093 
2094   // Don't attempt to schedule around any instruction that defines
2095   // a stack-oriented pointer, as it's unlikely to be profitable. This
2096   // saves compile time, because it doesn't require every single
2097   // stack slot reference to depend on the instruction that does the
2098   // modification.
2099   // Calls don't actually change the stack pointer, even if they have imp-defs.
2100   // No ARM calling conventions change the stack pointer. (X86 calling
2101   // conventions sometimes do).
2102   if (!MI.isCall() && MI.definesRegister(ARM::SP))
2103     return true;
2104 
2105   return false;
2106 }
2107 
2108 bool ARMBaseInstrInfo::
2109 isProfitableToIfCvt(MachineBasicBlock &MBB,
2110                     unsigned NumCycles, unsigned ExtraPredCycles,
2111                     BranchProbability Probability) const {
2112   if (!NumCycles)
2113     return false;
2114 
2115   // If we are optimizing for size, see if the branch in the predecessor can be
2116   // lowered to cbn?z by the constant island lowering pass, and return false if
2117   // so. This results in a shorter instruction sequence.
2118   if (MBB.getParent()->getFunction().hasOptSize()) {
2119     MachineBasicBlock *Pred = *MBB.pred_begin();
2120     if (!Pred->empty()) {
2121       MachineInstr *LastMI = &*Pred->rbegin();
2122       if (LastMI->getOpcode() == ARM::t2Bcc) {
2123         const TargetRegisterInfo *TRI = &getRegisterInfo();
2124         MachineInstr *CmpMI = findCMPToFoldIntoCBZ(LastMI, TRI);
2125         if (CmpMI)
2126           return false;
2127       }
2128     }
2129   }
2130   return isProfitableToIfCvt(MBB, NumCycles, ExtraPredCycles,
2131                              MBB, 0, 0, Probability);
2132 }
2133 
2134 bool ARMBaseInstrInfo::
2135 isProfitableToIfCvt(MachineBasicBlock &TBB,
2136                     unsigned TCycles, unsigned TExtra,
2137                     MachineBasicBlock &FBB,
2138                     unsigned FCycles, unsigned FExtra,
2139                     BranchProbability Probability) const {
2140   if (!TCycles)
2141     return false;
2142 
2143   // In thumb code we often end up trading one branch for a IT block, and
2144   // if we are cloning the instruction can increase code size. Prevent
2145   // blocks with multiple predecesors from being ifcvted to prevent this
2146   // cloning.
2147   if (Subtarget.isThumb2() && TBB.getParent()->getFunction().hasMinSize()) {
2148     if (TBB.pred_size() != 1 || FBB.pred_size() != 1)
2149       return false;
2150   }
2151 
2152   // Attempt to estimate the relative costs of predication versus branching.
2153   // Here we scale up each component of UnpredCost to avoid precision issue when
2154   // scaling TCycles/FCycles by Probability.
2155   const unsigned ScalingUpFactor = 1024;
2156 
2157   unsigned PredCost = (TCycles + FCycles + TExtra + FExtra) * ScalingUpFactor;
2158   unsigned UnpredCost;
2159   if (!Subtarget.hasBranchPredictor()) {
2160     // When we don't have a branch predictor it's always cheaper to not take a
2161     // branch than take it, so we have to take that into account.
2162     unsigned NotTakenBranchCost = 1;
2163     unsigned TakenBranchCost = Subtarget.getMispredictionPenalty();
2164     unsigned TUnpredCycles, FUnpredCycles;
2165     if (!FCycles) {
2166       // Triangle: TBB is the fallthrough
2167       TUnpredCycles = TCycles + NotTakenBranchCost;
2168       FUnpredCycles = TakenBranchCost;
2169     } else {
2170       // Diamond: TBB is the block that is branched to, FBB is the fallthrough
2171       TUnpredCycles = TCycles + TakenBranchCost;
2172       FUnpredCycles = FCycles + NotTakenBranchCost;
2173       // The branch at the end of FBB will disappear when it's predicated, so
2174       // discount it from PredCost.
2175       PredCost -= 1 * ScalingUpFactor;
2176     }
2177     // The total cost is the cost of each path scaled by their probabilites
2178     unsigned TUnpredCost = Probability.scale(TUnpredCycles * ScalingUpFactor);
2179     unsigned FUnpredCost = Probability.getCompl().scale(FUnpredCycles * ScalingUpFactor);
2180     UnpredCost = TUnpredCost + FUnpredCost;
2181     // When predicating assume that the first IT can be folded away but later
2182     // ones cost one cycle each
2183     if (Subtarget.isThumb2() && TCycles + FCycles > 4) {
2184       PredCost += ((TCycles + FCycles - 4) / 4) * ScalingUpFactor;
2185     }
2186   } else {
2187     unsigned TUnpredCost = Probability.scale(TCycles * ScalingUpFactor);
2188     unsigned FUnpredCost =
2189       Probability.getCompl().scale(FCycles * ScalingUpFactor);
2190     UnpredCost = TUnpredCost + FUnpredCost;
2191     UnpredCost += 1 * ScalingUpFactor; // The branch itself
2192     UnpredCost += Subtarget.getMispredictionPenalty() * ScalingUpFactor / 10;
2193   }
2194 
2195   return PredCost <= UnpredCost;
2196 }
2197 
2198 unsigned
2199 ARMBaseInstrInfo::extraSizeToPredicateInstructions(const MachineFunction &MF,
2200                                                    unsigned NumInsts) const {
2201   // Thumb2 needs a 2-byte IT instruction to predicate up to 4 instructions.
2202   // ARM has a condition code field in every predicable instruction, using it
2203   // doesn't change code size.
2204   if (!Subtarget.isThumb2())
2205     return 0;
2206 
2207   // It's possible that the size of the IT is restricted to a single block.
2208   unsigned MaxInsts = Subtarget.restrictIT() ? 1 : 4;
2209   return divideCeil(NumInsts, MaxInsts) * 2;
2210 }
2211 
2212 unsigned
2213 ARMBaseInstrInfo::predictBranchSizeForIfCvt(MachineInstr &MI) const {
2214   // If this branch is likely to be folded into the comparison to form a
2215   // CB(N)Z, then removing it won't reduce code size at all, because that will
2216   // just replace the CB(N)Z with a CMP.
2217   if (MI.getOpcode() == ARM::t2Bcc &&
2218       findCMPToFoldIntoCBZ(&MI, &getRegisterInfo()))
2219     return 0;
2220 
2221   unsigned Size = getInstSizeInBytes(MI);
2222 
2223   // For Thumb2, all branches are 32-bit instructions during the if conversion
2224   // pass, but may be replaced with 16-bit instructions during size reduction.
2225   // Since the branches considered by if conversion tend to be forward branches
2226   // over small basic blocks, they are very likely to be in range for the
2227   // narrow instructions, so we assume the final code size will be half what it
2228   // currently is.
2229   if (Subtarget.isThumb2())
2230     Size /= 2;
2231 
2232   return Size;
2233 }
2234 
2235 bool
2236 ARMBaseInstrInfo::isProfitableToUnpredicate(MachineBasicBlock &TMBB,
2237                                             MachineBasicBlock &FMBB) const {
2238   // Reduce false anti-dependencies to let the target's out-of-order execution
2239   // engine do its thing.
2240   return Subtarget.isProfitableToUnpredicate();
2241 }
2242 
2243 /// getInstrPredicate - If instruction is predicated, returns its predicate
2244 /// condition, otherwise returns AL. It also returns the condition code
2245 /// register by reference.
2246 ARMCC::CondCodes llvm::getInstrPredicate(const MachineInstr &MI,
2247                                          Register &PredReg) {
2248   int PIdx = MI.findFirstPredOperandIdx();
2249   if (PIdx == -1) {
2250     PredReg = 0;
2251     return ARMCC::AL;
2252   }
2253 
2254   PredReg = MI.getOperand(PIdx+1).getReg();
2255   return (ARMCC::CondCodes)MI.getOperand(PIdx).getImm();
2256 }
2257 
2258 unsigned llvm::getMatchingCondBranchOpcode(unsigned Opc) {
2259   if (Opc == ARM::B)
2260     return ARM::Bcc;
2261   if (Opc == ARM::tB)
2262     return ARM::tBcc;
2263   if (Opc == ARM::t2B)
2264     return ARM::t2Bcc;
2265 
2266   llvm_unreachable("Unknown unconditional branch opcode!");
2267 }
2268 
2269 MachineInstr *ARMBaseInstrInfo::commuteInstructionImpl(MachineInstr &MI,
2270                                                        bool NewMI,
2271                                                        unsigned OpIdx1,
2272                                                        unsigned OpIdx2) const {
2273   switch (MI.getOpcode()) {
2274   case ARM::MOVCCr:
2275   case ARM::t2MOVCCr: {
2276     // MOVCC can be commuted by inverting the condition.
2277     Register PredReg;
2278     ARMCC::CondCodes CC = getInstrPredicate(MI, PredReg);
2279     // MOVCC AL can't be inverted. Shouldn't happen.
2280     if (CC == ARMCC::AL || PredReg != ARM::CPSR)
2281       return nullptr;
2282     MachineInstr *CommutedMI =
2283         TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
2284     if (!CommutedMI)
2285       return nullptr;
2286     // After swapping the MOVCC operands, also invert the condition.
2287     CommutedMI->getOperand(CommutedMI->findFirstPredOperandIdx())
2288         .setImm(ARMCC::getOppositeCondition(CC));
2289     return CommutedMI;
2290   }
2291   }
2292   return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
2293 }
2294 
2295 /// Identify instructions that can be folded into a MOVCC instruction, and
2296 /// return the defining instruction.
2297 MachineInstr *
2298 ARMBaseInstrInfo::canFoldIntoMOVCC(Register Reg, const MachineRegisterInfo &MRI,
2299                                    const TargetInstrInfo *TII) const {
2300   if (!Reg.isVirtual())
2301     return nullptr;
2302   if (!MRI.hasOneNonDBGUse(Reg))
2303     return nullptr;
2304   MachineInstr *MI = MRI.getVRegDef(Reg);
2305   if (!MI)
2306     return nullptr;
2307   // Check if MI can be predicated and folded into the MOVCC.
2308   if (!isPredicable(*MI))
2309     return nullptr;
2310   // Check if MI has any non-dead defs or physreg uses. This also detects
2311   // predicated instructions which will be reading CPSR.
2312   for (const MachineOperand &MO : llvm::drop_begin(MI->operands(), 1)) {
2313     // Reject frame index operands, PEI can't handle the predicated pseudos.
2314     if (MO.isFI() || MO.isCPI() || MO.isJTI())
2315       return nullptr;
2316     if (!MO.isReg())
2317       continue;
2318     // MI can't have any tied operands, that would conflict with predication.
2319     if (MO.isTied())
2320       return nullptr;
2321     if (MO.getReg().isPhysical())
2322       return nullptr;
2323     if (MO.isDef() && !MO.isDead())
2324       return nullptr;
2325   }
2326   bool DontMoveAcrossStores = true;
2327   if (!MI->isSafeToMove(/* AliasAnalysis = */ nullptr, DontMoveAcrossStores))
2328     return nullptr;
2329   return MI;
2330 }
2331 
2332 bool ARMBaseInstrInfo::analyzeSelect(const MachineInstr &MI,
2333                                      SmallVectorImpl<MachineOperand> &Cond,
2334                                      unsigned &TrueOp, unsigned &FalseOp,
2335                                      bool &Optimizable) const {
2336   assert((MI.getOpcode() == ARM::MOVCCr || MI.getOpcode() == ARM::t2MOVCCr) &&
2337          "Unknown select instruction");
2338   // MOVCC operands:
2339   // 0: Def.
2340   // 1: True use.
2341   // 2: False use.
2342   // 3: Condition code.
2343   // 4: CPSR use.
2344   TrueOp = 1;
2345   FalseOp = 2;
2346   Cond.push_back(MI.getOperand(3));
2347   Cond.push_back(MI.getOperand(4));
2348   // We can always fold a def.
2349   Optimizable = true;
2350   return false;
2351 }
2352 
2353 MachineInstr *
2354 ARMBaseInstrInfo::optimizeSelect(MachineInstr &MI,
2355                                  SmallPtrSetImpl<MachineInstr *> &SeenMIs,
2356                                  bool PreferFalse) const {
2357   assert((MI.getOpcode() == ARM::MOVCCr || MI.getOpcode() == ARM::t2MOVCCr) &&
2358          "Unknown select instruction");
2359   MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
2360   MachineInstr *DefMI = canFoldIntoMOVCC(MI.getOperand(2).getReg(), MRI, this);
2361   bool Invert = !DefMI;
2362   if (!DefMI)
2363     DefMI = canFoldIntoMOVCC(MI.getOperand(1).getReg(), MRI, this);
2364   if (!DefMI)
2365     return nullptr;
2366 
2367   // Find new register class to use.
2368   MachineOperand FalseReg = MI.getOperand(Invert ? 2 : 1);
2369   MachineOperand TrueReg = MI.getOperand(Invert ? 1 : 2);
2370   Register DestReg = MI.getOperand(0).getReg();
2371   const TargetRegisterClass *FalseClass = MRI.getRegClass(FalseReg.getReg());
2372   const TargetRegisterClass *TrueClass = MRI.getRegClass(TrueReg.getReg());
2373   if (!MRI.constrainRegClass(DestReg, FalseClass))
2374     return nullptr;
2375   if (!MRI.constrainRegClass(DestReg, TrueClass))
2376     return nullptr;
2377 
2378   // Create a new predicated version of DefMI.
2379   // Rfalse is the first use.
2380   MachineInstrBuilder NewMI =
2381       BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), DefMI->getDesc(), DestReg);
2382 
2383   // Copy all the DefMI operands, excluding its (null) predicate.
2384   const MCInstrDesc &DefDesc = DefMI->getDesc();
2385   for (unsigned i = 1, e = DefDesc.getNumOperands();
2386        i != e && !DefDesc.operands()[i].isPredicate(); ++i)
2387     NewMI.add(DefMI->getOperand(i));
2388 
2389   unsigned CondCode = MI.getOperand(3).getImm();
2390   if (Invert)
2391     NewMI.addImm(ARMCC::getOppositeCondition(ARMCC::CondCodes(CondCode)));
2392   else
2393     NewMI.addImm(CondCode);
2394   NewMI.add(MI.getOperand(4));
2395 
2396   // DefMI is not the -S version that sets CPSR, so add an optional %noreg.
2397   if (NewMI->hasOptionalDef())
2398     NewMI.add(condCodeOp());
2399 
2400   // The output register value when the predicate is false is an implicit
2401   // register operand tied to the first def.
2402   // The tie makes the register allocator ensure the FalseReg is allocated the
2403   // same register as operand 0.
2404   FalseReg.setImplicit();
2405   NewMI.add(FalseReg);
2406   NewMI->tieOperands(0, NewMI->getNumOperands() - 1);
2407 
2408   // Update SeenMIs set: register newly created MI and erase removed DefMI.
2409   SeenMIs.insert(NewMI);
2410   SeenMIs.erase(DefMI);
2411 
2412   // If MI is inside a loop, and DefMI is outside the loop, then kill flags on
2413   // DefMI would be invalid when tranferred inside the loop.  Checking for a
2414   // loop is expensive, but at least remove kill flags if they are in different
2415   // BBs.
2416   if (DefMI->getParent() != MI.getParent())
2417     NewMI->clearKillInfo();
2418 
2419   // The caller will erase MI, but not DefMI.
2420   DefMI->eraseFromParent();
2421   return NewMI;
2422 }
2423 
2424 /// Map pseudo instructions that imply an 'S' bit onto real opcodes. Whether the
2425 /// instruction is encoded with an 'S' bit is determined by the optional CPSR
2426 /// def operand.
2427 ///
2428 /// This will go away once we can teach tblgen how to set the optional CPSR def
2429 /// operand itself.
2430 struct AddSubFlagsOpcodePair {
2431   uint16_t PseudoOpc;
2432   uint16_t MachineOpc;
2433 };
2434 
2435 static const AddSubFlagsOpcodePair AddSubFlagsOpcodeMap[] = {
2436   {ARM::ADDSri, ARM::ADDri},
2437   {ARM::ADDSrr, ARM::ADDrr},
2438   {ARM::ADDSrsi, ARM::ADDrsi},
2439   {ARM::ADDSrsr, ARM::ADDrsr},
2440 
2441   {ARM::SUBSri, ARM::SUBri},
2442   {ARM::SUBSrr, ARM::SUBrr},
2443   {ARM::SUBSrsi, ARM::SUBrsi},
2444   {ARM::SUBSrsr, ARM::SUBrsr},
2445 
2446   {ARM::RSBSri, ARM::RSBri},
2447   {ARM::RSBSrsi, ARM::RSBrsi},
2448   {ARM::RSBSrsr, ARM::RSBrsr},
2449 
2450   {ARM::tADDSi3, ARM::tADDi3},
2451   {ARM::tADDSi8, ARM::tADDi8},
2452   {ARM::tADDSrr, ARM::tADDrr},
2453   {ARM::tADCS, ARM::tADC},
2454 
2455   {ARM::tSUBSi3, ARM::tSUBi3},
2456   {ARM::tSUBSi8, ARM::tSUBi8},
2457   {ARM::tSUBSrr, ARM::tSUBrr},
2458   {ARM::tSBCS, ARM::tSBC},
2459   {ARM::tRSBS, ARM::tRSB},
2460   {ARM::tLSLSri, ARM::tLSLri},
2461 
2462   {ARM::t2ADDSri, ARM::t2ADDri},
2463   {ARM::t2ADDSrr, ARM::t2ADDrr},
2464   {ARM::t2ADDSrs, ARM::t2ADDrs},
2465 
2466   {ARM::t2SUBSri, ARM::t2SUBri},
2467   {ARM::t2SUBSrr, ARM::t2SUBrr},
2468   {ARM::t2SUBSrs, ARM::t2SUBrs},
2469 
2470   {ARM::t2RSBSri, ARM::t2RSBri},
2471   {ARM::t2RSBSrs, ARM::t2RSBrs},
2472 };
2473 
2474 unsigned llvm::convertAddSubFlagsOpcode(unsigned OldOpc) {
2475   for (const auto &Entry : AddSubFlagsOpcodeMap)
2476     if (OldOpc == Entry.PseudoOpc)
2477       return Entry.MachineOpc;
2478   return 0;
2479 }
2480 
2481 void llvm::emitARMRegPlusImmediate(MachineBasicBlock &MBB,
2482                                    MachineBasicBlock::iterator &MBBI,
2483                                    const DebugLoc &dl, Register DestReg,
2484                                    Register BaseReg, int NumBytes,
2485                                    ARMCC::CondCodes Pred, Register PredReg,
2486                                    const ARMBaseInstrInfo &TII,
2487                                    unsigned MIFlags) {
2488   if (NumBytes == 0 && DestReg != BaseReg) {
2489     BuildMI(MBB, MBBI, dl, TII.get(ARM::MOVr), DestReg)
2490         .addReg(BaseReg, RegState::Kill)
2491         .add(predOps(Pred, PredReg))
2492         .add(condCodeOp())
2493         .setMIFlags(MIFlags);
2494     return;
2495   }
2496 
2497   bool isSub = NumBytes < 0;
2498   if (isSub) NumBytes = -NumBytes;
2499 
2500   while (NumBytes) {
2501     unsigned RotAmt = ARM_AM::getSOImmValRotate(NumBytes);
2502     unsigned ThisVal = NumBytes & ARM_AM::rotr32(0xFF, RotAmt);
2503     assert(ThisVal && "Didn't extract field correctly");
2504 
2505     // We will handle these bits from offset, clear them.
2506     NumBytes &= ~ThisVal;
2507 
2508     assert(ARM_AM::getSOImmVal(ThisVal) != -1 && "Bit extraction didn't work?");
2509 
2510     // Build the new ADD / SUB.
2511     unsigned Opc = isSub ? ARM::SUBri : ARM::ADDri;
2512     BuildMI(MBB, MBBI, dl, TII.get(Opc), DestReg)
2513         .addReg(BaseReg, RegState::Kill)
2514         .addImm(ThisVal)
2515         .add(predOps(Pred, PredReg))
2516         .add(condCodeOp())
2517         .setMIFlags(MIFlags);
2518     BaseReg = DestReg;
2519   }
2520 }
2521 
2522 bool llvm::tryFoldSPUpdateIntoPushPop(const ARMSubtarget &Subtarget,
2523                                       MachineFunction &MF, MachineInstr *MI,
2524                                       unsigned NumBytes) {
2525   // This optimisation potentially adds lots of load and store
2526   // micro-operations, it's only really a great benefit to code-size.
2527   if (!Subtarget.hasMinSize())
2528     return false;
2529 
2530   // If only one register is pushed/popped, LLVM can use an LDR/STR
2531   // instead. We can't modify those so make sure we're dealing with an
2532   // instruction we understand.
2533   bool IsPop = isPopOpcode(MI->getOpcode());
2534   bool IsPush = isPushOpcode(MI->getOpcode());
2535   if (!IsPush && !IsPop)
2536     return false;
2537 
2538   bool IsVFPPushPop = MI->getOpcode() == ARM::VSTMDDB_UPD ||
2539                       MI->getOpcode() == ARM::VLDMDIA_UPD;
2540   bool IsT1PushPop = MI->getOpcode() == ARM::tPUSH ||
2541                      MI->getOpcode() == ARM::tPOP ||
2542                      MI->getOpcode() == ARM::tPOP_RET;
2543 
2544   assert((IsT1PushPop || (MI->getOperand(0).getReg() == ARM::SP &&
2545                           MI->getOperand(1).getReg() == ARM::SP)) &&
2546          "trying to fold sp update into non-sp-updating push/pop");
2547 
2548   // The VFP push & pop act on D-registers, so we can only fold an adjustment
2549   // by a multiple of 8 bytes in correctly. Similarly rN is 4-bytes. Don't try
2550   // if this is violated.
2551   if (NumBytes % (IsVFPPushPop ? 8 : 4) != 0)
2552     return false;
2553 
2554   // ARM and Thumb2 push/pop insts have explicit "sp, sp" operands (+
2555   // pred) so the list starts at 4. Thumb1 starts after the predicate.
2556   int RegListIdx = IsT1PushPop ? 2 : 4;
2557 
2558   // Calculate the space we'll need in terms of registers.
2559   unsigned RegsNeeded;
2560   const TargetRegisterClass *RegClass;
2561   if (IsVFPPushPop) {
2562     RegsNeeded = NumBytes / 8;
2563     RegClass = &ARM::DPRRegClass;
2564   } else {
2565     RegsNeeded = NumBytes / 4;
2566     RegClass = &ARM::GPRRegClass;
2567   }
2568 
2569   // We're going to have to strip all list operands off before
2570   // re-adding them since the order matters, so save the existing ones
2571   // for later.
2572   SmallVector<MachineOperand, 4> RegList;
2573 
2574   // We're also going to need the first register transferred by this
2575   // instruction, which won't necessarily be the first register in the list.
2576   unsigned FirstRegEnc = -1;
2577 
2578   const TargetRegisterInfo *TRI = MF.getRegInfo().getTargetRegisterInfo();
2579   for (int i = MI->getNumOperands() - 1; i >= RegListIdx; --i) {
2580     MachineOperand &MO = MI->getOperand(i);
2581     RegList.push_back(MO);
2582 
2583     if (MO.isReg() && !MO.isImplicit() &&
2584         TRI->getEncodingValue(MO.getReg()) < FirstRegEnc)
2585       FirstRegEnc = TRI->getEncodingValue(MO.getReg());
2586   }
2587 
2588   const MCPhysReg *CSRegs = TRI->getCalleeSavedRegs(&MF);
2589 
2590   // Now try to find enough space in the reglist to allocate NumBytes.
2591   for (int CurRegEnc = FirstRegEnc - 1; CurRegEnc >= 0 && RegsNeeded;
2592        --CurRegEnc) {
2593     unsigned CurReg = RegClass->getRegister(CurRegEnc);
2594     if (IsT1PushPop && CurRegEnc > TRI->getEncodingValue(ARM::R7))
2595       continue;
2596     if (!IsPop) {
2597       // Pushing any register is completely harmless, mark the register involved
2598       // as undef since we don't care about its value and must not restore it
2599       // during stack unwinding.
2600       RegList.push_back(MachineOperand::CreateReg(CurReg, false, false,
2601                                                   false, false, true));
2602       --RegsNeeded;
2603       continue;
2604     }
2605 
2606     // However, we can only pop an extra register if it's not live. For
2607     // registers live within the function we might clobber a return value
2608     // register; the other way a register can be live here is if it's
2609     // callee-saved.
2610     if (isCalleeSavedRegister(CurReg, CSRegs) ||
2611         MI->getParent()->computeRegisterLiveness(TRI, CurReg, MI) !=
2612         MachineBasicBlock::LQR_Dead) {
2613       // VFP pops don't allow holes in the register list, so any skip is fatal
2614       // for our transformation. GPR pops do, so we should just keep looking.
2615       if (IsVFPPushPop)
2616         return false;
2617       else
2618         continue;
2619     }
2620 
2621     // Mark the unimportant registers as <def,dead> in the POP.
2622     RegList.push_back(MachineOperand::CreateReg(CurReg, true, false, false,
2623                                                 true));
2624     --RegsNeeded;
2625   }
2626 
2627   if (RegsNeeded > 0)
2628     return false;
2629 
2630   // Finally we know we can profitably perform the optimisation so go
2631   // ahead: strip all existing registers off and add them back again
2632   // in the right order.
2633   for (int i = MI->getNumOperands() - 1; i >= RegListIdx; --i)
2634     MI->removeOperand(i);
2635 
2636   // Add the complete list back in.
2637   MachineInstrBuilder MIB(MF, &*MI);
2638   for (const MachineOperand &MO : llvm::reverse(RegList))
2639     MIB.add(MO);
2640 
2641   return true;
2642 }
2643 
2644 bool llvm::rewriteARMFrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
2645                                 Register FrameReg, int &Offset,
2646                                 const ARMBaseInstrInfo &TII) {
2647   unsigned Opcode = MI.getOpcode();
2648   const MCInstrDesc &Desc = MI.getDesc();
2649   unsigned AddrMode = (Desc.TSFlags & ARMII::AddrModeMask);
2650   bool isSub = false;
2651 
2652   // Memory operands in inline assembly always use AddrMode2.
2653   if (Opcode == ARM::INLINEASM || Opcode == ARM::INLINEASM_BR)
2654     AddrMode = ARMII::AddrMode2;
2655 
2656   if (Opcode == ARM::ADDri) {
2657     Offset += MI.getOperand(FrameRegIdx+1).getImm();
2658     if (Offset == 0) {
2659       // Turn it into a move.
2660       MI.setDesc(TII.get(ARM::MOVr));
2661       MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
2662       MI.removeOperand(FrameRegIdx+1);
2663       Offset = 0;
2664       return true;
2665     } else if (Offset < 0) {
2666       Offset = -Offset;
2667       isSub = true;
2668       MI.setDesc(TII.get(ARM::SUBri));
2669     }
2670 
2671     // Common case: small offset, fits into instruction.
2672     if (ARM_AM::getSOImmVal(Offset) != -1) {
2673       // Replace the FrameIndex with sp / fp
2674       MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
2675       MI.getOperand(FrameRegIdx+1).ChangeToImmediate(Offset);
2676       Offset = 0;
2677       return true;
2678     }
2679 
2680     // Otherwise, pull as much of the immedidate into this ADDri/SUBri
2681     // as possible.
2682     unsigned RotAmt = ARM_AM::getSOImmValRotate(Offset);
2683     unsigned ThisImmVal = Offset & ARM_AM::rotr32(0xFF, RotAmt);
2684 
2685     // We will handle these bits from offset, clear them.
2686     Offset &= ~ThisImmVal;
2687 
2688     // Get the properly encoded SOImmVal field.
2689     assert(ARM_AM::getSOImmVal(ThisImmVal) != -1 &&
2690            "Bit extraction didn't work?");
2691     MI.getOperand(FrameRegIdx+1).ChangeToImmediate(ThisImmVal);
2692  } else {
2693     unsigned ImmIdx = 0;
2694     int InstrOffs = 0;
2695     unsigned NumBits = 0;
2696     unsigned Scale = 1;
2697     switch (AddrMode) {
2698     case ARMII::AddrMode_i12:
2699       ImmIdx = FrameRegIdx + 1;
2700       InstrOffs = MI.getOperand(ImmIdx).getImm();
2701       NumBits = 12;
2702       break;
2703     case ARMII::AddrMode2:
2704       ImmIdx = FrameRegIdx+2;
2705       InstrOffs = ARM_AM::getAM2Offset(MI.getOperand(ImmIdx).getImm());
2706       if (ARM_AM::getAM2Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub)
2707         InstrOffs *= -1;
2708       NumBits = 12;
2709       break;
2710     case ARMII::AddrMode3:
2711       ImmIdx = FrameRegIdx+2;
2712       InstrOffs = ARM_AM::getAM3Offset(MI.getOperand(ImmIdx).getImm());
2713       if (ARM_AM::getAM3Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub)
2714         InstrOffs *= -1;
2715       NumBits = 8;
2716       break;
2717     case ARMII::AddrMode4:
2718     case ARMII::AddrMode6:
2719       // Can't fold any offset even if it's zero.
2720       return false;
2721     case ARMII::AddrMode5:
2722       ImmIdx = FrameRegIdx+1;
2723       InstrOffs = ARM_AM::getAM5Offset(MI.getOperand(ImmIdx).getImm());
2724       if (ARM_AM::getAM5Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub)
2725         InstrOffs *= -1;
2726       NumBits = 8;
2727       Scale = 4;
2728       break;
2729     case ARMII::AddrMode5FP16:
2730       ImmIdx = FrameRegIdx+1;
2731       InstrOffs = ARM_AM::getAM5Offset(MI.getOperand(ImmIdx).getImm());
2732       if (ARM_AM::getAM5Op(MI.getOperand(ImmIdx).getImm()) == ARM_AM::sub)
2733         InstrOffs *= -1;
2734       NumBits = 8;
2735       Scale = 2;
2736       break;
2737     case ARMII::AddrModeT2_i7:
2738     case ARMII::AddrModeT2_i7s2:
2739     case ARMII::AddrModeT2_i7s4:
2740       ImmIdx = FrameRegIdx+1;
2741       InstrOffs = MI.getOperand(ImmIdx).getImm();
2742       NumBits = 7;
2743       Scale = (AddrMode == ARMII::AddrModeT2_i7s2 ? 2 :
2744                AddrMode == ARMII::AddrModeT2_i7s4 ? 4 : 1);
2745       break;
2746     default:
2747       llvm_unreachable("Unsupported addressing mode!");
2748     }
2749 
2750     Offset += InstrOffs * Scale;
2751     assert((Offset & (Scale-1)) == 0 && "Can't encode this offset!");
2752     if (Offset < 0) {
2753       Offset = -Offset;
2754       isSub = true;
2755     }
2756 
2757     // Attempt to fold address comp. if opcode has offset bits
2758     if (NumBits > 0) {
2759       // Common case: small offset, fits into instruction.
2760       MachineOperand &ImmOp = MI.getOperand(ImmIdx);
2761       int ImmedOffset = Offset / Scale;
2762       unsigned Mask = (1 << NumBits) - 1;
2763       if ((unsigned)Offset <= Mask * Scale) {
2764         // Replace the FrameIndex with sp
2765         MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
2766         // FIXME: When addrmode2 goes away, this will simplify (like the
2767         // T2 version), as the LDR.i12 versions don't need the encoding
2768         // tricks for the offset value.
2769         if (isSub) {
2770           if (AddrMode == ARMII::AddrMode_i12)
2771             ImmedOffset = -ImmedOffset;
2772           else
2773             ImmedOffset |= 1 << NumBits;
2774         }
2775         ImmOp.ChangeToImmediate(ImmedOffset);
2776         Offset = 0;
2777         return true;
2778       }
2779 
2780       // Otherwise, it didn't fit. Pull in what we can to simplify the immed.
2781       ImmedOffset = ImmedOffset & Mask;
2782       if (isSub) {
2783         if (AddrMode == ARMII::AddrMode_i12)
2784           ImmedOffset = -ImmedOffset;
2785         else
2786           ImmedOffset |= 1 << NumBits;
2787       }
2788       ImmOp.ChangeToImmediate(ImmedOffset);
2789       Offset &= ~(Mask*Scale);
2790     }
2791   }
2792 
2793   Offset = (isSub) ? -Offset : Offset;
2794   return Offset == 0;
2795 }
2796 
2797 /// analyzeCompare - For a comparison instruction, return the source registers
2798 /// in SrcReg and SrcReg2 if having two register operands, and the value it
2799 /// compares against in CmpValue. Return true if the comparison instruction
2800 /// can be analyzed.
2801 bool ARMBaseInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
2802                                       Register &SrcReg2, int64_t &CmpMask,
2803                                       int64_t &CmpValue) const {
2804   switch (MI.getOpcode()) {
2805   default: break;
2806   case ARM::CMPri:
2807   case ARM::t2CMPri:
2808   case ARM::tCMPi8:
2809     SrcReg = MI.getOperand(0).getReg();
2810     SrcReg2 = 0;
2811     CmpMask = ~0;
2812     CmpValue = MI.getOperand(1).getImm();
2813     return true;
2814   case ARM::CMPrr:
2815   case ARM::t2CMPrr:
2816   case ARM::tCMPr:
2817     SrcReg = MI.getOperand(0).getReg();
2818     SrcReg2 = MI.getOperand(1).getReg();
2819     CmpMask = ~0;
2820     CmpValue = 0;
2821     return true;
2822   case ARM::TSTri:
2823   case ARM::t2TSTri:
2824     SrcReg = MI.getOperand(0).getReg();
2825     SrcReg2 = 0;
2826     CmpMask = MI.getOperand(1).getImm();
2827     CmpValue = 0;
2828     return true;
2829   }
2830 
2831   return false;
2832 }
2833 
2834 /// isSuitableForMask - Identify a suitable 'and' instruction that
2835 /// operates on the given source register and applies the same mask
2836 /// as a 'tst' instruction. Provide a limited look-through for copies.
2837 /// When successful, MI will hold the found instruction.
2838 static bool isSuitableForMask(MachineInstr *&MI, Register SrcReg,
2839                               int CmpMask, bool CommonUse) {
2840   switch (MI->getOpcode()) {
2841     case ARM::ANDri:
2842     case ARM::t2ANDri:
2843       if (CmpMask != MI->getOperand(2).getImm())
2844         return false;
2845       if (SrcReg == MI->getOperand(CommonUse ? 1 : 0).getReg())
2846         return true;
2847       break;
2848   }
2849 
2850   return false;
2851 }
2852 
2853 /// getCmpToAddCondition - assume the flags are set by CMP(a,b), return
2854 /// the condition code if we modify the instructions such that flags are
2855 /// set by ADD(a,b,X).
2856 inline static ARMCC::CondCodes getCmpToAddCondition(ARMCC::CondCodes CC) {
2857   switch (CC) {
2858   default: return ARMCC::AL;
2859   case ARMCC::HS: return ARMCC::LO;
2860   case ARMCC::LO: return ARMCC::HS;
2861   case ARMCC::VS: return ARMCC::VS;
2862   case ARMCC::VC: return ARMCC::VC;
2863   }
2864 }
2865 
2866 /// isRedundantFlagInstr - check whether the first instruction, whose only
2867 /// purpose is to update flags, can be made redundant.
2868 /// CMPrr can be made redundant by SUBrr if the operands are the same.
2869 /// CMPri can be made redundant by SUBri if the operands are the same.
2870 /// CMPrr(r0, r1) can be made redundant by ADDr[ri](r0, r1, X).
2871 /// This function can be extended later on.
2872 inline static bool isRedundantFlagInstr(const MachineInstr *CmpI,
2873                                         Register SrcReg, Register SrcReg2,
2874                                         int64_t ImmValue,
2875                                         const MachineInstr *OI,
2876                                         bool &IsThumb1) {
2877   if ((CmpI->getOpcode() == ARM::CMPrr || CmpI->getOpcode() == ARM::t2CMPrr) &&
2878       (OI->getOpcode() == ARM::SUBrr || OI->getOpcode() == ARM::t2SUBrr) &&
2879       ((OI->getOperand(1).getReg() == SrcReg &&
2880         OI->getOperand(2).getReg() == SrcReg2) ||
2881        (OI->getOperand(1).getReg() == SrcReg2 &&
2882         OI->getOperand(2).getReg() == SrcReg))) {
2883     IsThumb1 = false;
2884     return true;
2885   }
2886 
2887   if (CmpI->getOpcode() == ARM::tCMPr && OI->getOpcode() == ARM::tSUBrr &&
2888       ((OI->getOperand(2).getReg() == SrcReg &&
2889         OI->getOperand(3).getReg() == SrcReg2) ||
2890        (OI->getOperand(2).getReg() == SrcReg2 &&
2891         OI->getOperand(3).getReg() == SrcReg))) {
2892     IsThumb1 = true;
2893     return true;
2894   }
2895 
2896   if ((CmpI->getOpcode() == ARM::CMPri || CmpI->getOpcode() == ARM::t2CMPri) &&
2897       (OI->getOpcode() == ARM::SUBri || OI->getOpcode() == ARM::t2SUBri) &&
2898       OI->getOperand(1).getReg() == SrcReg &&
2899       OI->getOperand(2).getImm() == ImmValue) {
2900     IsThumb1 = false;
2901     return true;
2902   }
2903 
2904   if (CmpI->getOpcode() == ARM::tCMPi8 &&
2905       (OI->getOpcode() == ARM::tSUBi8 || OI->getOpcode() == ARM::tSUBi3) &&
2906       OI->getOperand(2).getReg() == SrcReg &&
2907       OI->getOperand(3).getImm() == ImmValue) {
2908     IsThumb1 = true;
2909     return true;
2910   }
2911 
2912   if ((CmpI->getOpcode() == ARM::CMPrr || CmpI->getOpcode() == ARM::t2CMPrr) &&
2913       (OI->getOpcode() == ARM::ADDrr || OI->getOpcode() == ARM::t2ADDrr ||
2914        OI->getOpcode() == ARM::ADDri || OI->getOpcode() == ARM::t2ADDri) &&
2915       OI->getOperand(0).isReg() && OI->getOperand(1).isReg() &&
2916       OI->getOperand(0).getReg() == SrcReg &&
2917       OI->getOperand(1).getReg() == SrcReg2) {
2918     IsThumb1 = false;
2919     return true;
2920   }
2921 
2922   if (CmpI->getOpcode() == ARM::tCMPr &&
2923       (OI->getOpcode() == ARM::tADDi3 || OI->getOpcode() == ARM::tADDi8 ||
2924        OI->getOpcode() == ARM::tADDrr) &&
2925       OI->getOperand(0).getReg() == SrcReg &&
2926       OI->getOperand(2).getReg() == SrcReg2) {
2927     IsThumb1 = true;
2928     return true;
2929   }
2930 
2931   return false;
2932 }
2933 
2934 static bool isOptimizeCompareCandidate(MachineInstr *MI, bool &IsThumb1) {
2935   switch (MI->getOpcode()) {
2936   default: return false;
2937   case ARM::tLSLri:
2938   case ARM::tLSRri:
2939   case ARM::tLSLrr:
2940   case ARM::tLSRrr:
2941   case ARM::tSUBrr:
2942   case ARM::tADDrr:
2943   case ARM::tADDi3:
2944   case ARM::tADDi8:
2945   case ARM::tSUBi3:
2946   case ARM::tSUBi8:
2947   case ARM::tMUL:
2948   case ARM::tADC:
2949   case ARM::tSBC:
2950   case ARM::tRSB:
2951   case ARM::tAND:
2952   case ARM::tORR:
2953   case ARM::tEOR:
2954   case ARM::tBIC:
2955   case ARM::tMVN:
2956   case ARM::tASRri:
2957   case ARM::tASRrr:
2958   case ARM::tROR:
2959     IsThumb1 = true;
2960     [[fallthrough]];
2961   case ARM::RSBrr:
2962   case ARM::RSBri:
2963   case ARM::RSCrr:
2964   case ARM::RSCri:
2965   case ARM::ADDrr:
2966   case ARM::ADDri:
2967   case ARM::ADCrr:
2968   case ARM::ADCri:
2969   case ARM::SUBrr:
2970   case ARM::SUBri:
2971   case ARM::SBCrr:
2972   case ARM::SBCri:
2973   case ARM::t2RSBri:
2974   case ARM::t2ADDrr:
2975   case ARM::t2ADDri:
2976   case ARM::t2ADCrr:
2977   case ARM::t2ADCri:
2978   case ARM::t2SUBrr:
2979   case ARM::t2SUBri:
2980   case ARM::t2SBCrr:
2981   case ARM::t2SBCri:
2982   case ARM::ANDrr:
2983   case ARM::ANDri:
2984   case ARM::ANDrsr:
2985   case ARM::ANDrsi:
2986   case ARM::t2ANDrr:
2987   case ARM::t2ANDri:
2988   case ARM::t2ANDrs:
2989   case ARM::ORRrr:
2990   case ARM::ORRri:
2991   case ARM::ORRrsr:
2992   case ARM::ORRrsi:
2993   case ARM::t2ORRrr:
2994   case ARM::t2ORRri:
2995   case ARM::t2ORRrs:
2996   case ARM::EORrr:
2997   case ARM::EORri:
2998   case ARM::EORrsr:
2999   case ARM::EORrsi:
3000   case ARM::t2EORrr:
3001   case ARM::t2EORri:
3002   case ARM::t2EORrs:
3003   case ARM::BICri:
3004   case ARM::BICrr:
3005   case ARM::BICrsi:
3006   case ARM::BICrsr:
3007   case ARM::t2BICri:
3008   case ARM::t2BICrr:
3009   case ARM::t2BICrs:
3010   case ARM::t2LSRri:
3011   case ARM::t2LSRrr:
3012   case ARM::t2LSLri:
3013   case ARM::t2LSLrr:
3014   case ARM::MOVsr:
3015   case ARM::MOVsi:
3016     return true;
3017   }
3018 }
3019 
3020 /// optimizeCompareInstr - Convert the instruction supplying the argument to the
3021 /// comparison into one that sets the zero bit in the flags register;
3022 /// Remove a redundant Compare instruction if an earlier instruction can set the
3023 /// flags in the same way as Compare.
3024 /// E.g. SUBrr(r1,r2) and CMPrr(r1,r2). We also handle the case where two
3025 /// operands are swapped: SUBrr(r1,r2) and CMPrr(r2,r1), by updating the
3026 /// condition code of instructions which use the flags.
3027 bool ARMBaseInstrInfo::optimizeCompareInstr(
3028     MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask,
3029     int64_t CmpValue, const MachineRegisterInfo *MRI) const {
3030   // Get the unique definition of SrcReg.
3031   MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
3032   if (!MI) return false;
3033 
3034   // Masked compares sometimes use the same register as the corresponding 'and'.
3035   if (CmpMask != ~0) {
3036     if (!isSuitableForMask(MI, SrcReg, CmpMask, false) || isPredicated(*MI)) {
3037       MI = nullptr;
3038       for (MachineRegisterInfo::use_instr_iterator
3039            UI = MRI->use_instr_begin(SrcReg), UE = MRI->use_instr_end();
3040            UI != UE; ++UI) {
3041         if (UI->getParent() != CmpInstr.getParent())
3042           continue;
3043         MachineInstr *PotentialAND = &*UI;
3044         if (!isSuitableForMask(PotentialAND, SrcReg, CmpMask, true) ||
3045             isPredicated(*PotentialAND))
3046           continue;
3047         MI = PotentialAND;
3048         break;
3049       }
3050       if (!MI) return false;
3051     }
3052   }
3053 
3054   // Get ready to iterate backward from CmpInstr.
3055   MachineBasicBlock::iterator I = CmpInstr, E = MI,
3056                               B = CmpInstr.getParent()->begin();
3057 
3058   // Early exit if CmpInstr is at the beginning of the BB.
3059   if (I == B) return false;
3060 
3061   // There are two possible candidates which can be changed to set CPSR:
3062   // One is MI, the other is a SUB or ADD instruction.
3063   // For CMPrr(r1,r2), we are looking for SUB(r1,r2), SUB(r2,r1), or
3064   // ADDr[ri](r1, r2, X).
3065   // For CMPri(r1, CmpValue), we are looking for SUBri(r1, CmpValue).
3066   MachineInstr *SubAdd = nullptr;
3067   if (SrcReg2 != 0)
3068     // MI is not a candidate for CMPrr.
3069     MI = nullptr;
3070   else if (MI->getParent() != CmpInstr.getParent() || CmpValue != 0) {
3071     // Conservatively refuse to convert an instruction which isn't in the same
3072     // BB as the comparison.
3073     // For CMPri w/ CmpValue != 0, a SubAdd may still be a candidate.
3074     // Thus we cannot return here.
3075     if (CmpInstr.getOpcode() == ARM::CMPri ||
3076         CmpInstr.getOpcode() == ARM::t2CMPri ||
3077         CmpInstr.getOpcode() == ARM::tCMPi8)
3078       MI = nullptr;
3079     else
3080       return false;
3081   }
3082 
3083   bool IsThumb1 = false;
3084   if (MI && !isOptimizeCompareCandidate(MI, IsThumb1))
3085     return false;
3086 
3087   // We also want to do this peephole for cases like this: if (a*b == 0),
3088   // and optimise away the CMP instruction from the generated code sequence:
3089   // MULS, MOVS, MOVS, CMP. Here the MOVS instructions load the boolean values
3090   // resulting from the select instruction, but these MOVS instructions for
3091   // Thumb1 (V6M) are flag setting and are thus preventing this optimisation.
3092   // However, if we only have MOVS instructions in between the CMP and the
3093   // other instruction (the MULS in this example), then the CPSR is dead so we
3094   // can safely reorder the sequence into: MOVS, MOVS, MULS, CMP. We do this
3095   // reordering and then continue the analysis hoping we can eliminate the
3096   // CMP. This peephole works on the vregs, so is still in SSA form. As a
3097   // consequence, the movs won't redefine/kill the MUL operands which would
3098   // make this reordering illegal.
3099   const TargetRegisterInfo *TRI = &getRegisterInfo();
3100   if (MI && IsThumb1) {
3101     --I;
3102     if (I != E && !MI->readsRegister(ARM::CPSR, TRI)) {
3103       bool CanReorder = true;
3104       for (; I != E; --I) {
3105         if (I->getOpcode() != ARM::tMOVi8) {
3106           CanReorder = false;
3107           break;
3108         }
3109       }
3110       if (CanReorder) {
3111         MI = MI->removeFromParent();
3112         E = CmpInstr;
3113         CmpInstr.getParent()->insert(E, MI);
3114       }
3115     }
3116     I = CmpInstr;
3117     E = MI;
3118   }
3119 
3120   // Check that CPSR isn't set between the comparison instruction and the one we
3121   // want to change. At the same time, search for SubAdd.
3122   bool SubAddIsThumb1 = false;
3123   do {
3124     const MachineInstr &Instr = *--I;
3125 
3126     // Check whether CmpInstr can be made redundant by the current instruction.
3127     if (isRedundantFlagInstr(&CmpInstr, SrcReg, SrcReg2, CmpValue, &Instr,
3128                              SubAddIsThumb1)) {
3129       SubAdd = &*I;
3130       break;
3131     }
3132 
3133     // Allow E (which was initially MI) to be SubAdd but do not search before E.
3134     if (I == E)
3135       break;
3136 
3137     if (Instr.modifiesRegister(ARM::CPSR, TRI) ||
3138         Instr.readsRegister(ARM::CPSR, TRI))
3139       // This instruction modifies or uses CPSR after the one we want to
3140       // change. We can't do this transformation.
3141       return false;
3142 
3143     if (I == B) {
3144       // In some cases, we scan the use-list of an instruction for an AND;
3145       // that AND is in the same BB, but may not be scheduled before the
3146       // corresponding TST.  In that case, bail out.
3147       //
3148       // FIXME: We could try to reschedule the AND.
3149       return false;
3150     }
3151   } while (true);
3152 
3153   // Return false if no candidates exist.
3154   if (!MI && !SubAdd)
3155     return false;
3156 
3157   // If we found a SubAdd, use it as it will be closer to the CMP
3158   if (SubAdd) {
3159     MI = SubAdd;
3160     IsThumb1 = SubAddIsThumb1;
3161   }
3162 
3163   // We can't use a predicated instruction - it doesn't always write the flags.
3164   if (isPredicated(*MI))
3165     return false;
3166 
3167   // Scan forward for the use of CPSR
3168   // When checking against MI: if it's a conditional code that requires
3169   // checking of the V bit or C bit, then this is not safe to do.
3170   // It is safe to remove CmpInstr if CPSR is redefined or killed.
3171   // If we are done with the basic block, we need to check whether CPSR is
3172   // live-out.
3173   SmallVector<std::pair<MachineOperand*, ARMCC::CondCodes>, 4>
3174       OperandsToUpdate;
3175   bool isSafe = false;
3176   I = CmpInstr;
3177   E = CmpInstr.getParent()->end();
3178   while (!isSafe && ++I != E) {
3179     const MachineInstr &Instr = *I;
3180     for (unsigned IO = 0, EO = Instr.getNumOperands();
3181          !isSafe && IO != EO; ++IO) {
3182       const MachineOperand &MO = Instr.getOperand(IO);
3183       if (MO.isRegMask() && MO.clobbersPhysReg(ARM::CPSR)) {
3184         isSafe = true;
3185         break;
3186       }
3187       if (!MO.isReg() || MO.getReg() != ARM::CPSR)
3188         continue;
3189       if (MO.isDef()) {
3190         isSafe = true;
3191         break;
3192       }
3193       // Condition code is after the operand before CPSR except for VSELs.
3194       ARMCC::CondCodes CC;
3195       bool IsInstrVSel = true;
3196       switch (Instr.getOpcode()) {
3197       default:
3198         IsInstrVSel = false;
3199         CC = (ARMCC::CondCodes)Instr.getOperand(IO - 1).getImm();
3200         break;
3201       case ARM::VSELEQD:
3202       case ARM::VSELEQS:
3203       case ARM::VSELEQH:
3204         CC = ARMCC::EQ;
3205         break;
3206       case ARM::VSELGTD:
3207       case ARM::VSELGTS:
3208       case ARM::VSELGTH:
3209         CC = ARMCC::GT;
3210         break;
3211       case ARM::VSELGED:
3212       case ARM::VSELGES:
3213       case ARM::VSELGEH:
3214         CC = ARMCC::GE;
3215         break;
3216       case ARM::VSELVSD:
3217       case ARM::VSELVSS:
3218       case ARM::VSELVSH:
3219         CC = ARMCC::VS;
3220         break;
3221       }
3222 
3223       if (SubAdd) {
3224         // If we have SUB(r1, r2) and CMP(r2, r1), the condition code based
3225         // on CMP needs to be updated to be based on SUB.
3226         // If we have ADD(r1, r2, X) and CMP(r1, r2), the condition code also
3227         // needs to be modified.
3228         // Push the condition code operands to OperandsToUpdate.
3229         // If it is safe to remove CmpInstr, the condition code of these
3230         // operands will be modified.
3231         unsigned Opc = SubAdd->getOpcode();
3232         bool IsSub = Opc == ARM::SUBrr || Opc == ARM::t2SUBrr ||
3233                      Opc == ARM::SUBri || Opc == ARM::t2SUBri ||
3234                      Opc == ARM::tSUBrr || Opc == ARM::tSUBi3 ||
3235                      Opc == ARM::tSUBi8;
3236         unsigned OpI = Opc != ARM::tSUBrr ? 1 : 2;
3237         if (!IsSub ||
3238             (SrcReg2 != 0 && SubAdd->getOperand(OpI).getReg() == SrcReg2 &&
3239              SubAdd->getOperand(OpI + 1).getReg() == SrcReg)) {
3240           // VSel doesn't support condition code update.
3241           if (IsInstrVSel)
3242             return false;
3243           // Ensure we can swap the condition.
3244           ARMCC::CondCodes NewCC = (IsSub ? getSwappedCondition(CC) : getCmpToAddCondition(CC));
3245           if (NewCC == ARMCC::AL)
3246             return false;
3247           OperandsToUpdate.push_back(
3248               std::make_pair(&((*I).getOperand(IO - 1)), NewCC));
3249         }
3250       } else {
3251         // No SubAdd, so this is x = <op> y, z; cmp x, 0.
3252         switch (CC) {
3253         case ARMCC::EQ: // Z
3254         case ARMCC::NE: // Z
3255         case ARMCC::MI: // N
3256         case ARMCC::PL: // N
3257         case ARMCC::AL: // none
3258           // CPSR can be used multiple times, we should continue.
3259           break;
3260         case ARMCC::HS: // C
3261         case ARMCC::LO: // C
3262         case ARMCC::VS: // V
3263         case ARMCC::VC: // V
3264         case ARMCC::HI: // C Z
3265         case ARMCC::LS: // C Z
3266         case ARMCC::GE: // N V
3267         case ARMCC::LT: // N V
3268         case ARMCC::GT: // Z N V
3269         case ARMCC::LE: // Z N V
3270           // The instruction uses the V bit or C bit which is not safe.
3271           return false;
3272         }
3273       }
3274     }
3275   }
3276 
3277   // If CPSR is not killed nor re-defined, we should check whether it is
3278   // live-out. If it is live-out, do not optimize.
3279   if (!isSafe) {
3280     MachineBasicBlock *MBB = CmpInstr.getParent();
3281     for (MachineBasicBlock *Succ : MBB->successors())
3282       if (Succ->isLiveIn(ARM::CPSR))
3283         return false;
3284   }
3285 
3286   // Toggle the optional operand to CPSR (if it exists - in Thumb1 we always
3287   // set CPSR so this is represented as an explicit output)
3288   if (!IsThumb1) {
3289     unsigned CPSRRegNum = MI->getNumExplicitOperands() - 1;
3290     MI->getOperand(CPSRRegNum).setReg(ARM::CPSR);
3291     MI->getOperand(CPSRRegNum).setIsDef(true);
3292   }
3293   assert(!isPredicated(*MI) && "Can't use flags from predicated instruction");
3294   CmpInstr.eraseFromParent();
3295 
3296   // Modify the condition code of operands in OperandsToUpdate.
3297   // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to
3298   // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
3299   for (unsigned i = 0, e = OperandsToUpdate.size(); i < e; i++)
3300     OperandsToUpdate[i].first->setImm(OperandsToUpdate[i].second);
3301 
3302   MI->clearRegisterDeads(ARM::CPSR);
3303 
3304   return true;
3305 }
3306 
3307 bool ARMBaseInstrInfo::shouldSink(const MachineInstr &MI) const {
3308   // Do not sink MI if it might be used to optimize a redundant compare.
3309   // We heuristically only look at the instruction immediately following MI to
3310   // avoid potentially searching the entire basic block.
3311   if (isPredicated(MI))
3312     return true;
3313   MachineBasicBlock::const_iterator Next = &MI;
3314   ++Next;
3315   Register SrcReg, SrcReg2;
3316   int64_t CmpMask, CmpValue;
3317   bool IsThumb1;
3318   if (Next != MI.getParent()->end() &&
3319       analyzeCompare(*Next, SrcReg, SrcReg2, CmpMask, CmpValue) &&
3320       isRedundantFlagInstr(&*Next, SrcReg, SrcReg2, CmpValue, &MI, IsThumb1))
3321     return false;
3322   return true;
3323 }
3324 
3325 bool ARMBaseInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
3326                                      Register Reg,
3327                                      MachineRegisterInfo *MRI) const {
3328   // Fold large immediates into add, sub, or, xor.
3329   unsigned DefOpc = DefMI.getOpcode();
3330   if (DefOpc != ARM::t2MOVi32imm && DefOpc != ARM::MOVi32imm)
3331     return false;
3332   if (!DefMI.getOperand(1).isImm())
3333     // Could be t2MOVi32imm @xx
3334     return false;
3335 
3336   if (!MRI->hasOneNonDBGUse(Reg))
3337     return false;
3338 
3339   const MCInstrDesc &DefMCID = DefMI.getDesc();
3340   if (DefMCID.hasOptionalDef()) {
3341     unsigned NumOps = DefMCID.getNumOperands();
3342     const MachineOperand &MO = DefMI.getOperand(NumOps - 1);
3343     if (MO.getReg() == ARM::CPSR && !MO.isDead())
3344       // If DefMI defines CPSR and it is not dead, it's obviously not safe
3345       // to delete DefMI.
3346       return false;
3347   }
3348 
3349   const MCInstrDesc &UseMCID = UseMI.getDesc();
3350   if (UseMCID.hasOptionalDef()) {
3351     unsigned NumOps = UseMCID.getNumOperands();
3352     if (UseMI.getOperand(NumOps - 1).getReg() == ARM::CPSR)
3353       // If the instruction sets the flag, do not attempt this optimization
3354       // since it may change the semantics of the code.
3355       return false;
3356   }
3357 
3358   unsigned UseOpc = UseMI.getOpcode();
3359   unsigned NewUseOpc = 0;
3360   uint32_t ImmVal = (uint32_t)DefMI.getOperand(1).getImm();
3361   uint32_t SOImmValV1 = 0, SOImmValV2 = 0;
3362   bool Commute = false;
3363   switch (UseOpc) {
3364   default: return false;
3365   case ARM::SUBrr:
3366   case ARM::ADDrr:
3367   case ARM::ORRrr:
3368   case ARM::EORrr:
3369   case ARM::t2SUBrr:
3370   case ARM::t2ADDrr:
3371   case ARM::t2ORRrr:
3372   case ARM::t2EORrr: {
3373     Commute = UseMI.getOperand(2).getReg() != Reg;
3374     switch (UseOpc) {
3375     default: break;
3376     case ARM::ADDrr:
3377     case ARM::SUBrr:
3378       if (UseOpc == ARM::SUBrr && Commute)
3379         return false;
3380 
3381       // ADD/SUB are special because they're essentially the same operation, so
3382       // we can handle a larger range of immediates.
3383       if (ARM_AM::isSOImmTwoPartVal(ImmVal))
3384         NewUseOpc = UseOpc == ARM::ADDrr ? ARM::ADDri : ARM::SUBri;
3385       else if (ARM_AM::isSOImmTwoPartVal(-ImmVal)) {
3386         ImmVal = -ImmVal;
3387         NewUseOpc = UseOpc == ARM::ADDrr ? ARM::SUBri : ARM::ADDri;
3388       } else
3389         return false;
3390       SOImmValV1 = (uint32_t)ARM_AM::getSOImmTwoPartFirst(ImmVal);
3391       SOImmValV2 = (uint32_t)ARM_AM::getSOImmTwoPartSecond(ImmVal);
3392       break;
3393     case ARM::ORRrr:
3394     case ARM::EORrr:
3395       if (!ARM_AM::isSOImmTwoPartVal(ImmVal))
3396         return false;
3397       SOImmValV1 = (uint32_t)ARM_AM::getSOImmTwoPartFirst(ImmVal);
3398       SOImmValV2 = (uint32_t)ARM_AM::getSOImmTwoPartSecond(ImmVal);
3399       switch (UseOpc) {
3400       default: break;
3401       case ARM::ORRrr: NewUseOpc = ARM::ORRri; break;
3402       case ARM::EORrr: NewUseOpc = ARM::EORri; break;
3403       }
3404       break;
3405     case ARM::t2ADDrr:
3406     case ARM::t2SUBrr: {
3407       if (UseOpc == ARM::t2SUBrr && Commute)
3408         return false;
3409 
3410       // ADD/SUB are special because they're essentially the same operation, so
3411       // we can handle a larger range of immediates.
3412       const bool ToSP = DefMI.getOperand(0).getReg() == ARM::SP;
3413       const unsigned t2ADD = ToSP ? ARM::t2ADDspImm : ARM::t2ADDri;
3414       const unsigned t2SUB = ToSP ? ARM::t2SUBspImm : ARM::t2SUBri;
3415       if (ARM_AM::isT2SOImmTwoPartVal(ImmVal))
3416         NewUseOpc = UseOpc == ARM::t2ADDrr ? t2ADD : t2SUB;
3417       else if (ARM_AM::isT2SOImmTwoPartVal(-ImmVal)) {
3418         ImmVal = -ImmVal;
3419         NewUseOpc = UseOpc == ARM::t2ADDrr ? t2SUB : t2ADD;
3420       } else
3421         return false;
3422       SOImmValV1 = (uint32_t)ARM_AM::getT2SOImmTwoPartFirst(ImmVal);
3423       SOImmValV2 = (uint32_t)ARM_AM::getT2SOImmTwoPartSecond(ImmVal);
3424       break;
3425     }
3426     case ARM::t2ORRrr:
3427     case ARM::t2EORrr:
3428       if (!ARM_AM::isT2SOImmTwoPartVal(ImmVal))
3429         return false;
3430       SOImmValV1 = (uint32_t)ARM_AM::getT2SOImmTwoPartFirst(ImmVal);
3431       SOImmValV2 = (uint32_t)ARM_AM::getT2SOImmTwoPartSecond(ImmVal);
3432       switch (UseOpc) {
3433       default: break;
3434       case ARM::t2ORRrr: NewUseOpc = ARM::t2ORRri; break;
3435       case ARM::t2EORrr: NewUseOpc = ARM::t2EORri; break;
3436       }
3437       break;
3438     }
3439   }
3440   }
3441 
3442   unsigned OpIdx = Commute ? 2 : 1;
3443   Register Reg1 = UseMI.getOperand(OpIdx).getReg();
3444   bool isKill = UseMI.getOperand(OpIdx).isKill();
3445   const TargetRegisterClass *TRC = MRI->getRegClass(Reg);
3446   Register NewReg = MRI->createVirtualRegister(TRC);
3447   BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), get(NewUseOpc),
3448           NewReg)
3449       .addReg(Reg1, getKillRegState(isKill))
3450       .addImm(SOImmValV1)
3451       .add(predOps(ARMCC::AL))
3452       .add(condCodeOp());
3453   UseMI.setDesc(get(NewUseOpc));
3454   UseMI.getOperand(1).setReg(NewReg);
3455   UseMI.getOperand(1).setIsKill();
3456   UseMI.getOperand(2).ChangeToImmediate(SOImmValV2);
3457   DefMI.eraseFromParent();
3458   // FIXME: t2ADDrr should be split, as different rulles apply when writing to SP.
3459   // Just as t2ADDri, that was split to [t2ADDri, t2ADDspImm].
3460   // Then the below code will not be needed, as the input/output register
3461   // classes will be rgpr or gprSP.
3462   // For now, we fix the UseMI operand explicitly here:
3463   switch(NewUseOpc){
3464     case ARM::t2ADDspImm:
3465     case ARM::t2SUBspImm:
3466     case ARM::t2ADDri:
3467     case ARM::t2SUBri:
3468       MRI->constrainRegClass(UseMI.getOperand(0).getReg(), TRC);
3469   }
3470   return true;
3471 }
3472 
3473 static unsigned getNumMicroOpsSwiftLdSt(const InstrItineraryData *ItinData,
3474                                         const MachineInstr &MI) {
3475   switch (MI.getOpcode()) {
3476   default: {
3477     const MCInstrDesc &Desc = MI.getDesc();
3478     int UOps = ItinData->getNumMicroOps(Desc.getSchedClass());
3479     assert(UOps >= 0 && "bad # UOps");
3480     return UOps;
3481   }
3482 
3483   case ARM::LDRrs:
3484   case ARM::LDRBrs:
3485   case ARM::STRrs:
3486   case ARM::STRBrs: {
3487     unsigned ShOpVal = MI.getOperand(3).getImm();
3488     bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
3489     unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
3490     if (!isSub &&
3491         (ShImm == 0 ||
3492          ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3493           ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
3494       return 1;
3495     return 2;
3496   }
3497 
3498   case ARM::LDRH:
3499   case ARM::STRH: {
3500     if (!MI.getOperand(2).getReg())
3501       return 1;
3502 
3503     unsigned ShOpVal = MI.getOperand(3).getImm();
3504     bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
3505     unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
3506     if (!isSub &&
3507         (ShImm == 0 ||
3508          ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3509           ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
3510       return 1;
3511     return 2;
3512   }
3513 
3514   case ARM::LDRSB:
3515   case ARM::LDRSH:
3516     return (ARM_AM::getAM3Op(MI.getOperand(3).getImm()) == ARM_AM::sub) ? 3 : 2;
3517 
3518   case ARM::LDRSB_POST:
3519   case ARM::LDRSH_POST: {
3520     Register Rt = MI.getOperand(0).getReg();
3521     Register Rm = MI.getOperand(3).getReg();
3522     return (Rt == Rm) ? 4 : 3;
3523   }
3524 
3525   case ARM::LDR_PRE_REG:
3526   case ARM::LDRB_PRE_REG: {
3527     Register Rt = MI.getOperand(0).getReg();
3528     Register Rm = MI.getOperand(3).getReg();
3529     if (Rt == Rm)
3530       return 3;
3531     unsigned ShOpVal = MI.getOperand(4).getImm();
3532     bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
3533     unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
3534     if (!isSub &&
3535         (ShImm == 0 ||
3536          ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3537           ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
3538       return 2;
3539     return 3;
3540   }
3541 
3542   case ARM::STR_PRE_REG:
3543   case ARM::STRB_PRE_REG: {
3544     unsigned ShOpVal = MI.getOperand(4).getImm();
3545     bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
3546     unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
3547     if (!isSub &&
3548         (ShImm == 0 ||
3549          ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3550           ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
3551       return 2;
3552     return 3;
3553   }
3554 
3555   case ARM::LDRH_PRE:
3556   case ARM::STRH_PRE: {
3557     Register Rt = MI.getOperand(0).getReg();
3558     Register Rm = MI.getOperand(3).getReg();
3559     if (!Rm)
3560       return 2;
3561     if (Rt == Rm)
3562       return 3;
3563     return (ARM_AM::getAM3Op(MI.getOperand(4).getImm()) == ARM_AM::sub) ? 3 : 2;
3564   }
3565 
3566   case ARM::LDR_POST_REG:
3567   case ARM::LDRB_POST_REG:
3568   case ARM::LDRH_POST: {
3569     Register Rt = MI.getOperand(0).getReg();
3570     Register Rm = MI.getOperand(3).getReg();
3571     return (Rt == Rm) ? 3 : 2;
3572   }
3573 
3574   case ARM::LDR_PRE_IMM:
3575   case ARM::LDRB_PRE_IMM:
3576   case ARM::LDR_POST_IMM:
3577   case ARM::LDRB_POST_IMM:
3578   case ARM::STRB_POST_IMM:
3579   case ARM::STRB_POST_REG:
3580   case ARM::STRB_PRE_IMM:
3581   case ARM::STRH_POST:
3582   case ARM::STR_POST_IMM:
3583   case ARM::STR_POST_REG:
3584   case ARM::STR_PRE_IMM:
3585     return 2;
3586 
3587   case ARM::LDRSB_PRE:
3588   case ARM::LDRSH_PRE: {
3589     Register Rm = MI.getOperand(3).getReg();
3590     if (Rm == 0)
3591       return 3;
3592     Register Rt = MI.getOperand(0).getReg();
3593     if (Rt == Rm)
3594       return 4;
3595     unsigned ShOpVal = MI.getOperand(4).getImm();
3596     bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
3597     unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
3598     if (!isSub &&
3599         (ShImm == 0 ||
3600          ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
3601           ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
3602       return 3;
3603     return 4;
3604   }
3605 
3606   case ARM::LDRD: {
3607     Register Rt = MI.getOperand(0).getReg();
3608     Register Rn = MI.getOperand(2).getReg();
3609     Register Rm = MI.getOperand(3).getReg();
3610     if (Rm)
3611       return (ARM_AM::getAM3Op(MI.getOperand(4).getImm()) == ARM_AM::sub) ? 4
3612                                                                           : 3;
3613     return (Rt == Rn) ? 3 : 2;
3614   }
3615 
3616   case ARM::STRD: {
3617     Register Rm = MI.getOperand(3).getReg();
3618     if (Rm)
3619       return (ARM_AM::getAM3Op(MI.getOperand(4).getImm()) == ARM_AM::sub) ? 4
3620                                                                           : 3;
3621     return 2;
3622   }
3623 
3624   case ARM::LDRD_POST:
3625   case ARM::t2LDRD_POST:
3626     return 3;
3627 
3628   case ARM::STRD_POST:
3629   case ARM::t2STRD_POST:
3630     return 4;
3631 
3632   case ARM::LDRD_PRE: {
3633     Register Rt = MI.getOperand(0).getReg();
3634     Register Rn = MI.getOperand(3).getReg();
3635     Register Rm = MI.getOperand(4).getReg();
3636     if (Rm)
3637       return (ARM_AM::getAM3Op(MI.getOperand(5).getImm()) == ARM_AM::sub) ? 5
3638                                                                           : 4;
3639     return (Rt == Rn) ? 4 : 3;
3640   }
3641 
3642   case ARM::t2LDRD_PRE: {
3643     Register Rt = MI.getOperand(0).getReg();
3644     Register Rn = MI.getOperand(3).getReg();
3645     return (Rt == Rn) ? 4 : 3;
3646   }
3647 
3648   case ARM::STRD_PRE: {
3649     Register Rm = MI.getOperand(4).getReg();
3650     if (Rm)
3651       return (ARM_AM::getAM3Op(MI.getOperand(5).getImm()) == ARM_AM::sub) ? 5
3652                                                                           : 4;
3653     return 3;
3654   }
3655 
3656   case ARM::t2STRD_PRE:
3657     return 3;
3658 
3659   case ARM::t2LDR_POST:
3660   case ARM::t2LDRB_POST:
3661   case ARM::t2LDRB_PRE:
3662   case ARM::t2LDRSBi12:
3663   case ARM::t2LDRSBi8:
3664   case ARM::t2LDRSBpci:
3665   case ARM::t2LDRSBs:
3666   case ARM::t2LDRH_POST:
3667   case ARM::t2LDRH_PRE:
3668   case ARM::t2LDRSBT:
3669   case ARM::t2LDRSB_POST:
3670   case ARM::t2LDRSB_PRE:
3671   case ARM::t2LDRSH_POST:
3672   case ARM::t2LDRSH_PRE:
3673   case ARM::t2LDRSHi12:
3674   case ARM::t2LDRSHi8:
3675   case ARM::t2LDRSHpci:
3676   case ARM::t2LDRSHs:
3677     return 2;
3678 
3679   case ARM::t2LDRDi8: {
3680     Register Rt = MI.getOperand(0).getReg();
3681     Register Rn = MI.getOperand(2).getReg();
3682     return (Rt == Rn) ? 3 : 2;
3683   }
3684 
3685   case ARM::t2STRB_POST:
3686   case ARM::t2STRB_PRE:
3687   case ARM::t2STRBs:
3688   case ARM::t2STRDi8:
3689   case ARM::t2STRH_POST:
3690   case ARM::t2STRH_PRE:
3691   case ARM::t2STRHs:
3692   case ARM::t2STR_POST:
3693   case ARM::t2STR_PRE:
3694   case ARM::t2STRs:
3695     return 2;
3696   }
3697 }
3698 
3699 // Return the number of 32-bit words loaded by LDM or stored by STM. If this
3700 // can't be easily determined return 0 (missing MachineMemOperand).
3701 //
3702 // FIXME: The current MachineInstr design does not support relying on machine
3703 // mem operands to determine the width of a memory access. Instead, we expect
3704 // the target to provide this information based on the instruction opcode and
3705 // operands. However, using MachineMemOperand is the best solution now for
3706 // two reasons:
3707 //
3708 // 1) getNumMicroOps tries to infer LDM memory width from the total number of MI
3709 // operands. This is much more dangerous than using the MachineMemOperand
3710 // sizes because CodeGen passes can insert/remove optional machine operands. In
3711 // fact, it's totally incorrect for preRA passes and appears to be wrong for
3712 // postRA passes as well.
3713 //
3714 // 2) getNumLDMAddresses is only used by the scheduling machine model and any
3715 // machine model that calls this should handle the unknown (zero size) case.
3716 //
3717 // Long term, we should require a target hook that verifies MachineMemOperand
3718 // sizes during MC lowering. That target hook should be local to MC lowering
3719 // because we can't ensure that it is aware of other MI forms. Doing this will
3720 // ensure that MachineMemOperands are correctly propagated through all passes.
3721 unsigned ARMBaseInstrInfo::getNumLDMAddresses(const MachineInstr &MI) const {
3722   unsigned Size = 0;
3723   for (MachineInstr::mmo_iterator I = MI.memoperands_begin(),
3724                                   E = MI.memoperands_end();
3725        I != E; ++I) {
3726     Size += (*I)->getSize();
3727   }
3728   // FIXME: The scheduler currently can't handle values larger than 16. But
3729   // the values can actually go up to 32 for floating-point load/store
3730   // multiple (VLDMIA etc.). Also, the way this code is reasoning about memory
3731   // operations isn't right; we could end up with "extra" memory operands for
3732   // various reasons, like tail merge merging two memory operations.
3733   return std::min(Size / 4, 16U);
3734 }
3735 
3736 static unsigned getNumMicroOpsSingleIssuePlusExtras(unsigned Opc,
3737                                                     unsigned NumRegs) {
3738   unsigned UOps = 1 + NumRegs; // 1 for address computation.
3739   switch (Opc) {
3740   default:
3741     break;
3742   case ARM::VLDMDIA_UPD:
3743   case ARM::VLDMDDB_UPD:
3744   case ARM::VLDMSIA_UPD:
3745   case ARM::VLDMSDB_UPD:
3746   case ARM::VSTMDIA_UPD:
3747   case ARM::VSTMDDB_UPD:
3748   case ARM::VSTMSIA_UPD:
3749   case ARM::VSTMSDB_UPD:
3750   case ARM::LDMIA_UPD:
3751   case ARM::LDMDA_UPD:
3752   case ARM::LDMDB_UPD:
3753   case ARM::LDMIB_UPD:
3754   case ARM::STMIA_UPD:
3755   case ARM::STMDA_UPD:
3756   case ARM::STMDB_UPD:
3757   case ARM::STMIB_UPD:
3758   case ARM::tLDMIA_UPD:
3759   case ARM::tSTMIA_UPD:
3760   case ARM::t2LDMIA_UPD:
3761   case ARM::t2LDMDB_UPD:
3762   case ARM::t2STMIA_UPD:
3763   case ARM::t2STMDB_UPD:
3764     ++UOps; // One for base register writeback.
3765     break;
3766   case ARM::LDMIA_RET:
3767   case ARM::tPOP_RET:
3768   case ARM::t2LDMIA_RET:
3769     UOps += 2; // One for base reg wb, one for write to pc.
3770     break;
3771   }
3772   return UOps;
3773 }
3774 
3775 unsigned ARMBaseInstrInfo::getNumMicroOps(const InstrItineraryData *ItinData,
3776                                           const MachineInstr &MI) const {
3777   if (!ItinData || ItinData->isEmpty())
3778     return 1;
3779 
3780   const MCInstrDesc &Desc = MI.getDesc();
3781   unsigned Class = Desc.getSchedClass();
3782   int ItinUOps = ItinData->getNumMicroOps(Class);
3783   if (ItinUOps >= 0) {
3784     if (Subtarget.isSwift() && (Desc.mayLoad() || Desc.mayStore()))
3785       return getNumMicroOpsSwiftLdSt(ItinData, MI);
3786 
3787     return ItinUOps;
3788   }
3789 
3790   unsigned Opc = MI.getOpcode();
3791   switch (Opc) {
3792   default:
3793     llvm_unreachable("Unexpected multi-uops instruction!");
3794   case ARM::VLDMQIA:
3795   case ARM::VSTMQIA:
3796     return 2;
3797 
3798   // The number of uOps for load / store multiple are determined by the number
3799   // registers.
3800   //
3801   // On Cortex-A8, each pair of register loads / stores can be scheduled on the
3802   // same cycle. The scheduling for the first load / store must be done
3803   // separately by assuming the address is not 64-bit aligned.
3804   //
3805   // On Cortex-A9, the formula is simply (#reg / 2) + (#reg % 2). If the address
3806   // is not 64-bit aligned, then AGU would take an extra cycle.  For VFP / NEON
3807   // load / store multiple, the formula is (#reg / 2) + (#reg % 2) + 1.
3808   case ARM::VLDMDIA:
3809   case ARM::VLDMDIA_UPD:
3810   case ARM::VLDMDDB_UPD:
3811   case ARM::VLDMSIA:
3812   case ARM::VLDMSIA_UPD:
3813   case ARM::VLDMSDB_UPD:
3814   case ARM::VSTMDIA:
3815   case ARM::VSTMDIA_UPD:
3816   case ARM::VSTMDDB_UPD:
3817   case ARM::VSTMSIA:
3818   case ARM::VSTMSIA_UPD:
3819   case ARM::VSTMSDB_UPD: {
3820     unsigned NumRegs = MI.getNumOperands() - Desc.getNumOperands();
3821     return (NumRegs / 2) + (NumRegs % 2) + 1;
3822   }
3823 
3824   case ARM::LDMIA_RET:
3825   case ARM::LDMIA:
3826   case ARM::LDMDA:
3827   case ARM::LDMDB:
3828   case ARM::LDMIB:
3829   case ARM::LDMIA_UPD:
3830   case ARM::LDMDA_UPD:
3831   case ARM::LDMDB_UPD:
3832   case ARM::LDMIB_UPD:
3833   case ARM::STMIA:
3834   case ARM::STMDA:
3835   case ARM::STMDB:
3836   case ARM::STMIB:
3837   case ARM::STMIA_UPD:
3838   case ARM::STMDA_UPD:
3839   case ARM::STMDB_UPD:
3840   case ARM::STMIB_UPD:
3841   case ARM::tLDMIA:
3842   case ARM::tLDMIA_UPD:
3843   case ARM::tSTMIA_UPD:
3844   case ARM::tPOP_RET:
3845   case ARM::tPOP:
3846   case ARM::tPUSH:
3847   case ARM::t2LDMIA_RET:
3848   case ARM::t2LDMIA:
3849   case ARM::t2LDMDB:
3850   case ARM::t2LDMIA_UPD:
3851   case ARM::t2LDMDB_UPD:
3852   case ARM::t2STMIA:
3853   case ARM::t2STMDB:
3854   case ARM::t2STMIA_UPD:
3855   case ARM::t2STMDB_UPD: {
3856     unsigned NumRegs = MI.getNumOperands() - Desc.getNumOperands() + 1;
3857     switch (Subtarget.getLdStMultipleTiming()) {
3858     case ARMSubtarget::SingleIssuePlusExtras:
3859       return getNumMicroOpsSingleIssuePlusExtras(Opc, NumRegs);
3860     case ARMSubtarget::SingleIssue:
3861       // Assume the worst.
3862       return NumRegs;
3863     case ARMSubtarget::DoubleIssue: {
3864       if (NumRegs < 4)
3865         return 2;
3866       // 4 registers would be issued: 2, 2.
3867       // 5 registers would be issued: 2, 2, 1.
3868       unsigned UOps = (NumRegs / 2);
3869       if (NumRegs % 2)
3870         ++UOps;
3871       return UOps;
3872     }
3873     case ARMSubtarget::DoubleIssueCheckUnalignedAccess: {
3874       unsigned UOps = (NumRegs / 2);
3875       // If there are odd number of registers or if it's not 64-bit aligned,
3876       // then it takes an extra AGU (Address Generation Unit) cycle.
3877       if ((NumRegs % 2) || !MI.hasOneMemOperand() ||
3878           (*MI.memoperands_begin())->getAlign() < Align(8))
3879         ++UOps;
3880       return UOps;
3881       }
3882     }
3883   }
3884   }
3885   llvm_unreachable("Didn't find the number of microops");
3886 }
3887 
3888 int
3889 ARMBaseInstrInfo::getVLDMDefCycle(const InstrItineraryData *ItinData,
3890                                   const MCInstrDesc &DefMCID,
3891                                   unsigned DefClass,
3892                                   unsigned DefIdx, unsigned DefAlign) const {
3893   int RegNo = (int)(DefIdx+1) - DefMCID.getNumOperands() + 1;
3894   if (RegNo <= 0)
3895     // Def is the address writeback.
3896     return ItinData->getOperandCycle(DefClass, DefIdx);
3897 
3898   int DefCycle;
3899   if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3900     // (regno / 2) + (regno % 2) + 1
3901     DefCycle = RegNo / 2 + 1;
3902     if (RegNo % 2)
3903       ++DefCycle;
3904   } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3905     DefCycle = RegNo;
3906     bool isSLoad = false;
3907 
3908     switch (DefMCID.getOpcode()) {
3909     default: break;
3910     case ARM::VLDMSIA:
3911     case ARM::VLDMSIA_UPD:
3912     case ARM::VLDMSDB_UPD:
3913       isSLoad = true;
3914       break;
3915     }
3916 
3917     // If there are odd number of 'S' registers or if it's not 64-bit aligned,
3918     // then it takes an extra cycle.
3919     if ((isSLoad && (RegNo % 2)) || DefAlign < 8)
3920       ++DefCycle;
3921   } else {
3922     // Assume the worst.
3923     DefCycle = RegNo + 2;
3924   }
3925 
3926   return DefCycle;
3927 }
3928 
3929 int
3930 ARMBaseInstrInfo::getLDMDefCycle(const InstrItineraryData *ItinData,
3931                                  const MCInstrDesc &DefMCID,
3932                                  unsigned DefClass,
3933                                  unsigned DefIdx, unsigned DefAlign) const {
3934   int RegNo = (int)(DefIdx+1) - DefMCID.getNumOperands() + 1;
3935   if (RegNo <= 0)
3936     // Def is the address writeback.
3937     return ItinData->getOperandCycle(DefClass, DefIdx);
3938 
3939   int DefCycle;
3940   if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3941     // 4 registers would be issued: 1, 2, 1.
3942     // 5 registers would be issued: 1, 2, 2.
3943     DefCycle = RegNo / 2;
3944     if (DefCycle < 1)
3945       DefCycle = 1;
3946     // Result latency is issue cycle + 2: E2.
3947     DefCycle += 2;
3948   } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3949     DefCycle = (RegNo / 2);
3950     // If there are odd number of registers or if it's not 64-bit aligned,
3951     // then it takes an extra AGU (Address Generation Unit) cycle.
3952     if ((RegNo % 2) || DefAlign < 8)
3953       ++DefCycle;
3954     // Result latency is AGU cycles + 2.
3955     DefCycle += 2;
3956   } else {
3957     // Assume the worst.
3958     DefCycle = RegNo + 2;
3959   }
3960 
3961   return DefCycle;
3962 }
3963 
3964 int
3965 ARMBaseInstrInfo::getVSTMUseCycle(const InstrItineraryData *ItinData,
3966                                   const MCInstrDesc &UseMCID,
3967                                   unsigned UseClass,
3968                                   unsigned UseIdx, unsigned UseAlign) const {
3969   int RegNo = (int)(UseIdx+1) - UseMCID.getNumOperands() + 1;
3970   if (RegNo <= 0)
3971     return ItinData->getOperandCycle(UseClass, UseIdx);
3972 
3973   int UseCycle;
3974   if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
3975     // (regno / 2) + (regno % 2) + 1
3976     UseCycle = RegNo / 2 + 1;
3977     if (RegNo % 2)
3978       ++UseCycle;
3979   } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
3980     UseCycle = RegNo;
3981     bool isSStore = false;
3982 
3983     switch (UseMCID.getOpcode()) {
3984     default: break;
3985     case ARM::VSTMSIA:
3986     case ARM::VSTMSIA_UPD:
3987     case ARM::VSTMSDB_UPD:
3988       isSStore = true;
3989       break;
3990     }
3991 
3992     // If there are odd number of 'S' registers or if it's not 64-bit aligned,
3993     // then it takes an extra cycle.
3994     if ((isSStore && (RegNo % 2)) || UseAlign < 8)
3995       ++UseCycle;
3996   } else {
3997     // Assume the worst.
3998     UseCycle = RegNo + 2;
3999   }
4000 
4001   return UseCycle;
4002 }
4003 
4004 int
4005 ARMBaseInstrInfo::getSTMUseCycle(const InstrItineraryData *ItinData,
4006                                  const MCInstrDesc &UseMCID,
4007                                  unsigned UseClass,
4008                                  unsigned UseIdx, unsigned UseAlign) const {
4009   int RegNo = (int)(UseIdx+1) - UseMCID.getNumOperands() + 1;
4010   if (RegNo <= 0)
4011     return ItinData->getOperandCycle(UseClass, UseIdx);
4012 
4013   int UseCycle;
4014   if (Subtarget.isCortexA8() || Subtarget.isCortexA7()) {
4015     UseCycle = RegNo / 2;
4016     if (UseCycle < 2)
4017       UseCycle = 2;
4018     // Read in E3.
4019     UseCycle += 2;
4020   } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
4021     UseCycle = (RegNo / 2);
4022     // If there are odd number of registers or if it's not 64-bit aligned,
4023     // then it takes an extra AGU (Address Generation Unit) cycle.
4024     if ((RegNo % 2) || UseAlign < 8)
4025       ++UseCycle;
4026   } else {
4027     // Assume the worst.
4028     UseCycle = 1;
4029   }
4030   return UseCycle;
4031 }
4032 
4033 int
4034 ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
4035                                     const MCInstrDesc &DefMCID,
4036                                     unsigned DefIdx, unsigned DefAlign,
4037                                     const MCInstrDesc &UseMCID,
4038                                     unsigned UseIdx, unsigned UseAlign) const {
4039   unsigned DefClass = DefMCID.getSchedClass();
4040   unsigned UseClass = UseMCID.getSchedClass();
4041 
4042   if (DefIdx < DefMCID.getNumDefs() && UseIdx < UseMCID.getNumOperands())
4043     return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx);
4044 
4045   // This may be a def / use of a variable_ops instruction, the operand
4046   // latency might be determinable dynamically. Let the target try to
4047   // figure it out.
4048   int DefCycle = -1;
4049   bool LdmBypass = false;
4050   switch (DefMCID.getOpcode()) {
4051   default:
4052     DefCycle = ItinData->getOperandCycle(DefClass, DefIdx);
4053     break;
4054 
4055   case ARM::VLDMDIA:
4056   case ARM::VLDMDIA_UPD:
4057   case ARM::VLDMDDB_UPD:
4058   case ARM::VLDMSIA:
4059   case ARM::VLDMSIA_UPD:
4060   case ARM::VLDMSDB_UPD:
4061     DefCycle = getVLDMDefCycle(ItinData, DefMCID, DefClass, DefIdx, DefAlign);
4062     break;
4063 
4064   case ARM::LDMIA_RET:
4065   case ARM::LDMIA:
4066   case ARM::LDMDA:
4067   case ARM::LDMDB:
4068   case ARM::LDMIB:
4069   case ARM::LDMIA_UPD:
4070   case ARM::LDMDA_UPD:
4071   case ARM::LDMDB_UPD:
4072   case ARM::LDMIB_UPD:
4073   case ARM::tLDMIA:
4074   case ARM::tLDMIA_UPD:
4075   case ARM::tPUSH:
4076   case ARM::t2LDMIA_RET:
4077   case ARM::t2LDMIA:
4078   case ARM::t2LDMDB:
4079   case ARM::t2LDMIA_UPD:
4080   case ARM::t2LDMDB_UPD:
4081     LdmBypass = true;
4082     DefCycle = getLDMDefCycle(ItinData, DefMCID, DefClass, DefIdx, DefAlign);
4083     break;
4084   }
4085 
4086   if (DefCycle == -1)
4087     // We can't seem to determine the result latency of the def, assume it's 2.
4088     DefCycle = 2;
4089 
4090   int UseCycle = -1;
4091   switch (UseMCID.getOpcode()) {
4092   default:
4093     UseCycle = ItinData->getOperandCycle(UseClass, UseIdx);
4094     break;
4095 
4096   case ARM::VSTMDIA:
4097   case ARM::VSTMDIA_UPD:
4098   case ARM::VSTMDDB_UPD:
4099   case ARM::VSTMSIA:
4100   case ARM::VSTMSIA_UPD:
4101   case ARM::VSTMSDB_UPD:
4102     UseCycle = getVSTMUseCycle(ItinData, UseMCID, UseClass, UseIdx, UseAlign);
4103     break;
4104 
4105   case ARM::STMIA:
4106   case ARM::STMDA:
4107   case ARM::STMDB:
4108   case ARM::STMIB:
4109   case ARM::STMIA_UPD:
4110   case ARM::STMDA_UPD:
4111   case ARM::STMDB_UPD:
4112   case ARM::STMIB_UPD:
4113   case ARM::tSTMIA_UPD:
4114   case ARM::tPOP_RET:
4115   case ARM::tPOP:
4116   case ARM::t2STMIA:
4117   case ARM::t2STMDB:
4118   case ARM::t2STMIA_UPD:
4119   case ARM::t2STMDB_UPD:
4120     UseCycle = getSTMUseCycle(ItinData, UseMCID, UseClass, UseIdx, UseAlign);
4121     break;
4122   }
4123 
4124   if (UseCycle == -1)
4125     // Assume it's read in the first stage.
4126     UseCycle = 1;
4127 
4128   UseCycle = DefCycle - UseCycle + 1;
4129   if (UseCycle > 0) {
4130     if (LdmBypass) {
4131       // It's a variable_ops instruction so we can't use DefIdx here. Just use
4132       // first def operand.
4133       if (ItinData->hasPipelineForwarding(DefClass, DefMCID.getNumOperands()-1,
4134                                           UseClass, UseIdx))
4135         --UseCycle;
4136     } else if (ItinData->hasPipelineForwarding(DefClass, DefIdx,
4137                                                UseClass, UseIdx)) {
4138       --UseCycle;
4139     }
4140   }
4141 
4142   return UseCycle;
4143 }
4144 
4145 static const MachineInstr *getBundledDefMI(const TargetRegisterInfo *TRI,
4146                                            const MachineInstr *MI, unsigned Reg,
4147                                            unsigned &DefIdx, unsigned &Dist) {
4148   Dist = 0;
4149 
4150   MachineBasicBlock::const_iterator I = MI; ++I;
4151   MachineBasicBlock::const_instr_iterator II = std::prev(I.getInstrIterator());
4152   assert(II->isInsideBundle() && "Empty bundle?");
4153 
4154   int Idx = -1;
4155   while (II->isInsideBundle()) {
4156     Idx = II->findRegisterDefOperandIdx(Reg, false, true, TRI);
4157     if (Idx != -1)
4158       break;
4159     --II;
4160     ++Dist;
4161   }
4162 
4163   assert(Idx != -1 && "Cannot find bundled definition!");
4164   DefIdx = Idx;
4165   return &*II;
4166 }
4167 
4168 static const MachineInstr *getBundledUseMI(const TargetRegisterInfo *TRI,
4169                                            const MachineInstr &MI, unsigned Reg,
4170                                            unsigned &UseIdx, unsigned &Dist) {
4171   Dist = 0;
4172 
4173   MachineBasicBlock::const_instr_iterator II = ++MI.getIterator();
4174   assert(II->isInsideBundle() && "Empty bundle?");
4175   MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
4176 
4177   // FIXME: This doesn't properly handle multiple uses.
4178   int Idx = -1;
4179   while (II != E && II->isInsideBundle()) {
4180     Idx = II->findRegisterUseOperandIdx(Reg, false, TRI);
4181     if (Idx != -1)
4182       break;
4183     if (II->getOpcode() != ARM::t2IT)
4184       ++Dist;
4185     ++II;
4186   }
4187 
4188   if (Idx == -1) {
4189     Dist = 0;
4190     return nullptr;
4191   }
4192 
4193   UseIdx = Idx;
4194   return &*II;
4195 }
4196 
4197 /// Return the number of cycles to add to (or subtract from) the static
4198 /// itinerary based on the def opcode and alignment. The caller will ensure that
4199 /// adjusted latency is at least one cycle.
4200 static int adjustDefLatency(const ARMSubtarget &Subtarget,
4201                             const MachineInstr &DefMI,
4202                             const MCInstrDesc &DefMCID, unsigned DefAlign) {
4203   int Adjust = 0;
4204   if (Subtarget.isCortexA8() || Subtarget.isLikeA9() || Subtarget.isCortexA7()) {
4205     // FIXME: Shifter op hack: no shift (i.e. [r +/- r]) or [r + r << 2]
4206     // variants are one cycle cheaper.
4207     switch (DefMCID.getOpcode()) {
4208     default: break;
4209     case ARM::LDRrs:
4210     case ARM::LDRBrs: {
4211       unsigned ShOpVal = DefMI.getOperand(3).getImm();
4212       unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
4213       if (ShImm == 0 ||
4214           (ShImm == 2 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))
4215         --Adjust;
4216       break;
4217     }
4218     case ARM::t2LDRs:
4219     case ARM::t2LDRBs:
4220     case ARM::t2LDRHs:
4221     case ARM::t2LDRSHs: {
4222       // Thumb2 mode: lsl only.
4223       unsigned ShAmt = DefMI.getOperand(3).getImm();
4224       if (ShAmt == 0 || ShAmt == 2)
4225         --Adjust;
4226       break;
4227     }
4228     }
4229   } else if (Subtarget.isSwift()) {
4230     // FIXME: Properly handle all of the latency adjustments for address
4231     // writeback.
4232     switch (DefMCID.getOpcode()) {
4233     default: break;
4234     case ARM::LDRrs:
4235     case ARM::LDRBrs: {
4236       unsigned ShOpVal = DefMI.getOperand(3).getImm();
4237       bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
4238       unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
4239       if (!isSub &&
4240           (ShImm == 0 ||
4241            ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
4242             ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
4243         Adjust -= 2;
4244       else if (!isSub &&
4245                ShImm == 1 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsr)
4246         --Adjust;
4247       break;
4248     }
4249     case ARM::t2LDRs:
4250     case ARM::t2LDRBs:
4251     case ARM::t2LDRHs:
4252     case ARM::t2LDRSHs: {
4253       // Thumb2 mode: lsl only.
4254       unsigned ShAmt = DefMI.getOperand(3).getImm();
4255       if (ShAmt == 0 || ShAmt == 1 || ShAmt == 2 || ShAmt == 3)
4256         Adjust -= 2;
4257       break;
4258     }
4259     }
4260   }
4261 
4262   if (DefAlign < 8 && Subtarget.checkVLDnAccessAlignment()) {
4263     switch (DefMCID.getOpcode()) {
4264     default: break;
4265     case ARM::VLD1q8:
4266     case ARM::VLD1q16:
4267     case ARM::VLD1q32:
4268     case ARM::VLD1q64:
4269     case ARM::VLD1q8wb_fixed:
4270     case ARM::VLD1q16wb_fixed:
4271     case ARM::VLD1q32wb_fixed:
4272     case ARM::VLD1q64wb_fixed:
4273     case ARM::VLD1q8wb_register:
4274     case ARM::VLD1q16wb_register:
4275     case ARM::VLD1q32wb_register:
4276     case ARM::VLD1q64wb_register:
4277     case ARM::VLD2d8:
4278     case ARM::VLD2d16:
4279     case ARM::VLD2d32:
4280     case ARM::VLD2q8:
4281     case ARM::VLD2q16:
4282     case ARM::VLD2q32:
4283     case ARM::VLD2d8wb_fixed:
4284     case ARM::VLD2d16wb_fixed:
4285     case ARM::VLD2d32wb_fixed:
4286     case ARM::VLD2q8wb_fixed:
4287     case ARM::VLD2q16wb_fixed:
4288     case ARM::VLD2q32wb_fixed:
4289     case ARM::VLD2d8wb_register:
4290     case ARM::VLD2d16wb_register:
4291     case ARM::VLD2d32wb_register:
4292     case ARM::VLD2q8wb_register:
4293     case ARM::VLD2q16wb_register:
4294     case ARM::VLD2q32wb_register:
4295     case ARM::VLD3d8:
4296     case ARM::VLD3d16:
4297     case ARM::VLD3d32:
4298     case ARM::VLD1d64T:
4299     case ARM::VLD3d8_UPD:
4300     case ARM::VLD3d16_UPD:
4301     case ARM::VLD3d32_UPD:
4302     case ARM::VLD1d64Twb_fixed:
4303     case ARM::VLD1d64Twb_register:
4304     case ARM::VLD3q8_UPD:
4305     case ARM::VLD3q16_UPD:
4306     case ARM::VLD3q32_UPD:
4307     case ARM::VLD4d8:
4308     case ARM::VLD4d16:
4309     case ARM::VLD4d32:
4310     case ARM::VLD1d64Q:
4311     case ARM::VLD4d8_UPD:
4312     case ARM::VLD4d16_UPD:
4313     case ARM::VLD4d32_UPD:
4314     case ARM::VLD1d64Qwb_fixed:
4315     case ARM::VLD1d64Qwb_register:
4316     case ARM::VLD4q8_UPD:
4317     case ARM::VLD4q16_UPD:
4318     case ARM::VLD4q32_UPD:
4319     case ARM::VLD1DUPq8:
4320     case ARM::VLD1DUPq16:
4321     case ARM::VLD1DUPq32:
4322     case ARM::VLD1DUPq8wb_fixed:
4323     case ARM::VLD1DUPq16wb_fixed:
4324     case ARM::VLD1DUPq32wb_fixed:
4325     case ARM::VLD1DUPq8wb_register:
4326     case ARM::VLD1DUPq16wb_register:
4327     case ARM::VLD1DUPq32wb_register:
4328     case ARM::VLD2DUPd8:
4329     case ARM::VLD2DUPd16:
4330     case ARM::VLD2DUPd32:
4331     case ARM::VLD2DUPd8wb_fixed:
4332     case ARM::VLD2DUPd16wb_fixed:
4333     case ARM::VLD2DUPd32wb_fixed:
4334     case ARM::VLD2DUPd8wb_register:
4335     case ARM::VLD2DUPd16wb_register:
4336     case ARM::VLD2DUPd32wb_register:
4337     case ARM::VLD4DUPd8:
4338     case ARM::VLD4DUPd16:
4339     case ARM::VLD4DUPd32:
4340     case ARM::VLD4DUPd8_UPD:
4341     case ARM::VLD4DUPd16_UPD:
4342     case ARM::VLD4DUPd32_UPD:
4343     case ARM::VLD1LNd8:
4344     case ARM::VLD1LNd16:
4345     case ARM::VLD1LNd32:
4346     case ARM::VLD1LNd8_UPD:
4347     case ARM::VLD1LNd16_UPD:
4348     case ARM::VLD1LNd32_UPD:
4349     case ARM::VLD2LNd8:
4350     case ARM::VLD2LNd16:
4351     case ARM::VLD2LNd32:
4352     case ARM::VLD2LNq16:
4353     case ARM::VLD2LNq32:
4354     case ARM::VLD2LNd8_UPD:
4355     case ARM::VLD2LNd16_UPD:
4356     case ARM::VLD2LNd32_UPD:
4357     case ARM::VLD2LNq16_UPD:
4358     case ARM::VLD2LNq32_UPD:
4359     case ARM::VLD4LNd8:
4360     case ARM::VLD4LNd16:
4361     case ARM::VLD4LNd32:
4362     case ARM::VLD4LNq16:
4363     case ARM::VLD4LNq32:
4364     case ARM::VLD4LNd8_UPD:
4365     case ARM::VLD4LNd16_UPD:
4366     case ARM::VLD4LNd32_UPD:
4367     case ARM::VLD4LNq16_UPD:
4368     case ARM::VLD4LNq32_UPD:
4369       // If the address is not 64-bit aligned, the latencies of these
4370       // instructions increases by one.
4371       ++Adjust;
4372       break;
4373     }
4374   }
4375   return Adjust;
4376 }
4377 
4378 int ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
4379                                         const MachineInstr &DefMI,
4380                                         unsigned DefIdx,
4381                                         const MachineInstr &UseMI,
4382                                         unsigned UseIdx) const {
4383   // No operand latency. The caller may fall back to getInstrLatency.
4384   if (!ItinData || ItinData->isEmpty())
4385     return -1;
4386 
4387   const MachineOperand &DefMO = DefMI.getOperand(DefIdx);
4388   Register Reg = DefMO.getReg();
4389 
4390   const MachineInstr *ResolvedDefMI = &DefMI;
4391   unsigned DefAdj = 0;
4392   if (DefMI.isBundle())
4393     ResolvedDefMI =
4394         getBundledDefMI(&getRegisterInfo(), &DefMI, Reg, DefIdx, DefAdj);
4395   if (ResolvedDefMI->isCopyLike() || ResolvedDefMI->isInsertSubreg() ||
4396       ResolvedDefMI->isRegSequence() || ResolvedDefMI->isImplicitDef()) {
4397     return 1;
4398   }
4399 
4400   const MachineInstr *ResolvedUseMI = &UseMI;
4401   unsigned UseAdj = 0;
4402   if (UseMI.isBundle()) {
4403     ResolvedUseMI =
4404         getBundledUseMI(&getRegisterInfo(), UseMI, Reg, UseIdx, UseAdj);
4405     if (!ResolvedUseMI)
4406       return -1;
4407   }
4408 
4409   return getOperandLatencyImpl(
4410       ItinData, *ResolvedDefMI, DefIdx, ResolvedDefMI->getDesc(), DefAdj, DefMO,
4411       Reg, *ResolvedUseMI, UseIdx, ResolvedUseMI->getDesc(), UseAdj);
4412 }
4413 
4414 int ARMBaseInstrInfo::getOperandLatencyImpl(
4415     const InstrItineraryData *ItinData, const MachineInstr &DefMI,
4416     unsigned DefIdx, const MCInstrDesc &DefMCID, unsigned DefAdj,
4417     const MachineOperand &DefMO, unsigned Reg, const MachineInstr &UseMI,
4418     unsigned UseIdx, const MCInstrDesc &UseMCID, unsigned UseAdj) const {
4419   if (Reg == ARM::CPSR) {
4420     if (DefMI.getOpcode() == ARM::FMSTAT) {
4421       // fpscr -> cpsr stalls over 20 cycles on A8 (and earlier?)
4422       return Subtarget.isLikeA9() ? 1 : 20;
4423     }
4424 
4425     // CPSR set and branch can be paired in the same cycle.
4426     if (UseMI.isBranch())
4427       return 0;
4428 
4429     // Otherwise it takes the instruction latency (generally one).
4430     unsigned Latency = getInstrLatency(ItinData, DefMI);
4431 
4432     // For Thumb2 and -Os, prefer scheduling CPSR setting instruction close to
4433     // its uses. Instructions which are otherwise scheduled between them may
4434     // incur a code size penalty (not able to use the CPSR setting 16-bit
4435     // instructions).
4436     if (Latency > 0 && Subtarget.isThumb2()) {
4437       const MachineFunction *MF = DefMI.getParent()->getParent();
4438       // FIXME: Use Function::hasOptSize().
4439       if (MF->getFunction().hasFnAttribute(Attribute::OptimizeForSize))
4440         --Latency;
4441     }
4442     return Latency;
4443   }
4444 
4445   if (DefMO.isImplicit() || UseMI.getOperand(UseIdx).isImplicit())
4446     return -1;
4447 
4448   unsigned DefAlign = DefMI.hasOneMemOperand()
4449                           ? (*DefMI.memoperands_begin())->getAlign().value()
4450                           : 0;
4451   unsigned UseAlign = UseMI.hasOneMemOperand()
4452                           ? (*UseMI.memoperands_begin())->getAlign().value()
4453                           : 0;
4454 
4455   // Get the itinerary's latency if possible, and handle variable_ops.
4456   int Latency = getOperandLatency(ItinData, DefMCID, DefIdx, DefAlign, UseMCID,
4457                                   UseIdx, UseAlign);
4458   // Unable to find operand latency. The caller may resort to getInstrLatency.
4459   if (Latency < 0)
4460     return Latency;
4461 
4462   // Adjust for IT block position.
4463   int Adj = DefAdj + UseAdj;
4464 
4465   // Adjust for dynamic def-side opcode variants not captured by the itinerary.
4466   Adj += adjustDefLatency(Subtarget, DefMI, DefMCID, DefAlign);
4467   if (Adj >= 0 || (int)Latency > -Adj) {
4468     return Latency + Adj;
4469   }
4470   // Return the itinerary latency, which may be zero but not less than zero.
4471   return Latency;
4472 }
4473 
4474 int
4475 ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
4476                                     SDNode *DefNode, unsigned DefIdx,
4477                                     SDNode *UseNode, unsigned UseIdx) const {
4478   if (!DefNode->isMachineOpcode())
4479     return 1;
4480 
4481   const MCInstrDesc &DefMCID = get(DefNode->getMachineOpcode());
4482 
4483   if (isZeroCost(DefMCID.Opcode))
4484     return 0;
4485 
4486   if (!ItinData || ItinData->isEmpty())
4487     return DefMCID.mayLoad() ? 3 : 1;
4488 
4489   if (!UseNode->isMachineOpcode()) {
4490     int Latency = ItinData->getOperandCycle(DefMCID.getSchedClass(), DefIdx);
4491     int Adj = Subtarget.getPreISelOperandLatencyAdjustment();
4492     int Threshold = 1 + Adj;
4493     return Latency <= Threshold ? 1 : Latency - Adj;
4494   }
4495 
4496   const MCInstrDesc &UseMCID = get(UseNode->getMachineOpcode());
4497   auto *DefMN = cast<MachineSDNode>(DefNode);
4498   unsigned DefAlign = !DefMN->memoperands_empty()
4499                           ? (*DefMN->memoperands_begin())->getAlign().value()
4500                           : 0;
4501   auto *UseMN = cast<MachineSDNode>(UseNode);
4502   unsigned UseAlign = !UseMN->memoperands_empty()
4503                           ? (*UseMN->memoperands_begin())->getAlign().value()
4504                           : 0;
4505   int Latency = getOperandLatency(ItinData, DefMCID, DefIdx, DefAlign,
4506                                   UseMCID, UseIdx, UseAlign);
4507 
4508   if (Latency > 1 &&
4509       (Subtarget.isCortexA8() || Subtarget.isLikeA9() ||
4510        Subtarget.isCortexA7())) {
4511     // FIXME: Shifter op hack: no shift (i.e. [r +/- r]) or [r + r << 2]
4512     // variants are one cycle cheaper.
4513     switch (DefMCID.getOpcode()) {
4514     default: break;
4515     case ARM::LDRrs:
4516     case ARM::LDRBrs: {
4517       unsigned ShOpVal =
4518         cast<ConstantSDNode>(DefNode->getOperand(2))->getZExtValue();
4519       unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
4520       if (ShImm == 0 ||
4521           (ShImm == 2 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))
4522         --Latency;
4523       break;
4524     }
4525     case ARM::t2LDRs:
4526     case ARM::t2LDRBs:
4527     case ARM::t2LDRHs:
4528     case ARM::t2LDRSHs: {
4529       // Thumb2 mode: lsl only.
4530       unsigned ShAmt =
4531         cast<ConstantSDNode>(DefNode->getOperand(2))->getZExtValue();
4532       if (ShAmt == 0 || ShAmt == 2)
4533         --Latency;
4534       break;
4535     }
4536     }
4537   } else if (DefIdx == 0 && Latency > 2 && Subtarget.isSwift()) {
4538     // FIXME: Properly handle all of the latency adjustments for address
4539     // writeback.
4540     switch (DefMCID.getOpcode()) {
4541     default: break;
4542     case ARM::LDRrs:
4543     case ARM::LDRBrs: {
4544       unsigned ShOpVal =
4545         cast<ConstantSDNode>(DefNode->getOperand(2))->getZExtValue();
4546       unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
4547       if (ShImm == 0 ||
4548           ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
4549            ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))
4550         Latency -= 2;
4551       else if (ShImm == 1 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsr)
4552         --Latency;
4553       break;
4554     }
4555     case ARM::t2LDRs:
4556     case ARM::t2LDRBs:
4557     case ARM::t2LDRHs:
4558     case ARM::t2LDRSHs:
4559       // Thumb2 mode: lsl 0-3 only.
4560       Latency -= 2;
4561       break;
4562     }
4563   }
4564 
4565   if (DefAlign < 8 && Subtarget.checkVLDnAccessAlignment())
4566     switch (DefMCID.getOpcode()) {
4567     default: break;
4568     case ARM::VLD1q8:
4569     case ARM::VLD1q16:
4570     case ARM::VLD1q32:
4571     case ARM::VLD1q64:
4572     case ARM::VLD1q8wb_register:
4573     case ARM::VLD1q16wb_register:
4574     case ARM::VLD1q32wb_register:
4575     case ARM::VLD1q64wb_register:
4576     case ARM::VLD1q8wb_fixed:
4577     case ARM::VLD1q16wb_fixed:
4578     case ARM::VLD1q32wb_fixed:
4579     case ARM::VLD1q64wb_fixed:
4580     case ARM::VLD2d8:
4581     case ARM::VLD2d16:
4582     case ARM::VLD2d32:
4583     case ARM::VLD2q8Pseudo:
4584     case ARM::VLD2q16Pseudo:
4585     case ARM::VLD2q32Pseudo:
4586     case ARM::VLD2d8wb_fixed:
4587     case ARM::VLD2d16wb_fixed:
4588     case ARM::VLD2d32wb_fixed:
4589     case ARM::VLD2q8PseudoWB_fixed:
4590     case ARM::VLD2q16PseudoWB_fixed:
4591     case ARM::VLD2q32PseudoWB_fixed:
4592     case ARM::VLD2d8wb_register:
4593     case ARM::VLD2d16wb_register:
4594     case ARM::VLD2d32wb_register:
4595     case ARM::VLD2q8PseudoWB_register:
4596     case ARM::VLD2q16PseudoWB_register:
4597     case ARM::VLD2q32PseudoWB_register:
4598     case ARM::VLD3d8Pseudo:
4599     case ARM::VLD3d16Pseudo:
4600     case ARM::VLD3d32Pseudo:
4601     case ARM::VLD1d8TPseudo:
4602     case ARM::VLD1d16TPseudo:
4603     case ARM::VLD1d32TPseudo:
4604     case ARM::VLD1d64TPseudo:
4605     case ARM::VLD1d64TPseudoWB_fixed:
4606     case ARM::VLD1d64TPseudoWB_register:
4607     case ARM::VLD3d8Pseudo_UPD:
4608     case ARM::VLD3d16Pseudo_UPD:
4609     case ARM::VLD3d32Pseudo_UPD:
4610     case ARM::VLD3q8Pseudo_UPD:
4611     case ARM::VLD3q16Pseudo_UPD:
4612     case ARM::VLD3q32Pseudo_UPD:
4613     case ARM::VLD3q8oddPseudo:
4614     case ARM::VLD3q16oddPseudo:
4615     case ARM::VLD3q32oddPseudo:
4616     case ARM::VLD3q8oddPseudo_UPD:
4617     case ARM::VLD3q16oddPseudo_UPD:
4618     case ARM::VLD3q32oddPseudo_UPD:
4619     case ARM::VLD4d8Pseudo:
4620     case ARM::VLD4d16Pseudo:
4621     case ARM::VLD4d32Pseudo:
4622     case ARM::VLD1d8QPseudo:
4623     case ARM::VLD1d16QPseudo:
4624     case ARM::VLD1d32QPseudo:
4625     case ARM::VLD1d64QPseudo:
4626     case ARM::VLD1d64QPseudoWB_fixed:
4627     case ARM::VLD1d64QPseudoWB_register:
4628     case ARM::VLD1q8HighQPseudo:
4629     case ARM::VLD1q8LowQPseudo_UPD:
4630     case ARM::VLD1q8HighTPseudo:
4631     case ARM::VLD1q8LowTPseudo_UPD:
4632     case ARM::VLD1q16HighQPseudo:
4633     case ARM::VLD1q16LowQPseudo_UPD:
4634     case ARM::VLD1q16HighTPseudo:
4635     case ARM::VLD1q16LowTPseudo_UPD:
4636     case ARM::VLD1q32HighQPseudo:
4637     case ARM::VLD1q32LowQPseudo_UPD:
4638     case ARM::VLD1q32HighTPseudo:
4639     case ARM::VLD1q32LowTPseudo_UPD:
4640     case ARM::VLD1q64HighQPseudo:
4641     case ARM::VLD1q64LowQPseudo_UPD:
4642     case ARM::VLD1q64HighTPseudo:
4643     case ARM::VLD1q64LowTPseudo_UPD:
4644     case ARM::VLD4d8Pseudo_UPD:
4645     case ARM::VLD4d16Pseudo_UPD:
4646     case ARM::VLD4d32Pseudo_UPD:
4647     case ARM::VLD4q8Pseudo_UPD:
4648     case ARM::VLD4q16Pseudo_UPD:
4649     case ARM::VLD4q32Pseudo_UPD:
4650     case ARM::VLD4q8oddPseudo:
4651     case ARM::VLD4q16oddPseudo:
4652     case ARM::VLD4q32oddPseudo:
4653     case ARM::VLD4q8oddPseudo_UPD:
4654     case ARM::VLD4q16oddPseudo_UPD:
4655     case ARM::VLD4q32oddPseudo_UPD:
4656     case ARM::VLD1DUPq8:
4657     case ARM::VLD1DUPq16:
4658     case ARM::VLD1DUPq32:
4659     case ARM::VLD1DUPq8wb_fixed:
4660     case ARM::VLD1DUPq16wb_fixed:
4661     case ARM::VLD1DUPq32wb_fixed:
4662     case ARM::VLD1DUPq8wb_register:
4663     case ARM::VLD1DUPq16wb_register:
4664     case ARM::VLD1DUPq32wb_register:
4665     case ARM::VLD2DUPd8:
4666     case ARM::VLD2DUPd16:
4667     case ARM::VLD2DUPd32:
4668     case ARM::VLD2DUPd8wb_fixed:
4669     case ARM::VLD2DUPd16wb_fixed:
4670     case ARM::VLD2DUPd32wb_fixed:
4671     case ARM::VLD2DUPd8wb_register:
4672     case ARM::VLD2DUPd16wb_register:
4673     case ARM::VLD2DUPd32wb_register:
4674     case ARM::VLD2DUPq8EvenPseudo:
4675     case ARM::VLD2DUPq8OddPseudo:
4676     case ARM::VLD2DUPq16EvenPseudo:
4677     case ARM::VLD2DUPq16OddPseudo:
4678     case ARM::VLD2DUPq32EvenPseudo:
4679     case ARM::VLD2DUPq32OddPseudo:
4680     case ARM::VLD3DUPq8EvenPseudo:
4681     case ARM::VLD3DUPq8OddPseudo:
4682     case ARM::VLD3DUPq16EvenPseudo:
4683     case ARM::VLD3DUPq16OddPseudo:
4684     case ARM::VLD3DUPq32EvenPseudo:
4685     case ARM::VLD3DUPq32OddPseudo:
4686     case ARM::VLD4DUPd8Pseudo:
4687     case ARM::VLD4DUPd16Pseudo:
4688     case ARM::VLD4DUPd32Pseudo:
4689     case ARM::VLD4DUPd8Pseudo_UPD:
4690     case ARM::VLD4DUPd16Pseudo_UPD:
4691     case ARM::VLD4DUPd32Pseudo_UPD:
4692     case ARM::VLD4DUPq8EvenPseudo:
4693     case ARM::VLD4DUPq8OddPseudo:
4694     case ARM::VLD4DUPq16EvenPseudo:
4695     case ARM::VLD4DUPq16OddPseudo:
4696     case ARM::VLD4DUPq32EvenPseudo:
4697     case ARM::VLD4DUPq32OddPseudo:
4698     case ARM::VLD1LNq8Pseudo:
4699     case ARM::VLD1LNq16Pseudo:
4700     case ARM::VLD1LNq32Pseudo:
4701     case ARM::VLD1LNq8Pseudo_UPD:
4702     case ARM::VLD1LNq16Pseudo_UPD:
4703     case ARM::VLD1LNq32Pseudo_UPD:
4704     case ARM::VLD2LNd8Pseudo:
4705     case ARM::VLD2LNd16Pseudo:
4706     case ARM::VLD2LNd32Pseudo:
4707     case ARM::VLD2LNq16Pseudo:
4708     case ARM::VLD2LNq32Pseudo:
4709     case ARM::VLD2LNd8Pseudo_UPD:
4710     case ARM::VLD2LNd16Pseudo_UPD:
4711     case ARM::VLD2LNd32Pseudo_UPD:
4712     case ARM::VLD2LNq16Pseudo_UPD:
4713     case ARM::VLD2LNq32Pseudo_UPD:
4714     case ARM::VLD4LNd8Pseudo:
4715     case ARM::VLD4LNd16Pseudo:
4716     case ARM::VLD4LNd32Pseudo:
4717     case ARM::VLD4LNq16Pseudo:
4718     case ARM::VLD4LNq32Pseudo:
4719     case ARM::VLD4LNd8Pseudo_UPD:
4720     case ARM::VLD4LNd16Pseudo_UPD:
4721     case ARM::VLD4LNd32Pseudo_UPD:
4722     case ARM::VLD4LNq16Pseudo_UPD:
4723     case ARM::VLD4LNq32Pseudo_UPD:
4724       // If the address is not 64-bit aligned, the latencies of these
4725       // instructions increases by one.
4726       ++Latency;
4727       break;
4728     }
4729 
4730   return Latency;
4731 }
4732 
4733 unsigned ARMBaseInstrInfo::getPredicationCost(const MachineInstr &MI) const {
4734   if (MI.isCopyLike() || MI.isInsertSubreg() || MI.isRegSequence() ||
4735       MI.isImplicitDef())
4736     return 0;
4737 
4738   if (MI.isBundle())
4739     return 0;
4740 
4741   const MCInstrDesc &MCID = MI.getDesc();
4742 
4743   if (MCID.isCall() || (MCID.hasImplicitDefOfPhysReg(ARM::CPSR) &&
4744                         !Subtarget.cheapPredicableCPSRDef())) {
4745     // When predicated, CPSR is an additional source operand for CPSR updating
4746     // instructions, this apparently increases their latencies.
4747     return 1;
4748   }
4749   return 0;
4750 }
4751 
4752 unsigned ARMBaseInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
4753                                            const MachineInstr &MI,
4754                                            unsigned *PredCost) const {
4755   if (MI.isCopyLike() || MI.isInsertSubreg() || MI.isRegSequence() ||
4756       MI.isImplicitDef())
4757     return 1;
4758 
4759   // An instruction scheduler typically runs on unbundled instructions, however
4760   // other passes may query the latency of a bundled instruction.
4761   if (MI.isBundle()) {
4762     unsigned Latency = 0;
4763     MachineBasicBlock::const_instr_iterator I = MI.getIterator();
4764     MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
4765     while (++I != E && I->isInsideBundle()) {
4766       if (I->getOpcode() != ARM::t2IT)
4767         Latency += getInstrLatency(ItinData, *I, PredCost);
4768     }
4769     return Latency;
4770   }
4771 
4772   const MCInstrDesc &MCID = MI.getDesc();
4773   if (PredCost && (MCID.isCall() || (MCID.hasImplicitDefOfPhysReg(ARM::CPSR) &&
4774                                      !Subtarget.cheapPredicableCPSRDef()))) {
4775     // When predicated, CPSR is an additional source operand for CPSR updating
4776     // instructions, this apparently increases their latencies.
4777     *PredCost = 1;
4778   }
4779   // Be sure to call getStageLatency for an empty itinerary in case it has a
4780   // valid MinLatency property.
4781   if (!ItinData)
4782     return MI.mayLoad() ? 3 : 1;
4783 
4784   unsigned Class = MCID.getSchedClass();
4785 
4786   // For instructions with variable uops, use uops as latency.
4787   if (!ItinData->isEmpty() && ItinData->getNumMicroOps(Class) < 0)
4788     return getNumMicroOps(ItinData, MI);
4789 
4790   // For the common case, fall back on the itinerary's latency.
4791   unsigned Latency = ItinData->getStageLatency(Class);
4792 
4793   // Adjust for dynamic def-side opcode variants not captured by the itinerary.
4794   unsigned DefAlign =
4795       MI.hasOneMemOperand() ? (*MI.memoperands_begin())->getAlign().value() : 0;
4796   int Adj = adjustDefLatency(Subtarget, MI, MCID, DefAlign);
4797   if (Adj >= 0 || (int)Latency > -Adj) {
4798     return Latency + Adj;
4799   }
4800   return Latency;
4801 }
4802 
4803 int ARMBaseInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
4804                                       SDNode *Node) const {
4805   if (!Node->isMachineOpcode())
4806     return 1;
4807 
4808   if (!ItinData || ItinData->isEmpty())
4809     return 1;
4810 
4811   unsigned Opcode = Node->getMachineOpcode();
4812   switch (Opcode) {
4813   default:
4814     return ItinData->getStageLatency(get(Opcode).getSchedClass());
4815   case ARM::VLDMQIA:
4816   case ARM::VSTMQIA:
4817     return 2;
4818   }
4819 }
4820 
4821 bool ARMBaseInstrInfo::hasHighOperandLatency(const TargetSchedModel &SchedModel,
4822                                              const MachineRegisterInfo *MRI,
4823                                              const MachineInstr &DefMI,
4824                                              unsigned DefIdx,
4825                                              const MachineInstr &UseMI,
4826                                              unsigned UseIdx) const {
4827   unsigned DDomain = DefMI.getDesc().TSFlags & ARMII::DomainMask;
4828   unsigned UDomain = UseMI.getDesc().TSFlags & ARMII::DomainMask;
4829   if (Subtarget.nonpipelinedVFP() &&
4830       (DDomain == ARMII::DomainVFP || UDomain == ARMII::DomainVFP))
4831     return true;
4832 
4833   // Hoist VFP / NEON instructions with 4 or higher latency.
4834   unsigned Latency =
4835       SchedModel.computeOperandLatency(&DefMI, DefIdx, &UseMI, UseIdx);
4836   if (Latency <= 3)
4837     return false;
4838   return DDomain == ARMII::DomainVFP || DDomain == ARMII::DomainNEON ||
4839          UDomain == ARMII::DomainVFP || UDomain == ARMII::DomainNEON;
4840 }
4841 
4842 bool ARMBaseInstrInfo::hasLowDefLatency(const TargetSchedModel &SchedModel,
4843                                         const MachineInstr &DefMI,
4844                                         unsigned DefIdx) const {
4845   const InstrItineraryData *ItinData = SchedModel.getInstrItineraries();
4846   if (!ItinData || ItinData->isEmpty())
4847     return false;
4848 
4849   unsigned DDomain = DefMI.getDesc().TSFlags & ARMII::DomainMask;
4850   if (DDomain == ARMII::DomainGeneral) {
4851     unsigned DefClass = DefMI.getDesc().getSchedClass();
4852     int DefCycle = ItinData->getOperandCycle(DefClass, DefIdx);
4853     return (DefCycle != -1 && DefCycle <= 2);
4854   }
4855   return false;
4856 }
4857 
4858 bool ARMBaseInstrInfo::verifyInstruction(const MachineInstr &MI,
4859                                          StringRef &ErrInfo) const {
4860   if (convertAddSubFlagsOpcode(MI.getOpcode())) {
4861     ErrInfo = "Pseudo flag setting opcodes only exist in Selection DAG";
4862     return false;
4863   }
4864   if (MI.getOpcode() == ARM::tMOVr && !Subtarget.hasV6Ops()) {
4865     // Make sure we don't generate a lo-lo mov that isn't supported.
4866     if (!ARM::hGPRRegClass.contains(MI.getOperand(0).getReg()) &&
4867         !ARM::hGPRRegClass.contains(MI.getOperand(1).getReg())) {
4868       ErrInfo = "Non-flag-setting Thumb1 mov is v6-only";
4869       return false;
4870     }
4871   }
4872   if (MI.getOpcode() == ARM::tPUSH ||
4873       MI.getOpcode() == ARM::tPOP ||
4874       MI.getOpcode() == ARM::tPOP_RET) {
4875     for (const MachineOperand &MO : llvm::drop_begin(MI.operands(), 2)) {
4876       if (MO.isImplicit() || !MO.isReg())
4877         continue;
4878       Register Reg = MO.getReg();
4879       if (Reg < ARM::R0 || Reg > ARM::R7) {
4880         if (!(MI.getOpcode() == ARM::tPUSH && Reg == ARM::LR) &&
4881             !(MI.getOpcode() == ARM::tPOP_RET && Reg == ARM::PC)) {
4882           ErrInfo = "Unsupported register in Thumb1 push/pop";
4883           return false;
4884         }
4885       }
4886     }
4887   }
4888   if (MI.getOpcode() == ARM::MVE_VMOV_q_rr) {
4889     assert(MI.getOperand(4).isImm() && MI.getOperand(5).isImm());
4890     if ((MI.getOperand(4).getImm() != 2 && MI.getOperand(4).getImm() != 3) ||
4891         MI.getOperand(4).getImm() != MI.getOperand(5).getImm() + 2) {
4892       ErrInfo = "Incorrect array index for MVE_VMOV_q_rr";
4893       return false;
4894     }
4895   }
4896 
4897   // Check the address model by taking the first Imm operand and checking it is
4898   // legal for that addressing mode.
4899   ARMII::AddrMode AddrMode =
4900       (ARMII::AddrMode)(MI.getDesc().TSFlags & ARMII::AddrModeMask);
4901   switch (AddrMode) {
4902   default:
4903     break;
4904   case ARMII::AddrModeT2_i7:
4905   case ARMII::AddrModeT2_i7s2:
4906   case ARMII::AddrModeT2_i7s4:
4907   case ARMII::AddrModeT2_i8:
4908   case ARMII::AddrModeT2_i8pos:
4909   case ARMII::AddrModeT2_i8neg:
4910   case ARMII::AddrModeT2_i8s4:
4911   case ARMII::AddrModeT2_i12: {
4912     uint32_t Imm = 0;
4913     for (auto Op : MI.operands()) {
4914       if (Op.isImm()) {
4915         Imm = Op.getImm();
4916         break;
4917       }
4918     }
4919     if (!isLegalAddressImm(MI.getOpcode(), Imm, this)) {
4920       ErrInfo = "Incorrect AddrMode Imm for instruction";
4921       return false;
4922     }
4923     break;
4924   }
4925   }
4926   return true;
4927 }
4928 
4929 void ARMBaseInstrInfo::expandLoadStackGuardBase(MachineBasicBlock::iterator MI,
4930                                                 unsigned LoadImmOpc,
4931                                                 unsigned LoadOpc) const {
4932   assert(!Subtarget.isROPI() && !Subtarget.isRWPI() &&
4933          "ROPI/RWPI not currently supported with stack guard");
4934 
4935   MachineBasicBlock &MBB = *MI->getParent();
4936   DebugLoc DL = MI->getDebugLoc();
4937   Register Reg = MI->getOperand(0).getReg();
4938   MachineInstrBuilder MIB;
4939   unsigned int Offset = 0;
4940 
4941   if (LoadImmOpc == ARM::MRC || LoadImmOpc == ARM::t2MRC) {
4942     assert(Subtarget.isReadTPHard() &&
4943            "TLS stack protector requires hardware TLS register");
4944 
4945     BuildMI(MBB, MI, DL, get(LoadImmOpc), Reg)
4946         .addImm(15)
4947         .addImm(0)
4948         .addImm(13)
4949         .addImm(0)
4950         .addImm(3)
4951         .add(predOps(ARMCC::AL));
4952 
4953     Module &M = *MBB.getParent()->getFunction().getParent();
4954     Offset = M.getStackProtectorGuardOffset();
4955     if (Offset & ~0xfffU) {
4956       // The offset won't fit in the LDR's 12-bit immediate field, so emit an
4957       // extra ADD to cover the delta. This gives us a guaranteed 8 additional
4958       // bits, resulting in a range of 0 to +1 MiB for the guard offset.
4959       unsigned AddOpc = (LoadImmOpc == ARM::MRC) ? ARM::ADDri : ARM::t2ADDri;
4960       BuildMI(MBB, MI, DL, get(AddOpc), Reg)
4961           .addReg(Reg, RegState::Kill)
4962           .addImm(Offset & ~0xfffU)
4963           .add(predOps(ARMCC::AL))
4964           .addReg(0);
4965       Offset &= 0xfffU;
4966     }
4967   } else {
4968     const GlobalValue *GV =
4969         cast<GlobalValue>((*MI->memoperands_begin())->getValue());
4970     bool IsIndirect = Subtarget.isGVIndirectSymbol(GV);
4971 
4972     unsigned TargetFlags = ARMII::MO_NO_FLAG;
4973     if (Subtarget.isTargetMachO()) {
4974       TargetFlags |= ARMII::MO_NONLAZY;
4975     } else if (Subtarget.isTargetCOFF()) {
4976       if (GV->hasDLLImportStorageClass())
4977         TargetFlags |= ARMII::MO_DLLIMPORT;
4978       else if (IsIndirect)
4979         TargetFlags |= ARMII::MO_COFFSTUB;
4980     } else if (Subtarget.isGVInGOT(GV)) {
4981       TargetFlags |= ARMII::MO_GOT;
4982     }
4983 
4984     BuildMI(MBB, MI, DL, get(LoadImmOpc), Reg)
4985         .addGlobalAddress(GV, 0, TargetFlags);
4986 
4987     if (IsIndirect) {
4988       MIB = BuildMI(MBB, MI, DL, get(LoadOpc), Reg);
4989       MIB.addReg(Reg, RegState::Kill).addImm(0);
4990       auto Flags = MachineMemOperand::MOLoad |
4991                    MachineMemOperand::MODereferenceable |
4992                    MachineMemOperand::MOInvariant;
4993       MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand(
4994           MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 4, Align(4));
4995       MIB.addMemOperand(MMO).add(predOps(ARMCC::AL));
4996     }
4997   }
4998 
4999   MIB = BuildMI(MBB, MI, DL, get(LoadOpc), Reg);
5000   MIB.addReg(Reg, RegState::Kill)
5001       .addImm(Offset)
5002       .cloneMemRefs(*MI)
5003       .add(predOps(ARMCC::AL));
5004 }
5005 
5006 bool
5007 ARMBaseInstrInfo::isFpMLxInstruction(unsigned Opcode, unsigned &MulOpc,
5008                                      unsigned &AddSubOpc,
5009                                      bool &NegAcc, bool &HasLane) const {
5010   DenseMap<unsigned, unsigned>::const_iterator I = MLxEntryMap.find(Opcode);
5011   if (I == MLxEntryMap.end())
5012     return false;
5013 
5014   const ARM_MLxEntry &Entry = ARM_MLxTable[I->second];
5015   MulOpc = Entry.MulOpc;
5016   AddSubOpc = Entry.AddSubOpc;
5017   NegAcc = Entry.NegAcc;
5018   HasLane = Entry.HasLane;
5019   return true;
5020 }
5021 
5022 //===----------------------------------------------------------------------===//
5023 // Execution domains.
5024 //===----------------------------------------------------------------------===//
5025 //
5026 // Some instructions go down the NEON pipeline, some go down the VFP pipeline,
5027 // and some can go down both.  The vmov instructions go down the VFP pipeline,
5028 // but they can be changed to vorr equivalents that are executed by the NEON
5029 // pipeline.
5030 //
5031 // We use the following execution domain numbering:
5032 //
5033 enum ARMExeDomain {
5034   ExeGeneric = 0,
5035   ExeVFP = 1,
5036   ExeNEON = 2
5037 };
5038 
5039 //
5040 // Also see ARMInstrFormats.td and Domain* enums in ARMBaseInfo.h
5041 //
5042 std::pair<uint16_t, uint16_t>
5043 ARMBaseInstrInfo::getExecutionDomain(const MachineInstr &MI) const {
5044   // If we don't have access to NEON instructions then we won't be able
5045   // to swizzle anything to the NEON domain. Check to make sure.
5046   if (Subtarget.hasNEON()) {
5047     // VMOVD, VMOVRS and VMOVSR are VFP instructions, but can be changed to NEON
5048     // if they are not predicated.
5049     if (MI.getOpcode() == ARM::VMOVD && !isPredicated(MI))
5050       return std::make_pair(ExeVFP, (1 << ExeVFP) | (1 << ExeNEON));
5051 
5052     // CortexA9 is particularly picky about mixing the two and wants these
5053     // converted.
5054     if (Subtarget.useNEONForFPMovs() && !isPredicated(MI) &&
5055         (MI.getOpcode() == ARM::VMOVRS || MI.getOpcode() == ARM::VMOVSR ||
5056          MI.getOpcode() == ARM::VMOVS))
5057       return std::make_pair(ExeVFP, (1 << ExeVFP) | (1 << ExeNEON));
5058   }
5059   // No other instructions can be swizzled, so just determine their domain.
5060   unsigned Domain = MI.getDesc().TSFlags & ARMII::DomainMask;
5061 
5062   if (Domain & ARMII::DomainNEON)
5063     return std::make_pair(ExeNEON, 0);
5064 
5065   // Certain instructions can go either way on Cortex-A8.
5066   // Treat them as NEON instructions.
5067   if ((Domain & ARMII::DomainNEONA8) && Subtarget.isCortexA8())
5068     return std::make_pair(ExeNEON, 0);
5069 
5070   if (Domain & ARMII::DomainVFP)
5071     return std::make_pair(ExeVFP, 0);
5072 
5073   return std::make_pair(ExeGeneric, 0);
5074 }
5075 
5076 static unsigned getCorrespondingDRegAndLane(const TargetRegisterInfo *TRI,
5077                                             unsigned SReg, unsigned &Lane) {
5078   unsigned DReg = TRI->getMatchingSuperReg(SReg, ARM::ssub_0, &ARM::DPRRegClass);
5079   Lane = 0;
5080 
5081   if (DReg != ARM::NoRegister)
5082    return DReg;
5083 
5084   Lane = 1;
5085   DReg = TRI->getMatchingSuperReg(SReg, ARM::ssub_1, &ARM::DPRRegClass);
5086 
5087   assert(DReg && "S-register with no D super-register?");
5088   return DReg;
5089 }
5090 
5091 /// getImplicitSPRUseForDPRUse - Given a use of a DPR register and lane,
5092 /// set ImplicitSReg to a register number that must be marked as implicit-use or
5093 /// zero if no register needs to be defined as implicit-use.
5094 ///
5095 /// If the function cannot determine if an SPR should be marked implicit use or
5096 /// not, it returns false.
5097 ///
5098 /// This function handles cases where an instruction is being modified from taking
5099 /// an SPR to a DPR[Lane]. A use of the DPR is being added, which may conflict
5100 /// with an earlier def of an SPR corresponding to DPR[Lane^1] (i.e. the other
5101 /// lane of the DPR).
5102 ///
5103 /// If the other SPR is defined, an implicit-use of it should be added. Else,
5104 /// (including the case where the DPR itself is defined), it should not.
5105 ///
5106 static bool getImplicitSPRUseForDPRUse(const TargetRegisterInfo *TRI,
5107                                        MachineInstr &MI, unsigned DReg,
5108                                        unsigned Lane, unsigned &ImplicitSReg) {
5109   // If the DPR is defined or used already, the other SPR lane will be chained
5110   // correctly, so there is nothing to be done.
5111   if (MI.definesRegister(DReg, TRI) || MI.readsRegister(DReg, TRI)) {
5112     ImplicitSReg = 0;
5113     return true;
5114   }
5115 
5116   // Otherwise we need to go searching to see if the SPR is set explicitly.
5117   ImplicitSReg = TRI->getSubReg(DReg,
5118                                 (Lane & 1) ? ARM::ssub_0 : ARM::ssub_1);
5119   MachineBasicBlock::LivenessQueryResult LQR =
5120       MI.getParent()->computeRegisterLiveness(TRI, ImplicitSReg, MI);
5121 
5122   if (LQR == MachineBasicBlock::LQR_Live)
5123     return true;
5124   else if (LQR == MachineBasicBlock::LQR_Unknown)
5125     return false;
5126 
5127   // If the register is known not to be live, there is no need to add an
5128   // implicit-use.
5129   ImplicitSReg = 0;
5130   return true;
5131 }
5132 
5133 void ARMBaseInstrInfo::setExecutionDomain(MachineInstr &MI,
5134                                           unsigned Domain) const {
5135   unsigned DstReg, SrcReg, DReg;
5136   unsigned Lane;
5137   MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI);
5138   const TargetRegisterInfo *TRI = &getRegisterInfo();
5139   switch (MI.getOpcode()) {
5140   default:
5141     llvm_unreachable("cannot handle opcode!");
5142     break;
5143   case ARM::VMOVD:
5144     if (Domain != ExeNEON)
5145       break;
5146 
5147     // Zap the predicate operands.
5148     assert(!isPredicated(MI) && "Cannot predicate a VORRd");
5149 
5150     // Make sure we've got NEON instructions.
5151     assert(Subtarget.hasNEON() && "VORRd requires NEON");
5152 
5153     // Source instruction is %DDst = VMOVD %DSrc, 14, %noreg (; implicits)
5154     DstReg = MI.getOperand(0).getReg();
5155     SrcReg = MI.getOperand(1).getReg();
5156 
5157     for (unsigned i = MI.getDesc().getNumOperands(); i; --i)
5158       MI.removeOperand(i - 1);
5159 
5160     // Change to a %DDst = VORRd %DSrc, %DSrc, 14, %noreg (; implicits)
5161     MI.setDesc(get(ARM::VORRd));
5162     MIB.addReg(DstReg, RegState::Define)
5163         .addReg(SrcReg)
5164         .addReg(SrcReg)
5165         .add(predOps(ARMCC::AL));
5166     break;
5167   case ARM::VMOVRS:
5168     if (Domain != ExeNEON)
5169       break;
5170     assert(!isPredicated(MI) && "Cannot predicate a VGETLN");
5171 
5172     // Source instruction is %RDst = VMOVRS %SSrc, 14, %noreg (; implicits)
5173     DstReg = MI.getOperand(0).getReg();
5174     SrcReg = MI.getOperand(1).getReg();
5175 
5176     for (unsigned i = MI.getDesc().getNumOperands(); i; --i)
5177       MI.removeOperand(i - 1);
5178 
5179     DReg = getCorrespondingDRegAndLane(TRI, SrcReg, Lane);
5180 
5181     // Convert to %RDst = VGETLNi32 %DSrc, Lane, 14, %noreg (; imps)
5182     // Note that DSrc has been widened and the other lane may be undef, which
5183     // contaminates the entire register.
5184     MI.setDesc(get(ARM::VGETLNi32));
5185     MIB.addReg(DstReg, RegState::Define)
5186         .addReg(DReg, RegState::Undef)
5187         .addImm(Lane)
5188         .add(predOps(ARMCC::AL));
5189 
5190     // The old source should be an implicit use, otherwise we might think it
5191     // was dead before here.
5192     MIB.addReg(SrcReg, RegState::Implicit);
5193     break;
5194   case ARM::VMOVSR: {
5195     if (Domain != ExeNEON)
5196       break;
5197     assert(!isPredicated(MI) && "Cannot predicate a VSETLN");
5198 
5199     // Source instruction is %SDst = VMOVSR %RSrc, 14, %noreg (; implicits)
5200     DstReg = MI.getOperand(0).getReg();
5201     SrcReg = MI.getOperand(1).getReg();
5202 
5203     DReg = getCorrespondingDRegAndLane(TRI, DstReg, Lane);
5204 
5205     unsigned ImplicitSReg;
5206     if (!getImplicitSPRUseForDPRUse(TRI, MI, DReg, Lane, ImplicitSReg))
5207       break;
5208 
5209     for (unsigned i = MI.getDesc().getNumOperands(); i; --i)
5210       MI.removeOperand(i - 1);
5211 
5212     // Convert to %DDst = VSETLNi32 %DDst, %RSrc, Lane, 14, %noreg (; imps)
5213     // Again DDst may be undefined at the beginning of this instruction.
5214     MI.setDesc(get(ARM::VSETLNi32));
5215     MIB.addReg(DReg, RegState::Define)
5216         .addReg(DReg, getUndefRegState(!MI.readsRegister(DReg, TRI)))
5217         .addReg(SrcReg)
5218         .addImm(Lane)
5219         .add(predOps(ARMCC::AL));
5220 
5221     // The narrower destination must be marked as set to keep previous chains
5222     // in place.
5223     MIB.addReg(DstReg, RegState::Define | RegState::Implicit);
5224     if (ImplicitSReg != 0)
5225       MIB.addReg(ImplicitSReg, RegState::Implicit);
5226     break;
5227     }
5228     case ARM::VMOVS: {
5229       if (Domain != ExeNEON)
5230         break;
5231 
5232       // Source instruction is %SDst = VMOVS %SSrc, 14, %noreg (; implicits)
5233       DstReg = MI.getOperand(0).getReg();
5234       SrcReg = MI.getOperand(1).getReg();
5235 
5236       unsigned DstLane = 0, SrcLane = 0, DDst, DSrc;
5237       DDst = getCorrespondingDRegAndLane(TRI, DstReg, DstLane);
5238       DSrc = getCorrespondingDRegAndLane(TRI, SrcReg, SrcLane);
5239 
5240       unsigned ImplicitSReg;
5241       if (!getImplicitSPRUseForDPRUse(TRI, MI, DSrc, SrcLane, ImplicitSReg))
5242         break;
5243 
5244       for (unsigned i = MI.getDesc().getNumOperands(); i; --i)
5245         MI.removeOperand(i - 1);
5246 
5247       if (DSrc == DDst) {
5248         // Destination can be:
5249         //     %DDst = VDUPLN32d %DDst, Lane, 14, %noreg (; implicits)
5250         MI.setDesc(get(ARM::VDUPLN32d));
5251         MIB.addReg(DDst, RegState::Define)
5252             .addReg(DDst, getUndefRegState(!MI.readsRegister(DDst, TRI)))
5253             .addImm(SrcLane)
5254             .add(predOps(ARMCC::AL));
5255 
5256         // Neither the source or the destination are naturally represented any
5257         // more, so add them in manually.
5258         MIB.addReg(DstReg, RegState::Implicit | RegState::Define);
5259         MIB.addReg(SrcReg, RegState::Implicit);
5260         if (ImplicitSReg != 0)
5261           MIB.addReg(ImplicitSReg, RegState::Implicit);
5262         break;
5263       }
5264 
5265       // In general there's no single instruction that can perform an S <-> S
5266       // move in NEON space, but a pair of VEXT instructions *can* do the
5267       // job. It turns out that the VEXTs needed will only use DSrc once, with
5268       // the position based purely on the combination of lane-0 and lane-1
5269       // involved. For example
5270       //     vmov s0, s2 -> vext.32 d0, d0, d1, #1  vext.32 d0, d0, d0, #1
5271       //     vmov s1, s3 -> vext.32 d0, d1, d0, #1  vext.32 d0, d0, d0, #1
5272       //     vmov s0, s3 -> vext.32 d0, d0, d0, #1  vext.32 d0, d1, d0, #1
5273       //     vmov s1, s2 -> vext.32 d0, d0, d0, #1  vext.32 d0, d0, d1, #1
5274       //
5275       // Pattern of the MachineInstrs is:
5276       //     %DDst = VEXTd32 %DSrc1, %DSrc2, Lane, 14, %noreg (;implicits)
5277       MachineInstrBuilder NewMIB;
5278       NewMIB = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(ARM::VEXTd32),
5279                        DDst);
5280 
5281       // On the first instruction, both DSrc and DDst may be undef if present.
5282       // Specifically when the original instruction didn't have them as an
5283       // <imp-use>.
5284       unsigned CurReg = SrcLane == 1 && DstLane == 1 ? DSrc : DDst;
5285       bool CurUndef = !MI.readsRegister(CurReg, TRI);
5286       NewMIB.addReg(CurReg, getUndefRegState(CurUndef));
5287 
5288       CurReg = SrcLane == 0 && DstLane == 0 ? DSrc : DDst;
5289       CurUndef = !MI.readsRegister(CurReg, TRI);
5290       NewMIB.addReg(CurReg, getUndefRegState(CurUndef))
5291             .addImm(1)
5292             .add(predOps(ARMCC::AL));
5293 
5294       if (SrcLane == DstLane)
5295         NewMIB.addReg(SrcReg, RegState::Implicit);
5296 
5297       MI.setDesc(get(ARM::VEXTd32));
5298       MIB.addReg(DDst, RegState::Define);
5299 
5300       // On the second instruction, DDst has definitely been defined above, so
5301       // it is not undef. DSrc, if present, can be undef as above.
5302       CurReg = SrcLane == 1 && DstLane == 0 ? DSrc : DDst;
5303       CurUndef = CurReg == DSrc && !MI.readsRegister(CurReg, TRI);
5304       MIB.addReg(CurReg, getUndefRegState(CurUndef));
5305 
5306       CurReg = SrcLane == 0 && DstLane == 1 ? DSrc : DDst;
5307       CurUndef = CurReg == DSrc && !MI.readsRegister(CurReg, TRI);
5308       MIB.addReg(CurReg, getUndefRegState(CurUndef))
5309          .addImm(1)
5310          .add(predOps(ARMCC::AL));
5311 
5312       if (SrcLane != DstLane)
5313         MIB.addReg(SrcReg, RegState::Implicit);
5314 
5315       // As before, the original destination is no longer represented, add it
5316       // implicitly.
5317       MIB.addReg(DstReg, RegState::Define | RegState::Implicit);
5318       if (ImplicitSReg != 0)
5319         MIB.addReg(ImplicitSReg, RegState::Implicit);
5320       break;
5321     }
5322   }
5323 }
5324 
5325 //===----------------------------------------------------------------------===//
5326 // Partial register updates
5327 //===----------------------------------------------------------------------===//
5328 //
5329 // Swift renames NEON registers with 64-bit granularity.  That means any
5330 // instruction writing an S-reg implicitly reads the containing D-reg.  The
5331 // problem is mostly avoided by translating f32 operations to v2f32 operations
5332 // on D-registers, but f32 loads are still a problem.
5333 //
5334 // These instructions can load an f32 into a NEON register:
5335 //
5336 // VLDRS - Only writes S, partial D update.
5337 // VLD1LNd32 - Writes all D-regs, explicit partial D update, 2 uops.
5338 // VLD1DUPd32 - Writes all D-regs, no partial reg update, 2 uops.
5339 //
5340 // FCONSTD can be used as a dependency-breaking instruction.
5341 unsigned ARMBaseInstrInfo::getPartialRegUpdateClearance(
5342     const MachineInstr &MI, unsigned OpNum,
5343     const TargetRegisterInfo *TRI) const {
5344   auto PartialUpdateClearance = Subtarget.getPartialUpdateClearance();
5345   if (!PartialUpdateClearance)
5346     return 0;
5347 
5348   assert(TRI && "Need TRI instance");
5349 
5350   const MachineOperand &MO = MI.getOperand(OpNum);
5351   if (MO.readsReg())
5352     return 0;
5353   Register Reg = MO.getReg();
5354   int UseOp = -1;
5355 
5356   switch (MI.getOpcode()) {
5357   // Normal instructions writing only an S-register.
5358   case ARM::VLDRS:
5359   case ARM::FCONSTS:
5360   case ARM::VMOVSR:
5361   case ARM::VMOVv8i8:
5362   case ARM::VMOVv4i16:
5363   case ARM::VMOVv2i32:
5364   case ARM::VMOVv2f32:
5365   case ARM::VMOVv1i64:
5366     UseOp = MI.findRegisterUseOperandIdx(Reg, false, TRI);
5367     break;
5368 
5369     // Explicitly reads the dependency.
5370   case ARM::VLD1LNd32:
5371     UseOp = 3;
5372     break;
5373   default:
5374     return 0;
5375   }
5376 
5377   // If this instruction actually reads a value from Reg, there is no unwanted
5378   // dependency.
5379   if (UseOp != -1 && MI.getOperand(UseOp).readsReg())
5380     return 0;
5381 
5382   // We must be able to clobber the whole D-reg.
5383   if (Reg.isVirtual()) {
5384     // Virtual register must be a def undef foo:ssub_0 operand.
5385     if (!MO.getSubReg() || MI.readsVirtualRegister(Reg))
5386       return 0;
5387   } else if (ARM::SPRRegClass.contains(Reg)) {
5388     // Physical register: MI must define the full D-reg.
5389     unsigned DReg = TRI->getMatchingSuperReg(Reg, ARM::ssub_0,
5390                                              &ARM::DPRRegClass);
5391     if (!DReg || !MI.definesRegister(DReg, TRI))
5392       return 0;
5393   }
5394 
5395   // MI has an unwanted D-register dependency.
5396   // Avoid defs in the previous N instructrions.
5397   return PartialUpdateClearance;
5398 }
5399 
5400 // Break a partial register dependency after getPartialRegUpdateClearance
5401 // returned non-zero.
5402 void ARMBaseInstrInfo::breakPartialRegDependency(
5403     MachineInstr &MI, unsigned OpNum, const TargetRegisterInfo *TRI) const {
5404   assert(OpNum < MI.getDesc().getNumDefs() && "OpNum is not a def");
5405   assert(TRI && "Need TRI instance");
5406 
5407   const MachineOperand &MO = MI.getOperand(OpNum);
5408   Register Reg = MO.getReg();
5409   assert(Reg.isPhysical() && "Can't break virtual register dependencies.");
5410   unsigned DReg = Reg;
5411 
5412   // If MI defines an S-reg, find the corresponding D super-register.
5413   if (ARM::SPRRegClass.contains(Reg)) {
5414     DReg = ARM::D0 + (Reg - ARM::S0) / 2;
5415     assert(TRI->isSuperRegister(Reg, DReg) && "Register enums broken");
5416   }
5417 
5418   assert(ARM::DPRRegClass.contains(DReg) && "Can only break D-reg deps");
5419   assert(MI.definesRegister(DReg, TRI) && "MI doesn't clobber full D-reg");
5420 
5421   // FIXME: In some cases, VLDRS can be changed to a VLD1DUPd32 which defines
5422   // the full D-register by loading the same value to both lanes.  The
5423   // instruction is micro-coded with 2 uops, so don't do this until we can
5424   // properly schedule micro-coded instructions.  The dispatcher stalls cause
5425   // too big regressions.
5426 
5427   // Insert the dependency-breaking FCONSTD before MI.
5428   // 96 is the encoding of 0.5, but the actual value doesn't matter here.
5429   BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(ARM::FCONSTD), DReg)
5430       .addImm(96)
5431       .add(predOps(ARMCC::AL));
5432   MI.addRegisterKilled(DReg, TRI, true);
5433 }
5434 
5435 bool ARMBaseInstrInfo::hasNOP() const {
5436   return Subtarget.getFeatureBits()[ARM::HasV6KOps];
5437 }
5438 
5439 bool ARMBaseInstrInfo::isSwiftFastImmShift(const MachineInstr *MI) const {
5440   if (MI->getNumOperands() < 4)
5441     return true;
5442   unsigned ShOpVal = MI->getOperand(3).getImm();
5443   unsigned ShImm = ARM_AM::getSORegOffset(ShOpVal);
5444   // Swift supports faster shifts for: lsl 2, lsl 1, and lsr 1.
5445   if ((ShImm == 1 && ARM_AM::getSORegShOp(ShOpVal) == ARM_AM::lsr) ||
5446       ((ShImm == 1 || ShImm == 2) &&
5447        ARM_AM::getSORegShOp(ShOpVal) == ARM_AM::lsl))
5448     return true;
5449 
5450   return false;
5451 }
5452 
5453 bool ARMBaseInstrInfo::getRegSequenceLikeInputs(
5454     const MachineInstr &MI, unsigned DefIdx,
5455     SmallVectorImpl<RegSubRegPairAndIdx> &InputRegs) const {
5456   assert(DefIdx < MI.getDesc().getNumDefs() && "Invalid definition index");
5457   assert(MI.isRegSequenceLike() && "Invalid kind of instruction");
5458 
5459   switch (MI.getOpcode()) {
5460   case ARM::VMOVDRR:
5461     // dX = VMOVDRR rY, rZ
5462     // is the same as:
5463     // dX = REG_SEQUENCE rY, ssub_0, rZ, ssub_1
5464     // Populate the InputRegs accordingly.
5465     // rY
5466     const MachineOperand *MOReg = &MI.getOperand(1);
5467     if (!MOReg->isUndef())
5468       InputRegs.push_back(RegSubRegPairAndIdx(MOReg->getReg(),
5469                                               MOReg->getSubReg(), ARM::ssub_0));
5470     // rZ
5471     MOReg = &MI.getOperand(2);
5472     if (!MOReg->isUndef())
5473       InputRegs.push_back(RegSubRegPairAndIdx(MOReg->getReg(),
5474                                               MOReg->getSubReg(), ARM::ssub_1));
5475     return true;
5476   }
5477   llvm_unreachable("Target dependent opcode missing");
5478 }
5479 
5480 bool ARMBaseInstrInfo::getExtractSubregLikeInputs(
5481     const MachineInstr &MI, unsigned DefIdx,
5482     RegSubRegPairAndIdx &InputReg) const {
5483   assert(DefIdx < MI.getDesc().getNumDefs() && "Invalid definition index");
5484   assert(MI.isExtractSubregLike() && "Invalid kind of instruction");
5485 
5486   switch (MI.getOpcode()) {
5487   case ARM::VMOVRRD:
5488     // rX, rY = VMOVRRD dZ
5489     // is the same as:
5490     // rX = EXTRACT_SUBREG dZ, ssub_0
5491     // rY = EXTRACT_SUBREG dZ, ssub_1
5492     const MachineOperand &MOReg = MI.getOperand(2);
5493     if (MOReg.isUndef())
5494       return false;
5495     InputReg.Reg = MOReg.getReg();
5496     InputReg.SubReg = MOReg.getSubReg();
5497     InputReg.SubIdx = DefIdx == 0 ? ARM::ssub_0 : ARM::ssub_1;
5498     return true;
5499   }
5500   llvm_unreachable("Target dependent opcode missing");
5501 }
5502 
5503 bool ARMBaseInstrInfo::getInsertSubregLikeInputs(
5504     const MachineInstr &MI, unsigned DefIdx, RegSubRegPair &BaseReg,
5505     RegSubRegPairAndIdx &InsertedReg) const {
5506   assert(DefIdx < MI.getDesc().getNumDefs() && "Invalid definition index");
5507   assert(MI.isInsertSubregLike() && "Invalid kind of instruction");
5508 
5509   switch (MI.getOpcode()) {
5510   case ARM::VSETLNi32:
5511   case ARM::MVE_VMOV_to_lane_32:
5512     // dX = VSETLNi32 dY, rZ, imm
5513     // qX = MVE_VMOV_to_lane_32 qY, rZ, imm
5514     const MachineOperand &MOBaseReg = MI.getOperand(1);
5515     const MachineOperand &MOInsertedReg = MI.getOperand(2);
5516     if (MOInsertedReg.isUndef())
5517       return false;
5518     const MachineOperand &MOIndex = MI.getOperand(3);
5519     BaseReg.Reg = MOBaseReg.getReg();
5520     BaseReg.SubReg = MOBaseReg.getSubReg();
5521 
5522     InsertedReg.Reg = MOInsertedReg.getReg();
5523     InsertedReg.SubReg = MOInsertedReg.getSubReg();
5524     InsertedReg.SubIdx = ARM::ssub_0 + MOIndex.getImm();
5525     return true;
5526   }
5527   llvm_unreachable("Target dependent opcode missing");
5528 }
5529 
5530 std::pair<unsigned, unsigned>
5531 ARMBaseInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
5532   const unsigned Mask = ARMII::MO_OPTION_MASK;
5533   return std::make_pair(TF & Mask, TF & ~Mask);
5534 }
5535 
5536 ArrayRef<std::pair<unsigned, const char *>>
5537 ARMBaseInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
5538   using namespace ARMII;
5539 
5540   static const std::pair<unsigned, const char *> TargetFlags[] = {
5541       {MO_LO16, "arm-lo16"}, {MO_HI16, "arm-hi16"}};
5542   return ArrayRef(TargetFlags);
5543 }
5544 
5545 ArrayRef<std::pair<unsigned, const char *>>
5546 ARMBaseInstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const {
5547   using namespace ARMII;
5548 
5549   static const std::pair<unsigned, const char *> TargetFlags[] = {
5550       {MO_COFFSTUB, "arm-coffstub"},
5551       {MO_GOT, "arm-got"},
5552       {MO_SBREL, "arm-sbrel"},
5553       {MO_DLLIMPORT, "arm-dllimport"},
5554       {MO_SECREL, "arm-secrel"},
5555       {MO_NONLAZY, "arm-nonlazy"}};
5556   return ArrayRef(TargetFlags);
5557 }
5558 
5559 std::optional<RegImmPair>
5560 ARMBaseInstrInfo::isAddImmediate(const MachineInstr &MI, Register Reg) const {
5561   int Sign = 1;
5562   unsigned Opcode = MI.getOpcode();
5563   int64_t Offset = 0;
5564 
5565   // TODO: Handle cases where Reg is a super- or sub-register of the
5566   // destination register.
5567   const MachineOperand &Op0 = MI.getOperand(0);
5568   if (!Op0.isReg() || Reg != Op0.getReg())
5569     return std::nullopt;
5570 
5571   // We describe SUBri or ADDri instructions.
5572   if (Opcode == ARM::SUBri)
5573     Sign = -1;
5574   else if (Opcode != ARM::ADDri)
5575     return std::nullopt;
5576 
5577   // TODO: Third operand can be global address (usually some string). Since
5578   //       strings can be relocated we cannot calculate their offsets for
5579   //       now.
5580   if (!MI.getOperand(1).isReg() || !MI.getOperand(2).isImm())
5581     return std::nullopt;
5582 
5583   Offset = MI.getOperand(2).getImm() * Sign;
5584   return RegImmPair{MI.getOperand(1).getReg(), Offset};
5585 }
5586 
5587 bool llvm::registerDefinedBetween(unsigned Reg,
5588                                   MachineBasicBlock::iterator From,
5589                                   MachineBasicBlock::iterator To,
5590                                   const TargetRegisterInfo *TRI) {
5591   for (auto I = From; I != To; ++I)
5592     if (I->modifiesRegister(Reg, TRI))
5593       return true;
5594   return false;
5595 }
5596 
5597 MachineInstr *llvm::findCMPToFoldIntoCBZ(MachineInstr *Br,
5598                                          const TargetRegisterInfo *TRI) {
5599   // Search backwards to the instruction that defines CSPR. This may or not
5600   // be a CMP, we check that after this loop. If we find another instruction
5601   // that reads cpsr, we return nullptr.
5602   MachineBasicBlock::iterator CmpMI = Br;
5603   while (CmpMI != Br->getParent()->begin()) {
5604     --CmpMI;
5605     if (CmpMI->modifiesRegister(ARM::CPSR, TRI))
5606       break;
5607     if (CmpMI->readsRegister(ARM::CPSR, TRI))
5608       break;
5609   }
5610 
5611   // Check that this inst is a CMP r[0-7], #0 and that the register
5612   // is not redefined between the cmp and the br.
5613   if (CmpMI->getOpcode() != ARM::tCMPi8 && CmpMI->getOpcode() != ARM::t2CMPri)
5614     return nullptr;
5615   Register Reg = CmpMI->getOperand(0).getReg();
5616   Register PredReg;
5617   ARMCC::CondCodes Pred = getInstrPredicate(*CmpMI, PredReg);
5618   if (Pred != ARMCC::AL || CmpMI->getOperand(1).getImm() != 0)
5619     return nullptr;
5620   if (!isARMLowRegister(Reg))
5621     return nullptr;
5622   if (registerDefinedBetween(Reg, CmpMI->getNextNode(), Br, TRI))
5623     return nullptr;
5624 
5625   return &*CmpMI;
5626 }
5627 
5628 unsigned llvm::ConstantMaterializationCost(unsigned Val,
5629                                            const ARMSubtarget *Subtarget,
5630                                            bool ForCodesize) {
5631   if (Subtarget->isThumb()) {
5632     if (Val <= 255) // MOV
5633       return ForCodesize ? 2 : 1;
5634     if (Subtarget->hasV6T2Ops() && (Val <= 0xffff ||                    // MOV
5635                                     ARM_AM::getT2SOImmVal(Val) != -1 || // MOVW
5636                                     ARM_AM::getT2SOImmVal(~Val) != -1)) // MVN
5637       return ForCodesize ? 4 : 1;
5638     if (Val <= 510) // MOV + ADDi8
5639       return ForCodesize ? 4 : 2;
5640     if (~Val <= 255) // MOV + MVN
5641       return ForCodesize ? 4 : 2;
5642     if (ARM_AM::isThumbImmShiftedVal(Val)) // MOV + LSL
5643       return ForCodesize ? 4 : 2;
5644   } else {
5645     if (ARM_AM::getSOImmVal(Val) != -1) // MOV
5646       return ForCodesize ? 4 : 1;
5647     if (ARM_AM::getSOImmVal(~Val) != -1) // MVN
5648       return ForCodesize ? 4 : 1;
5649     if (Subtarget->hasV6T2Ops() && Val <= 0xffff) // MOVW
5650       return ForCodesize ? 4 : 1;
5651     if (ARM_AM::isSOImmTwoPartVal(Val)) // two instrs
5652       return ForCodesize ? 8 : 2;
5653     if (ARM_AM::isSOImmTwoPartValNeg(Val)) // two instrs
5654       return ForCodesize ? 8 : 2;
5655   }
5656   if (Subtarget->useMovt()) // MOVW + MOVT
5657     return ForCodesize ? 8 : 2;
5658   return ForCodesize ? 8 : 3; // Literal pool load
5659 }
5660 
5661 bool llvm::HasLowerConstantMaterializationCost(unsigned Val1, unsigned Val2,
5662                                                const ARMSubtarget *Subtarget,
5663                                                bool ForCodesize) {
5664   // Check with ForCodesize
5665   unsigned Cost1 = ConstantMaterializationCost(Val1, Subtarget, ForCodesize);
5666   unsigned Cost2 = ConstantMaterializationCost(Val2, Subtarget, ForCodesize);
5667   if (Cost1 < Cost2)
5668     return true;
5669   if (Cost1 > Cost2)
5670     return false;
5671 
5672   // If they are equal, try with !ForCodesize
5673   return ConstantMaterializationCost(Val1, Subtarget, !ForCodesize) <
5674          ConstantMaterializationCost(Val2, Subtarget, !ForCodesize);
5675 }
5676 
5677 /// Constants defining how certain sequences should be outlined.
5678 /// This encompasses how an outlined function should be called, and what kind of
5679 /// frame should be emitted for that outlined function.
5680 ///
5681 /// \p MachineOutlinerTailCall implies that the function is being created from
5682 /// a sequence of instructions ending in a return.
5683 ///
5684 /// That is,
5685 ///
5686 /// I1                                OUTLINED_FUNCTION:
5687 /// I2    --> B OUTLINED_FUNCTION     I1
5688 /// BX LR                             I2
5689 ///                                   BX LR
5690 ///
5691 /// +-------------------------+--------+-----+
5692 /// |                         | Thumb2 | ARM |
5693 /// +-------------------------+--------+-----+
5694 /// | Call overhead in Bytes  |      4 |   4 |
5695 /// | Frame overhead in Bytes |      0 |   0 |
5696 /// | Stack fixup required    |     No |  No |
5697 /// +-------------------------+--------+-----+
5698 ///
5699 /// \p MachineOutlinerThunk implies that the function is being created from
5700 /// a sequence of instructions ending in a call. The outlined function is
5701 /// called with a BL instruction, and the outlined function tail-calls the
5702 /// original call destination.
5703 ///
5704 /// That is,
5705 ///
5706 /// I1                                OUTLINED_FUNCTION:
5707 /// I2   --> BL OUTLINED_FUNCTION     I1
5708 /// BL f                              I2
5709 ///                                   B f
5710 ///
5711 /// +-------------------------+--------+-----+
5712 /// |                         | Thumb2 | ARM |
5713 /// +-------------------------+--------+-----+
5714 /// | Call overhead in Bytes  |      4 |   4 |
5715 /// | Frame overhead in Bytes |      0 |   0 |
5716 /// | Stack fixup required    |     No |  No |
5717 /// +-------------------------+--------+-----+
5718 ///
5719 /// \p MachineOutlinerNoLRSave implies that the function should be called using
5720 /// a BL instruction, but doesn't require LR to be saved and restored. This
5721 /// happens when LR is known to be dead.
5722 ///
5723 /// That is,
5724 ///
5725 /// I1                                OUTLINED_FUNCTION:
5726 /// I2 --> BL OUTLINED_FUNCTION       I1
5727 /// I3                                I2
5728 ///                                   I3
5729 ///                                   BX LR
5730 ///
5731 /// +-------------------------+--------+-----+
5732 /// |                         | Thumb2 | ARM |
5733 /// +-------------------------+--------+-----+
5734 /// | Call overhead in Bytes  |      4 |   4 |
5735 /// | Frame overhead in Bytes |      2 |   4 |
5736 /// | Stack fixup required    |     No |  No |
5737 /// +-------------------------+--------+-----+
5738 ///
5739 /// \p MachineOutlinerRegSave implies that the function should be called with a
5740 /// save and restore of LR to an available register. This allows us to avoid
5741 /// stack fixups. Note that this outlining variant is compatible with the
5742 /// NoLRSave case.
5743 ///
5744 /// That is,
5745 ///
5746 /// I1     Save LR                    OUTLINED_FUNCTION:
5747 /// I2 --> BL OUTLINED_FUNCTION       I1
5748 /// I3     Restore LR                 I2
5749 ///                                   I3
5750 ///                                   BX LR
5751 ///
5752 /// +-------------------------+--------+-----+
5753 /// |                         | Thumb2 | ARM |
5754 /// +-------------------------+--------+-----+
5755 /// | Call overhead in Bytes  |      8 |  12 |
5756 /// | Frame overhead in Bytes |      2 |   4 |
5757 /// | Stack fixup required    |     No |  No |
5758 /// +-------------------------+--------+-----+
5759 ///
5760 /// \p MachineOutlinerDefault implies that the function should be called with
5761 /// a save and restore of LR to the stack.
5762 ///
5763 /// That is,
5764 ///
5765 /// I1     Save LR                    OUTLINED_FUNCTION:
5766 /// I2 --> BL OUTLINED_FUNCTION       I1
5767 /// I3     Restore LR                 I2
5768 ///                                   I3
5769 ///                                   BX LR
5770 ///
5771 /// +-------------------------+--------+-----+
5772 /// |                         | Thumb2 | ARM |
5773 /// +-------------------------+--------+-----+
5774 /// | Call overhead in Bytes  |      8 |  12 |
5775 /// | Frame overhead in Bytes |      2 |   4 |
5776 /// | Stack fixup required    |    Yes | Yes |
5777 /// +-------------------------+--------+-----+
5778 
5779 enum MachineOutlinerClass {
5780   MachineOutlinerTailCall,
5781   MachineOutlinerThunk,
5782   MachineOutlinerNoLRSave,
5783   MachineOutlinerRegSave,
5784   MachineOutlinerDefault
5785 };
5786 
5787 enum MachineOutlinerMBBFlags {
5788   LRUnavailableSomewhere = 0x2,
5789   HasCalls = 0x4,
5790   UnsafeRegsDead = 0x8
5791 };
5792 
5793 struct OutlinerCosts {
5794   int CallTailCall;
5795   int FrameTailCall;
5796   int CallThunk;
5797   int FrameThunk;
5798   int CallNoLRSave;
5799   int FrameNoLRSave;
5800   int CallRegSave;
5801   int FrameRegSave;
5802   int CallDefault;
5803   int FrameDefault;
5804   int SaveRestoreLROnStack;
5805 
5806   OutlinerCosts(const ARMSubtarget &target)
5807       : CallTailCall(target.isThumb() ? 4 : 4),
5808         FrameTailCall(target.isThumb() ? 0 : 0),
5809         CallThunk(target.isThumb() ? 4 : 4),
5810         FrameThunk(target.isThumb() ? 0 : 0),
5811         CallNoLRSave(target.isThumb() ? 4 : 4),
5812         FrameNoLRSave(target.isThumb() ? 2 : 4),
5813         CallRegSave(target.isThumb() ? 8 : 12),
5814         FrameRegSave(target.isThumb() ? 2 : 4),
5815         CallDefault(target.isThumb() ? 8 : 12),
5816         FrameDefault(target.isThumb() ? 2 : 4),
5817         SaveRestoreLROnStack(target.isThumb() ? 8 : 8) {}
5818 };
5819 
5820 Register
5821 ARMBaseInstrInfo::findRegisterToSaveLRTo(outliner::Candidate &C) const {
5822   MachineFunction *MF = C.getMF();
5823   const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
5824   const ARMBaseRegisterInfo *ARI =
5825       static_cast<const ARMBaseRegisterInfo *>(&TRI);
5826 
5827   BitVector regsReserved = ARI->getReservedRegs(*MF);
5828   // Check if there is an available register across the sequence that we can
5829   // use.
5830   for (Register Reg : ARM::rGPRRegClass) {
5831     if (!(Reg < regsReserved.size() && regsReserved.test(Reg)) &&
5832         Reg != ARM::LR &&  // LR is not reserved, but don't use it.
5833         Reg != ARM::R12 && // R12 is not guaranteed to be preserved.
5834         C.isAvailableAcrossAndOutOfSeq(Reg, TRI) &&
5835         C.isAvailableInsideSeq(Reg, TRI))
5836       return Reg;
5837   }
5838   return Register();
5839 }
5840 
5841 // Compute liveness of LR at the point after the interval [I, E), which
5842 // denotes a *backward* iteration through instructions. Used only for return
5843 // basic blocks, which do not end with a tail call.
5844 static bool isLRAvailable(const TargetRegisterInfo &TRI,
5845                           MachineBasicBlock::reverse_iterator I,
5846                           MachineBasicBlock::reverse_iterator E) {
5847   // At the end of the function LR dead.
5848   bool Live = false;
5849   for (; I != E; ++I) {
5850     const MachineInstr &MI = *I;
5851 
5852     // Check defs of LR.
5853     if (MI.modifiesRegister(ARM::LR, &TRI))
5854       Live = false;
5855 
5856     // Check uses of LR.
5857     unsigned Opcode = MI.getOpcode();
5858     if (Opcode == ARM::BX_RET || Opcode == ARM::MOVPCLR ||
5859         Opcode == ARM::SUBS_PC_LR || Opcode == ARM::tBX_RET ||
5860         Opcode == ARM::tBXNS_RET) {
5861       // These instructions use LR, but it's not an (explicit or implicit)
5862       // operand.
5863       Live = true;
5864       continue;
5865     }
5866     if (MI.readsRegister(ARM::LR, &TRI))
5867       Live = true;
5868   }
5869   return !Live;
5870 }
5871 
5872 outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo(
5873     std::vector<outliner::Candidate> &RepeatedSequenceLocs) const {
5874   outliner::Candidate &FirstCand = RepeatedSequenceLocs[0];
5875   unsigned SequenceSize =
5876       std::accumulate(FirstCand.front(), std::next(FirstCand.back()), 0,
5877                       [this](unsigned Sum, const MachineInstr &MI) {
5878                         return Sum + getInstSizeInBytes(MI);
5879                       });
5880 
5881   // Properties about candidate MBBs that hold for all of them.
5882   unsigned FlagsSetInAll = 0xF;
5883 
5884   // Compute liveness information for each candidate, and set FlagsSetInAll.
5885   const TargetRegisterInfo &TRI = getRegisterInfo();
5886   for (outliner::Candidate &C : RepeatedSequenceLocs)
5887     FlagsSetInAll &= C.Flags;
5888 
5889   // According to the ARM Procedure Call Standard, the following are
5890   // undefined on entry/exit from a function call:
5891   //
5892   // * Register R12(IP),
5893   // * Condition codes (and thus the CPSR register)
5894   //
5895   // Since we control the instructions which are part of the outlined regions
5896   // we don't need to be fully compliant with the AAPCS, but we have to
5897   // guarantee that if a veneer is inserted at link time the code is still
5898   // correct.  Because of this, we can't outline any sequence of instructions
5899   // where one of these registers is live into/across it. Thus, we need to
5900   // delete those candidates.
5901   auto CantGuaranteeValueAcrossCall = [&TRI](outliner::Candidate &C) {
5902     // If the unsafe registers in this block are all dead, then we don't need
5903     // to compute liveness here.
5904     if (C.Flags & UnsafeRegsDead)
5905       return false;
5906     return C.isAnyUnavailableAcrossOrOutOfSeq({ARM::R12, ARM::CPSR}, TRI);
5907   };
5908 
5909   // Are there any candidates where those registers are live?
5910   if (!(FlagsSetInAll & UnsafeRegsDead)) {
5911     // Erase every candidate that violates the restrictions above. (It could be
5912     // true that we have viable candidates, so it's not worth bailing out in
5913     // the case that, say, 1 out of 20 candidates violate the restructions.)
5914     llvm::erase_if(RepeatedSequenceLocs, CantGuaranteeValueAcrossCall);
5915 
5916     // If the sequence doesn't have enough candidates left, then we're done.
5917     if (RepeatedSequenceLocs.size() < 2)
5918       return outliner::OutlinedFunction();
5919   }
5920 
5921   // We expect the majority of the outlining candidates to be in consensus with
5922   // regard to return address sign and authentication, and branch target
5923   // enforcement, in other words, partitioning according to all the four
5924   // possible combinations of PAC-RET and BTI is going to yield one big subset
5925   // and three small (likely empty) subsets. That allows us to cull incompatible
5926   // candidates separately for PAC-RET and BTI.
5927 
5928   // Partition the candidates in two sets: one with BTI enabled and one with BTI
5929   // disabled. Remove the candidates from the smaller set. If they are the same
5930   // number prefer the non-BTI ones for outlining, since they have less
5931   // overhead.
5932   auto NoBTI =
5933       llvm::partition(RepeatedSequenceLocs, [](const outliner::Candidate &C) {
5934         const ARMFunctionInfo &AFI = *C.getMF()->getInfo<ARMFunctionInfo>();
5935         return AFI.branchTargetEnforcement();
5936       });
5937   if (std::distance(RepeatedSequenceLocs.begin(), NoBTI) >
5938       std::distance(NoBTI, RepeatedSequenceLocs.end()))
5939     RepeatedSequenceLocs.erase(NoBTI, RepeatedSequenceLocs.end());
5940   else
5941     RepeatedSequenceLocs.erase(RepeatedSequenceLocs.begin(), NoBTI);
5942 
5943   if (RepeatedSequenceLocs.size() < 2)
5944     return outliner::OutlinedFunction();
5945 
5946   // Likewise, partition the candidates according to PAC-RET enablement.
5947   auto NoPAC =
5948       llvm::partition(RepeatedSequenceLocs, [](const outliner::Candidate &C) {
5949         const ARMFunctionInfo &AFI = *C.getMF()->getInfo<ARMFunctionInfo>();
5950         // If the function happens to not spill the LR, do not disqualify it
5951         // from the outlining.
5952         return AFI.shouldSignReturnAddress(true);
5953       });
5954   if (std::distance(RepeatedSequenceLocs.begin(), NoPAC) >
5955       std::distance(NoPAC, RepeatedSequenceLocs.end()))
5956     RepeatedSequenceLocs.erase(NoPAC, RepeatedSequenceLocs.end());
5957   else
5958     RepeatedSequenceLocs.erase(RepeatedSequenceLocs.begin(), NoPAC);
5959 
5960   if (RepeatedSequenceLocs.size() < 2)
5961     return outliner::OutlinedFunction();
5962 
5963   // At this point, we have only "safe" candidates to outline. Figure out
5964   // frame + call instruction information.
5965 
5966   unsigned LastInstrOpcode = RepeatedSequenceLocs[0].back()->getOpcode();
5967 
5968   // Helper lambda which sets call information for every candidate.
5969   auto SetCandidateCallInfo =
5970       [&RepeatedSequenceLocs](unsigned CallID, unsigned NumBytesForCall) {
5971         for (outliner::Candidate &C : RepeatedSequenceLocs)
5972           C.setCallInfo(CallID, NumBytesForCall);
5973       };
5974 
5975   OutlinerCosts Costs(Subtarget);
5976 
5977   const auto &SomeMFI =
5978       *RepeatedSequenceLocs.front().getMF()->getInfo<ARMFunctionInfo>();
5979   // Adjust costs to account for the BTI instructions.
5980   if (SomeMFI.branchTargetEnforcement()) {
5981     Costs.FrameDefault += 4;
5982     Costs.FrameNoLRSave += 4;
5983     Costs.FrameRegSave += 4;
5984     Costs.FrameTailCall += 4;
5985     Costs.FrameThunk += 4;
5986   }
5987 
5988   // Adjust costs to account for sign and authentication instructions.
5989   if (SomeMFI.shouldSignReturnAddress(true)) {
5990     Costs.CallDefault += 8;          // +PAC instr, +AUT instr
5991     Costs.SaveRestoreLROnStack += 8; // +PAC instr, +AUT instr
5992   }
5993 
5994   unsigned FrameID = MachineOutlinerDefault;
5995   unsigned NumBytesToCreateFrame = Costs.FrameDefault;
5996 
5997   // If the last instruction in any candidate is a terminator, then we should
5998   // tail call all of the candidates.
5999   if (RepeatedSequenceLocs[0].back()->isTerminator()) {
6000     FrameID = MachineOutlinerTailCall;
6001     NumBytesToCreateFrame = Costs.FrameTailCall;
6002     SetCandidateCallInfo(MachineOutlinerTailCall, Costs.CallTailCall);
6003   } else if (LastInstrOpcode == ARM::BL || LastInstrOpcode == ARM::BLX ||
6004              LastInstrOpcode == ARM::BLX_noip || LastInstrOpcode == ARM::tBL ||
6005              LastInstrOpcode == ARM::tBLXr ||
6006              LastInstrOpcode == ARM::tBLXr_noip ||
6007              LastInstrOpcode == ARM::tBLXi) {
6008     FrameID = MachineOutlinerThunk;
6009     NumBytesToCreateFrame = Costs.FrameThunk;
6010     SetCandidateCallInfo(MachineOutlinerThunk, Costs.CallThunk);
6011   } else {
6012     // We need to decide how to emit calls + frames. We can always emit the same
6013     // frame if we don't need to save to the stack. If we have to save to the
6014     // stack, then we need a different frame.
6015     unsigned NumBytesNoStackCalls = 0;
6016     std::vector<outliner::Candidate> CandidatesWithoutStackFixups;
6017 
6018     for (outliner::Candidate &C : RepeatedSequenceLocs) {
6019       // LR liveness is overestimated in return blocks, unless they end with a
6020       // tail call.
6021       const auto Last = C.getMBB()->rbegin();
6022       const bool LRIsAvailable =
6023           C.getMBB()->isReturnBlock() && !Last->isCall()
6024               ? isLRAvailable(TRI, Last,
6025                               (MachineBasicBlock::reverse_iterator)C.front())
6026               : C.isAvailableAcrossAndOutOfSeq(ARM::LR, TRI);
6027       if (LRIsAvailable) {
6028         FrameID = MachineOutlinerNoLRSave;
6029         NumBytesNoStackCalls += Costs.CallNoLRSave;
6030         C.setCallInfo(MachineOutlinerNoLRSave, Costs.CallNoLRSave);
6031         CandidatesWithoutStackFixups.push_back(C);
6032       }
6033 
6034       // Is an unused register available? If so, we won't modify the stack, so
6035       // we can outline with the same frame type as those that don't save LR.
6036       else if (findRegisterToSaveLRTo(C)) {
6037         FrameID = MachineOutlinerRegSave;
6038         NumBytesNoStackCalls += Costs.CallRegSave;
6039         C.setCallInfo(MachineOutlinerRegSave, Costs.CallRegSave);
6040         CandidatesWithoutStackFixups.push_back(C);
6041       }
6042 
6043       // Is SP used in the sequence at all? If not, we don't have to modify
6044       // the stack, so we are guaranteed to get the same frame.
6045       else if (C.isAvailableInsideSeq(ARM::SP, TRI)) {
6046         NumBytesNoStackCalls += Costs.CallDefault;
6047         C.setCallInfo(MachineOutlinerDefault, Costs.CallDefault);
6048         CandidatesWithoutStackFixups.push_back(C);
6049       }
6050 
6051       // If we outline this, we need to modify the stack. Pretend we don't
6052       // outline this by saving all of its bytes.
6053       else
6054         NumBytesNoStackCalls += SequenceSize;
6055     }
6056 
6057     // If there are no places where we have to save LR, then note that we don't
6058     // have to update the stack. Otherwise, give every candidate the default
6059     // call type
6060     if (NumBytesNoStackCalls <=
6061         RepeatedSequenceLocs.size() * Costs.CallDefault) {
6062       RepeatedSequenceLocs = CandidatesWithoutStackFixups;
6063       FrameID = MachineOutlinerNoLRSave;
6064     } else
6065       SetCandidateCallInfo(MachineOutlinerDefault, Costs.CallDefault);
6066   }
6067 
6068   // Does every candidate's MBB contain a call?  If so, then we might have a
6069   // call in the range.
6070   if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
6071     // check if the range contains a call.  These require a save + restore of
6072     // the link register.
6073     if (std::any_of(FirstCand.front(), FirstCand.back(),
6074                     [](const MachineInstr &MI) { return MI.isCall(); }))
6075       NumBytesToCreateFrame += Costs.SaveRestoreLROnStack;
6076 
6077     // Handle the last instruction separately.  If it is tail call, then the
6078     // last instruction is a call, we don't want to save + restore in this
6079     // case.  However, it could be possible that the last instruction is a
6080     // call without it being valid to tail call this sequence.  We should
6081     // consider this as well.
6082     else if (FrameID != MachineOutlinerThunk &&
6083              FrameID != MachineOutlinerTailCall && FirstCand.back()->isCall())
6084       NumBytesToCreateFrame += Costs.SaveRestoreLROnStack;
6085   }
6086 
6087   return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize,
6088                                     NumBytesToCreateFrame, FrameID);
6089 }
6090 
6091 bool ARMBaseInstrInfo::checkAndUpdateStackOffset(MachineInstr *MI,
6092                                                  int64_t Fixup,
6093                                                  bool Updt) const {
6094   int SPIdx = MI->findRegisterUseOperandIdx(ARM::SP);
6095   unsigned AddrMode = (MI->getDesc().TSFlags & ARMII::AddrModeMask);
6096   if (SPIdx < 0)
6097     // No SP operand
6098     return true;
6099   else if (SPIdx != 1 && (AddrMode != ARMII::AddrModeT2_i8s4 || SPIdx != 2))
6100     // If SP is not the base register we can't do much
6101     return false;
6102 
6103   // Stack might be involved but addressing mode doesn't handle any offset.
6104   // Rq: AddrModeT1_[1|2|4] don't operate on SP
6105   if (AddrMode == ARMII::AddrMode1 ||       // Arithmetic instructions
6106       AddrMode == ARMII::AddrMode4 ||       // Load/Store Multiple
6107       AddrMode == ARMII::AddrMode6 ||       // Neon Load/Store Multiple
6108       AddrMode == ARMII::AddrModeT2_so ||   // SP can't be used as based register
6109       AddrMode == ARMII::AddrModeT2_pc ||   // PCrel access
6110       AddrMode == ARMII::AddrMode2 ||       // Used by PRE and POST indexed LD/ST
6111       AddrMode == ARMII::AddrModeT2_i7 ||   // v8.1-M MVE
6112       AddrMode == ARMII::AddrModeT2_i7s2 || // v8.1-M MVE
6113       AddrMode == ARMII::AddrModeT2_i7s4 || // v8.1-M sys regs VLDR/VSTR
6114       AddrMode == ARMII::AddrModeNone ||
6115       AddrMode == ARMII::AddrModeT2_i8 ||   // Pre/Post inc instructions
6116       AddrMode == ARMII::AddrModeT2_i8neg)  // Always negative imm
6117     return false;
6118 
6119   unsigned NumOps = MI->getDesc().getNumOperands();
6120   unsigned ImmIdx = NumOps - 3;
6121 
6122   const MachineOperand &Offset = MI->getOperand(ImmIdx);
6123   assert(Offset.isImm() && "Is not an immediate");
6124   int64_t OffVal = Offset.getImm();
6125 
6126   if (OffVal < 0)
6127     // Don't override data if the are below SP.
6128     return false;
6129 
6130   unsigned NumBits = 0;
6131   unsigned Scale = 1;
6132 
6133   switch (AddrMode) {
6134   case ARMII::AddrMode3:
6135     if (ARM_AM::getAM3Op(OffVal) == ARM_AM::sub)
6136       return false;
6137     OffVal = ARM_AM::getAM3Offset(OffVal);
6138     NumBits = 8;
6139     break;
6140   case ARMII::AddrMode5:
6141     if (ARM_AM::getAM5Op(OffVal) == ARM_AM::sub)
6142       return false;
6143     OffVal = ARM_AM::getAM5Offset(OffVal);
6144     NumBits = 8;
6145     Scale = 4;
6146     break;
6147   case ARMII::AddrMode5FP16:
6148     if (ARM_AM::getAM5FP16Op(OffVal) == ARM_AM::sub)
6149       return false;
6150     OffVal = ARM_AM::getAM5FP16Offset(OffVal);
6151     NumBits = 8;
6152     Scale = 2;
6153     break;
6154   case ARMII::AddrModeT2_i8pos:
6155     NumBits = 8;
6156     break;
6157   case ARMII::AddrModeT2_i8s4:
6158     // FIXME: Values are already scaled in this addressing mode.
6159     assert((Fixup & 3) == 0 && "Can't encode this offset!");
6160     NumBits = 10;
6161     break;
6162   case ARMII::AddrModeT2_ldrex:
6163     NumBits = 8;
6164     Scale = 4;
6165     break;
6166   case ARMII::AddrModeT2_i12:
6167   case ARMII::AddrMode_i12:
6168     NumBits = 12;
6169     break;
6170   case ARMII::AddrModeT1_s: // SP-relative LD/ST
6171     NumBits = 8;
6172     Scale = 4;
6173     break;
6174   default:
6175     llvm_unreachable("Unsupported addressing mode!");
6176   }
6177   // Make sure the offset is encodable for instructions that scale the
6178   // immediate.
6179   assert(((OffVal * Scale + Fixup) & (Scale - 1)) == 0 &&
6180          "Can't encode this offset!");
6181   OffVal += Fixup / Scale;
6182 
6183   unsigned Mask = (1 << NumBits) - 1;
6184 
6185   if (OffVal <= Mask) {
6186     if (Updt)
6187       MI->getOperand(ImmIdx).setImm(OffVal);
6188     return true;
6189   }
6190 
6191   return false;
6192 }
6193 
6194 void ARMBaseInstrInfo::mergeOutliningCandidateAttributes(
6195     Function &F, std::vector<outliner::Candidate> &Candidates) const {
6196   outliner::Candidate &C = Candidates.front();
6197   // branch-target-enforcement is guaranteed to be consistent between all
6198   // candidates, so we only need to look at one.
6199   const Function &CFn = C.getMF()->getFunction();
6200   if (CFn.hasFnAttribute("branch-target-enforcement"))
6201     F.addFnAttr(CFn.getFnAttribute("branch-target-enforcement"));
6202 
6203   ARMGenInstrInfo::mergeOutliningCandidateAttributes(F, Candidates);
6204 }
6205 
6206 bool ARMBaseInstrInfo::isFunctionSafeToOutlineFrom(
6207     MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
6208   const Function &F = MF.getFunction();
6209 
6210   // Can F be deduplicated by the linker? If it can, don't outline from it.
6211   if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
6212     return false;
6213 
6214   // Don't outline from functions with section markings; the program could
6215   // expect that all the code is in the named section.
6216   // FIXME: Allow outlining from multiple functions with the same section
6217   // marking.
6218   if (F.hasSection())
6219     return false;
6220 
6221   // FIXME: Thumb1 outlining is not handled
6222   if (MF.getInfo<ARMFunctionInfo>()->isThumb1OnlyFunction())
6223     return false;
6224 
6225   // It's safe to outline from MF.
6226   return true;
6227 }
6228 
6229 bool ARMBaseInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
6230                                               unsigned &Flags) const {
6231   // Check if LR is available through all of the MBB. If it's not, then set
6232   // a flag.
6233   assert(MBB.getParent()->getRegInfo().tracksLiveness() &&
6234          "Suitable Machine Function for outlining must track liveness");
6235 
6236   LiveRegUnits LRU(getRegisterInfo());
6237 
6238   for (MachineInstr &MI : llvm::reverse(MBB))
6239     LRU.accumulate(MI);
6240 
6241   // Check if each of the unsafe registers are available...
6242   bool R12AvailableInBlock = LRU.available(ARM::R12);
6243   bool CPSRAvailableInBlock = LRU.available(ARM::CPSR);
6244 
6245   // If all of these are dead (and not live out), we know we don't have to check
6246   // them later.
6247   if (R12AvailableInBlock && CPSRAvailableInBlock)
6248     Flags |= MachineOutlinerMBBFlags::UnsafeRegsDead;
6249 
6250   // Now, add the live outs to the set.
6251   LRU.addLiveOuts(MBB);
6252 
6253   // If any of these registers is available in the MBB, but also a live out of
6254   // the block, then we know outlining is unsafe.
6255   if (R12AvailableInBlock && !LRU.available(ARM::R12))
6256     return false;
6257   if (CPSRAvailableInBlock && !LRU.available(ARM::CPSR))
6258     return false;
6259 
6260   // Check if there's a call inside this MachineBasicBlock.  If there is, then
6261   // set a flag.
6262   if (any_of(MBB, [](MachineInstr &MI) { return MI.isCall(); }))
6263     Flags |= MachineOutlinerMBBFlags::HasCalls;
6264 
6265   // LR liveness is overestimated in return blocks.
6266 
6267   bool LRIsAvailable =
6268       MBB.isReturnBlock() && !MBB.back().isCall()
6269           ? isLRAvailable(getRegisterInfo(), MBB.rbegin(), MBB.rend())
6270           : LRU.available(ARM::LR);
6271   if (!LRIsAvailable)
6272     Flags |= MachineOutlinerMBBFlags::LRUnavailableSomewhere;
6273 
6274   return true;
6275 }
6276 
6277 outliner::InstrType
6278 ARMBaseInstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT,
6279                                    unsigned Flags) const {
6280   MachineInstr &MI = *MIT;
6281   const TargetRegisterInfo *TRI = &getRegisterInfo();
6282 
6283   // Be conservative with inline ASM
6284   if (MI.isInlineAsm())
6285     return outliner::InstrType::Illegal;
6286 
6287   // Don't allow debug values to impact outlining type.
6288   if (MI.isDebugInstr() || MI.isIndirectDebugValue())
6289     return outliner::InstrType::Invisible;
6290 
6291   // At this point, KILL or IMPLICIT_DEF instructions don't really tell us much
6292   // so we can go ahead and skip over them.
6293   if (MI.isKill() || MI.isImplicitDef())
6294     return outliner::InstrType::Invisible;
6295 
6296   // PIC instructions contain labels, outlining them would break offset
6297   // computing.  unsigned Opc = MI.getOpcode();
6298   unsigned Opc = MI.getOpcode();
6299   if (Opc == ARM::tPICADD || Opc == ARM::PICADD || Opc == ARM::PICSTR ||
6300       Opc == ARM::PICSTRB || Opc == ARM::PICSTRH || Opc == ARM::PICLDR ||
6301       Opc == ARM::PICLDRB || Opc == ARM::PICLDRH || Opc == ARM::PICLDRSB ||
6302       Opc == ARM::PICLDRSH || Opc == ARM::t2LDRpci_pic ||
6303       Opc == ARM::t2MOVi16_ga_pcrel || Opc == ARM::t2MOVTi16_ga_pcrel ||
6304       Opc == ARM::t2MOV_ga_pcrel)
6305     return outliner::InstrType::Illegal;
6306 
6307   // Be conservative with ARMv8.1 MVE instructions.
6308   if (Opc == ARM::t2BF_LabelPseudo || Opc == ARM::t2DoLoopStart ||
6309       Opc == ARM::t2DoLoopStartTP || Opc == ARM::t2WhileLoopStart ||
6310       Opc == ARM::t2WhileLoopStartLR || Opc == ARM::t2WhileLoopStartTP ||
6311       Opc == ARM::t2LoopDec || Opc == ARM::t2LoopEnd ||
6312       Opc == ARM::t2LoopEndDec)
6313     return outliner::InstrType::Illegal;
6314 
6315   const MCInstrDesc &MCID = MI.getDesc();
6316   uint64_t MIFlags = MCID.TSFlags;
6317   if ((MIFlags & ARMII::DomainMask) == ARMII::DomainMVE)
6318     return outliner::InstrType::Illegal;
6319 
6320   // Is this a terminator for a basic block?
6321   if (MI.isTerminator()) {
6322     // Don't outline if the branch is not unconditional.
6323     if (isPredicated(MI))
6324       return outliner::InstrType::Illegal;
6325 
6326     // Is this the end of a function?
6327     if (MI.getParent()->succ_empty())
6328       return outliner::InstrType::Legal;
6329 
6330     // It's not, so don't outline it.
6331     return outliner::InstrType::Illegal;
6332   }
6333 
6334   // Make sure none of the operands are un-outlinable.
6335   for (const MachineOperand &MOP : MI.operands()) {
6336     if (MOP.isCPI() || MOP.isJTI() || MOP.isCFIIndex() || MOP.isFI() ||
6337         MOP.isTargetIndex())
6338       return outliner::InstrType::Illegal;
6339   }
6340 
6341   // Don't outline if link register or program counter value are used.
6342   if (MI.readsRegister(ARM::LR, TRI) || MI.readsRegister(ARM::PC, TRI))
6343     return outliner::InstrType::Illegal;
6344 
6345   if (MI.isCall()) {
6346     // Get the function associated with the call.  Look at each operand and find
6347     // the one that represents the calle and get its name.
6348     const Function *Callee = nullptr;
6349     for (const MachineOperand &MOP : MI.operands()) {
6350       if (MOP.isGlobal()) {
6351         Callee = dyn_cast<Function>(MOP.getGlobal());
6352         break;
6353       }
6354     }
6355 
6356     // Dont't outline calls to "mcount" like functions, in particular Linux
6357     // kernel function tracing relies on it.
6358     if (Callee &&
6359         (Callee->getName() == "\01__gnu_mcount_nc" ||
6360          Callee->getName() == "\01mcount" || Callee->getName() == "__mcount"))
6361       return outliner::InstrType::Illegal;
6362 
6363     // If we don't know anything about the callee, assume it depends on the
6364     // stack layout of the caller. In that case, it's only legal to outline
6365     // as a tail-call. Explicitly list the call instructions we know about so
6366     // we don't get unexpected results with call pseudo-instructions.
6367     auto UnknownCallOutlineType = outliner::InstrType::Illegal;
6368     if (Opc == ARM::BL || Opc == ARM::tBL || Opc == ARM::BLX ||
6369         Opc == ARM::BLX_noip || Opc == ARM::tBLXr || Opc == ARM::tBLXr_noip ||
6370         Opc == ARM::tBLXi)
6371       UnknownCallOutlineType = outliner::InstrType::LegalTerminator;
6372 
6373     if (!Callee)
6374       return UnknownCallOutlineType;
6375 
6376     // We have a function we have information about.  Check if it's something we
6377     // can safely outline.
6378     MachineFunction *MF = MI.getParent()->getParent();
6379     MachineFunction *CalleeMF = MF->getMMI().getMachineFunction(*Callee);
6380 
6381     // We don't know what's going on with the callee at all.  Don't touch it.
6382     if (!CalleeMF)
6383       return UnknownCallOutlineType;
6384 
6385     // Check if we know anything about the callee saves on the function. If we
6386     // don't, then don't touch it, since that implies that we haven't computed
6387     // anything about its stack frame yet.
6388     MachineFrameInfo &MFI = CalleeMF->getFrameInfo();
6389     if (!MFI.isCalleeSavedInfoValid() || MFI.getStackSize() > 0 ||
6390         MFI.getNumObjects() > 0)
6391       return UnknownCallOutlineType;
6392 
6393     // At this point, we can say that CalleeMF ought to not pass anything on the
6394     // stack. Therefore, we can outline it.
6395     return outliner::InstrType::Legal;
6396   }
6397 
6398   // Since calls are handled, don't touch LR or PC
6399   if (MI.modifiesRegister(ARM::LR, TRI) || MI.modifiesRegister(ARM::PC, TRI))
6400     return outliner::InstrType::Illegal;
6401 
6402   // Does this use the stack?
6403   if (MI.modifiesRegister(ARM::SP, TRI) || MI.readsRegister(ARM::SP, TRI)) {
6404     // True if there is no chance that any outlined candidate from this range
6405     // could require stack fixups. That is, both
6406     // * LR is available in the range (No save/restore around call)
6407     // * The range doesn't include calls (No save/restore in outlined frame)
6408     // are true.
6409     // These conditions also ensure correctness of the return address
6410     // authentication - we insert sign and authentication instructions only if
6411     // we save/restore LR on stack, but then this condition ensures that the
6412     // outlined range does not modify the SP, therefore the SP value used for
6413     // signing is the same as the one used for authentication.
6414     // FIXME: This is very restrictive; the flags check the whole block,
6415     // not just the bit we will try to outline.
6416     bool MightNeedStackFixUp =
6417         (Flags & (MachineOutlinerMBBFlags::LRUnavailableSomewhere |
6418                   MachineOutlinerMBBFlags::HasCalls));
6419 
6420     if (!MightNeedStackFixUp)
6421       return outliner::InstrType::Legal;
6422 
6423     // Any modification of SP will break our code to save/restore LR.
6424     // FIXME: We could handle some instructions which add a constant offset to
6425     // SP, with a bit more work.
6426     if (MI.modifiesRegister(ARM::SP, TRI))
6427       return outliner::InstrType::Illegal;
6428 
6429     // At this point, we have a stack instruction that we might need to fix up.
6430     // up. We'll handle it if it's a load or store.
6431     if (checkAndUpdateStackOffset(&MI, Subtarget.getStackAlignment().value(),
6432                                   false))
6433       return outliner::InstrType::Legal;
6434 
6435     // We can't fix it up, so don't outline it.
6436     return outliner::InstrType::Illegal;
6437   }
6438 
6439   // Be conservative with IT blocks.
6440   if (MI.readsRegister(ARM::ITSTATE, TRI) ||
6441       MI.modifiesRegister(ARM::ITSTATE, TRI))
6442     return outliner::InstrType::Illegal;
6443 
6444   // Don't outline positions.
6445   if (MI.isPosition())
6446     return outliner::InstrType::Illegal;
6447 
6448   return outliner::InstrType::Legal;
6449 }
6450 
6451 void ARMBaseInstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const {
6452   for (MachineInstr &MI : MBB) {
6453     checkAndUpdateStackOffset(&MI, Subtarget.getStackAlignment().value(), true);
6454   }
6455 }
6456 
6457 void ARMBaseInstrInfo::saveLROnStack(MachineBasicBlock &MBB,
6458                                      MachineBasicBlock::iterator It, bool CFI,
6459                                      bool Auth) const {
6460   int Align = std::max(Subtarget.getStackAlignment().value(), uint64_t(8));
6461   assert(Align >= 8 && Align <= 256);
6462   if (Auth) {
6463     assert(Subtarget.isThumb2());
6464     // Compute PAC in R12. Outlining ensures R12 is dead across the outlined
6465     // sequence.
6466     BuildMI(MBB, It, DebugLoc(), get(ARM::t2PAC))
6467         .setMIFlags(MachineInstr::FrameSetup);
6468     BuildMI(MBB, It, DebugLoc(), get(ARM::t2STRD_PRE), ARM::SP)
6469         .addReg(ARM::R12, RegState::Kill)
6470         .addReg(ARM::LR, RegState::Kill)
6471         .addReg(ARM::SP)
6472         .addImm(-Align)
6473         .add(predOps(ARMCC::AL))
6474         .setMIFlags(MachineInstr::FrameSetup);
6475   } else {
6476     unsigned Opc = Subtarget.isThumb() ? ARM::t2STR_PRE : ARM::STR_PRE_IMM;
6477     BuildMI(MBB, It, DebugLoc(), get(Opc), ARM::SP)
6478         .addReg(ARM::LR, RegState::Kill)
6479         .addReg(ARM::SP)
6480         .addImm(-Align)
6481         .add(predOps(ARMCC::AL))
6482         .setMIFlags(MachineInstr::FrameSetup);
6483   }
6484 
6485   if (!CFI)
6486     return;
6487 
6488   MachineFunction &MF = *MBB.getParent();
6489 
6490   // Add a CFI, saying CFA is offset by Align bytes from SP.
6491   int64_t StackPosEntry =
6492       MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, Align));
6493   BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
6494       .addCFIIndex(StackPosEntry)
6495       .setMIFlags(MachineInstr::FrameSetup);
6496 
6497   // Add a CFI saying that the LR that we want to find is now higher than
6498   // before.
6499   int LROffset = Auth ? Align - 4 : Align;
6500   const MCRegisterInfo *MRI = Subtarget.getRegisterInfo();
6501   unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true);
6502   int64_t LRPosEntry = MF.addFrameInst(
6503       MCCFIInstruction::createOffset(nullptr, DwarfLR, -LROffset));
6504   BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
6505       .addCFIIndex(LRPosEntry)
6506       .setMIFlags(MachineInstr::FrameSetup);
6507   if (Auth) {
6508     // Add a CFI for the location of the return adddress PAC.
6509     unsigned DwarfRAC = MRI->getDwarfRegNum(ARM::RA_AUTH_CODE, true);
6510     int64_t RACPosEntry = MF.addFrameInst(
6511         MCCFIInstruction::createOffset(nullptr, DwarfRAC, -Align));
6512     BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
6513         .addCFIIndex(RACPosEntry)
6514         .setMIFlags(MachineInstr::FrameSetup);
6515   }
6516 }
6517 
6518 void ARMBaseInstrInfo::emitCFIForLRSaveToReg(MachineBasicBlock &MBB,
6519                                              MachineBasicBlock::iterator It,
6520                                              Register Reg) const {
6521   MachineFunction &MF = *MBB.getParent();
6522   const MCRegisterInfo *MRI = Subtarget.getRegisterInfo();
6523   unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true);
6524   unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true);
6525 
6526   int64_t LRPosEntry = MF.addFrameInst(
6527       MCCFIInstruction::createRegister(nullptr, DwarfLR, DwarfReg));
6528   BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
6529       .addCFIIndex(LRPosEntry)
6530       .setMIFlags(MachineInstr::FrameSetup);
6531 }
6532 
6533 void ARMBaseInstrInfo::restoreLRFromStack(MachineBasicBlock &MBB,
6534                                           MachineBasicBlock::iterator It,
6535                                           bool CFI, bool Auth) const {
6536   int Align = Subtarget.getStackAlignment().value();
6537   if (Auth) {
6538     assert(Subtarget.isThumb2());
6539     // Restore return address PAC and LR.
6540     BuildMI(MBB, It, DebugLoc(), get(ARM::t2LDRD_POST))
6541         .addReg(ARM::R12, RegState::Define)
6542         .addReg(ARM::LR, RegState::Define)
6543         .addReg(ARM::SP, RegState::Define)
6544         .addReg(ARM::SP)
6545         .addImm(Align)
6546         .add(predOps(ARMCC::AL))
6547         .setMIFlags(MachineInstr::FrameDestroy);
6548     // LR authentication is after the CFI instructions, below.
6549   } else {
6550     unsigned Opc = Subtarget.isThumb() ? ARM::t2LDR_POST : ARM::LDR_POST_IMM;
6551     MachineInstrBuilder MIB = BuildMI(MBB, It, DebugLoc(), get(Opc), ARM::LR)
6552                                   .addReg(ARM::SP, RegState::Define)
6553                                   .addReg(ARM::SP);
6554     if (!Subtarget.isThumb())
6555       MIB.addReg(0);
6556     MIB.addImm(Subtarget.getStackAlignment().value())
6557         .add(predOps(ARMCC::AL))
6558         .setMIFlags(MachineInstr::FrameDestroy);
6559   }
6560 
6561   if (CFI) {
6562     // Now stack has moved back up...
6563     MachineFunction &MF = *MBB.getParent();
6564     const MCRegisterInfo *MRI = Subtarget.getRegisterInfo();
6565     unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true);
6566     int64_t StackPosEntry =
6567         MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, 0));
6568     BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
6569         .addCFIIndex(StackPosEntry)
6570         .setMIFlags(MachineInstr::FrameDestroy);
6571 
6572     // ... and we have restored LR.
6573     int64_t LRPosEntry =
6574         MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, DwarfLR));
6575     BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
6576         .addCFIIndex(LRPosEntry)
6577         .setMIFlags(MachineInstr::FrameDestroy);
6578 
6579     if (Auth) {
6580       unsigned DwarfRAC = MRI->getDwarfRegNum(ARM::RA_AUTH_CODE, true);
6581       int64_t Entry =
6582           MF.addFrameInst(MCCFIInstruction::createUndefined(nullptr, DwarfRAC));
6583       BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
6584           .addCFIIndex(Entry)
6585           .setMIFlags(MachineInstr::FrameDestroy);
6586     }
6587   }
6588 
6589   if (Auth)
6590     BuildMI(MBB, It, DebugLoc(), get(ARM::t2AUT));
6591 }
6592 
6593 void ARMBaseInstrInfo::emitCFIForLRRestoreFromReg(
6594     MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const {
6595   MachineFunction &MF = *MBB.getParent();
6596   const MCRegisterInfo *MRI = Subtarget.getRegisterInfo();
6597   unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true);
6598 
6599   int64_t LRPosEntry =
6600       MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, DwarfLR));
6601   BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION))
6602       .addCFIIndex(LRPosEntry)
6603       .setMIFlags(MachineInstr::FrameDestroy);
6604 }
6605 
6606 void ARMBaseInstrInfo::buildOutlinedFrame(
6607     MachineBasicBlock &MBB, MachineFunction &MF,
6608     const outliner::OutlinedFunction &OF) const {
6609   // For thunk outlining, rewrite the last instruction from a call to a
6610   // tail-call.
6611   if (OF.FrameConstructionID == MachineOutlinerThunk) {
6612     MachineInstr *Call = &*--MBB.instr_end();
6613     bool isThumb = Subtarget.isThumb();
6614     unsigned FuncOp = isThumb ? 2 : 0;
6615     unsigned Opc = Call->getOperand(FuncOp).isReg()
6616                        ? isThumb ? ARM::tTAILJMPr : ARM::TAILJMPr
6617                        : isThumb ? Subtarget.isTargetMachO() ? ARM::tTAILJMPd
6618                                                              : ARM::tTAILJMPdND
6619                                  : ARM::TAILJMPd;
6620     MachineInstrBuilder MIB = BuildMI(MBB, MBB.end(), DebugLoc(), get(Opc))
6621                                   .add(Call->getOperand(FuncOp));
6622     if (isThumb && !Call->getOperand(FuncOp).isReg())
6623       MIB.add(predOps(ARMCC::AL));
6624     Call->eraseFromParent();
6625   }
6626 
6627   // Is there a call in the outlined range?
6628   auto IsNonTailCall = [](MachineInstr &MI) {
6629     return MI.isCall() && !MI.isReturn();
6630   };
6631   if (llvm::any_of(MBB.instrs(), IsNonTailCall)) {
6632     MachineBasicBlock::iterator It = MBB.begin();
6633     MachineBasicBlock::iterator Et = MBB.end();
6634 
6635     if (OF.FrameConstructionID == MachineOutlinerTailCall ||
6636         OF.FrameConstructionID == MachineOutlinerThunk)
6637       Et = std::prev(MBB.end());
6638 
6639     // We have to save and restore LR, we need to add it to the liveins if it
6640     // is not already part of the set.  This is suffient since outlined
6641     // functions only have one block.
6642     if (!MBB.isLiveIn(ARM::LR))
6643       MBB.addLiveIn(ARM::LR);
6644 
6645     // Insert a save before the outlined region
6646     bool Auth = OF.Candidates.front()
6647                     .getMF()
6648                     ->getInfo<ARMFunctionInfo>()
6649                     ->shouldSignReturnAddress(true);
6650     saveLROnStack(MBB, It, true, Auth);
6651 
6652     // Fix up the instructions in the range, since we're going to modify the
6653     // stack.
6654     assert(OF.FrameConstructionID != MachineOutlinerDefault &&
6655            "Can only fix up stack references once");
6656     fixupPostOutline(MBB);
6657 
6658     // Insert a restore before the terminator for the function.  Restore LR.
6659     restoreLRFromStack(MBB, Et, true, Auth);
6660   }
6661 
6662   // If this is a tail call outlined function, then there's already a return.
6663   if (OF.FrameConstructionID == MachineOutlinerTailCall ||
6664       OF.FrameConstructionID == MachineOutlinerThunk)
6665     return;
6666 
6667   // Here we have to insert the return ourselves.  Get the correct opcode from
6668   // current feature set.
6669   BuildMI(MBB, MBB.end(), DebugLoc(), get(Subtarget.getReturnOpcode()))
6670       .add(predOps(ARMCC::AL));
6671 
6672   // Did we have to modify the stack by saving the link register?
6673   if (OF.FrameConstructionID != MachineOutlinerDefault &&
6674       OF.Candidates[0].CallConstructionID != MachineOutlinerDefault)
6675     return;
6676 
6677   // We modified the stack.
6678   // Walk over the basic block and fix up all the stack accesses.
6679   fixupPostOutline(MBB);
6680 }
6681 
6682 MachineBasicBlock::iterator ARMBaseInstrInfo::insertOutlinedCall(
6683     Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
6684     MachineFunction &MF, outliner::Candidate &C) const {
6685   MachineInstrBuilder MIB;
6686   MachineBasicBlock::iterator CallPt;
6687   unsigned Opc;
6688   bool isThumb = Subtarget.isThumb();
6689 
6690   // Are we tail calling?
6691   if (C.CallConstructionID == MachineOutlinerTailCall) {
6692     // If yes, then we can just branch to the label.
6693     Opc = isThumb
6694               ? Subtarget.isTargetMachO() ? ARM::tTAILJMPd : ARM::tTAILJMPdND
6695               : ARM::TAILJMPd;
6696     MIB = BuildMI(MF, DebugLoc(), get(Opc))
6697               .addGlobalAddress(M.getNamedValue(MF.getName()));
6698     if (isThumb)
6699       MIB.add(predOps(ARMCC::AL));
6700     It = MBB.insert(It, MIB);
6701     return It;
6702   }
6703 
6704   // Create the call instruction.
6705   Opc = isThumb ? ARM::tBL : ARM::BL;
6706   MachineInstrBuilder CallMIB = BuildMI(MF, DebugLoc(), get(Opc));
6707   if (isThumb)
6708     CallMIB.add(predOps(ARMCC::AL));
6709   CallMIB.addGlobalAddress(M.getNamedValue(MF.getName()));
6710 
6711   if (C.CallConstructionID == MachineOutlinerNoLRSave ||
6712       C.CallConstructionID == MachineOutlinerThunk) {
6713     // No, so just insert the call.
6714     It = MBB.insert(It, CallMIB);
6715     return It;
6716   }
6717 
6718   const ARMFunctionInfo &AFI = *C.getMF()->getInfo<ARMFunctionInfo>();
6719   // Can we save to a register?
6720   if (C.CallConstructionID == MachineOutlinerRegSave) {
6721     Register Reg = findRegisterToSaveLRTo(C);
6722     assert(Reg != 0 && "No callee-saved register available?");
6723 
6724     // Save and restore LR from that register.
6725     copyPhysReg(MBB, It, DebugLoc(), Reg, ARM::LR, true);
6726     if (!AFI.isLRSpilled())
6727       emitCFIForLRSaveToReg(MBB, It, Reg);
6728     CallPt = MBB.insert(It, CallMIB);
6729     copyPhysReg(MBB, It, DebugLoc(), ARM::LR, Reg, true);
6730     if (!AFI.isLRSpilled())
6731       emitCFIForLRRestoreFromReg(MBB, It);
6732     It--;
6733     return CallPt;
6734   }
6735   // We have the default case. Save and restore from SP.
6736   if (!MBB.isLiveIn(ARM::LR))
6737     MBB.addLiveIn(ARM::LR);
6738   bool Auth = !AFI.isLRSpilled() && AFI.shouldSignReturnAddress(true);
6739   saveLROnStack(MBB, It, !AFI.isLRSpilled(), Auth);
6740   CallPt = MBB.insert(It, CallMIB);
6741   restoreLRFromStack(MBB, It, !AFI.isLRSpilled(), Auth);
6742   It--;
6743   return CallPt;
6744 }
6745 
6746 bool ARMBaseInstrInfo::shouldOutlineFromFunctionByDefault(
6747     MachineFunction &MF) const {
6748   return Subtarget.isMClass() && MF.getFunction().hasMinSize();
6749 }
6750 
6751 bool ARMBaseInstrInfo::isReallyTriviallyReMaterializable(
6752     const MachineInstr &MI) const {
6753   // Try hard to rematerialize any VCTPs because if we spill P0, it will block
6754   // the tail predication conversion. This means that the element count
6755   // register has to be live for longer, but that has to be better than
6756   // spill/restore and VPT predication.
6757   return isVCTP(&MI) && !isPredicated(MI);
6758 }
6759 
6760 unsigned llvm::getBLXOpcode(const MachineFunction &MF) {
6761   return (MF.getSubtarget<ARMSubtarget>().hardenSlsBlr()) ? ARM::BLX_noip
6762                                                           : ARM::BLX;
6763 }
6764 
6765 unsigned llvm::gettBLXrOpcode(const MachineFunction &MF) {
6766   return (MF.getSubtarget<ARMSubtarget>().hardenSlsBlr()) ? ARM::tBLXr_noip
6767                                                           : ARM::tBLXr;
6768 }
6769 
6770 unsigned llvm::getBLXpredOpcode(const MachineFunction &MF) {
6771   return (MF.getSubtarget<ARMSubtarget>().hardenSlsBlr()) ? ARM::BLX_pred_noip
6772                                                           : ARM::BLX_pred;
6773 }
6774 
6775 namespace {
6776 class ARMPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
6777   MachineInstr *EndLoop, *LoopCount;
6778   MachineFunction *MF;
6779   const TargetInstrInfo *TII;
6780 
6781   // Bitset[0 .. MAX_STAGES-1] ... iterations needed
6782   //       [LAST_IS_USE] : last reference to register in schedule is a use
6783   //       [SEEN_AS_LIVE] : Normal pressure algorithm believes register is live
6784   static int constexpr MAX_STAGES = 30;
6785   static int constexpr LAST_IS_USE = MAX_STAGES;
6786   static int constexpr SEEN_AS_LIVE = MAX_STAGES + 1;
6787   typedef std::bitset<MAX_STAGES + 2> IterNeed;
6788   typedef std::map<unsigned, IterNeed> IterNeeds;
6789 
6790   void bumpCrossIterationPressure(RegPressureTracker &RPT,
6791                                   const IterNeeds &CIN);
6792   bool tooMuchRegisterPressure(SwingSchedulerDAG &SSD, SMSchedule &SMS);
6793 
6794   // Meanings of the various stuff with loop types:
6795   // t2Bcc:
6796   //   EndLoop = branch at end of original BB that will become a kernel
6797   //   LoopCount = CC setter live into branch
6798   // t2LoopEnd:
6799   //   EndLoop = branch at end of original BB
6800   //   LoopCount = t2LoopDec
6801 public:
6802   ARMPipelinerLoopInfo(MachineInstr *EndLoop, MachineInstr *LoopCount)
6803       : EndLoop(EndLoop), LoopCount(LoopCount),
6804         MF(EndLoop->getParent()->getParent()),
6805         TII(MF->getSubtarget().getInstrInfo()) {}
6806 
6807   bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
6808     // Only ignore the terminator.
6809     return MI == EndLoop || MI == LoopCount;
6810   }
6811 
6812   bool shouldUseSchedule(SwingSchedulerDAG &SSD, SMSchedule &SMS) override {
6813     if (tooMuchRegisterPressure(SSD, SMS))
6814       return false;
6815 
6816     return true;
6817   }
6818 
6819   std::optional<bool> createTripCountGreaterCondition(
6820       int TC, MachineBasicBlock &MBB,
6821       SmallVectorImpl<MachineOperand> &Cond) override {
6822 
6823     if (isCondBranchOpcode(EndLoop->getOpcode())) {
6824       Cond.push_back(EndLoop->getOperand(1));
6825       Cond.push_back(EndLoop->getOperand(2));
6826       if (EndLoop->getOperand(0).getMBB() == EndLoop->getParent()) {
6827         TII->reverseBranchCondition(Cond);
6828       }
6829       return {};
6830     } else if (EndLoop->getOpcode() == ARM::t2LoopEnd) {
6831       // General case just lets the unrolled t2LoopDec do the subtraction and
6832       // therefore just needs to check if zero has been reached.
6833       MachineInstr *LoopDec = nullptr;
6834       for (auto &I : MBB.instrs())
6835         if (I.getOpcode() == ARM::t2LoopDec)
6836           LoopDec = &I;
6837       assert(LoopDec && "Unable to find copied LoopDec");
6838       // Check if we're done with the loop.
6839       BuildMI(&MBB, LoopDec->getDebugLoc(), TII->get(ARM::t2CMPri))
6840           .addReg(LoopDec->getOperand(0).getReg())
6841           .addImm(0)
6842           .addImm(ARMCC::AL)
6843           .addReg(ARM::NoRegister);
6844       Cond.push_back(MachineOperand::CreateImm(ARMCC::EQ));
6845       Cond.push_back(MachineOperand::CreateReg(ARM::CPSR, false));
6846       return {};
6847     } else
6848       llvm_unreachable("Unknown EndLoop");
6849   }
6850 
6851   void setPreheader(MachineBasicBlock *NewPreheader) override {}
6852 
6853   void adjustTripCount(int TripCountAdjust) override {}
6854 
6855   void disposed() override {}
6856 };
6857 
6858 void ARMPipelinerLoopInfo::bumpCrossIterationPressure(RegPressureTracker &RPT,
6859                                                       const IterNeeds &CIN) {
6860   // Increase pressure by the amounts in CrossIterationNeeds
6861   for (const auto &N : CIN) {
6862     int Cnt = N.second.count() - N.second[SEEN_AS_LIVE] * 2;
6863     for (int I = 0; I < Cnt; ++I)
6864       RPT.increaseRegPressure(Register(N.first), LaneBitmask::getNone(),
6865                               LaneBitmask::getAll());
6866   }
6867   // Decrease pressure by the amounts in CrossIterationNeeds
6868   for (const auto &N : CIN) {
6869     int Cnt = N.second.count() - N.second[SEEN_AS_LIVE] * 2;
6870     for (int I = 0; I < Cnt; ++I)
6871       RPT.decreaseRegPressure(Register(N.first), LaneBitmask::getAll(),
6872                               LaneBitmask::getNone());
6873   }
6874 }
6875 
6876 bool ARMPipelinerLoopInfo::tooMuchRegisterPressure(SwingSchedulerDAG &SSD,
6877                                                    SMSchedule &SMS) {
6878   IterNeeds CrossIterationNeeds;
6879 
6880   // Determine which values will be loop-carried after the schedule is
6881   // applied
6882 
6883   for (auto &SU : SSD.SUnits) {
6884     const MachineInstr *MI = SU.getInstr();
6885     int Stg = SMS.stageScheduled(const_cast<SUnit *>(&SU));
6886     for (auto &S : SU.Succs)
6887       if (MI->isPHI() && S.getKind() == SDep::Anti) {
6888         Register Reg = S.getReg();
6889         if (Reg.isVirtual())
6890           CrossIterationNeeds.insert(std::make_pair(Reg.id(), IterNeed()))
6891               .first->second.set(0);
6892       } else if (S.isAssignedRegDep()) {
6893         int OStg = SMS.stageScheduled(S.getSUnit());
6894         if (OStg >= 0 && OStg != Stg) {
6895           Register Reg = S.getReg();
6896           if (Reg.isVirtual())
6897             CrossIterationNeeds.insert(std::make_pair(Reg.id(), IterNeed()))
6898                 .first->second |= ((1 << (OStg - Stg)) - 1);
6899         }
6900       }
6901   }
6902 
6903   // Determine more-or-less what the proposed schedule (reversed) is going to
6904   // be; it might not be quite the same because the within-cycle ordering
6905   // created by SMSchedule depends upon changes to help with address offsets and
6906   // the like.
6907   std::vector<SUnit *> ProposedSchedule;
6908   for (int Cycle = SMS.getFinalCycle(); Cycle >= SMS.getFirstCycle(); --Cycle)
6909     for (int Stage = 0, StageEnd = SMS.getMaxStageCount(); Stage <= StageEnd;
6910          ++Stage) {
6911       std::deque<SUnit *> Instrs =
6912           SMS.getInstructions(Cycle + Stage * SMS.getInitiationInterval());
6913       std::sort(Instrs.begin(), Instrs.end(),
6914                 [](SUnit *A, SUnit *B) { return A->NodeNum > B->NodeNum; });
6915       for (SUnit *SU : Instrs)
6916         ProposedSchedule.push_back(SU);
6917     }
6918 
6919   // Learn whether the last use/def of each cross-iteration register is a use or
6920   // def. If it is a def, RegisterPressure will implicitly increase max pressure
6921   // and we do not have to add the pressure.
6922   for (auto *SU : ProposedSchedule)
6923     for (ConstMIBundleOperands OperI(*SU->getInstr()); OperI.isValid();
6924          ++OperI) {
6925       auto MO = *OperI;
6926       if (!MO.isReg() || !MO.getReg())
6927         continue;
6928       Register Reg = MO.getReg();
6929       auto CIter = CrossIterationNeeds.find(Reg.id());
6930       if (CIter == CrossIterationNeeds.end() || CIter->second[LAST_IS_USE] ||
6931           CIter->second[SEEN_AS_LIVE])
6932         continue;
6933       if (MO.isDef() && !MO.isDead())
6934         CIter->second.set(SEEN_AS_LIVE);
6935       else if (MO.isUse())
6936         CIter->second.set(LAST_IS_USE);
6937     }
6938   for (auto &CI : CrossIterationNeeds)
6939     CI.second.reset(LAST_IS_USE);
6940 
6941   RegionPressure RecRegPressure;
6942   RegPressureTracker RPTracker(RecRegPressure);
6943   RegisterClassInfo RegClassInfo;
6944   RegClassInfo.runOnMachineFunction(*MF);
6945   RPTracker.init(MF, &RegClassInfo, nullptr, EndLoop->getParent(),
6946                  EndLoop->getParent()->end(), false, false);
6947   const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
6948 
6949   bumpCrossIterationPressure(RPTracker, CrossIterationNeeds);
6950 
6951   for (auto *SU : ProposedSchedule) {
6952     MachineBasicBlock::const_iterator CurInstI = SU->getInstr();
6953     RPTracker.setPos(std::next(CurInstI));
6954     RPTracker.recede();
6955 
6956     // Track what cross-iteration registers would be seen as live
6957     for (ConstMIBundleOperands OperI(*CurInstI); OperI.isValid(); ++OperI) {
6958       auto MO = *OperI;
6959       if (!MO.isReg() || !MO.getReg())
6960         continue;
6961       Register Reg = MO.getReg();
6962       if (MO.isDef() && !MO.isDead()) {
6963         auto CIter = CrossIterationNeeds.find(Reg.id());
6964         if (CIter != CrossIterationNeeds.end()) {
6965           CIter->second.reset(0);
6966           CIter->second.reset(SEEN_AS_LIVE);
6967         }
6968       }
6969     }
6970     for (auto &S : SU->Preds) {
6971       auto Stg = SMS.stageScheduled(SU);
6972       if (S.isAssignedRegDep()) {
6973         Register Reg = S.getReg();
6974         auto CIter = CrossIterationNeeds.find(Reg.id());
6975         if (CIter != CrossIterationNeeds.end()) {
6976           auto Stg2 = SMS.stageScheduled(const_cast<SUnit *>(S.getSUnit()));
6977           assert(Stg2 <= Stg && "Data dependence upon earlier stage");
6978           if (Stg - Stg2 < MAX_STAGES)
6979             CIter->second.set(Stg - Stg2);
6980           CIter->second.set(SEEN_AS_LIVE);
6981         }
6982       }
6983     }
6984 
6985     bumpCrossIterationPressure(RPTracker, CrossIterationNeeds);
6986   }
6987 
6988   auto &P = RPTracker.getPressure().MaxSetPressure;
6989   for (unsigned I = 0, E = P.size(); I < E; ++I)
6990     if (P[I] > TRI->getRegPressureSetLimit(*MF, I)) {
6991       return true;
6992     }
6993   return false;
6994 }
6995 
6996 } // namespace
6997 
6998 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
6999 ARMBaseInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
7000   MachineBasicBlock::iterator I = LoopBB->getFirstTerminator();
7001   MachineBasicBlock *Preheader = *LoopBB->pred_begin();
7002   if (Preheader == LoopBB)
7003     Preheader = *std::next(LoopBB->pred_begin());
7004 
7005   if (I != LoopBB->end() && I->getOpcode() == ARM::t2Bcc) {
7006     // If the branch is a Bcc, then the CPSR should be set somewhere within the
7007     // block.  We need to determine the reaching definition of CPSR so that
7008     // it can be marked as non-pipelineable, allowing the pipeliner to force
7009     // it into stage 0 or give up if it cannot or will not do so.
7010     MachineInstr *CCSetter = nullptr;
7011     for (auto &L : LoopBB->instrs()) {
7012       if (L.isCall())
7013         return nullptr;
7014       if (isCPSRDefined(L))
7015         CCSetter = &L;
7016     }
7017     if (CCSetter)
7018       return std::make_unique<ARMPipelinerLoopInfo>(&*I, CCSetter);
7019     else
7020       return nullptr; // Unable to find the CC setter, so unable to guarantee
7021                       // that pipeline will work
7022   }
7023 
7024   // Recognize:
7025   //   preheader:
7026   //     %1 = t2DoopLoopStart %0
7027   //   loop:
7028   //     %2 = phi %1, <not loop>, %..., %loop
7029   //     %3 = t2LoopDec %2, <imm>
7030   //     t2LoopEnd %3, %loop
7031 
7032   if (I != LoopBB->end() && I->getOpcode() == ARM::t2LoopEnd) {
7033     for (auto &L : LoopBB->instrs())
7034       if (L.isCall())
7035         return nullptr;
7036       else if (isVCTP(&L))
7037         return nullptr;
7038     Register LoopDecResult = I->getOperand(0).getReg();
7039     MachineRegisterInfo &MRI = LoopBB->getParent()->getRegInfo();
7040     MachineInstr *LoopDec = MRI.getUniqueVRegDef(LoopDecResult);
7041     if (!LoopDec || LoopDec->getOpcode() != ARM::t2LoopDec)
7042       return nullptr;
7043     MachineInstr *LoopStart = nullptr;
7044     for (auto &J : Preheader->instrs())
7045       if (J.getOpcode() == ARM::t2DoLoopStart)
7046         LoopStart = &J;
7047     if (!LoopStart)
7048       return nullptr;
7049     return std::make_unique<ARMPipelinerLoopInfo>(&*I, LoopDec);
7050   }
7051   return nullptr;
7052 }
7053