xref: /freebsd/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp (revision e1e636193db45630c7881246d25902e57c43d24e)
1 //===-- PPCInstrInfo.cpp - PowerPC Instruction Information ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the PowerPC implementation of the TargetInstrInfo class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "PPCInstrInfo.h"
14 #include "MCTargetDesc/PPCPredicates.h"
15 #include "PPC.h"
16 #include "PPCHazardRecognizers.h"
17 #include "PPCInstrBuilder.h"
18 #include "PPCMachineFunctionInfo.h"
19 #include "PPCTargetMachine.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/Statistic.h"
22 #include "llvm/Analysis/AliasAnalysis.h"
23 #include "llvm/CodeGen/LiveIntervals.h"
24 #include "llvm/CodeGen/LivePhysRegs.h"
25 #include "llvm/CodeGen/MachineCombinerPattern.h"
26 #include "llvm/CodeGen/MachineConstantPool.h"
27 #include "llvm/CodeGen/MachineFrameInfo.h"
28 #include "llvm/CodeGen/MachineFunctionPass.h"
29 #include "llvm/CodeGen/MachineInstrBuilder.h"
30 #include "llvm/CodeGen/MachineMemOperand.h"
31 #include "llvm/CodeGen/MachineRegisterInfo.h"
32 #include "llvm/CodeGen/PseudoSourceValue.h"
33 #include "llvm/CodeGen/RegisterClassInfo.h"
34 #include "llvm/CodeGen/RegisterPressure.h"
35 #include "llvm/CodeGen/ScheduleDAG.h"
36 #include "llvm/CodeGen/SlotIndexes.h"
37 #include "llvm/CodeGen/StackMaps.h"
38 #include "llvm/MC/MCAsmInfo.h"
39 #include "llvm/MC/MCInst.h"
40 #include "llvm/MC/TargetRegistry.h"
41 #include "llvm/Support/CommandLine.h"
42 #include "llvm/Support/Debug.h"
43 #include "llvm/Support/ErrorHandling.h"
44 #include "llvm/Support/raw_ostream.h"
45 
46 using namespace llvm;
47 
48 #define DEBUG_TYPE "ppc-instr-info"
49 
50 #define GET_INSTRMAP_INFO
51 #define GET_INSTRINFO_CTOR_DTOR
52 #include "PPCGenInstrInfo.inc"
53 
54 STATISTIC(NumStoreSPILLVSRRCAsVec,
55           "Number of spillvsrrc spilled to stack as vec");
56 STATISTIC(NumStoreSPILLVSRRCAsGpr,
57           "Number of spillvsrrc spilled to stack as gpr");
58 STATISTIC(NumGPRtoVSRSpill, "Number of gpr spills to spillvsrrc");
59 STATISTIC(CmpIselsConverted,
60           "Number of ISELs that depend on comparison of constants converted");
61 STATISTIC(MissedConvertibleImmediateInstrs,
62           "Number of compare-immediate instructions fed by constants");
63 STATISTIC(NumRcRotatesConvertedToRcAnd,
64           "Number of record-form rotates converted to record-form andi");
65 
66 static cl::
67 opt<bool> DisableCTRLoopAnal("disable-ppc-ctrloop-analysis", cl::Hidden,
68             cl::desc("Disable analysis for CTR loops"));
69 
70 static cl::opt<bool> DisableCmpOpt("disable-ppc-cmp-opt",
71 cl::desc("Disable compare instruction optimization"), cl::Hidden);
72 
73 static cl::opt<bool> VSXSelfCopyCrash("crash-on-ppc-vsx-self-copy",
74 cl::desc("Causes the backend to crash instead of generating a nop VSX copy"),
75 cl::Hidden);
76 
77 static cl::opt<bool>
78 UseOldLatencyCalc("ppc-old-latency-calc", cl::Hidden,
79   cl::desc("Use the old (incorrect) instruction latency calculation"));
80 
81 static cl::opt<float>
82     FMARPFactor("ppc-fma-rp-factor", cl::Hidden, cl::init(1.5),
83                 cl::desc("register pressure factor for the transformations."));
84 
85 static cl::opt<bool> EnableFMARegPressureReduction(
86     "ppc-fma-rp-reduction", cl::Hidden, cl::init(true),
87     cl::desc("enable register pressure reduce in machine combiner pass."));
88 
89 // Pin the vtable to this file.
90 void PPCInstrInfo::anchor() {}
91 
92 PPCInstrInfo::PPCInstrInfo(PPCSubtarget &STI)
93     : PPCGenInstrInfo(PPC::ADJCALLSTACKDOWN, PPC::ADJCALLSTACKUP,
94                       /* CatchRetOpcode */ -1,
95                       STI.isPPC64() ? PPC::BLR8 : PPC::BLR),
96       Subtarget(STI), RI(STI.getTargetMachine()) {}
97 
98 /// CreateTargetHazardRecognizer - Return the hazard recognizer to use for
99 /// this target when scheduling the DAG.
100 ScheduleHazardRecognizer *
101 PPCInstrInfo::CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI,
102                                            const ScheduleDAG *DAG) const {
103   unsigned Directive =
104       static_cast<const PPCSubtarget *>(STI)->getCPUDirective();
105   if (Directive == PPC::DIR_440 || Directive == PPC::DIR_A2 ||
106       Directive == PPC::DIR_E500mc || Directive == PPC::DIR_E5500) {
107     const InstrItineraryData *II =
108         static_cast<const PPCSubtarget *>(STI)->getInstrItineraryData();
109     return new ScoreboardHazardRecognizer(II, DAG);
110   }
111 
112   return TargetInstrInfo::CreateTargetHazardRecognizer(STI, DAG);
113 }
114 
115 /// CreateTargetPostRAHazardRecognizer - Return the postRA hazard recognizer
116 /// to use for this target when scheduling the DAG.
117 ScheduleHazardRecognizer *
118 PPCInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
119                                                  const ScheduleDAG *DAG) const {
120   unsigned Directive =
121       DAG->MF.getSubtarget<PPCSubtarget>().getCPUDirective();
122 
123   // FIXME: Leaving this as-is until we have POWER9 scheduling info
124   if (Directive == PPC::DIR_PWR7 || Directive == PPC::DIR_PWR8)
125     return new PPCDispatchGroupSBHazardRecognizer(II, DAG);
126 
127   // Most subtargets use a PPC970 recognizer.
128   if (Directive != PPC::DIR_440 && Directive != PPC::DIR_A2 &&
129       Directive != PPC::DIR_E500mc && Directive != PPC::DIR_E5500) {
130     assert(DAG->TII && "No InstrInfo?");
131 
132     return new PPCHazardRecognizer970(*DAG);
133   }
134 
135   return new ScoreboardHazardRecognizer(II, DAG);
136 }
137 
138 unsigned PPCInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
139                                        const MachineInstr &MI,
140                                        unsigned *PredCost) const {
141   if (!ItinData || UseOldLatencyCalc)
142     return PPCGenInstrInfo::getInstrLatency(ItinData, MI, PredCost);
143 
144   // The default implementation of getInstrLatency calls getStageLatency, but
145   // getStageLatency does not do the right thing for us. While we have
146   // itinerary, most cores are fully pipelined, and so the itineraries only
147   // express the first part of the pipeline, not every stage. Instead, we need
148   // to use the listed output operand cycle number (using operand 0 here, which
149   // is an output).
150 
151   unsigned Latency = 1;
152   unsigned DefClass = MI.getDesc().getSchedClass();
153   for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
154     const MachineOperand &MO = MI.getOperand(i);
155     if (!MO.isReg() || !MO.isDef() || MO.isImplicit())
156       continue;
157 
158     std::optional<unsigned> Cycle = ItinData->getOperandCycle(DefClass, i);
159     if (!Cycle)
160       continue;
161 
162     Latency = std::max(Latency, *Cycle);
163   }
164 
165   return Latency;
166 }
167 
168 std::optional<unsigned> PPCInstrInfo::getOperandLatency(
169     const InstrItineraryData *ItinData, const MachineInstr &DefMI,
170     unsigned DefIdx, const MachineInstr &UseMI, unsigned UseIdx) const {
171   std::optional<unsigned> Latency = PPCGenInstrInfo::getOperandLatency(
172       ItinData, DefMI, DefIdx, UseMI, UseIdx);
173 
174   if (!DefMI.getParent())
175     return Latency;
176 
177   const MachineOperand &DefMO = DefMI.getOperand(DefIdx);
178   Register Reg = DefMO.getReg();
179 
180   bool IsRegCR;
181   if (Reg.isVirtual()) {
182     const MachineRegisterInfo *MRI =
183         &DefMI.getParent()->getParent()->getRegInfo();
184     IsRegCR = MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRRCRegClass) ||
185               MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRBITRCRegClass);
186   } else {
187     IsRegCR = PPC::CRRCRegClass.contains(Reg) ||
188               PPC::CRBITRCRegClass.contains(Reg);
189   }
190 
191   if (UseMI.isBranch() && IsRegCR) {
192     if (!Latency)
193       Latency = getInstrLatency(ItinData, DefMI);
194 
195     // On some cores, there is an additional delay between writing to a condition
196     // register, and using it from a branch.
197     unsigned Directive = Subtarget.getCPUDirective();
198     switch (Directive) {
199     default: break;
200     case PPC::DIR_7400:
201     case PPC::DIR_750:
202     case PPC::DIR_970:
203     case PPC::DIR_E5500:
204     case PPC::DIR_PWR4:
205     case PPC::DIR_PWR5:
206     case PPC::DIR_PWR5X:
207     case PPC::DIR_PWR6:
208     case PPC::DIR_PWR6X:
209     case PPC::DIR_PWR7:
210     case PPC::DIR_PWR8:
211     // FIXME: Is this needed for POWER9?
212     Latency = *Latency + 2;
213     break;
214     }
215   }
216 
217   return Latency;
218 }
219 
220 void PPCInstrInfo::setSpecialOperandAttr(MachineInstr &MI,
221                                          uint32_t Flags) const {
222   MI.setFlags(Flags);
223   MI.clearFlag(MachineInstr::MIFlag::NoSWrap);
224   MI.clearFlag(MachineInstr::MIFlag::NoUWrap);
225   MI.clearFlag(MachineInstr::MIFlag::IsExact);
226 }
227 
228 // This function does not list all associative and commutative operations, but
229 // only those worth feeding through the machine combiner in an attempt to
230 // reduce the critical path. Mostly, this means floating-point operations,
231 // because they have high latencies(>=5) (compared to other operations, such as
232 // and/or, which are also associative and commutative, but have low latencies).
233 bool PPCInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst,
234                                                bool Invert) const {
235   if (Invert)
236     return false;
237   switch (Inst.getOpcode()) {
238   // Floating point:
239   // FP Add:
240   case PPC::FADD:
241   case PPC::FADDS:
242   // FP Multiply:
243   case PPC::FMUL:
244   case PPC::FMULS:
245   // Altivec Add:
246   case PPC::VADDFP:
247   // VSX Add:
248   case PPC::XSADDDP:
249   case PPC::XVADDDP:
250   case PPC::XVADDSP:
251   case PPC::XSADDSP:
252   // VSX Multiply:
253   case PPC::XSMULDP:
254   case PPC::XVMULDP:
255   case PPC::XVMULSP:
256   case PPC::XSMULSP:
257     return Inst.getFlag(MachineInstr::MIFlag::FmReassoc) &&
258            Inst.getFlag(MachineInstr::MIFlag::FmNsz);
259   // Fixed point:
260   // Multiply:
261   case PPC::MULHD:
262   case PPC::MULLD:
263   case PPC::MULHW:
264   case PPC::MULLW:
265     return true;
266   default:
267     return false;
268   }
269 }
270 
271 #define InfoArrayIdxFMAInst 0
272 #define InfoArrayIdxFAddInst 1
273 #define InfoArrayIdxFMULInst 2
274 #define InfoArrayIdxAddOpIdx 3
275 #define InfoArrayIdxMULOpIdx 4
276 #define InfoArrayIdxFSubInst 5
277 // Array keeps info for FMA instructions:
278 // Index 0(InfoArrayIdxFMAInst): FMA instruction;
279 // Index 1(InfoArrayIdxFAddInst): ADD instruction associated with FMA;
280 // Index 2(InfoArrayIdxFMULInst): MUL instruction associated with FMA;
281 // Index 3(InfoArrayIdxAddOpIdx): ADD operand index in FMA operands;
282 // Index 4(InfoArrayIdxMULOpIdx): first MUL operand index in FMA operands;
283 //                                second MUL operand index is plus 1;
284 // Index 5(InfoArrayIdxFSubInst): SUB instruction associated with FMA.
285 static const uint16_t FMAOpIdxInfo[][6] = {
286     // FIXME: Add more FMA instructions like XSNMADDADP and so on.
287     {PPC::XSMADDADP, PPC::XSADDDP, PPC::XSMULDP, 1, 2, PPC::XSSUBDP},
288     {PPC::XSMADDASP, PPC::XSADDSP, PPC::XSMULSP, 1, 2, PPC::XSSUBSP},
289     {PPC::XVMADDADP, PPC::XVADDDP, PPC::XVMULDP, 1, 2, PPC::XVSUBDP},
290     {PPC::XVMADDASP, PPC::XVADDSP, PPC::XVMULSP, 1, 2, PPC::XVSUBSP},
291     {PPC::FMADD, PPC::FADD, PPC::FMUL, 3, 1, PPC::FSUB},
292     {PPC::FMADDS, PPC::FADDS, PPC::FMULS, 3, 1, PPC::FSUBS}};
293 
294 // Check if an opcode is a FMA instruction. If it is, return the index in array
295 // FMAOpIdxInfo. Otherwise, return -1.
296 int16_t PPCInstrInfo::getFMAOpIdxInfo(unsigned Opcode) const {
297   for (unsigned I = 0; I < std::size(FMAOpIdxInfo); I++)
298     if (FMAOpIdxInfo[I][InfoArrayIdxFMAInst] == Opcode)
299       return I;
300   return -1;
301 }
302 
303 // On PowerPC target, we have two kinds of patterns related to FMA:
304 // 1: Improve ILP.
305 // Try to reassociate FMA chains like below:
306 //
307 // Pattern 1:
308 //   A =  FADD X,  Y          (Leaf)
309 //   B =  FMA  A,  M21,  M22  (Prev)
310 //   C =  FMA  B,  M31,  M32  (Root)
311 // -->
312 //   A =  FMA  X,  M21,  M22
313 //   B =  FMA  Y,  M31,  M32
314 //   C =  FADD A,  B
315 //
316 // Pattern 2:
317 //   A =  FMA  X,  M11,  M12  (Leaf)
318 //   B =  FMA  A,  M21,  M22  (Prev)
319 //   C =  FMA  B,  M31,  M32  (Root)
320 // -->
321 //   A =  FMUL M11,  M12
322 //   B =  FMA  X,  M21,  M22
323 //   D =  FMA  A,  M31,  M32
324 //   C =  FADD B,  D
325 //
326 // breaking the dependency between A and B, allowing FMA to be executed in
327 // parallel (or back-to-back in a pipeline) instead of depending on each other.
328 //
329 // 2: Reduce register pressure.
330 // Try to reassociate FMA with FSUB and a constant like below:
331 // C is a floating point const.
332 //
333 // Pattern 1:
334 //   A = FSUB  X,  Y      (Leaf)
335 //   D = FMA   B,  C,  A  (Root)
336 // -->
337 //   A = FMA   B,  Y,  -C
338 //   D = FMA   A,  X,  C
339 //
340 // Pattern 2:
341 //   A = FSUB  X,  Y      (Leaf)
342 //   D = FMA   B,  A,  C  (Root)
343 // -->
344 //   A = FMA   B,  Y,  -C
345 //   D = FMA   A,  X,  C
346 //
347 //  Before the transformation, A must be assigned with different hardware
348 //  register with D. After the transformation, A and D must be assigned with
349 //  same hardware register due to TIE attribute of FMA instructions.
350 //
351 bool PPCInstrInfo::getFMAPatterns(
352     MachineInstr &Root, SmallVectorImpl<MachineCombinerPattern> &Patterns,
353     bool DoRegPressureReduce) const {
354   MachineBasicBlock *MBB = Root.getParent();
355   const MachineRegisterInfo *MRI = &MBB->getParent()->getRegInfo();
356   const TargetRegisterInfo *TRI = &getRegisterInfo();
357 
358   auto IsAllOpsVirtualReg = [](const MachineInstr &Instr) {
359     for (const auto &MO : Instr.explicit_operands())
360       if (!(MO.isReg() && MO.getReg().isVirtual()))
361         return false;
362     return true;
363   };
364 
365   auto IsReassociableAddOrSub = [&](const MachineInstr &Instr,
366                                     unsigned OpType) {
367     if (Instr.getOpcode() !=
368         FMAOpIdxInfo[getFMAOpIdxInfo(Root.getOpcode())][OpType])
369       return false;
370 
371     // Instruction can be reassociated.
372     // fast math flags may prohibit reassociation.
373     if (!(Instr.getFlag(MachineInstr::MIFlag::FmReassoc) &&
374           Instr.getFlag(MachineInstr::MIFlag::FmNsz)))
375       return false;
376 
377     // Instruction operands are virtual registers for reassociation.
378     if (!IsAllOpsVirtualReg(Instr))
379       return false;
380 
381     // For register pressure reassociation, the FSub must have only one use as
382     // we want to delete the sub to save its def.
383     if (OpType == InfoArrayIdxFSubInst &&
384         !MRI->hasOneNonDBGUse(Instr.getOperand(0).getReg()))
385       return false;
386 
387     return true;
388   };
389 
390   auto IsReassociableFMA = [&](const MachineInstr &Instr, int16_t &AddOpIdx,
391                                int16_t &MulOpIdx, bool IsLeaf) {
392     int16_t Idx = getFMAOpIdxInfo(Instr.getOpcode());
393     if (Idx < 0)
394       return false;
395 
396     // Instruction can be reassociated.
397     // fast math flags may prohibit reassociation.
398     if (!(Instr.getFlag(MachineInstr::MIFlag::FmReassoc) &&
399           Instr.getFlag(MachineInstr::MIFlag::FmNsz)))
400       return false;
401 
402     // Instruction operands are virtual registers for reassociation.
403     if (!IsAllOpsVirtualReg(Instr))
404       return false;
405 
406     MulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
407     if (IsLeaf)
408       return true;
409 
410     AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx];
411 
412     const MachineOperand &OpAdd = Instr.getOperand(AddOpIdx);
413     MachineInstr *MIAdd = MRI->getUniqueVRegDef(OpAdd.getReg());
414     // If 'add' operand's def is not in current block, don't do ILP related opt.
415     if (!MIAdd || MIAdd->getParent() != MBB)
416       return false;
417 
418     // If this is not Leaf FMA Instr, its 'add' operand should only have one use
419     // as this fma will be changed later.
420     return IsLeaf ? true : MRI->hasOneNonDBGUse(OpAdd.getReg());
421   };
422 
423   int16_t AddOpIdx = -1;
424   int16_t MulOpIdx = -1;
425 
426   bool IsUsedOnceL = false;
427   bool IsUsedOnceR = false;
428   MachineInstr *MULInstrL = nullptr;
429   MachineInstr *MULInstrR = nullptr;
430 
431   auto IsRPReductionCandidate = [&]() {
432     // Currently, we only support float and double.
433     // FIXME: add support for other types.
434     unsigned Opcode = Root.getOpcode();
435     if (Opcode != PPC::XSMADDASP && Opcode != PPC::XSMADDADP)
436       return false;
437 
438     // Root must be a valid FMA like instruction.
439     // Treat it as leaf as we don't care its add operand.
440     if (IsReassociableFMA(Root, AddOpIdx, MulOpIdx, true)) {
441       assert((MulOpIdx >= 0) && "mul operand index not right!");
442       Register MULRegL = TRI->lookThruSingleUseCopyChain(
443           Root.getOperand(MulOpIdx).getReg(), MRI);
444       Register MULRegR = TRI->lookThruSingleUseCopyChain(
445           Root.getOperand(MulOpIdx + 1).getReg(), MRI);
446       if (!MULRegL && !MULRegR)
447         return false;
448 
449       if (MULRegL && !MULRegR) {
450         MULRegR =
451             TRI->lookThruCopyLike(Root.getOperand(MulOpIdx + 1).getReg(), MRI);
452         IsUsedOnceL = true;
453       } else if (!MULRegL && MULRegR) {
454         MULRegL =
455             TRI->lookThruCopyLike(Root.getOperand(MulOpIdx).getReg(), MRI);
456         IsUsedOnceR = true;
457       } else {
458         IsUsedOnceL = true;
459         IsUsedOnceR = true;
460       }
461 
462       if (!MULRegL.isVirtual() || !MULRegR.isVirtual())
463         return false;
464 
465       MULInstrL = MRI->getVRegDef(MULRegL);
466       MULInstrR = MRI->getVRegDef(MULRegR);
467       return true;
468     }
469     return false;
470   };
471 
472   // Register pressure fma reassociation patterns.
473   if (DoRegPressureReduce && IsRPReductionCandidate()) {
474     assert((MULInstrL && MULInstrR) && "wrong register preduction candidate!");
475     // Register pressure pattern 1
476     if (isLoadFromConstantPool(MULInstrL) && IsUsedOnceR &&
477         IsReassociableAddOrSub(*MULInstrR, InfoArrayIdxFSubInst)) {
478       LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_BCA\n");
479       Patterns.push_back(MachineCombinerPattern::REASSOC_XY_BCA);
480       return true;
481     }
482 
483     // Register pressure pattern 2
484     if ((isLoadFromConstantPool(MULInstrR) && IsUsedOnceL &&
485          IsReassociableAddOrSub(*MULInstrL, InfoArrayIdxFSubInst))) {
486       LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_BAC\n");
487       Patterns.push_back(MachineCombinerPattern::REASSOC_XY_BAC);
488       return true;
489     }
490   }
491 
492   // ILP fma reassociation patterns.
493   // Root must be a valid FMA like instruction.
494   AddOpIdx = -1;
495   if (!IsReassociableFMA(Root, AddOpIdx, MulOpIdx, false))
496     return false;
497 
498   assert((AddOpIdx >= 0) && "add operand index not right!");
499 
500   Register RegB = Root.getOperand(AddOpIdx).getReg();
501   MachineInstr *Prev = MRI->getUniqueVRegDef(RegB);
502 
503   // Prev must be a valid FMA like instruction.
504   AddOpIdx = -1;
505   if (!IsReassociableFMA(*Prev, AddOpIdx, MulOpIdx, false))
506     return false;
507 
508   assert((AddOpIdx >= 0) && "add operand index not right!");
509 
510   Register RegA = Prev->getOperand(AddOpIdx).getReg();
511   MachineInstr *Leaf = MRI->getUniqueVRegDef(RegA);
512   AddOpIdx = -1;
513   if (IsReassociableFMA(*Leaf, AddOpIdx, MulOpIdx, true)) {
514     Patterns.push_back(MachineCombinerPattern::REASSOC_XMM_AMM_BMM);
515     LLVM_DEBUG(dbgs() << "add pattern REASSOC_XMM_AMM_BMM\n");
516     return true;
517   }
518   if (IsReassociableAddOrSub(*Leaf, InfoArrayIdxFAddInst)) {
519     Patterns.push_back(MachineCombinerPattern::REASSOC_XY_AMM_BMM);
520     LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_AMM_BMM\n");
521     return true;
522   }
523   return false;
524 }
525 
526 void PPCInstrInfo::finalizeInsInstrs(
527     MachineInstr &Root, MachineCombinerPattern &P,
528     SmallVectorImpl<MachineInstr *> &InsInstrs) const {
529   assert(!InsInstrs.empty() && "Instructions set to be inserted is empty!");
530 
531   MachineFunction *MF = Root.getMF();
532   MachineRegisterInfo *MRI = &MF->getRegInfo();
533   const TargetRegisterInfo *TRI = &getRegisterInfo();
534   MachineConstantPool *MCP = MF->getConstantPool();
535 
536   int16_t Idx = getFMAOpIdxInfo(Root.getOpcode());
537   if (Idx < 0)
538     return;
539 
540   uint16_t FirstMulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
541 
542   // For now we only need to fix up placeholder for register pressure reduce
543   // patterns.
544   Register ConstReg = 0;
545   switch (P) {
546   case MachineCombinerPattern::REASSOC_XY_BCA:
547     ConstReg =
548         TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx).getReg(), MRI);
549     break;
550   case MachineCombinerPattern::REASSOC_XY_BAC:
551     ConstReg =
552         TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx + 1).getReg(), MRI);
553     break;
554   default:
555     // Not register pressure reduce patterns.
556     return;
557   }
558 
559   MachineInstr *ConstDefInstr = MRI->getVRegDef(ConstReg);
560   // Get const value from const pool.
561   const Constant *C = getConstantFromConstantPool(ConstDefInstr);
562   assert(isa<llvm::ConstantFP>(C) && "not a valid constant!");
563 
564   // Get negative fp const.
565   APFloat F1((dyn_cast<ConstantFP>(C))->getValueAPF());
566   F1.changeSign();
567   Constant *NegC = ConstantFP::get(dyn_cast<ConstantFP>(C)->getContext(), F1);
568   Align Alignment = MF->getDataLayout().getPrefTypeAlign(C->getType());
569 
570   // Put negative fp const into constant pool.
571   unsigned ConstPoolIdx = MCP->getConstantPoolIndex(NegC, Alignment);
572 
573   MachineOperand *Placeholder = nullptr;
574   // Record the placeholder PPC::ZERO8 we add in reassociateFMA.
575   for (auto *Inst : InsInstrs) {
576     for (MachineOperand &Operand : Inst->explicit_operands()) {
577       assert(Operand.isReg() && "Invalid instruction in InsInstrs!");
578       if (Operand.getReg() == PPC::ZERO8) {
579         Placeholder = &Operand;
580         break;
581       }
582     }
583   }
584 
585   assert(Placeholder && "Placeholder does not exist!");
586 
587   // Generate instructions to load the const fp from constant pool.
588   // We only support PPC64 and medium code model.
589   Register LoadNewConst =
590       generateLoadForNewConst(ConstPoolIdx, &Root, C->getType(), InsInstrs);
591 
592   // Fill the placeholder with the new load from constant pool.
593   Placeholder->setReg(LoadNewConst);
594 }
595 
596 bool PPCInstrInfo::shouldReduceRegisterPressure(
597     const MachineBasicBlock *MBB, const RegisterClassInfo *RegClassInfo) const {
598 
599   if (!EnableFMARegPressureReduction)
600     return false;
601 
602   // Currently, we only enable register pressure reducing in machine combiner
603   // for: 1: PPC64; 2: Code Model is Medium; 3: Power9 which also has vector
604   // support.
605   //
606   // So we need following instructions to access a TOC entry:
607   //
608   // %6:g8rc_and_g8rc_nox0 = ADDIStocHA8 $x2, %const.0
609   // %7:vssrc = DFLOADf32 target-flags(ppc-toc-lo) %const.0,
610   //   killed %6:g8rc_and_g8rc_nox0, implicit $x2 :: (load 4 from constant-pool)
611   //
612   // FIXME: add more supported targets, like Small and Large code model, PPC32,
613   // AIX.
614   if (!(Subtarget.isPPC64() && Subtarget.hasP9Vector() &&
615         Subtarget.getTargetMachine().getCodeModel() == CodeModel::Medium))
616     return false;
617 
618   const TargetRegisterInfo *TRI = &getRegisterInfo();
619   const MachineFunction *MF = MBB->getParent();
620   const MachineRegisterInfo *MRI = &MF->getRegInfo();
621 
622   auto GetMBBPressure =
623       [&](const MachineBasicBlock *MBB) -> std::vector<unsigned> {
624     RegionPressure Pressure;
625     RegPressureTracker RPTracker(Pressure);
626 
627     // Initialize the register pressure tracker.
628     RPTracker.init(MBB->getParent(), RegClassInfo, nullptr, MBB, MBB->end(),
629                    /*TrackLaneMasks*/ false, /*TrackUntiedDefs=*/true);
630 
631     for (const auto &MI : reverse(*MBB)) {
632       if (MI.isDebugValue() || MI.isDebugLabel())
633         continue;
634       RegisterOperands RegOpers;
635       RegOpers.collect(MI, *TRI, *MRI, false, false);
636       RPTracker.recedeSkipDebugValues();
637       assert(&*RPTracker.getPos() == &MI && "RPTracker sync error!");
638       RPTracker.recede(RegOpers);
639     }
640 
641     // Close the RPTracker to finalize live ins.
642     RPTracker.closeRegion();
643 
644     return RPTracker.getPressure().MaxSetPressure;
645   };
646 
647   // For now we only care about float and double type fma.
648   unsigned VSSRCLimit = TRI->getRegPressureSetLimit(
649       *MBB->getParent(), PPC::RegisterPressureSets::VSSRC);
650 
651   // Only reduce register pressure when pressure is high.
652   return GetMBBPressure(MBB)[PPC::RegisterPressureSets::VSSRC] >
653          (float)VSSRCLimit * FMARPFactor;
654 }
655 
656 bool PPCInstrInfo::isLoadFromConstantPool(MachineInstr *I) const {
657   // I has only one memory operand which is load from constant pool.
658   if (!I->hasOneMemOperand())
659     return false;
660 
661   MachineMemOperand *Op = I->memoperands()[0];
662   return Op->isLoad() && Op->getPseudoValue() &&
663          Op->getPseudoValue()->kind() == PseudoSourceValue::ConstantPool;
664 }
665 
666 Register PPCInstrInfo::generateLoadForNewConst(
667     unsigned Idx, MachineInstr *MI, Type *Ty,
668     SmallVectorImpl<MachineInstr *> &InsInstrs) const {
669   // Now we only support PPC64, Medium code model and P9 with vector.
670   // We have immutable pattern to access const pool. See function
671   // shouldReduceRegisterPressure.
672   assert((Subtarget.isPPC64() && Subtarget.hasP9Vector() &&
673           Subtarget.getTargetMachine().getCodeModel() == CodeModel::Medium) &&
674          "Target not supported!\n");
675 
676   MachineFunction *MF = MI->getMF();
677   MachineRegisterInfo *MRI = &MF->getRegInfo();
678 
679   // Generate ADDIStocHA8
680   Register VReg1 = MRI->createVirtualRegister(&PPC::G8RC_and_G8RC_NOX0RegClass);
681   MachineInstrBuilder TOCOffset =
682       BuildMI(*MF, MI->getDebugLoc(), get(PPC::ADDIStocHA8), VReg1)
683           .addReg(PPC::X2)
684           .addConstantPoolIndex(Idx);
685 
686   assert((Ty->isFloatTy() || Ty->isDoubleTy()) &&
687          "Only float and double are supported!");
688 
689   unsigned LoadOpcode;
690   // Should be float type or double type.
691   if (Ty->isFloatTy())
692     LoadOpcode = PPC::DFLOADf32;
693   else
694     LoadOpcode = PPC::DFLOADf64;
695 
696   const TargetRegisterClass *RC = MRI->getRegClass(MI->getOperand(0).getReg());
697   Register VReg2 = MRI->createVirtualRegister(RC);
698   MachineMemOperand *MMO = MF->getMachineMemOperand(
699       MachinePointerInfo::getConstantPool(*MF), MachineMemOperand::MOLoad,
700       Ty->getScalarSizeInBits() / 8, MF->getDataLayout().getPrefTypeAlign(Ty));
701 
702   // Generate Load from constant pool.
703   MachineInstrBuilder Load =
704       BuildMI(*MF, MI->getDebugLoc(), get(LoadOpcode), VReg2)
705           .addConstantPoolIndex(Idx)
706           .addReg(VReg1, getKillRegState(true))
707           .addMemOperand(MMO);
708 
709   Load->getOperand(1).setTargetFlags(PPCII::MO_TOC_LO);
710 
711   // Insert the toc load instructions into InsInstrs.
712   InsInstrs.insert(InsInstrs.begin(), Load);
713   InsInstrs.insert(InsInstrs.begin(), TOCOffset);
714   return VReg2;
715 }
716 
717 // This function returns the const value in constant pool if the \p I is a load
718 // from constant pool.
719 const Constant *
720 PPCInstrInfo::getConstantFromConstantPool(MachineInstr *I) const {
721   MachineFunction *MF = I->getMF();
722   MachineRegisterInfo *MRI = &MF->getRegInfo();
723   MachineConstantPool *MCP = MF->getConstantPool();
724   assert(I->mayLoad() && "Should be a load instruction.\n");
725   for (auto MO : I->uses()) {
726     if (!MO.isReg())
727       continue;
728     Register Reg = MO.getReg();
729     if (Reg == 0 || !Reg.isVirtual())
730       continue;
731     // Find the toc address.
732     MachineInstr *DefMI = MRI->getVRegDef(Reg);
733     for (auto MO2 : DefMI->uses())
734       if (MO2.isCPI())
735         return (MCP->getConstants())[MO2.getIndex()].Val.ConstVal;
736   }
737   return nullptr;
738 }
739 
740 bool PPCInstrInfo::getMachineCombinerPatterns(
741     MachineInstr &Root, SmallVectorImpl<MachineCombinerPattern> &Patterns,
742     bool DoRegPressureReduce) const {
743   // Using the machine combiner in this way is potentially expensive, so
744   // restrict to when aggressive optimizations are desired.
745   if (Subtarget.getTargetMachine().getOptLevel() != CodeGenOptLevel::Aggressive)
746     return false;
747 
748   if (getFMAPatterns(Root, Patterns, DoRegPressureReduce))
749     return true;
750 
751   return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
752                                                      DoRegPressureReduce);
753 }
754 
755 void PPCInstrInfo::genAlternativeCodeSequence(
756     MachineInstr &Root, MachineCombinerPattern Pattern,
757     SmallVectorImpl<MachineInstr *> &InsInstrs,
758     SmallVectorImpl<MachineInstr *> &DelInstrs,
759     DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
760   switch (Pattern) {
761   case MachineCombinerPattern::REASSOC_XY_AMM_BMM:
762   case MachineCombinerPattern::REASSOC_XMM_AMM_BMM:
763   case MachineCombinerPattern::REASSOC_XY_BCA:
764   case MachineCombinerPattern::REASSOC_XY_BAC:
765     reassociateFMA(Root, Pattern, InsInstrs, DelInstrs, InstrIdxForVirtReg);
766     break;
767   default:
768     // Reassociate default patterns.
769     TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
770                                                 DelInstrs, InstrIdxForVirtReg);
771     break;
772   }
773 }
774 
775 void PPCInstrInfo::reassociateFMA(
776     MachineInstr &Root, MachineCombinerPattern Pattern,
777     SmallVectorImpl<MachineInstr *> &InsInstrs,
778     SmallVectorImpl<MachineInstr *> &DelInstrs,
779     DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
780   MachineFunction *MF = Root.getMF();
781   MachineRegisterInfo &MRI = MF->getRegInfo();
782   const TargetRegisterInfo *TRI = &getRegisterInfo();
783   MachineOperand &OpC = Root.getOperand(0);
784   Register RegC = OpC.getReg();
785   const TargetRegisterClass *RC = MRI.getRegClass(RegC);
786   MRI.constrainRegClass(RegC, RC);
787 
788   unsigned FmaOp = Root.getOpcode();
789   int16_t Idx = getFMAOpIdxInfo(FmaOp);
790   assert(Idx >= 0 && "Root must be a FMA instruction");
791 
792   bool IsILPReassociate =
793       (Pattern == MachineCombinerPattern::REASSOC_XY_AMM_BMM) ||
794       (Pattern == MachineCombinerPattern::REASSOC_XMM_AMM_BMM);
795 
796   uint16_t AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx];
797   uint16_t FirstMulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
798 
799   MachineInstr *Prev = nullptr;
800   MachineInstr *Leaf = nullptr;
801   switch (Pattern) {
802   default:
803     llvm_unreachable("not recognized pattern!");
804   case MachineCombinerPattern::REASSOC_XY_AMM_BMM:
805   case MachineCombinerPattern::REASSOC_XMM_AMM_BMM:
806     Prev = MRI.getUniqueVRegDef(Root.getOperand(AddOpIdx).getReg());
807     Leaf = MRI.getUniqueVRegDef(Prev->getOperand(AddOpIdx).getReg());
808     break;
809   case MachineCombinerPattern::REASSOC_XY_BAC: {
810     Register MULReg =
811         TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx).getReg(), &MRI);
812     Leaf = MRI.getVRegDef(MULReg);
813     break;
814   }
815   case MachineCombinerPattern::REASSOC_XY_BCA: {
816     Register MULReg = TRI->lookThruCopyLike(
817         Root.getOperand(FirstMulOpIdx + 1).getReg(), &MRI);
818     Leaf = MRI.getVRegDef(MULReg);
819     break;
820   }
821   }
822 
823   uint32_t IntersectedFlags = 0;
824   if (IsILPReassociate)
825     IntersectedFlags = Root.getFlags() & Prev->getFlags() & Leaf->getFlags();
826   else
827     IntersectedFlags = Root.getFlags() & Leaf->getFlags();
828 
829   auto GetOperandInfo = [&](const MachineOperand &Operand, Register &Reg,
830                             bool &KillFlag) {
831     Reg = Operand.getReg();
832     MRI.constrainRegClass(Reg, RC);
833     KillFlag = Operand.isKill();
834   };
835 
836   auto GetFMAInstrInfo = [&](const MachineInstr &Instr, Register &MulOp1,
837                              Register &MulOp2, Register &AddOp,
838                              bool &MulOp1KillFlag, bool &MulOp2KillFlag,
839                              bool &AddOpKillFlag) {
840     GetOperandInfo(Instr.getOperand(FirstMulOpIdx), MulOp1, MulOp1KillFlag);
841     GetOperandInfo(Instr.getOperand(FirstMulOpIdx + 1), MulOp2, MulOp2KillFlag);
842     GetOperandInfo(Instr.getOperand(AddOpIdx), AddOp, AddOpKillFlag);
843   };
844 
845   Register RegM11, RegM12, RegX, RegY, RegM21, RegM22, RegM31, RegM32, RegA11,
846       RegA21, RegB;
847   bool KillX = false, KillY = false, KillM11 = false, KillM12 = false,
848        KillM21 = false, KillM22 = false, KillM31 = false, KillM32 = false,
849        KillA11 = false, KillA21 = false, KillB = false;
850 
851   GetFMAInstrInfo(Root, RegM31, RegM32, RegB, KillM31, KillM32, KillB);
852 
853   if (IsILPReassociate)
854     GetFMAInstrInfo(*Prev, RegM21, RegM22, RegA21, KillM21, KillM22, KillA21);
855 
856   if (Pattern == MachineCombinerPattern::REASSOC_XMM_AMM_BMM) {
857     GetFMAInstrInfo(*Leaf, RegM11, RegM12, RegA11, KillM11, KillM12, KillA11);
858     GetOperandInfo(Leaf->getOperand(AddOpIdx), RegX, KillX);
859   } else if (Pattern == MachineCombinerPattern::REASSOC_XY_AMM_BMM) {
860     GetOperandInfo(Leaf->getOperand(1), RegX, KillX);
861     GetOperandInfo(Leaf->getOperand(2), RegY, KillY);
862   } else {
863     // Get FSUB instruction info.
864     GetOperandInfo(Leaf->getOperand(1), RegX, KillX);
865     GetOperandInfo(Leaf->getOperand(2), RegY, KillY);
866   }
867 
868   // Create new virtual registers for the new results instead of
869   // recycling legacy ones because the MachineCombiner's computation of the
870   // critical path requires a new register definition rather than an existing
871   // one.
872   // For register pressure reassociation, we only need create one virtual
873   // register for the new fma.
874   Register NewVRA = MRI.createVirtualRegister(RC);
875   InstrIdxForVirtReg.insert(std::make_pair(NewVRA, 0));
876 
877   Register NewVRB = 0;
878   if (IsILPReassociate) {
879     NewVRB = MRI.createVirtualRegister(RC);
880     InstrIdxForVirtReg.insert(std::make_pair(NewVRB, 1));
881   }
882 
883   Register NewVRD = 0;
884   if (Pattern == MachineCombinerPattern::REASSOC_XMM_AMM_BMM) {
885     NewVRD = MRI.createVirtualRegister(RC);
886     InstrIdxForVirtReg.insert(std::make_pair(NewVRD, 2));
887   }
888 
889   auto AdjustOperandOrder = [&](MachineInstr *MI, Register RegAdd, bool KillAdd,
890                                 Register RegMul1, bool KillRegMul1,
891                                 Register RegMul2, bool KillRegMul2) {
892     MI->getOperand(AddOpIdx).setReg(RegAdd);
893     MI->getOperand(AddOpIdx).setIsKill(KillAdd);
894     MI->getOperand(FirstMulOpIdx).setReg(RegMul1);
895     MI->getOperand(FirstMulOpIdx).setIsKill(KillRegMul1);
896     MI->getOperand(FirstMulOpIdx + 1).setReg(RegMul2);
897     MI->getOperand(FirstMulOpIdx + 1).setIsKill(KillRegMul2);
898   };
899 
900   MachineInstrBuilder NewARegPressure, NewCRegPressure;
901   switch (Pattern) {
902   default:
903     llvm_unreachable("not recognized pattern!");
904   case MachineCombinerPattern::REASSOC_XY_AMM_BMM: {
905     // Create new instructions for insertion.
906     MachineInstrBuilder MINewB =
907         BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB)
908             .addReg(RegX, getKillRegState(KillX))
909             .addReg(RegM21, getKillRegState(KillM21))
910             .addReg(RegM22, getKillRegState(KillM22));
911     MachineInstrBuilder MINewA =
912         BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRA)
913             .addReg(RegY, getKillRegState(KillY))
914             .addReg(RegM31, getKillRegState(KillM31))
915             .addReg(RegM32, getKillRegState(KillM32));
916     // If AddOpIdx is not 1, adjust the order.
917     if (AddOpIdx != 1) {
918       AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
919       AdjustOperandOrder(MINewA, RegY, KillY, RegM31, KillM31, RegM32, KillM32);
920     }
921 
922     MachineInstrBuilder MINewC =
923         BuildMI(*MF, Root.getDebugLoc(),
924                 get(FMAOpIdxInfo[Idx][InfoArrayIdxFAddInst]), RegC)
925             .addReg(NewVRB, getKillRegState(true))
926             .addReg(NewVRA, getKillRegState(true));
927 
928     // Update flags for newly created instructions.
929     setSpecialOperandAttr(*MINewA, IntersectedFlags);
930     setSpecialOperandAttr(*MINewB, IntersectedFlags);
931     setSpecialOperandAttr(*MINewC, IntersectedFlags);
932 
933     // Record new instructions for insertion.
934     InsInstrs.push_back(MINewA);
935     InsInstrs.push_back(MINewB);
936     InsInstrs.push_back(MINewC);
937     break;
938   }
939   case MachineCombinerPattern::REASSOC_XMM_AMM_BMM: {
940     assert(NewVRD && "new FMA register not created!");
941     // Create new instructions for insertion.
942     MachineInstrBuilder MINewA =
943         BuildMI(*MF, Leaf->getDebugLoc(),
944                 get(FMAOpIdxInfo[Idx][InfoArrayIdxFMULInst]), NewVRA)
945             .addReg(RegM11, getKillRegState(KillM11))
946             .addReg(RegM12, getKillRegState(KillM12));
947     MachineInstrBuilder MINewB =
948         BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB)
949             .addReg(RegX, getKillRegState(KillX))
950             .addReg(RegM21, getKillRegState(KillM21))
951             .addReg(RegM22, getKillRegState(KillM22));
952     MachineInstrBuilder MINewD =
953         BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRD)
954             .addReg(NewVRA, getKillRegState(true))
955             .addReg(RegM31, getKillRegState(KillM31))
956             .addReg(RegM32, getKillRegState(KillM32));
957     // If AddOpIdx is not 1, adjust the order.
958     if (AddOpIdx != 1) {
959       AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
960       AdjustOperandOrder(MINewD, NewVRA, true, RegM31, KillM31, RegM32,
961                          KillM32);
962     }
963 
964     MachineInstrBuilder MINewC =
965         BuildMI(*MF, Root.getDebugLoc(),
966                 get(FMAOpIdxInfo[Idx][InfoArrayIdxFAddInst]), RegC)
967             .addReg(NewVRB, getKillRegState(true))
968             .addReg(NewVRD, getKillRegState(true));
969 
970     // Update flags for newly created instructions.
971     setSpecialOperandAttr(*MINewA, IntersectedFlags);
972     setSpecialOperandAttr(*MINewB, IntersectedFlags);
973     setSpecialOperandAttr(*MINewD, IntersectedFlags);
974     setSpecialOperandAttr(*MINewC, IntersectedFlags);
975 
976     // Record new instructions for insertion.
977     InsInstrs.push_back(MINewA);
978     InsInstrs.push_back(MINewB);
979     InsInstrs.push_back(MINewD);
980     InsInstrs.push_back(MINewC);
981     break;
982   }
983   case MachineCombinerPattern::REASSOC_XY_BAC:
984   case MachineCombinerPattern::REASSOC_XY_BCA: {
985     Register VarReg;
986     bool KillVarReg = false;
987     if (Pattern == MachineCombinerPattern::REASSOC_XY_BCA) {
988       VarReg = RegM31;
989       KillVarReg = KillM31;
990     } else {
991       VarReg = RegM32;
992       KillVarReg = KillM32;
993     }
994     // We don't want to get negative const from memory pool too early, as the
995     // created entry will not be deleted even if it has no users. Since all
996     // operand of Leaf and Root are virtual register, we use zero register
997     // here as a placeholder. When the InsInstrs is selected in
998     // MachineCombiner, we call finalizeInsInstrs to replace the zero register
999     // with a virtual register which is a load from constant pool.
1000     NewARegPressure = BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRA)
1001                           .addReg(RegB, getKillRegState(RegB))
1002                           .addReg(RegY, getKillRegState(KillY))
1003                           .addReg(PPC::ZERO8);
1004     NewCRegPressure = BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), RegC)
1005                           .addReg(NewVRA, getKillRegState(true))
1006                           .addReg(RegX, getKillRegState(KillX))
1007                           .addReg(VarReg, getKillRegState(KillVarReg));
1008     // For now, we only support xsmaddadp/xsmaddasp, their add operand are
1009     // both at index 1, no need to adjust.
1010     // FIXME: when add more fma instructions support, like fma/fmas, adjust
1011     // the operand index here.
1012     break;
1013   }
1014   }
1015 
1016   if (!IsILPReassociate) {
1017     setSpecialOperandAttr(*NewARegPressure, IntersectedFlags);
1018     setSpecialOperandAttr(*NewCRegPressure, IntersectedFlags);
1019 
1020     InsInstrs.push_back(NewARegPressure);
1021     InsInstrs.push_back(NewCRegPressure);
1022   }
1023 
1024   assert(!InsInstrs.empty() &&
1025          "Insertion instructions set should not be empty!");
1026 
1027   // Record old instructions for deletion.
1028   DelInstrs.push_back(Leaf);
1029   if (IsILPReassociate)
1030     DelInstrs.push_back(Prev);
1031   DelInstrs.push_back(&Root);
1032 }
1033 
1034 // Detect 32 -> 64-bit extensions where we may reuse the low sub-register.
1035 bool PPCInstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
1036                                          Register &SrcReg, Register &DstReg,
1037                                          unsigned &SubIdx) const {
1038   switch (MI.getOpcode()) {
1039   default: return false;
1040   case PPC::EXTSW:
1041   case PPC::EXTSW_32:
1042   case PPC::EXTSW_32_64:
1043     SrcReg = MI.getOperand(1).getReg();
1044     DstReg = MI.getOperand(0).getReg();
1045     SubIdx = PPC::sub_32;
1046     return true;
1047   }
1048 }
1049 
1050 unsigned PPCInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
1051                                            int &FrameIndex) const {
1052   if (llvm::is_contained(getLoadOpcodesForSpillArray(), MI.getOpcode())) {
1053     // Check for the operands added by addFrameReference (the immediate is the
1054     // offset which defaults to 0).
1055     if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() &&
1056         MI.getOperand(2).isFI()) {
1057       FrameIndex = MI.getOperand(2).getIndex();
1058       return MI.getOperand(0).getReg();
1059     }
1060   }
1061   return 0;
1062 }
1063 
1064 // For opcodes with the ReMaterializable flag set, this function is called to
1065 // verify the instruction is really rematable.
1066 bool PPCInstrInfo::isReallyTriviallyReMaterializable(
1067     const MachineInstr &MI) const {
1068   switch (MI.getOpcode()) {
1069   default:
1070     // Let base implementaion decide.
1071     break;
1072   case PPC::LI:
1073   case PPC::LI8:
1074   case PPC::PLI:
1075   case PPC::PLI8:
1076   case PPC::LIS:
1077   case PPC::LIS8:
1078   case PPC::ADDIStocHA:
1079   case PPC::ADDIStocHA8:
1080   case PPC::ADDItocL:
1081   case PPC::LOAD_STACK_GUARD:
1082   case PPC::XXLXORz:
1083   case PPC::XXLXORspz:
1084   case PPC::XXLXORdpz:
1085   case PPC::XXLEQVOnes:
1086   case PPC::XXSPLTI32DX:
1087   case PPC::XXSPLTIW:
1088   case PPC::XXSPLTIDP:
1089   case PPC::V_SET0B:
1090   case PPC::V_SET0H:
1091   case PPC::V_SET0:
1092   case PPC::V_SETALLONESB:
1093   case PPC::V_SETALLONESH:
1094   case PPC::V_SETALLONES:
1095   case PPC::CRSET:
1096   case PPC::CRUNSET:
1097   case PPC::XXSETACCZ:
1098   case PPC::XXSETACCZW:
1099     return true;
1100   }
1101   return TargetInstrInfo::isReallyTriviallyReMaterializable(MI);
1102 }
1103 
1104 unsigned PPCInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
1105                                           int &FrameIndex) const {
1106   if (llvm::is_contained(getStoreOpcodesForSpillArray(), MI.getOpcode())) {
1107     if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() &&
1108         MI.getOperand(2).isFI()) {
1109       FrameIndex = MI.getOperand(2).getIndex();
1110       return MI.getOperand(0).getReg();
1111     }
1112   }
1113   return 0;
1114 }
1115 
1116 MachineInstr *PPCInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
1117                                                    unsigned OpIdx1,
1118                                                    unsigned OpIdx2) const {
1119   MachineFunction &MF = *MI.getParent()->getParent();
1120 
1121   // Normal instructions can be commuted the obvious way.
1122   if (MI.getOpcode() != PPC::RLWIMI && MI.getOpcode() != PPC::RLWIMI_rec)
1123     return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
1124   // Note that RLWIMI can be commuted as a 32-bit instruction, but not as a
1125   // 64-bit instruction (so we don't handle PPC::RLWIMI8 here), because
1126   // changing the relative order of the mask operands might change what happens
1127   // to the high-bits of the mask (and, thus, the result).
1128 
1129   // Cannot commute if it has a non-zero rotate count.
1130   if (MI.getOperand(3).getImm() != 0)
1131     return nullptr;
1132 
1133   // If we have a zero rotate count, we have:
1134   //   M = mask(MB,ME)
1135   //   Op0 = (Op1 & ~M) | (Op2 & M)
1136   // Change this to:
1137   //   M = mask((ME+1)&31, (MB-1)&31)
1138   //   Op0 = (Op2 & ~M) | (Op1 & M)
1139 
1140   // Swap op1/op2
1141   assert(((OpIdx1 == 1 && OpIdx2 == 2) || (OpIdx1 == 2 && OpIdx2 == 1)) &&
1142          "Only the operands 1 and 2 can be swapped in RLSIMI/RLWIMI_rec.");
1143   Register Reg0 = MI.getOperand(0).getReg();
1144   Register Reg1 = MI.getOperand(1).getReg();
1145   Register Reg2 = MI.getOperand(2).getReg();
1146   unsigned SubReg1 = MI.getOperand(1).getSubReg();
1147   unsigned SubReg2 = MI.getOperand(2).getSubReg();
1148   bool Reg1IsKill = MI.getOperand(1).isKill();
1149   bool Reg2IsKill = MI.getOperand(2).isKill();
1150   bool ChangeReg0 = false;
1151   // If machine instrs are no longer in two-address forms, update
1152   // destination register as well.
1153   if (Reg0 == Reg1) {
1154     // Must be two address instruction (i.e. op1 is tied to op0).
1155     assert(MI.getDesc().getOperandConstraint(1, MCOI::TIED_TO) == 0 &&
1156            "Expecting a two-address instruction!");
1157     assert(MI.getOperand(0).getSubReg() == SubReg1 && "Tied subreg mismatch");
1158     Reg2IsKill = false;
1159     ChangeReg0 = true;
1160   }
1161 
1162   // Masks.
1163   unsigned MB = MI.getOperand(4).getImm();
1164   unsigned ME = MI.getOperand(5).getImm();
1165 
1166   // We can't commute a trivial mask (there is no way to represent an all-zero
1167   // mask).
1168   if (MB == 0 && ME == 31)
1169     return nullptr;
1170 
1171   if (NewMI) {
1172     // Create a new instruction.
1173     Register Reg0 = ChangeReg0 ? Reg2 : MI.getOperand(0).getReg();
1174     bool Reg0IsDead = MI.getOperand(0).isDead();
1175     return BuildMI(MF, MI.getDebugLoc(), MI.getDesc())
1176         .addReg(Reg0, RegState::Define | getDeadRegState(Reg0IsDead))
1177         .addReg(Reg2, getKillRegState(Reg2IsKill))
1178         .addReg(Reg1, getKillRegState(Reg1IsKill))
1179         .addImm((ME + 1) & 31)
1180         .addImm((MB - 1) & 31);
1181   }
1182 
1183   if (ChangeReg0) {
1184     MI.getOperand(0).setReg(Reg2);
1185     MI.getOperand(0).setSubReg(SubReg2);
1186   }
1187   MI.getOperand(2).setReg(Reg1);
1188   MI.getOperand(1).setReg(Reg2);
1189   MI.getOperand(2).setSubReg(SubReg1);
1190   MI.getOperand(1).setSubReg(SubReg2);
1191   MI.getOperand(2).setIsKill(Reg1IsKill);
1192   MI.getOperand(1).setIsKill(Reg2IsKill);
1193 
1194   // Swap the mask around.
1195   MI.getOperand(4).setImm((ME + 1) & 31);
1196   MI.getOperand(5).setImm((MB - 1) & 31);
1197   return &MI;
1198 }
1199 
1200 bool PPCInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
1201                                          unsigned &SrcOpIdx1,
1202                                          unsigned &SrcOpIdx2) const {
1203   // For VSX A-Type FMA instructions, it is the first two operands that can be
1204   // commuted, however, because the non-encoded tied input operand is listed
1205   // first, the operands to swap are actually the second and third.
1206 
1207   int AltOpc = PPC::getAltVSXFMAOpcode(MI.getOpcode());
1208   if (AltOpc == -1)
1209     return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
1210 
1211   // The commutable operand indices are 2 and 3. Return them in SrcOpIdx1
1212   // and SrcOpIdx2.
1213   return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);
1214 }
1215 
1216 void PPCInstrInfo::insertNoop(MachineBasicBlock &MBB,
1217                               MachineBasicBlock::iterator MI) const {
1218   // This function is used for scheduling, and the nop wanted here is the type
1219   // that terminates dispatch groups on the POWER cores.
1220   unsigned Directive = Subtarget.getCPUDirective();
1221   unsigned Opcode;
1222   switch (Directive) {
1223   default:            Opcode = PPC::NOP; break;
1224   case PPC::DIR_PWR6: Opcode = PPC::NOP_GT_PWR6; break;
1225   case PPC::DIR_PWR7: Opcode = PPC::NOP_GT_PWR7; break;
1226   case PPC::DIR_PWR8: Opcode = PPC::NOP_GT_PWR7; break; /* FIXME: Update when P8 InstrScheduling model is ready */
1227   // FIXME: Update when POWER9 scheduling model is ready.
1228   case PPC::DIR_PWR9: Opcode = PPC::NOP_GT_PWR7; break;
1229   }
1230 
1231   DebugLoc DL;
1232   BuildMI(MBB, MI, DL, get(Opcode));
1233 }
1234 
1235 /// Return the noop instruction to use for a noop.
1236 MCInst PPCInstrInfo::getNop() const {
1237   MCInst Nop;
1238   Nop.setOpcode(PPC::NOP);
1239   return Nop;
1240 }
1241 
1242 // Branch analysis.
1243 // Note: If the condition register is set to CTR or CTR8 then this is a
1244 // BDNZ (imm == 1) or BDZ (imm == 0) branch.
1245 bool PPCInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
1246                                  MachineBasicBlock *&TBB,
1247                                  MachineBasicBlock *&FBB,
1248                                  SmallVectorImpl<MachineOperand> &Cond,
1249                                  bool AllowModify) const {
1250   bool isPPC64 = Subtarget.isPPC64();
1251 
1252   // If the block has no terminators, it just falls into the block after it.
1253   MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1254   if (I == MBB.end())
1255     return false;
1256 
1257   if (!isUnpredicatedTerminator(*I))
1258     return false;
1259 
1260   if (AllowModify) {
1261     // If the BB ends with an unconditional branch to the fallthrough BB,
1262     // we eliminate the branch instruction.
1263     if (I->getOpcode() == PPC::B &&
1264         MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) {
1265       I->eraseFromParent();
1266 
1267       // We update iterator after deleting the last branch.
1268       I = MBB.getLastNonDebugInstr();
1269       if (I == MBB.end() || !isUnpredicatedTerminator(*I))
1270         return false;
1271     }
1272   }
1273 
1274   // Get the last instruction in the block.
1275   MachineInstr &LastInst = *I;
1276 
1277   // If there is only one terminator instruction, process it.
1278   if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
1279     if (LastInst.getOpcode() == PPC::B) {
1280       if (!LastInst.getOperand(0).isMBB())
1281         return true;
1282       TBB = LastInst.getOperand(0).getMBB();
1283       return false;
1284     } else if (LastInst.getOpcode() == PPC::BCC) {
1285       if (!LastInst.getOperand(2).isMBB())
1286         return true;
1287       // Block ends with fall-through condbranch.
1288       TBB = LastInst.getOperand(2).getMBB();
1289       Cond.push_back(LastInst.getOperand(0));
1290       Cond.push_back(LastInst.getOperand(1));
1291       return false;
1292     } else if (LastInst.getOpcode() == PPC::BC) {
1293       if (!LastInst.getOperand(1).isMBB())
1294         return true;
1295       // Block ends with fall-through condbranch.
1296       TBB = LastInst.getOperand(1).getMBB();
1297       Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET));
1298       Cond.push_back(LastInst.getOperand(0));
1299       return false;
1300     } else if (LastInst.getOpcode() == PPC::BCn) {
1301       if (!LastInst.getOperand(1).isMBB())
1302         return true;
1303       // Block ends with fall-through condbranch.
1304       TBB = LastInst.getOperand(1).getMBB();
1305       Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_UNSET));
1306       Cond.push_back(LastInst.getOperand(0));
1307       return false;
1308     } else if (LastInst.getOpcode() == PPC::BDNZ8 ||
1309                LastInst.getOpcode() == PPC::BDNZ) {
1310       if (!LastInst.getOperand(0).isMBB())
1311         return true;
1312       if (DisableCTRLoopAnal)
1313         return true;
1314       TBB = LastInst.getOperand(0).getMBB();
1315       Cond.push_back(MachineOperand::CreateImm(1));
1316       Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1317                                                true));
1318       return false;
1319     } else if (LastInst.getOpcode() == PPC::BDZ8 ||
1320                LastInst.getOpcode() == PPC::BDZ) {
1321       if (!LastInst.getOperand(0).isMBB())
1322         return true;
1323       if (DisableCTRLoopAnal)
1324         return true;
1325       TBB = LastInst.getOperand(0).getMBB();
1326       Cond.push_back(MachineOperand::CreateImm(0));
1327       Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1328                                                true));
1329       return false;
1330     }
1331 
1332     // Otherwise, don't know what this is.
1333     return true;
1334   }
1335 
1336   // Get the instruction before it if it's a terminator.
1337   MachineInstr &SecondLastInst = *I;
1338 
1339   // If there are three terminators, we don't know what sort of block this is.
1340   if (I != MBB.begin() && isUnpredicatedTerminator(*--I))
1341     return true;
1342 
1343   // If the block ends with PPC::B and PPC:BCC, handle it.
1344   if (SecondLastInst.getOpcode() == PPC::BCC &&
1345       LastInst.getOpcode() == PPC::B) {
1346     if (!SecondLastInst.getOperand(2).isMBB() ||
1347         !LastInst.getOperand(0).isMBB())
1348       return true;
1349     TBB = SecondLastInst.getOperand(2).getMBB();
1350     Cond.push_back(SecondLastInst.getOperand(0));
1351     Cond.push_back(SecondLastInst.getOperand(1));
1352     FBB = LastInst.getOperand(0).getMBB();
1353     return false;
1354   } else if (SecondLastInst.getOpcode() == PPC::BC &&
1355              LastInst.getOpcode() == PPC::B) {
1356     if (!SecondLastInst.getOperand(1).isMBB() ||
1357         !LastInst.getOperand(0).isMBB())
1358       return true;
1359     TBB = SecondLastInst.getOperand(1).getMBB();
1360     Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET));
1361     Cond.push_back(SecondLastInst.getOperand(0));
1362     FBB = LastInst.getOperand(0).getMBB();
1363     return false;
1364   } else if (SecondLastInst.getOpcode() == PPC::BCn &&
1365              LastInst.getOpcode() == PPC::B) {
1366     if (!SecondLastInst.getOperand(1).isMBB() ||
1367         !LastInst.getOperand(0).isMBB())
1368       return true;
1369     TBB = SecondLastInst.getOperand(1).getMBB();
1370     Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_UNSET));
1371     Cond.push_back(SecondLastInst.getOperand(0));
1372     FBB = LastInst.getOperand(0).getMBB();
1373     return false;
1374   } else if ((SecondLastInst.getOpcode() == PPC::BDNZ8 ||
1375               SecondLastInst.getOpcode() == PPC::BDNZ) &&
1376              LastInst.getOpcode() == PPC::B) {
1377     if (!SecondLastInst.getOperand(0).isMBB() ||
1378         !LastInst.getOperand(0).isMBB())
1379       return true;
1380     if (DisableCTRLoopAnal)
1381       return true;
1382     TBB = SecondLastInst.getOperand(0).getMBB();
1383     Cond.push_back(MachineOperand::CreateImm(1));
1384     Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1385                                              true));
1386     FBB = LastInst.getOperand(0).getMBB();
1387     return false;
1388   } else if ((SecondLastInst.getOpcode() == PPC::BDZ8 ||
1389               SecondLastInst.getOpcode() == PPC::BDZ) &&
1390              LastInst.getOpcode() == PPC::B) {
1391     if (!SecondLastInst.getOperand(0).isMBB() ||
1392         !LastInst.getOperand(0).isMBB())
1393       return true;
1394     if (DisableCTRLoopAnal)
1395       return true;
1396     TBB = SecondLastInst.getOperand(0).getMBB();
1397     Cond.push_back(MachineOperand::CreateImm(0));
1398     Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1399                                              true));
1400     FBB = LastInst.getOperand(0).getMBB();
1401     return false;
1402   }
1403 
1404   // If the block ends with two PPC:Bs, handle it.  The second one is not
1405   // executed, so remove it.
1406   if (SecondLastInst.getOpcode() == PPC::B && LastInst.getOpcode() == PPC::B) {
1407     if (!SecondLastInst.getOperand(0).isMBB())
1408       return true;
1409     TBB = SecondLastInst.getOperand(0).getMBB();
1410     I = LastInst;
1411     if (AllowModify)
1412       I->eraseFromParent();
1413     return false;
1414   }
1415 
1416   // Otherwise, can't handle this.
1417   return true;
1418 }
1419 
1420 unsigned PPCInstrInfo::removeBranch(MachineBasicBlock &MBB,
1421                                     int *BytesRemoved) const {
1422   assert(!BytesRemoved && "code size not handled");
1423 
1424   MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1425   if (I == MBB.end())
1426     return 0;
1427 
1428   if (I->getOpcode() != PPC::B && I->getOpcode() != PPC::BCC &&
1429       I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
1430       I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
1431       I->getOpcode() != PPC::BDZ8  && I->getOpcode() != PPC::BDZ)
1432     return 0;
1433 
1434   // Remove the branch.
1435   I->eraseFromParent();
1436 
1437   I = MBB.end();
1438 
1439   if (I == MBB.begin()) return 1;
1440   --I;
1441   if (I->getOpcode() != PPC::BCC &&
1442       I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
1443       I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
1444       I->getOpcode() != PPC::BDZ8  && I->getOpcode() != PPC::BDZ)
1445     return 1;
1446 
1447   // Remove the branch.
1448   I->eraseFromParent();
1449   return 2;
1450 }
1451 
1452 unsigned PPCInstrInfo::insertBranch(MachineBasicBlock &MBB,
1453                                     MachineBasicBlock *TBB,
1454                                     MachineBasicBlock *FBB,
1455                                     ArrayRef<MachineOperand> Cond,
1456                                     const DebugLoc &DL,
1457                                     int *BytesAdded) const {
1458   // Shouldn't be a fall through.
1459   assert(TBB && "insertBranch must not be told to insert a fallthrough");
1460   assert((Cond.size() == 2 || Cond.size() == 0) &&
1461          "PPC branch conditions have two components!");
1462   assert(!BytesAdded && "code size not handled");
1463 
1464   bool isPPC64 = Subtarget.isPPC64();
1465 
1466   // One-way branch.
1467   if (!FBB) {
1468     if (Cond.empty())   // Unconditional branch
1469       BuildMI(&MBB, DL, get(PPC::B)).addMBB(TBB);
1470     else if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1471       BuildMI(&MBB, DL, get(Cond[0].getImm() ?
1472                               (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
1473                               (isPPC64 ? PPC::BDZ8  : PPC::BDZ))).addMBB(TBB);
1474     else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
1475       BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB);
1476     else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
1477       BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB);
1478     else                // Conditional branch
1479       BuildMI(&MBB, DL, get(PPC::BCC))
1480           .addImm(Cond[0].getImm())
1481           .add(Cond[1])
1482           .addMBB(TBB);
1483     return 1;
1484   }
1485 
1486   // Two-way Conditional Branch.
1487   if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1488     BuildMI(&MBB, DL, get(Cond[0].getImm() ?
1489                             (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
1490                             (isPPC64 ? PPC::BDZ8  : PPC::BDZ))).addMBB(TBB);
1491   else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
1492     BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB);
1493   else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
1494     BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB);
1495   else
1496     BuildMI(&MBB, DL, get(PPC::BCC))
1497         .addImm(Cond[0].getImm())
1498         .add(Cond[1])
1499         .addMBB(TBB);
1500   BuildMI(&MBB, DL, get(PPC::B)).addMBB(FBB);
1501   return 2;
1502 }
1503 
1504 // Select analysis.
1505 bool PPCInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
1506                                    ArrayRef<MachineOperand> Cond,
1507                                    Register DstReg, Register TrueReg,
1508                                    Register FalseReg, int &CondCycles,
1509                                    int &TrueCycles, int &FalseCycles) const {
1510   if (!Subtarget.hasISEL())
1511     return false;
1512 
1513   if (Cond.size() != 2)
1514     return false;
1515 
1516   // If this is really a bdnz-like condition, then it cannot be turned into a
1517   // select.
1518   if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1519     return false;
1520 
1521   // If the conditional branch uses a physical register, then it cannot be
1522   // turned into a select.
1523   if (Cond[1].getReg().isPhysical())
1524     return false;
1525 
1526   // Check register classes.
1527   const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1528   const TargetRegisterClass *RC =
1529     RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
1530   if (!RC)
1531     return false;
1532 
1533   // isel is for regular integer GPRs only.
1534   if (!PPC::GPRCRegClass.hasSubClassEq(RC) &&
1535       !PPC::GPRC_NOR0RegClass.hasSubClassEq(RC) &&
1536       !PPC::G8RCRegClass.hasSubClassEq(RC) &&
1537       !PPC::G8RC_NOX0RegClass.hasSubClassEq(RC))
1538     return false;
1539 
1540   // FIXME: These numbers are for the A2, how well they work for other cores is
1541   // an open question. On the A2, the isel instruction has a 2-cycle latency
1542   // but single-cycle throughput. These numbers are used in combination with
1543   // the MispredictPenalty setting from the active SchedMachineModel.
1544   CondCycles = 1;
1545   TrueCycles = 1;
1546   FalseCycles = 1;
1547 
1548   return true;
1549 }
1550 
1551 void PPCInstrInfo::insertSelect(MachineBasicBlock &MBB,
1552                                 MachineBasicBlock::iterator MI,
1553                                 const DebugLoc &dl, Register DestReg,
1554                                 ArrayRef<MachineOperand> Cond, Register TrueReg,
1555                                 Register FalseReg) const {
1556   assert(Cond.size() == 2 &&
1557          "PPC branch conditions have two components!");
1558 
1559   // Get the register classes.
1560   MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1561   const TargetRegisterClass *RC =
1562     RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
1563   assert(RC && "TrueReg and FalseReg must have overlapping register classes");
1564 
1565   bool Is64Bit = PPC::G8RCRegClass.hasSubClassEq(RC) ||
1566                  PPC::G8RC_NOX0RegClass.hasSubClassEq(RC);
1567   assert((Is64Bit ||
1568           PPC::GPRCRegClass.hasSubClassEq(RC) ||
1569           PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) &&
1570          "isel is for regular integer GPRs only");
1571 
1572   unsigned OpCode = Is64Bit ? PPC::ISEL8 : PPC::ISEL;
1573   auto SelectPred = static_cast<PPC::Predicate>(Cond[0].getImm());
1574 
1575   unsigned SubIdx = 0;
1576   bool SwapOps = false;
1577   switch (SelectPred) {
1578   case PPC::PRED_EQ:
1579   case PPC::PRED_EQ_MINUS:
1580   case PPC::PRED_EQ_PLUS:
1581       SubIdx = PPC::sub_eq; SwapOps = false; break;
1582   case PPC::PRED_NE:
1583   case PPC::PRED_NE_MINUS:
1584   case PPC::PRED_NE_PLUS:
1585       SubIdx = PPC::sub_eq; SwapOps = true; break;
1586   case PPC::PRED_LT:
1587   case PPC::PRED_LT_MINUS:
1588   case PPC::PRED_LT_PLUS:
1589       SubIdx = PPC::sub_lt; SwapOps = false; break;
1590   case PPC::PRED_GE:
1591   case PPC::PRED_GE_MINUS:
1592   case PPC::PRED_GE_PLUS:
1593       SubIdx = PPC::sub_lt; SwapOps = true; break;
1594   case PPC::PRED_GT:
1595   case PPC::PRED_GT_MINUS:
1596   case PPC::PRED_GT_PLUS:
1597       SubIdx = PPC::sub_gt; SwapOps = false; break;
1598   case PPC::PRED_LE:
1599   case PPC::PRED_LE_MINUS:
1600   case PPC::PRED_LE_PLUS:
1601       SubIdx = PPC::sub_gt; SwapOps = true; break;
1602   case PPC::PRED_UN:
1603   case PPC::PRED_UN_MINUS:
1604   case PPC::PRED_UN_PLUS:
1605       SubIdx = PPC::sub_un; SwapOps = false; break;
1606   case PPC::PRED_NU:
1607   case PPC::PRED_NU_MINUS:
1608   case PPC::PRED_NU_PLUS:
1609       SubIdx = PPC::sub_un; SwapOps = true; break;
1610   case PPC::PRED_BIT_SET:   SubIdx = 0; SwapOps = false; break;
1611   case PPC::PRED_BIT_UNSET: SubIdx = 0; SwapOps = true; break;
1612   }
1613 
1614   Register FirstReg =  SwapOps ? FalseReg : TrueReg,
1615            SecondReg = SwapOps ? TrueReg  : FalseReg;
1616 
1617   // The first input register of isel cannot be r0. If it is a member
1618   // of a register class that can be r0, then copy it first (the
1619   // register allocator should eliminate the copy).
1620   if (MRI.getRegClass(FirstReg)->contains(PPC::R0) ||
1621       MRI.getRegClass(FirstReg)->contains(PPC::X0)) {
1622     const TargetRegisterClass *FirstRC =
1623       MRI.getRegClass(FirstReg)->contains(PPC::X0) ?
1624         &PPC::G8RC_NOX0RegClass : &PPC::GPRC_NOR0RegClass;
1625     Register OldFirstReg = FirstReg;
1626     FirstReg = MRI.createVirtualRegister(FirstRC);
1627     BuildMI(MBB, MI, dl, get(TargetOpcode::COPY), FirstReg)
1628       .addReg(OldFirstReg);
1629   }
1630 
1631   BuildMI(MBB, MI, dl, get(OpCode), DestReg)
1632     .addReg(FirstReg).addReg(SecondReg)
1633     .addReg(Cond[1].getReg(), 0, SubIdx);
1634 }
1635 
1636 static unsigned getCRBitValue(unsigned CRBit) {
1637   unsigned Ret = 4;
1638   if (CRBit == PPC::CR0LT || CRBit == PPC::CR1LT ||
1639       CRBit == PPC::CR2LT || CRBit == PPC::CR3LT ||
1640       CRBit == PPC::CR4LT || CRBit == PPC::CR5LT ||
1641       CRBit == PPC::CR6LT || CRBit == PPC::CR7LT)
1642     Ret = 3;
1643   if (CRBit == PPC::CR0GT || CRBit == PPC::CR1GT ||
1644       CRBit == PPC::CR2GT || CRBit == PPC::CR3GT ||
1645       CRBit == PPC::CR4GT || CRBit == PPC::CR5GT ||
1646       CRBit == PPC::CR6GT || CRBit == PPC::CR7GT)
1647     Ret = 2;
1648   if (CRBit == PPC::CR0EQ || CRBit == PPC::CR1EQ ||
1649       CRBit == PPC::CR2EQ || CRBit == PPC::CR3EQ ||
1650       CRBit == PPC::CR4EQ || CRBit == PPC::CR5EQ ||
1651       CRBit == PPC::CR6EQ || CRBit == PPC::CR7EQ)
1652     Ret = 1;
1653   if (CRBit == PPC::CR0UN || CRBit == PPC::CR1UN ||
1654       CRBit == PPC::CR2UN || CRBit == PPC::CR3UN ||
1655       CRBit == PPC::CR4UN || CRBit == PPC::CR5UN ||
1656       CRBit == PPC::CR6UN || CRBit == PPC::CR7UN)
1657     Ret = 0;
1658 
1659   assert(Ret != 4 && "Invalid CR bit register");
1660   return Ret;
1661 }
1662 
1663 void PPCInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
1664                                MachineBasicBlock::iterator I,
1665                                const DebugLoc &DL, MCRegister DestReg,
1666                                MCRegister SrcReg, bool KillSrc) const {
1667   // We can end up with self copies and similar things as a result of VSX copy
1668   // legalization. Promote them here.
1669   const TargetRegisterInfo *TRI = &getRegisterInfo();
1670   if (PPC::F8RCRegClass.contains(DestReg) &&
1671       PPC::VSRCRegClass.contains(SrcReg)) {
1672     MCRegister SuperReg =
1673         TRI->getMatchingSuperReg(DestReg, PPC::sub_64, &PPC::VSRCRegClass);
1674 
1675     if (VSXSelfCopyCrash && SrcReg == SuperReg)
1676       llvm_unreachable("nop VSX copy");
1677 
1678     DestReg = SuperReg;
1679   } else if (PPC::F8RCRegClass.contains(SrcReg) &&
1680              PPC::VSRCRegClass.contains(DestReg)) {
1681     MCRegister SuperReg =
1682         TRI->getMatchingSuperReg(SrcReg, PPC::sub_64, &PPC::VSRCRegClass);
1683 
1684     if (VSXSelfCopyCrash && DestReg == SuperReg)
1685       llvm_unreachable("nop VSX copy");
1686 
1687     SrcReg = SuperReg;
1688   }
1689 
1690   // Different class register copy
1691   if (PPC::CRBITRCRegClass.contains(SrcReg) &&
1692       PPC::GPRCRegClass.contains(DestReg)) {
1693     MCRegister CRReg = getCRFromCRBit(SrcReg);
1694     BuildMI(MBB, I, DL, get(PPC::MFOCRF), DestReg).addReg(CRReg);
1695     getKillRegState(KillSrc);
1696     // Rotate the CR bit in the CR fields to be the least significant bit and
1697     // then mask with 0x1 (MB = ME = 31).
1698     BuildMI(MBB, I, DL, get(PPC::RLWINM), DestReg)
1699        .addReg(DestReg, RegState::Kill)
1700        .addImm(TRI->getEncodingValue(CRReg) * 4 + (4 - getCRBitValue(SrcReg)))
1701        .addImm(31)
1702        .addImm(31);
1703     return;
1704   } else if (PPC::CRRCRegClass.contains(SrcReg) &&
1705              (PPC::G8RCRegClass.contains(DestReg) ||
1706               PPC::GPRCRegClass.contains(DestReg))) {
1707     bool Is64Bit = PPC::G8RCRegClass.contains(DestReg);
1708     unsigned MvCode = Is64Bit ? PPC::MFOCRF8 : PPC::MFOCRF;
1709     unsigned ShCode = Is64Bit ? PPC::RLWINM8 : PPC::RLWINM;
1710     unsigned CRNum = TRI->getEncodingValue(SrcReg);
1711     BuildMI(MBB, I, DL, get(MvCode), DestReg).addReg(SrcReg);
1712     getKillRegState(KillSrc);
1713     if (CRNum == 7)
1714       return;
1715     // Shift the CR bits to make the CR field in the lowest 4 bits of GRC.
1716     BuildMI(MBB, I, DL, get(ShCode), DestReg)
1717         .addReg(DestReg, RegState::Kill)
1718         .addImm(CRNum * 4 + 4)
1719         .addImm(28)
1720         .addImm(31);
1721     return;
1722   } else if (PPC::G8RCRegClass.contains(SrcReg) &&
1723              PPC::VSFRCRegClass.contains(DestReg)) {
1724     assert(Subtarget.hasDirectMove() &&
1725            "Subtarget doesn't support directmove, don't know how to copy.");
1726     BuildMI(MBB, I, DL, get(PPC::MTVSRD), DestReg).addReg(SrcReg);
1727     NumGPRtoVSRSpill++;
1728     getKillRegState(KillSrc);
1729     return;
1730   } else if (PPC::VSFRCRegClass.contains(SrcReg) &&
1731              PPC::G8RCRegClass.contains(DestReg)) {
1732     assert(Subtarget.hasDirectMove() &&
1733            "Subtarget doesn't support directmove, don't know how to copy.");
1734     BuildMI(MBB, I, DL, get(PPC::MFVSRD), DestReg).addReg(SrcReg);
1735     getKillRegState(KillSrc);
1736     return;
1737   } else if (PPC::SPERCRegClass.contains(SrcReg) &&
1738              PPC::GPRCRegClass.contains(DestReg)) {
1739     BuildMI(MBB, I, DL, get(PPC::EFSCFD), DestReg).addReg(SrcReg);
1740     getKillRegState(KillSrc);
1741     return;
1742   } else if (PPC::GPRCRegClass.contains(SrcReg) &&
1743              PPC::SPERCRegClass.contains(DestReg)) {
1744     BuildMI(MBB, I, DL, get(PPC::EFDCFS), DestReg).addReg(SrcReg);
1745     getKillRegState(KillSrc);
1746     return;
1747   }
1748 
1749   unsigned Opc;
1750   if (PPC::GPRCRegClass.contains(DestReg, SrcReg))
1751     Opc = PPC::OR;
1752   else if (PPC::G8RCRegClass.contains(DestReg, SrcReg))
1753     Opc = PPC::OR8;
1754   else if (PPC::F4RCRegClass.contains(DestReg, SrcReg))
1755     Opc = PPC::FMR;
1756   else if (PPC::CRRCRegClass.contains(DestReg, SrcReg))
1757     Opc = PPC::MCRF;
1758   else if (PPC::VRRCRegClass.contains(DestReg, SrcReg))
1759     Opc = PPC::VOR;
1760   else if (PPC::VSRCRegClass.contains(DestReg, SrcReg))
1761     // There are two different ways this can be done:
1762     //   1. xxlor : This has lower latency (on the P7), 2 cycles, but can only
1763     //      issue in VSU pipeline 0.
1764     //   2. xmovdp/xmovsp: This has higher latency (on the P7), 6 cycles, but
1765     //      can go to either pipeline.
1766     // We'll always use xxlor here, because in practically all cases where
1767     // copies are generated, they are close enough to some use that the
1768     // lower-latency form is preferable.
1769     Opc = PPC::XXLOR;
1770   else if (PPC::VSFRCRegClass.contains(DestReg, SrcReg) ||
1771            PPC::VSSRCRegClass.contains(DestReg, SrcReg))
1772     Opc = (Subtarget.hasP9Vector()) ? PPC::XSCPSGNDP : PPC::XXLORf;
1773   else if (Subtarget.pairedVectorMemops() &&
1774            PPC::VSRpRCRegClass.contains(DestReg, SrcReg)) {
1775     if (SrcReg > PPC::VSRp15)
1776       SrcReg = PPC::V0 + (SrcReg - PPC::VSRp16) * 2;
1777     else
1778       SrcReg = PPC::VSL0 + (SrcReg - PPC::VSRp0) * 2;
1779     if (DestReg > PPC::VSRp15)
1780       DestReg = PPC::V0 + (DestReg - PPC::VSRp16) * 2;
1781     else
1782       DestReg = PPC::VSL0 + (DestReg - PPC::VSRp0) * 2;
1783     BuildMI(MBB, I, DL, get(PPC::XXLOR), DestReg).
1784       addReg(SrcReg).addReg(SrcReg, getKillRegState(KillSrc));
1785     BuildMI(MBB, I, DL, get(PPC::XXLOR), DestReg + 1).
1786       addReg(SrcReg + 1).addReg(SrcReg + 1, getKillRegState(KillSrc));
1787     return;
1788   }
1789   else if (PPC::CRBITRCRegClass.contains(DestReg, SrcReg))
1790     Opc = PPC::CROR;
1791   else if (PPC::SPERCRegClass.contains(DestReg, SrcReg))
1792     Opc = PPC::EVOR;
1793   else if ((PPC::ACCRCRegClass.contains(DestReg) ||
1794             PPC::UACCRCRegClass.contains(DestReg)) &&
1795            (PPC::ACCRCRegClass.contains(SrcReg) ||
1796             PPC::UACCRCRegClass.contains(SrcReg))) {
1797     // If primed, de-prime the source register, copy the individual registers
1798     // and prime the destination if needed. The vector subregisters are
1799     // vs[(u)acc * 4] - vs[(u)acc * 4 + 3]. If the copy is not a kill and the
1800     // source is primed, we need to re-prime it after the copy as well.
1801     PPCRegisterInfo::emitAccCopyInfo(MBB, DestReg, SrcReg);
1802     bool DestPrimed = PPC::ACCRCRegClass.contains(DestReg);
1803     bool SrcPrimed = PPC::ACCRCRegClass.contains(SrcReg);
1804     MCRegister VSLSrcReg =
1805         PPC::VSL0 + (SrcReg - (SrcPrimed ? PPC::ACC0 : PPC::UACC0)) * 4;
1806     MCRegister VSLDestReg =
1807         PPC::VSL0 + (DestReg - (DestPrimed ? PPC::ACC0 : PPC::UACC0)) * 4;
1808     if (SrcPrimed)
1809       BuildMI(MBB, I, DL, get(PPC::XXMFACC), SrcReg).addReg(SrcReg);
1810     for (unsigned Idx = 0; Idx < 4; Idx++)
1811       BuildMI(MBB, I, DL, get(PPC::XXLOR), VSLDestReg + Idx)
1812           .addReg(VSLSrcReg + Idx)
1813           .addReg(VSLSrcReg + Idx, getKillRegState(KillSrc));
1814     if (DestPrimed)
1815       BuildMI(MBB, I, DL, get(PPC::XXMTACC), DestReg).addReg(DestReg);
1816     if (SrcPrimed && !KillSrc)
1817       BuildMI(MBB, I, DL, get(PPC::XXMTACC), SrcReg).addReg(SrcReg);
1818     return;
1819   } else if (PPC::G8pRCRegClass.contains(DestReg) &&
1820              PPC::G8pRCRegClass.contains(SrcReg)) {
1821     // TODO: Handle G8RC to G8pRC (and vice versa) copy.
1822     unsigned DestRegIdx = DestReg - PPC::G8p0;
1823     MCRegister DestRegSub0 = PPC::X0 + 2 * DestRegIdx;
1824     MCRegister DestRegSub1 = PPC::X0 + 2 * DestRegIdx + 1;
1825     unsigned SrcRegIdx = SrcReg - PPC::G8p0;
1826     MCRegister SrcRegSub0 = PPC::X0 + 2 * SrcRegIdx;
1827     MCRegister SrcRegSub1 = PPC::X0 + 2 * SrcRegIdx + 1;
1828     BuildMI(MBB, I, DL, get(PPC::OR8), DestRegSub0)
1829         .addReg(SrcRegSub0)
1830         .addReg(SrcRegSub0, getKillRegState(KillSrc));
1831     BuildMI(MBB, I, DL, get(PPC::OR8), DestRegSub1)
1832         .addReg(SrcRegSub1)
1833         .addReg(SrcRegSub1, getKillRegState(KillSrc));
1834     return;
1835   } else
1836     llvm_unreachable("Impossible reg-to-reg copy");
1837 
1838   const MCInstrDesc &MCID = get(Opc);
1839   if (MCID.getNumOperands() == 3)
1840     BuildMI(MBB, I, DL, MCID, DestReg)
1841       .addReg(SrcReg).addReg(SrcReg, getKillRegState(KillSrc));
1842   else
1843     BuildMI(MBB, I, DL, MCID, DestReg).addReg(SrcReg, getKillRegState(KillSrc));
1844 }
1845 
1846 unsigned PPCInstrInfo::getSpillIndex(const TargetRegisterClass *RC) const {
1847   int OpcodeIndex = 0;
1848 
1849   if (PPC::GPRCRegClass.hasSubClassEq(RC) ||
1850       PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) {
1851     OpcodeIndex = SOK_Int4Spill;
1852   } else if (PPC::G8RCRegClass.hasSubClassEq(RC) ||
1853              PPC::G8RC_NOX0RegClass.hasSubClassEq(RC)) {
1854     OpcodeIndex = SOK_Int8Spill;
1855   } else if (PPC::F8RCRegClass.hasSubClassEq(RC)) {
1856     OpcodeIndex = SOK_Float8Spill;
1857   } else if (PPC::F4RCRegClass.hasSubClassEq(RC)) {
1858     OpcodeIndex = SOK_Float4Spill;
1859   } else if (PPC::SPERCRegClass.hasSubClassEq(RC)) {
1860     OpcodeIndex = SOK_SPESpill;
1861   } else if (PPC::CRRCRegClass.hasSubClassEq(RC)) {
1862     OpcodeIndex = SOK_CRSpill;
1863   } else if (PPC::CRBITRCRegClass.hasSubClassEq(RC)) {
1864     OpcodeIndex = SOK_CRBitSpill;
1865   } else if (PPC::VRRCRegClass.hasSubClassEq(RC)) {
1866     OpcodeIndex = SOK_VRVectorSpill;
1867   } else if (PPC::VSRCRegClass.hasSubClassEq(RC)) {
1868     OpcodeIndex = SOK_VSXVectorSpill;
1869   } else if (PPC::VSFRCRegClass.hasSubClassEq(RC)) {
1870     OpcodeIndex = SOK_VectorFloat8Spill;
1871   } else if (PPC::VSSRCRegClass.hasSubClassEq(RC)) {
1872     OpcodeIndex = SOK_VectorFloat4Spill;
1873   } else if (PPC::SPILLTOVSRRCRegClass.hasSubClassEq(RC)) {
1874     OpcodeIndex = SOK_SpillToVSR;
1875   } else if (PPC::ACCRCRegClass.hasSubClassEq(RC)) {
1876     assert(Subtarget.pairedVectorMemops() &&
1877            "Register unexpected when paired memops are disabled.");
1878     OpcodeIndex = SOK_AccumulatorSpill;
1879   } else if (PPC::UACCRCRegClass.hasSubClassEq(RC)) {
1880     assert(Subtarget.pairedVectorMemops() &&
1881            "Register unexpected when paired memops are disabled.");
1882     OpcodeIndex = SOK_UAccumulatorSpill;
1883   } else if (PPC::WACCRCRegClass.hasSubClassEq(RC)) {
1884     assert(Subtarget.pairedVectorMemops() &&
1885            "Register unexpected when paired memops are disabled.");
1886     OpcodeIndex = SOK_WAccumulatorSpill;
1887   } else if (PPC::VSRpRCRegClass.hasSubClassEq(RC)) {
1888     assert(Subtarget.pairedVectorMemops() &&
1889            "Register unexpected when paired memops are disabled.");
1890     OpcodeIndex = SOK_PairedVecSpill;
1891   } else if (PPC::G8pRCRegClass.hasSubClassEq(RC)) {
1892     OpcodeIndex = SOK_PairedG8Spill;
1893   } else {
1894     llvm_unreachable("Unknown regclass!");
1895   }
1896   return OpcodeIndex;
1897 }
1898 
1899 unsigned
1900 PPCInstrInfo::getStoreOpcodeForSpill(const TargetRegisterClass *RC) const {
1901   ArrayRef<unsigned> OpcodesForSpill = getStoreOpcodesForSpillArray();
1902   return OpcodesForSpill[getSpillIndex(RC)];
1903 }
1904 
1905 unsigned
1906 PPCInstrInfo::getLoadOpcodeForSpill(const TargetRegisterClass *RC) const {
1907   ArrayRef<unsigned> OpcodesForSpill = getLoadOpcodesForSpillArray();
1908   return OpcodesForSpill[getSpillIndex(RC)];
1909 }
1910 
1911 void PPCInstrInfo::StoreRegToStackSlot(
1912     MachineFunction &MF, unsigned SrcReg, bool isKill, int FrameIdx,
1913     const TargetRegisterClass *RC,
1914     SmallVectorImpl<MachineInstr *> &NewMIs) const {
1915   unsigned Opcode = getStoreOpcodeForSpill(RC);
1916   DebugLoc DL;
1917 
1918   PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
1919   FuncInfo->setHasSpills();
1920 
1921   NewMIs.push_back(addFrameReference(
1922       BuildMI(MF, DL, get(Opcode)).addReg(SrcReg, getKillRegState(isKill)),
1923       FrameIdx));
1924 
1925   if (PPC::CRRCRegClass.hasSubClassEq(RC) ||
1926       PPC::CRBITRCRegClass.hasSubClassEq(RC))
1927     FuncInfo->setSpillsCR();
1928 
1929   if (isXFormMemOp(Opcode))
1930     FuncInfo->setHasNonRISpills();
1931 }
1932 
1933 void PPCInstrInfo::storeRegToStackSlotNoUpd(
1934     MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned SrcReg,
1935     bool isKill, int FrameIdx, const TargetRegisterClass *RC,
1936     const TargetRegisterInfo *TRI) const {
1937   MachineFunction &MF = *MBB.getParent();
1938   SmallVector<MachineInstr *, 4> NewMIs;
1939 
1940   StoreRegToStackSlot(MF, SrcReg, isKill, FrameIdx, RC, NewMIs);
1941 
1942   for (unsigned i = 0, e = NewMIs.size(); i != e; ++i)
1943     MBB.insert(MI, NewMIs[i]);
1944 
1945   const MachineFrameInfo &MFI = MF.getFrameInfo();
1946   MachineMemOperand *MMO = MF.getMachineMemOperand(
1947       MachinePointerInfo::getFixedStack(MF, FrameIdx),
1948       MachineMemOperand::MOStore, MFI.getObjectSize(FrameIdx),
1949       MFI.getObjectAlign(FrameIdx));
1950   NewMIs.back()->addMemOperand(MF, MMO);
1951 }
1952 
1953 void PPCInstrInfo::storeRegToStackSlot(
1954     MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
1955     bool isKill, int FrameIdx, const TargetRegisterClass *RC,
1956     const TargetRegisterInfo *TRI, Register VReg) const {
1957   // We need to avoid a situation in which the value from a VRRC register is
1958   // spilled using an Altivec instruction and reloaded into a VSRC register
1959   // using a VSX instruction. The issue with this is that the VSX
1960   // load/store instructions swap the doublewords in the vector and the Altivec
1961   // ones don't. The register classes on the spill/reload may be different if
1962   // the register is defined using an Altivec instruction and is then used by a
1963   // VSX instruction.
1964   RC = updatedRC(RC);
1965   storeRegToStackSlotNoUpd(MBB, MI, SrcReg, isKill, FrameIdx, RC, TRI);
1966 }
1967 
1968 void PPCInstrInfo::LoadRegFromStackSlot(MachineFunction &MF, const DebugLoc &DL,
1969                                         unsigned DestReg, int FrameIdx,
1970                                         const TargetRegisterClass *RC,
1971                                         SmallVectorImpl<MachineInstr *> &NewMIs)
1972                                         const {
1973   unsigned Opcode = getLoadOpcodeForSpill(RC);
1974   NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(Opcode), DestReg),
1975                                      FrameIdx));
1976 }
1977 
1978 void PPCInstrInfo::loadRegFromStackSlotNoUpd(
1979     MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned DestReg,
1980     int FrameIdx, const TargetRegisterClass *RC,
1981     const TargetRegisterInfo *TRI) const {
1982   MachineFunction &MF = *MBB.getParent();
1983   SmallVector<MachineInstr*, 4> NewMIs;
1984   DebugLoc DL;
1985   if (MI != MBB.end()) DL = MI->getDebugLoc();
1986 
1987   LoadRegFromStackSlot(MF, DL, DestReg, FrameIdx, RC, NewMIs);
1988 
1989   for (unsigned i = 0, e = NewMIs.size(); i != e; ++i)
1990     MBB.insert(MI, NewMIs[i]);
1991 
1992   const MachineFrameInfo &MFI = MF.getFrameInfo();
1993   MachineMemOperand *MMO = MF.getMachineMemOperand(
1994       MachinePointerInfo::getFixedStack(MF, FrameIdx),
1995       MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIdx),
1996       MFI.getObjectAlign(FrameIdx));
1997   NewMIs.back()->addMemOperand(MF, MMO);
1998 }
1999 
2000 void PPCInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
2001                                         MachineBasicBlock::iterator MI,
2002                                         Register DestReg, int FrameIdx,
2003                                         const TargetRegisterClass *RC,
2004                                         const TargetRegisterInfo *TRI,
2005                                         Register VReg) const {
2006   // We need to avoid a situation in which the value from a VRRC register is
2007   // spilled using an Altivec instruction and reloaded into a VSRC register
2008   // using a VSX instruction. The issue with this is that the VSX
2009   // load/store instructions swap the doublewords in the vector and the Altivec
2010   // ones don't. The register classes on the spill/reload may be different if
2011   // the register is defined using an Altivec instruction and is then used by a
2012   // VSX instruction.
2013   RC = updatedRC(RC);
2014 
2015   loadRegFromStackSlotNoUpd(MBB, MI, DestReg, FrameIdx, RC, TRI);
2016 }
2017 
2018 bool PPCInstrInfo::
2019 reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
2020   assert(Cond.size() == 2 && "Invalid PPC branch opcode!");
2021   if (Cond[1].getReg() == PPC::CTR8 || Cond[1].getReg() == PPC::CTR)
2022     Cond[0].setImm(Cond[0].getImm() == 0 ? 1 : 0);
2023   else
2024     // Leave the CR# the same, but invert the condition.
2025     Cond[0].setImm(PPC::InvertPredicate((PPC::Predicate)Cond[0].getImm()));
2026   return false;
2027 }
2028 
2029 // For some instructions, it is legal to fold ZERO into the RA register field.
2030 // This function performs that fold by replacing the operand with PPC::ZERO,
2031 // it does not consider whether the load immediate zero is no longer in use.
2032 bool PPCInstrInfo::onlyFoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
2033                                      Register Reg) const {
2034   // A zero immediate should always be loaded with a single li.
2035   unsigned DefOpc = DefMI.getOpcode();
2036   if (DefOpc != PPC::LI && DefOpc != PPC::LI8)
2037     return false;
2038   if (!DefMI.getOperand(1).isImm())
2039     return false;
2040   if (DefMI.getOperand(1).getImm() != 0)
2041     return false;
2042 
2043   // Note that we cannot here invert the arguments of an isel in order to fold
2044   // a ZERO into what is presented as the second argument. All we have here
2045   // is the condition bit, and that might come from a CR-logical bit operation.
2046 
2047   const MCInstrDesc &UseMCID = UseMI.getDesc();
2048 
2049   // Only fold into real machine instructions.
2050   if (UseMCID.isPseudo())
2051     return false;
2052 
2053   // We need to find which of the User's operands is to be folded, that will be
2054   // the operand that matches the given register ID.
2055   unsigned UseIdx;
2056   for (UseIdx = 0; UseIdx < UseMI.getNumOperands(); ++UseIdx)
2057     if (UseMI.getOperand(UseIdx).isReg() &&
2058         UseMI.getOperand(UseIdx).getReg() == Reg)
2059       break;
2060 
2061   assert(UseIdx < UseMI.getNumOperands() && "Cannot find Reg in UseMI");
2062   assert(UseIdx < UseMCID.getNumOperands() && "No operand description for Reg");
2063 
2064   const MCOperandInfo *UseInfo = &UseMCID.operands()[UseIdx];
2065 
2066   // We can fold the zero if this register requires a GPRC_NOR0/G8RC_NOX0
2067   // register (which might also be specified as a pointer class kind).
2068   if (UseInfo->isLookupPtrRegClass()) {
2069     if (UseInfo->RegClass /* Kind */ != 1)
2070       return false;
2071   } else {
2072     if (UseInfo->RegClass != PPC::GPRC_NOR0RegClassID &&
2073         UseInfo->RegClass != PPC::G8RC_NOX0RegClassID)
2074       return false;
2075   }
2076 
2077   // Make sure this is not tied to an output register (or otherwise
2078   // constrained). This is true for ST?UX registers, for example, which
2079   // are tied to their output registers.
2080   if (UseInfo->Constraints != 0)
2081     return false;
2082 
2083   MCRegister ZeroReg;
2084   if (UseInfo->isLookupPtrRegClass()) {
2085     bool isPPC64 = Subtarget.isPPC64();
2086     ZeroReg = isPPC64 ? PPC::ZERO8 : PPC::ZERO;
2087   } else {
2088     ZeroReg = UseInfo->RegClass == PPC::G8RC_NOX0RegClassID ?
2089               PPC::ZERO8 : PPC::ZERO;
2090   }
2091 
2092   LLVM_DEBUG(dbgs() << "Folded immediate zero for: ");
2093   LLVM_DEBUG(UseMI.dump());
2094   UseMI.getOperand(UseIdx).setReg(ZeroReg);
2095   LLVM_DEBUG(dbgs() << "Into: ");
2096   LLVM_DEBUG(UseMI.dump());
2097   return true;
2098 }
2099 
2100 // Folds zero into instructions which have a load immediate zero as an operand
2101 // but also recognize zero as immediate zero. If the definition of the load
2102 // has no more users it is deleted.
2103 bool PPCInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
2104                                  Register Reg, MachineRegisterInfo *MRI) const {
2105   bool Changed = onlyFoldImmediate(UseMI, DefMI, Reg);
2106   if (MRI->use_nodbg_empty(Reg))
2107     DefMI.eraseFromParent();
2108   return Changed;
2109 }
2110 
2111 static bool MBBDefinesCTR(MachineBasicBlock &MBB) {
2112   for (MachineInstr &MI : MBB)
2113     if (MI.definesRegister(PPC::CTR) || MI.definesRegister(PPC::CTR8))
2114       return true;
2115   return false;
2116 }
2117 
2118 // We should make sure that, if we're going to predicate both sides of a
2119 // condition (a diamond), that both sides don't define the counter register. We
2120 // can predicate counter-decrement-based branches, but while that predicates
2121 // the branching, it does not predicate the counter decrement. If we tried to
2122 // merge the triangle into one predicated block, we'd decrement the counter
2123 // twice.
2124 bool PPCInstrInfo::isProfitableToIfCvt(MachineBasicBlock &TMBB,
2125                      unsigned NumT, unsigned ExtraT,
2126                      MachineBasicBlock &FMBB,
2127                      unsigned NumF, unsigned ExtraF,
2128                      BranchProbability Probability) const {
2129   return !(MBBDefinesCTR(TMBB) && MBBDefinesCTR(FMBB));
2130 }
2131 
2132 
2133 bool PPCInstrInfo::isPredicated(const MachineInstr &MI) const {
2134   // The predicated branches are identified by their type, not really by the
2135   // explicit presence of a predicate. Furthermore, some of them can be
2136   // predicated more than once. Because if conversion won't try to predicate
2137   // any instruction which already claims to be predicated (by returning true
2138   // here), always return false. In doing so, we let isPredicable() be the
2139   // final word on whether not the instruction can be (further) predicated.
2140 
2141   return false;
2142 }
2143 
2144 bool PPCInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
2145                                         const MachineBasicBlock *MBB,
2146                                         const MachineFunction &MF) const {
2147   switch (MI.getOpcode()) {
2148   default:
2149     break;
2150   // Set MFFS and MTFSF as scheduling boundary to avoid unexpected code motion
2151   // across them, since some FP operations may change content of FPSCR.
2152   // TODO: Model FPSCR in PPC instruction definitions and remove the workaround
2153   case PPC::MFFS:
2154   case PPC::MTFSF:
2155   case PPC::FENCE:
2156     return true;
2157   }
2158   return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF);
2159 }
2160 
2161 bool PPCInstrInfo::PredicateInstruction(MachineInstr &MI,
2162                                         ArrayRef<MachineOperand> Pred) const {
2163   unsigned OpC = MI.getOpcode();
2164   if (OpC == PPC::BLR || OpC == PPC::BLR8) {
2165     if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
2166       bool isPPC64 = Subtarget.isPPC64();
2167       MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZLR8 : PPC::BDNZLR)
2168                                       : (isPPC64 ? PPC::BDZLR8 : PPC::BDZLR)));
2169       // Need add Def and Use for CTR implicit operand.
2170       MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2171           .addReg(Pred[1].getReg(), RegState::Implicit)
2172           .addReg(Pred[1].getReg(), RegState::ImplicitDefine);
2173     } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2174       MI.setDesc(get(PPC::BCLR));
2175       MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2176     } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2177       MI.setDesc(get(PPC::BCLRn));
2178       MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2179     } else {
2180       MI.setDesc(get(PPC::BCCLR));
2181       MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2182           .addImm(Pred[0].getImm())
2183           .add(Pred[1]);
2184     }
2185 
2186     return true;
2187   } else if (OpC == PPC::B) {
2188     if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
2189       bool isPPC64 = Subtarget.isPPC64();
2190       MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ)
2191                                       : (isPPC64 ? PPC::BDZ8 : PPC::BDZ)));
2192       // Need add Def and Use for CTR implicit operand.
2193       MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2194           .addReg(Pred[1].getReg(), RegState::Implicit)
2195           .addReg(Pred[1].getReg(), RegState::ImplicitDefine);
2196     } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2197       MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
2198       MI.removeOperand(0);
2199 
2200       MI.setDesc(get(PPC::BC));
2201       MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2202           .add(Pred[1])
2203           .addMBB(MBB);
2204     } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2205       MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
2206       MI.removeOperand(0);
2207 
2208       MI.setDesc(get(PPC::BCn));
2209       MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2210           .add(Pred[1])
2211           .addMBB(MBB);
2212     } else {
2213       MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
2214       MI.removeOperand(0);
2215 
2216       MI.setDesc(get(PPC::BCC));
2217       MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2218           .addImm(Pred[0].getImm())
2219           .add(Pred[1])
2220           .addMBB(MBB);
2221     }
2222 
2223     return true;
2224   } else if (OpC == PPC::BCTR || OpC == PPC::BCTR8 || OpC == PPC::BCTRL ||
2225              OpC == PPC::BCTRL8 || OpC == PPC::BCTRL_RM ||
2226              OpC == PPC::BCTRL8_RM) {
2227     if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR)
2228       llvm_unreachable("Cannot predicate bctr[l] on the ctr register");
2229 
2230     bool setLR = OpC == PPC::BCTRL || OpC == PPC::BCTRL8 ||
2231                  OpC == PPC::BCTRL_RM || OpC == PPC::BCTRL8_RM;
2232     bool isPPC64 = Subtarget.isPPC64();
2233 
2234     if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2235       MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8 : PPC::BCCTR8)
2236                              : (setLR ? PPC::BCCTRL : PPC::BCCTR)));
2237       MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2238     } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2239       MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8n : PPC::BCCTR8n)
2240                              : (setLR ? PPC::BCCTRLn : PPC::BCCTRn)));
2241       MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2242     } else {
2243       MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCCTRL8 : PPC::BCCCTR8)
2244                              : (setLR ? PPC::BCCCTRL : PPC::BCCCTR)));
2245       MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2246           .addImm(Pred[0].getImm())
2247           .add(Pred[1]);
2248     }
2249 
2250     // Need add Def and Use for LR implicit operand.
2251     if (setLR)
2252       MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2253           .addReg(isPPC64 ? PPC::LR8 : PPC::LR, RegState::Implicit)
2254           .addReg(isPPC64 ? PPC::LR8 : PPC::LR, RegState::ImplicitDefine);
2255     if (OpC == PPC::BCTRL_RM || OpC == PPC::BCTRL8_RM)
2256       MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2257           .addReg(PPC::RM, RegState::ImplicitDefine);
2258 
2259     return true;
2260   }
2261 
2262   return false;
2263 }
2264 
2265 bool PPCInstrInfo::SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
2266                                      ArrayRef<MachineOperand> Pred2) const {
2267   assert(Pred1.size() == 2 && "Invalid PPC first predicate");
2268   assert(Pred2.size() == 2 && "Invalid PPC second predicate");
2269 
2270   if (Pred1[1].getReg() == PPC::CTR8 || Pred1[1].getReg() == PPC::CTR)
2271     return false;
2272   if (Pred2[1].getReg() == PPC::CTR8 || Pred2[1].getReg() == PPC::CTR)
2273     return false;
2274 
2275   // P1 can only subsume P2 if they test the same condition register.
2276   if (Pred1[1].getReg() != Pred2[1].getReg())
2277     return false;
2278 
2279   PPC::Predicate P1 = (PPC::Predicate) Pred1[0].getImm();
2280   PPC::Predicate P2 = (PPC::Predicate) Pred2[0].getImm();
2281 
2282   if (P1 == P2)
2283     return true;
2284 
2285   // Does P1 subsume P2, e.g. GE subsumes GT.
2286   if (P1 == PPC::PRED_LE &&
2287       (P2 == PPC::PRED_LT || P2 == PPC::PRED_EQ))
2288     return true;
2289   if (P1 == PPC::PRED_GE &&
2290       (P2 == PPC::PRED_GT || P2 == PPC::PRED_EQ))
2291     return true;
2292 
2293   return false;
2294 }
2295 
2296 bool PPCInstrInfo::ClobbersPredicate(MachineInstr &MI,
2297                                      std::vector<MachineOperand> &Pred,
2298                                      bool SkipDead) const {
2299   // Note: At the present time, the contents of Pred from this function is
2300   // unused by IfConversion. This implementation follows ARM by pushing the
2301   // CR-defining operand. Because the 'DZ' and 'DNZ' count as types of
2302   // predicate, instructions defining CTR or CTR8 are also included as
2303   // predicate-defining instructions.
2304 
2305   const TargetRegisterClass *RCs[] =
2306     { &PPC::CRRCRegClass, &PPC::CRBITRCRegClass,
2307       &PPC::CTRRCRegClass, &PPC::CTRRC8RegClass };
2308 
2309   bool Found = false;
2310   for (const MachineOperand &MO : MI.operands()) {
2311     for (unsigned c = 0; c < std::size(RCs) && !Found; ++c) {
2312       const TargetRegisterClass *RC = RCs[c];
2313       if (MO.isReg()) {
2314         if (MO.isDef() && RC->contains(MO.getReg())) {
2315           Pred.push_back(MO);
2316           Found = true;
2317         }
2318       } else if (MO.isRegMask()) {
2319         for (MCPhysReg R : *RC)
2320           if (MO.clobbersPhysReg(R)) {
2321             Pred.push_back(MO);
2322             Found = true;
2323           }
2324       }
2325     }
2326   }
2327 
2328   return Found;
2329 }
2330 
2331 bool PPCInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
2332                                   Register &SrcReg2, int64_t &Mask,
2333                                   int64_t &Value) const {
2334   unsigned Opc = MI.getOpcode();
2335 
2336   switch (Opc) {
2337   default: return false;
2338   case PPC::CMPWI:
2339   case PPC::CMPLWI:
2340   case PPC::CMPDI:
2341   case PPC::CMPLDI:
2342     SrcReg = MI.getOperand(1).getReg();
2343     SrcReg2 = 0;
2344     Value = MI.getOperand(2).getImm();
2345     Mask = 0xFFFF;
2346     return true;
2347   case PPC::CMPW:
2348   case PPC::CMPLW:
2349   case PPC::CMPD:
2350   case PPC::CMPLD:
2351   case PPC::FCMPUS:
2352   case PPC::FCMPUD:
2353     SrcReg = MI.getOperand(1).getReg();
2354     SrcReg2 = MI.getOperand(2).getReg();
2355     Value = 0;
2356     Mask = 0;
2357     return true;
2358   }
2359 }
2360 
2361 bool PPCInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
2362                                         Register SrcReg2, int64_t Mask,
2363                                         int64_t Value,
2364                                         const MachineRegisterInfo *MRI) const {
2365   if (DisableCmpOpt)
2366     return false;
2367 
2368   int OpC = CmpInstr.getOpcode();
2369   Register CRReg = CmpInstr.getOperand(0).getReg();
2370 
2371   // FP record forms set CR1 based on the exception status bits, not a
2372   // comparison with zero.
2373   if (OpC == PPC::FCMPUS || OpC == PPC::FCMPUD)
2374     return false;
2375 
2376   const TargetRegisterInfo *TRI = &getRegisterInfo();
2377   // The record forms set the condition register based on a signed comparison
2378   // with zero (so says the ISA manual). This is not as straightforward as it
2379   // seems, however, because this is always a 64-bit comparison on PPC64, even
2380   // for instructions that are 32-bit in nature (like slw for example).
2381   // So, on PPC32, for unsigned comparisons, we can use the record forms only
2382   // for equality checks (as those don't depend on the sign). On PPC64,
2383   // we are restricted to equality for unsigned 64-bit comparisons and for
2384   // signed 32-bit comparisons the applicability is more restricted.
2385   bool isPPC64 = Subtarget.isPPC64();
2386   bool is32BitSignedCompare   = OpC ==  PPC::CMPWI || OpC == PPC::CMPW;
2387   bool is32BitUnsignedCompare = OpC == PPC::CMPLWI || OpC == PPC::CMPLW;
2388   bool is64BitUnsignedCompare = OpC == PPC::CMPLDI || OpC == PPC::CMPLD;
2389 
2390   // Look through copies unless that gets us to a physical register.
2391   Register ActualSrc = TRI->lookThruCopyLike(SrcReg, MRI);
2392   if (ActualSrc.isVirtual())
2393     SrcReg = ActualSrc;
2394 
2395   // Get the unique definition of SrcReg.
2396   MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
2397   if (!MI) return false;
2398 
2399   bool equalityOnly = false;
2400   bool noSub = false;
2401   if (isPPC64) {
2402     if (is32BitSignedCompare) {
2403       // We can perform this optimization only if SrcReg is sign-extending.
2404       if (isSignExtended(SrcReg, MRI))
2405         noSub = true;
2406       else
2407         return false;
2408     } else if (is32BitUnsignedCompare) {
2409       // We can perform this optimization, equality only, if SrcReg is
2410       // zero-extending.
2411       if (isZeroExtended(SrcReg, MRI)) {
2412         noSub = true;
2413         equalityOnly = true;
2414       } else
2415         return false;
2416     } else
2417       equalityOnly = is64BitUnsignedCompare;
2418   } else
2419     equalityOnly = is32BitUnsignedCompare;
2420 
2421   if (equalityOnly) {
2422     // We need to check the uses of the condition register in order to reject
2423     // non-equality comparisons.
2424     for (MachineRegisterInfo::use_instr_iterator
2425          I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end();
2426          I != IE; ++I) {
2427       MachineInstr *UseMI = &*I;
2428       if (UseMI->getOpcode() == PPC::BCC) {
2429         PPC::Predicate Pred = (PPC::Predicate)UseMI->getOperand(0).getImm();
2430         unsigned PredCond = PPC::getPredicateCondition(Pred);
2431         // We ignore hint bits when checking for non-equality comparisons.
2432         if (PredCond != PPC::PRED_EQ && PredCond != PPC::PRED_NE)
2433           return false;
2434       } else if (UseMI->getOpcode() == PPC::ISEL ||
2435                  UseMI->getOpcode() == PPC::ISEL8) {
2436         unsigned SubIdx = UseMI->getOperand(3).getSubReg();
2437         if (SubIdx != PPC::sub_eq)
2438           return false;
2439       } else
2440         return false;
2441     }
2442   }
2443 
2444   MachineBasicBlock::iterator I = CmpInstr;
2445 
2446   // Scan forward to find the first use of the compare.
2447   for (MachineBasicBlock::iterator EL = CmpInstr.getParent()->end(); I != EL;
2448        ++I) {
2449     bool FoundUse = false;
2450     for (MachineRegisterInfo::use_instr_iterator
2451          J = MRI->use_instr_begin(CRReg), JE = MRI->use_instr_end();
2452          J != JE; ++J)
2453       if (&*J == &*I) {
2454         FoundUse = true;
2455         break;
2456       }
2457 
2458     if (FoundUse)
2459       break;
2460   }
2461 
2462   SmallVector<std::pair<MachineOperand*, PPC::Predicate>, 4> PredsToUpdate;
2463   SmallVector<std::pair<MachineOperand*, unsigned>, 4> SubRegsToUpdate;
2464 
2465   // There are two possible candidates which can be changed to set CR[01].
2466   // One is MI, the other is a SUB instruction.
2467   // For CMPrr(r1,r2), we are looking for SUB(r1,r2) or SUB(r2,r1).
2468   MachineInstr *Sub = nullptr;
2469   if (SrcReg2 != 0)
2470     // MI is not a candidate for CMPrr.
2471     MI = nullptr;
2472   // FIXME: Conservatively refuse to convert an instruction which isn't in the
2473   // same BB as the comparison. This is to allow the check below to avoid calls
2474   // (and other explicit clobbers); instead we should really check for these
2475   // more explicitly (in at least a few predecessors).
2476   else if (MI->getParent() != CmpInstr.getParent())
2477     return false;
2478   else if (Value != 0) {
2479     // The record-form instructions set CR bit based on signed comparison
2480     // against 0. We try to convert a compare against 1 or -1 into a compare
2481     // against 0 to exploit record-form instructions. For example, we change
2482     // the condition "greater than -1" into "greater than or equal to 0"
2483     // and "less than 1" into "less than or equal to 0".
2484 
2485     // Since we optimize comparison based on a specific branch condition,
2486     // we don't optimize if condition code is used by more than once.
2487     if (equalityOnly || !MRI->hasOneUse(CRReg))
2488       return false;
2489 
2490     MachineInstr *UseMI = &*MRI->use_instr_begin(CRReg);
2491     if (UseMI->getOpcode() != PPC::BCC)
2492       return false;
2493 
2494     PPC::Predicate Pred = (PPC::Predicate)UseMI->getOperand(0).getImm();
2495     unsigned PredCond = PPC::getPredicateCondition(Pred);
2496     unsigned PredHint = PPC::getPredicateHint(Pred);
2497     int16_t Immed = (int16_t)Value;
2498 
2499     // When modifying the condition in the predicate, we propagate hint bits
2500     // from the original predicate to the new one.
2501     if (Immed == -1 && PredCond == PPC::PRED_GT)
2502       // We convert "greater than -1" into "greater than or equal to 0",
2503       // since we are assuming signed comparison by !equalityOnly
2504       Pred = PPC::getPredicate(PPC::PRED_GE, PredHint);
2505     else if (Immed == -1 && PredCond == PPC::PRED_LE)
2506       // We convert "less than or equal to -1" into "less than 0".
2507       Pred = PPC::getPredicate(PPC::PRED_LT, PredHint);
2508     else if (Immed == 1 && PredCond == PPC::PRED_LT)
2509       // We convert "less than 1" into "less than or equal to 0".
2510       Pred = PPC::getPredicate(PPC::PRED_LE, PredHint);
2511     else if (Immed == 1 && PredCond == PPC::PRED_GE)
2512       // We convert "greater than or equal to 1" into "greater than 0".
2513       Pred = PPC::getPredicate(PPC::PRED_GT, PredHint);
2514     else
2515       return false;
2516 
2517     // Convert the comparison and its user to a compare against zero with the
2518     // appropriate predicate on the branch. Zero comparison might provide
2519     // optimization opportunities post-RA (see optimization in
2520     // PPCPreEmitPeephole.cpp).
2521     UseMI->getOperand(0).setImm(Pred);
2522     CmpInstr.getOperand(2).setImm(0);
2523   }
2524 
2525   // Search for Sub.
2526   --I;
2527 
2528   // Get ready to iterate backward from CmpInstr.
2529   MachineBasicBlock::iterator E = MI, B = CmpInstr.getParent()->begin();
2530 
2531   for (; I != E && !noSub; --I) {
2532     const MachineInstr &Instr = *I;
2533     unsigned IOpC = Instr.getOpcode();
2534 
2535     if (&*I != &CmpInstr && (Instr.modifiesRegister(PPC::CR0, TRI) ||
2536                              Instr.readsRegister(PPC::CR0, TRI)))
2537       // This instruction modifies or uses the record condition register after
2538       // the one we want to change. While we could do this transformation, it
2539       // would likely not be profitable. This transformation removes one
2540       // instruction, and so even forcing RA to generate one move probably
2541       // makes it unprofitable.
2542       return false;
2543 
2544     // Check whether CmpInstr can be made redundant by the current instruction.
2545     if ((OpC == PPC::CMPW || OpC == PPC::CMPLW ||
2546          OpC == PPC::CMPD || OpC == PPC::CMPLD) &&
2547         (IOpC == PPC::SUBF || IOpC == PPC::SUBF8) &&
2548         ((Instr.getOperand(1).getReg() == SrcReg &&
2549           Instr.getOperand(2).getReg() == SrcReg2) ||
2550         (Instr.getOperand(1).getReg() == SrcReg2 &&
2551          Instr.getOperand(2).getReg() == SrcReg))) {
2552       Sub = &*I;
2553       break;
2554     }
2555 
2556     if (I == B)
2557       // The 'and' is below the comparison instruction.
2558       return false;
2559   }
2560 
2561   // Return false if no candidates exist.
2562   if (!MI && !Sub)
2563     return false;
2564 
2565   // The single candidate is called MI.
2566   if (!MI) MI = Sub;
2567 
2568   int NewOpC = -1;
2569   int MIOpC = MI->getOpcode();
2570   if (MIOpC == PPC::ANDI_rec || MIOpC == PPC::ANDI8_rec ||
2571       MIOpC == PPC::ANDIS_rec || MIOpC == PPC::ANDIS8_rec)
2572     NewOpC = MIOpC;
2573   else {
2574     NewOpC = PPC::getRecordFormOpcode(MIOpC);
2575     if (NewOpC == -1 && PPC::getNonRecordFormOpcode(MIOpC) != -1)
2576       NewOpC = MIOpC;
2577   }
2578 
2579   // FIXME: On the non-embedded POWER architectures, only some of the record
2580   // forms are fast, and we should use only the fast ones.
2581 
2582   // The defining instruction has a record form (or is already a record
2583   // form). It is possible, however, that we'll need to reverse the condition
2584   // code of the users.
2585   if (NewOpC == -1)
2586     return false;
2587 
2588   // This transformation should not be performed if `nsw` is missing and is not
2589   // `equalityOnly` comparison. Since if there is overflow, sub_lt, sub_gt in
2590   // CRReg do not reflect correct order. If `equalityOnly` is true, sub_eq in
2591   // CRReg can reflect if compared values are equal, this optz is still valid.
2592   if (!equalityOnly && (NewOpC == PPC::SUBF_rec || NewOpC == PPC::SUBF8_rec) &&
2593       Sub && !Sub->getFlag(MachineInstr::NoSWrap))
2594     return false;
2595 
2596   // If we have SUB(r1, r2) and CMP(r2, r1), the condition code based on CMP
2597   // needs to be updated to be based on SUB.  Push the condition code
2598   // operands to OperandsToUpdate.  If it is safe to remove CmpInstr, the
2599   // condition code of these operands will be modified.
2600   // Here, Value == 0 means we haven't converted comparison against 1 or -1 to
2601   // comparison against 0, which may modify predicate.
2602   bool ShouldSwap = false;
2603   if (Sub && Value == 0) {
2604     ShouldSwap = SrcReg2 != 0 && Sub->getOperand(1).getReg() == SrcReg2 &&
2605       Sub->getOperand(2).getReg() == SrcReg;
2606 
2607     // The operands to subf are the opposite of sub, so only in the fixed-point
2608     // case, invert the order.
2609     ShouldSwap = !ShouldSwap;
2610   }
2611 
2612   if (ShouldSwap)
2613     for (MachineRegisterInfo::use_instr_iterator
2614          I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end();
2615          I != IE; ++I) {
2616       MachineInstr *UseMI = &*I;
2617       if (UseMI->getOpcode() == PPC::BCC) {
2618         PPC::Predicate Pred = (PPC::Predicate) UseMI->getOperand(0).getImm();
2619         unsigned PredCond = PPC::getPredicateCondition(Pred);
2620         assert((!equalityOnly ||
2621                 PredCond == PPC::PRED_EQ || PredCond == PPC::PRED_NE) &&
2622                "Invalid predicate for equality-only optimization");
2623         (void)PredCond; // To suppress warning in release build.
2624         PredsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(0)),
2625                                 PPC::getSwappedPredicate(Pred)));
2626       } else if (UseMI->getOpcode() == PPC::ISEL ||
2627                  UseMI->getOpcode() == PPC::ISEL8) {
2628         unsigned NewSubReg = UseMI->getOperand(3).getSubReg();
2629         assert((!equalityOnly || NewSubReg == PPC::sub_eq) &&
2630                "Invalid CR bit for equality-only optimization");
2631 
2632         if (NewSubReg == PPC::sub_lt)
2633           NewSubReg = PPC::sub_gt;
2634         else if (NewSubReg == PPC::sub_gt)
2635           NewSubReg = PPC::sub_lt;
2636 
2637         SubRegsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(3)),
2638                                                  NewSubReg));
2639       } else // We need to abort on a user we don't understand.
2640         return false;
2641     }
2642   assert(!(Value != 0 && ShouldSwap) &&
2643          "Non-zero immediate support and ShouldSwap"
2644          "may conflict in updating predicate");
2645 
2646   // Create a new virtual register to hold the value of the CR set by the
2647   // record-form instruction. If the instruction was not previously in
2648   // record form, then set the kill flag on the CR.
2649   CmpInstr.eraseFromParent();
2650 
2651   MachineBasicBlock::iterator MII = MI;
2652   BuildMI(*MI->getParent(), std::next(MII), MI->getDebugLoc(),
2653           get(TargetOpcode::COPY), CRReg)
2654     .addReg(PPC::CR0, MIOpC != NewOpC ? RegState::Kill : 0);
2655 
2656   // Even if CR0 register were dead before, it is alive now since the
2657   // instruction we just built uses it.
2658   MI->clearRegisterDeads(PPC::CR0);
2659 
2660   if (MIOpC != NewOpC) {
2661     // We need to be careful here: we're replacing one instruction with
2662     // another, and we need to make sure that we get all of the right
2663     // implicit uses and defs. On the other hand, the caller may be holding
2664     // an iterator to this instruction, and so we can't delete it (this is
2665     // specifically the case if this is the instruction directly after the
2666     // compare).
2667 
2668     // Rotates are expensive instructions. If we're emitting a record-form
2669     // rotate that can just be an andi/andis, we should just emit that.
2670     if (MIOpC == PPC::RLWINM || MIOpC == PPC::RLWINM8) {
2671       Register GPRRes = MI->getOperand(0).getReg();
2672       int64_t SH = MI->getOperand(2).getImm();
2673       int64_t MB = MI->getOperand(3).getImm();
2674       int64_t ME = MI->getOperand(4).getImm();
2675       // We can only do this if both the start and end of the mask are in the
2676       // same halfword.
2677       bool MBInLoHWord = MB >= 16;
2678       bool MEInLoHWord = ME >= 16;
2679       uint64_t Mask = ~0LLU;
2680 
2681       if (MB <= ME && MBInLoHWord == MEInLoHWord && SH == 0) {
2682         Mask = ((1LLU << (32 - MB)) - 1) & ~((1LLU << (31 - ME)) - 1);
2683         // The mask value needs to shift right 16 if we're emitting andis.
2684         Mask >>= MBInLoHWord ? 0 : 16;
2685         NewOpC = MIOpC == PPC::RLWINM
2686                      ? (MBInLoHWord ? PPC::ANDI_rec : PPC::ANDIS_rec)
2687                      : (MBInLoHWord ? PPC::ANDI8_rec : PPC::ANDIS8_rec);
2688       } else if (MRI->use_empty(GPRRes) && (ME == 31) &&
2689                  (ME - MB + 1 == SH) && (MB >= 16)) {
2690         // If we are rotating by the exact number of bits as are in the mask
2691         // and the mask is in the least significant bits of the register,
2692         // that's just an andis. (as long as the GPR result has no uses).
2693         Mask = ((1LLU << 32) - 1) & ~((1LLU << (32 - SH)) - 1);
2694         Mask >>= 16;
2695         NewOpC = MIOpC == PPC::RLWINM ? PPC::ANDIS_rec : PPC::ANDIS8_rec;
2696       }
2697       // If we've set the mask, we can transform.
2698       if (Mask != ~0LLU) {
2699         MI->removeOperand(4);
2700         MI->removeOperand(3);
2701         MI->getOperand(2).setImm(Mask);
2702         NumRcRotatesConvertedToRcAnd++;
2703       }
2704     } else if (MIOpC == PPC::RLDICL && MI->getOperand(2).getImm() == 0) {
2705       int64_t MB = MI->getOperand(3).getImm();
2706       if (MB >= 48) {
2707         uint64_t Mask = (1LLU << (63 - MB + 1)) - 1;
2708         NewOpC = PPC::ANDI8_rec;
2709         MI->removeOperand(3);
2710         MI->getOperand(2).setImm(Mask);
2711         NumRcRotatesConvertedToRcAnd++;
2712       }
2713     }
2714 
2715     const MCInstrDesc &NewDesc = get(NewOpC);
2716     MI->setDesc(NewDesc);
2717 
2718     for (MCPhysReg ImpDef : NewDesc.implicit_defs()) {
2719       if (!MI->definesRegister(ImpDef)) {
2720         MI->addOperand(*MI->getParent()->getParent(),
2721                        MachineOperand::CreateReg(ImpDef, true, true));
2722       }
2723     }
2724     for (MCPhysReg ImpUse : NewDesc.implicit_uses()) {
2725       if (!MI->readsRegister(ImpUse)) {
2726         MI->addOperand(*MI->getParent()->getParent(),
2727                        MachineOperand::CreateReg(ImpUse, false, true));
2728       }
2729     }
2730   }
2731   assert(MI->definesRegister(PPC::CR0) &&
2732          "Record-form instruction does not define cr0?");
2733 
2734   // Modify the condition code of operands in OperandsToUpdate.
2735   // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to
2736   // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
2737   for (unsigned i = 0, e = PredsToUpdate.size(); i < e; i++)
2738     PredsToUpdate[i].first->setImm(PredsToUpdate[i].second);
2739 
2740   for (unsigned i = 0, e = SubRegsToUpdate.size(); i < e; i++)
2741     SubRegsToUpdate[i].first->setSubReg(SubRegsToUpdate[i].second);
2742 
2743   return true;
2744 }
2745 
2746 bool PPCInstrInfo::optimizeCmpPostRA(MachineInstr &CmpMI) const {
2747   MachineRegisterInfo *MRI = &CmpMI.getParent()->getParent()->getRegInfo();
2748   if (MRI->isSSA())
2749     return false;
2750 
2751   Register SrcReg, SrcReg2;
2752   int64_t CmpMask, CmpValue;
2753   if (!analyzeCompare(CmpMI, SrcReg, SrcReg2, CmpMask, CmpValue))
2754     return false;
2755 
2756   // Try to optimize the comparison against 0.
2757   if (CmpValue || !CmpMask || SrcReg2)
2758     return false;
2759 
2760   // The record forms set the condition register based on a signed comparison
2761   // with zero (see comments in optimizeCompareInstr). Since we can't do the
2762   // equality checks in post-RA, we are more restricted on a unsigned
2763   // comparison.
2764   unsigned Opc = CmpMI.getOpcode();
2765   if (Opc == PPC::CMPLWI || Opc == PPC::CMPLDI)
2766     return false;
2767 
2768   // The record forms are always based on a 64-bit comparison on PPC64
2769   // (similary, a 32-bit comparison on PPC32), while the CMPWI is a 32-bit
2770   // comparison. Since we can't do the equality checks in post-RA, we bail out
2771   // the case.
2772   if (Subtarget.isPPC64() && Opc == PPC::CMPWI)
2773     return false;
2774 
2775   // CmpMI can't be deleted if it has implicit def.
2776   if (CmpMI.hasImplicitDef())
2777     return false;
2778 
2779   bool SrcRegHasOtherUse = false;
2780   MachineInstr *SrcMI = getDefMIPostRA(SrcReg, CmpMI, SrcRegHasOtherUse);
2781   if (!SrcMI || !SrcMI->definesRegister(SrcReg))
2782     return false;
2783 
2784   MachineOperand RegMO = CmpMI.getOperand(0);
2785   Register CRReg = RegMO.getReg();
2786   if (CRReg != PPC::CR0)
2787     return false;
2788 
2789   // Make sure there is no def/use of CRReg between SrcMI and CmpMI.
2790   bool SeenUseOfCRReg = false;
2791   bool IsCRRegKilled = false;
2792   if (!isRegElgibleForForwarding(RegMO, *SrcMI, CmpMI, false, IsCRRegKilled,
2793                                  SeenUseOfCRReg) ||
2794       SrcMI->definesRegister(CRReg) || SeenUseOfCRReg)
2795     return false;
2796 
2797   int SrcMIOpc = SrcMI->getOpcode();
2798   int NewOpC = PPC::getRecordFormOpcode(SrcMIOpc);
2799   if (NewOpC == -1)
2800     return false;
2801 
2802   LLVM_DEBUG(dbgs() << "Replace Instr: ");
2803   LLVM_DEBUG(SrcMI->dump());
2804 
2805   const MCInstrDesc &NewDesc = get(NewOpC);
2806   SrcMI->setDesc(NewDesc);
2807   MachineInstrBuilder(*SrcMI->getParent()->getParent(), SrcMI)
2808       .addReg(CRReg, RegState::ImplicitDefine);
2809   SrcMI->clearRegisterDeads(CRReg);
2810 
2811   assert(SrcMI->definesRegister(PPC::CR0) &&
2812          "Record-form instruction does not define cr0?");
2813 
2814   LLVM_DEBUG(dbgs() << "with: ");
2815   LLVM_DEBUG(SrcMI->dump());
2816   LLVM_DEBUG(dbgs() << "Delete dead instruction: ");
2817   LLVM_DEBUG(CmpMI.dump());
2818   return true;
2819 }
2820 
2821 bool PPCInstrInfo::getMemOperandsWithOffsetWidth(
2822     const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
2823     int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
2824     const TargetRegisterInfo *TRI) const {
2825   const MachineOperand *BaseOp;
2826   OffsetIsScalable = false;
2827   if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, TRI))
2828     return false;
2829   BaseOps.push_back(BaseOp);
2830   return true;
2831 }
2832 
2833 static bool isLdStSafeToCluster(const MachineInstr &LdSt,
2834                                 const TargetRegisterInfo *TRI) {
2835   // If this is a volatile load/store, don't mess with it.
2836   if (LdSt.hasOrderedMemoryRef() || LdSt.getNumExplicitOperands() != 3)
2837     return false;
2838 
2839   if (LdSt.getOperand(2).isFI())
2840     return true;
2841 
2842   assert(LdSt.getOperand(2).isReg() && "Expected a reg operand.");
2843   // Can't cluster if the instruction modifies the base register
2844   // or it is update form. e.g. ld r2,3(r2)
2845   if (LdSt.modifiesRegister(LdSt.getOperand(2).getReg(), TRI))
2846     return false;
2847 
2848   return true;
2849 }
2850 
2851 // Only cluster instruction pair that have the same opcode, and they are
2852 // clusterable according to PowerPC specification.
2853 static bool isClusterableLdStOpcPair(unsigned FirstOpc, unsigned SecondOpc,
2854                                      const PPCSubtarget &Subtarget) {
2855   switch (FirstOpc) {
2856   default:
2857     return false;
2858   case PPC::STD:
2859   case PPC::STFD:
2860   case PPC::STXSD:
2861   case PPC::DFSTOREf64:
2862     return FirstOpc == SecondOpc;
2863   // PowerPC backend has opcode STW/STW8 for instruction "stw" to deal with
2864   // 32bit and 64bit instruction selection. They are clusterable pair though
2865   // they are different opcode.
2866   case PPC::STW:
2867   case PPC::STW8:
2868     return SecondOpc == PPC::STW || SecondOpc == PPC::STW8;
2869   }
2870 }
2871 
2872 bool PPCInstrInfo::shouldClusterMemOps(
2873     ArrayRef<const MachineOperand *> BaseOps1, int64_t OpOffset1,
2874     bool OffsetIsScalable1, ArrayRef<const MachineOperand *> BaseOps2,
2875     int64_t OpOffset2, bool OffsetIsScalable2, unsigned ClusterSize,
2876     unsigned NumBytes) const {
2877 
2878   assert(BaseOps1.size() == 1 && BaseOps2.size() == 1);
2879   const MachineOperand &BaseOp1 = *BaseOps1.front();
2880   const MachineOperand &BaseOp2 = *BaseOps2.front();
2881   assert((BaseOp1.isReg() || BaseOp1.isFI()) &&
2882          "Only base registers and frame indices are supported.");
2883 
2884   // ClusterSize means the number of memory operations that will have been
2885   // clustered if this hook returns true.
2886   // Don't cluster memory op if there are already two ops clustered at least.
2887   if (ClusterSize > 2)
2888     return false;
2889 
2890   // Cluster the load/store only when they have the same base
2891   // register or FI.
2892   if ((BaseOp1.isReg() != BaseOp2.isReg()) ||
2893       (BaseOp1.isReg() && BaseOp1.getReg() != BaseOp2.getReg()) ||
2894       (BaseOp1.isFI() && BaseOp1.getIndex() != BaseOp2.getIndex()))
2895     return false;
2896 
2897   // Check if the load/store are clusterable according to the PowerPC
2898   // specification.
2899   const MachineInstr &FirstLdSt = *BaseOp1.getParent();
2900   const MachineInstr &SecondLdSt = *BaseOp2.getParent();
2901   unsigned FirstOpc = FirstLdSt.getOpcode();
2902   unsigned SecondOpc = SecondLdSt.getOpcode();
2903   const TargetRegisterInfo *TRI = &getRegisterInfo();
2904   // Cluster the load/store only when they have the same opcode, and they are
2905   // clusterable opcode according to PowerPC specification.
2906   if (!isClusterableLdStOpcPair(FirstOpc, SecondOpc, Subtarget))
2907     return false;
2908 
2909   // Can't cluster load/store that have ordered or volatile memory reference.
2910   if (!isLdStSafeToCluster(FirstLdSt, TRI) ||
2911       !isLdStSafeToCluster(SecondLdSt, TRI))
2912     return false;
2913 
2914   int64_t Offset1 = 0, Offset2 = 0;
2915   unsigned Width1 = 0, Width2 = 0;
2916   const MachineOperand *Base1 = nullptr, *Base2 = nullptr;
2917   if (!getMemOperandWithOffsetWidth(FirstLdSt, Base1, Offset1, Width1, TRI) ||
2918       !getMemOperandWithOffsetWidth(SecondLdSt, Base2, Offset2, Width2, TRI) ||
2919       Width1 != Width2)
2920     return false;
2921 
2922   assert(Base1 == &BaseOp1 && Base2 == &BaseOp2 &&
2923          "getMemOperandWithOffsetWidth return incorrect base op");
2924   // The caller should already have ordered FirstMemOp/SecondMemOp by offset.
2925   assert(Offset1 <= Offset2 && "Caller should have ordered offsets.");
2926   return Offset1 + Width1 == Offset2;
2927 }
2928 
2929 /// GetInstSize - Return the number of bytes of code the specified
2930 /// instruction may be.  This returns the maximum number of bytes.
2931 ///
2932 unsigned PPCInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
2933   unsigned Opcode = MI.getOpcode();
2934 
2935   if (Opcode == PPC::INLINEASM || Opcode == PPC::INLINEASM_BR) {
2936     const MachineFunction *MF = MI.getParent()->getParent();
2937     const char *AsmStr = MI.getOperand(0).getSymbolName();
2938     return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo());
2939   } else if (Opcode == TargetOpcode::STACKMAP) {
2940     StackMapOpers Opers(&MI);
2941     return Opers.getNumPatchBytes();
2942   } else if (Opcode == TargetOpcode::PATCHPOINT) {
2943     PatchPointOpers Opers(&MI);
2944     return Opers.getNumPatchBytes();
2945   } else {
2946     return get(Opcode).getSize();
2947   }
2948 }
2949 
2950 std::pair<unsigned, unsigned>
2951 PPCInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
2952   // PPC always uses a direct mask.
2953   return std::make_pair(TF, 0u);
2954 }
2955 
2956 ArrayRef<std::pair<unsigned, const char *>>
2957 PPCInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
2958   using namespace PPCII;
2959   static const std::pair<unsigned, const char *> TargetFlags[] = {
2960       {MO_PLT, "ppc-plt"},
2961       {MO_PIC_FLAG, "ppc-pic"},
2962       {MO_PCREL_FLAG, "ppc-pcrel"},
2963       {MO_GOT_FLAG, "ppc-got"},
2964       {MO_PCREL_OPT_FLAG, "ppc-opt-pcrel"},
2965       {MO_TLSGD_FLAG, "ppc-tlsgd"},
2966       {MO_TPREL_FLAG, "ppc-tprel"},
2967       {MO_TLSLD_FLAG, "ppc-tlsld"},
2968       {MO_TLSGDM_FLAG, "ppc-tlsgdm"},
2969       {MO_GOT_TLSGD_PCREL_FLAG, "ppc-got-tlsgd-pcrel"},
2970       {MO_GOT_TLSLD_PCREL_FLAG, "ppc-got-tlsld-pcrel"},
2971       {MO_GOT_TPREL_PCREL_FLAG, "ppc-got-tprel-pcrel"},
2972       {MO_LO, "ppc-lo"},
2973       {MO_HA, "ppc-ha"},
2974       {MO_TPREL_LO, "ppc-tprel-lo"},
2975       {MO_TPREL_HA, "ppc-tprel-ha"},
2976       {MO_DTPREL_LO, "ppc-dtprel-lo"},
2977       {MO_TLSLD_LO, "ppc-tlsld-lo"},
2978       {MO_TOC_LO, "ppc-toc-lo"},
2979       {MO_TLS, "ppc-tls"},
2980       {MO_PIC_HA_FLAG, "ppc-ha-pic"},
2981       {MO_PIC_LO_FLAG, "ppc-lo-pic"},
2982       {MO_TPREL_PCREL_FLAG, "ppc-tprel-pcrel"},
2983       {MO_TLS_PCREL_FLAG, "ppc-tls-pcrel"},
2984       {MO_GOT_PCREL_FLAG, "ppc-got-pcrel"},
2985   };
2986   return ArrayRef(TargetFlags);
2987 }
2988 
2989 // Expand VSX Memory Pseudo instruction to either a VSX or a FP instruction.
2990 // The VSX versions have the advantage of a full 64-register target whereas
2991 // the FP ones have the advantage of lower latency and higher throughput. So
2992 // what we are after is using the faster instructions in low register pressure
2993 // situations and using the larger register file in high register pressure
2994 // situations.
2995 bool PPCInstrInfo::expandVSXMemPseudo(MachineInstr &MI) const {
2996     unsigned UpperOpcode, LowerOpcode;
2997     switch (MI.getOpcode()) {
2998     case PPC::DFLOADf32:
2999       UpperOpcode = PPC::LXSSP;
3000       LowerOpcode = PPC::LFS;
3001       break;
3002     case PPC::DFLOADf64:
3003       UpperOpcode = PPC::LXSD;
3004       LowerOpcode = PPC::LFD;
3005       break;
3006     case PPC::DFSTOREf32:
3007       UpperOpcode = PPC::STXSSP;
3008       LowerOpcode = PPC::STFS;
3009       break;
3010     case PPC::DFSTOREf64:
3011       UpperOpcode = PPC::STXSD;
3012       LowerOpcode = PPC::STFD;
3013       break;
3014     case PPC::XFLOADf32:
3015       UpperOpcode = PPC::LXSSPX;
3016       LowerOpcode = PPC::LFSX;
3017       break;
3018     case PPC::XFLOADf64:
3019       UpperOpcode = PPC::LXSDX;
3020       LowerOpcode = PPC::LFDX;
3021       break;
3022     case PPC::XFSTOREf32:
3023       UpperOpcode = PPC::STXSSPX;
3024       LowerOpcode = PPC::STFSX;
3025       break;
3026     case PPC::XFSTOREf64:
3027       UpperOpcode = PPC::STXSDX;
3028       LowerOpcode = PPC::STFDX;
3029       break;
3030     case PPC::LIWAX:
3031       UpperOpcode = PPC::LXSIWAX;
3032       LowerOpcode = PPC::LFIWAX;
3033       break;
3034     case PPC::LIWZX:
3035       UpperOpcode = PPC::LXSIWZX;
3036       LowerOpcode = PPC::LFIWZX;
3037       break;
3038     case PPC::STIWX:
3039       UpperOpcode = PPC::STXSIWX;
3040       LowerOpcode = PPC::STFIWX;
3041       break;
3042     default:
3043       llvm_unreachable("Unknown Operation!");
3044     }
3045 
3046     Register TargetReg = MI.getOperand(0).getReg();
3047     unsigned Opcode;
3048     if ((TargetReg >= PPC::F0 && TargetReg <= PPC::F31) ||
3049         (TargetReg >= PPC::VSL0 && TargetReg <= PPC::VSL31))
3050       Opcode = LowerOpcode;
3051     else
3052       Opcode = UpperOpcode;
3053     MI.setDesc(get(Opcode));
3054     return true;
3055 }
3056 
3057 static bool isAnImmediateOperand(const MachineOperand &MO) {
3058   return MO.isCPI() || MO.isGlobal() || MO.isImm();
3059 }
3060 
3061 bool PPCInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
3062   auto &MBB = *MI.getParent();
3063   auto DL = MI.getDebugLoc();
3064 
3065   switch (MI.getOpcode()) {
3066   case PPC::BUILD_UACC: {
3067     MCRegister ACC = MI.getOperand(0).getReg();
3068     MCRegister UACC = MI.getOperand(1).getReg();
3069     if (ACC - PPC::ACC0 != UACC - PPC::UACC0) {
3070       MCRegister SrcVSR = PPC::VSL0 + (UACC - PPC::UACC0) * 4;
3071       MCRegister DstVSR = PPC::VSL0 + (ACC - PPC::ACC0) * 4;
3072       // FIXME: This can easily be improved to look up to the top of the MBB
3073       // to see if the inputs are XXLOR's. If they are and SrcReg is killed,
3074       // we can just re-target any such XXLOR's to DstVSR + offset.
3075       for (int VecNo = 0; VecNo < 4; VecNo++)
3076         BuildMI(MBB, MI, DL, get(PPC::XXLOR), DstVSR + VecNo)
3077             .addReg(SrcVSR + VecNo)
3078             .addReg(SrcVSR + VecNo);
3079     }
3080     // BUILD_UACC is expanded to 4 copies of the underlying vsx registers.
3081     // So after building the 4 copies, we can replace the BUILD_UACC instruction
3082     // with a NOP.
3083     [[fallthrough]];
3084   }
3085   case PPC::KILL_PAIR: {
3086     MI.setDesc(get(PPC::UNENCODED_NOP));
3087     MI.removeOperand(1);
3088     MI.removeOperand(0);
3089     return true;
3090   }
3091   case TargetOpcode::LOAD_STACK_GUARD: {
3092     assert(Subtarget.isTargetLinux() &&
3093            "Only Linux target is expected to contain LOAD_STACK_GUARD");
3094     const int64_t Offset = Subtarget.isPPC64() ? -0x7010 : -0x7008;
3095     const unsigned Reg = Subtarget.isPPC64() ? PPC::X13 : PPC::R2;
3096     MI.setDesc(get(Subtarget.isPPC64() ? PPC::LD : PPC::LWZ));
3097     MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3098         .addImm(Offset)
3099         .addReg(Reg);
3100     return true;
3101   }
3102   case PPC::DFLOADf32:
3103   case PPC::DFLOADf64:
3104   case PPC::DFSTOREf32:
3105   case PPC::DFSTOREf64: {
3106     assert(Subtarget.hasP9Vector() &&
3107            "Invalid D-Form Pseudo-ops on Pre-P9 target.");
3108     assert(MI.getOperand(2).isReg() &&
3109            isAnImmediateOperand(MI.getOperand(1)) &&
3110            "D-form op must have register and immediate operands");
3111     return expandVSXMemPseudo(MI);
3112   }
3113   case PPC::XFLOADf32:
3114   case PPC::XFSTOREf32:
3115   case PPC::LIWAX:
3116   case PPC::LIWZX:
3117   case PPC::STIWX: {
3118     assert(Subtarget.hasP8Vector() &&
3119            "Invalid X-Form Pseudo-ops on Pre-P8 target.");
3120     assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
3121            "X-form op must have register and register operands");
3122     return expandVSXMemPseudo(MI);
3123   }
3124   case PPC::XFLOADf64:
3125   case PPC::XFSTOREf64: {
3126     assert(Subtarget.hasVSX() &&
3127            "Invalid X-Form Pseudo-ops on target that has no VSX.");
3128     assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
3129            "X-form op must have register and register operands");
3130     return expandVSXMemPseudo(MI);
3131   }
3132   case PPC::SPILLTOVSR_LD: {
3133     Register TargetReg = MI.getOperand(0).getReg();
3134     if (PPC::VSFRCRegClass.contains(TargetReg)) {
3135       MI.setDesc(get(PPC::DFLOADf64));
3136       return expandPostRAPseudo(MI);
3137     }
3138     else
3139       MI.setDesc(get(PPC::LD));
3140     return true;
3141   }
3142   case PPC::SPILLTOVSR_ST: {
3143     Register SrcReg = MI.getOperand(0).getReg();
3144     if (PPC::VSFRCRegClass.contains(SrcReg)) {
3145       NumStoreSPILLVSRRCAsVec++;
3146       MI.setDesc(get(PPC::DFSTOREf64));
3147       return expandPostRAPseudo(MI);
3148     } else {
3149       NumStoreSPILLVSRRCAsGpr++;
3150       MI.setDesc(get(PPC::STD));
3151     }
3152     return true;
3153   }
3154   case PPC::SPILLTOVSR_LDX: {
3155     Register TargetReg = MI.getOperand(0).getReg();
3156     if (PPC::VSFRCRegClass.contains(TargetReg))
3157       MI.setDesc(get(PPC::LXSDX));
3158     else
3159       MI.setDesc(get(PPC::LDX));
3160     return true;
3161   }
3162   case PPC::SPILLTOVSR_STX: {
3163     Register SrcReg = MI.getOperand(0).getReg();
3164     if (PPC::VSFRCRegClass.contains(SrcReg)) {
3165       NumStoreSPILLVSRRCAsVec++;
3166       MI.setDesc(get(PPC::STXSDX));
3167     } else {
3168       NumStoreSPILLVSRRCAsGpr++;
3169       MI.setDesc(get(PPC::STDX));
3170     }
3171     return true;
3172   }
3173 
3174     // FIXME: Maybe we can expand it in 'PowerPC Expand Atomic' pass.
3175   case PPC::CFENCE:
3176   case PPC::CFENCE8: {
3177     auto Val = MI.getOperand(0).getReg();
3178     unsigned CmpOp = Subtarget.isPPC64() ? PPC::CMPD : PPC::CMPW;
3179     BuildMI(MBB, MI, DL, get(CmpOp), PPC::CR7).addReg(Val).addReg(Val);
3180     BuildMI(MBB, MI, DL, get(PPC::CTRL_DEP))
3181         .addImm(PPC::PRED_NE_MINUS)
3182         .addReg(PPC::CR7)
3183         .addImm(1);
3184     MI.setDesc(get(PPC::ISYNC));
3185     MI.removeOperand(0);
3186     return true;
3187   }
3188   }
3189   return false;
3190 }
3191 
3192 // Essentially a compile-time implementation of a compare->isel sequence.
3193 // It takes two constants to compare, along with the true/false registers
3194 // and the comparison type (as a subreg to a CR field) and returns one
3195 // of the true/false registers, depending on the comparison results.
3196 static unsigned selectReg(int64_t Imm1, int64_t Imm2, unsigned CompareOpc,
3197                           unsigned TrueReg, unsigned FalseReg,
3198                           unsigned CRSubReg) {
3199   // Signed comparisons. The immediates are assumed to be sign-extended.
3200   if (CompareOpc == PPC::CMPWI || CompareOpc == PPC::CMPDI) {
3201     switch (CRSubReg) {
3202     default: llvm_unreachable("Unknown integer comparison type.");
3203     case PPC::sub_lt:
3204       return Imm1 < Imm2 ? TrueReg : FalseReg;
3205     case PPC::sub_gt:
3206       return Imm1 > Imm2 ? TrueReg : FalseReg;
3207     case PPC::sub_eq:
3208       return Imm1 == Imm2 ? TrueReg : FalseReg;
3209     }
3210   }
3211   // Unsigned comparisons.
3212   else if (CompareOpc == PPC::CMPLWI || CompareOpc == PPC::CMPLDI) {
3213     switch (CRSubReg) {
3214     default: llvm_unreachable("Unknown integer comparison type.");
3215     case PPC::sub_lt:
3216       return (uint64_t)Imm1 < (uint64_t)Imm2 ? TrueReg : FalseReg;
3217     case PPC::sub_gt:
3218       return (uint64_t)Imm1 > (uint64_t)Imm2 ? TrueReg : FalseReg;
3219     case PPC::sub_eq:
3220       return Imm1 == Imm2 ? TrueReg : FalseReg;
3221     }
3222   }
3223   return PPC::NoRegister;
3224 }
3225 
3226 void PPCInstrInfo::replaceInstrOperandWithImm(MachineInstr &MI,
3227                                               unsigned OpNo,
3228                                               int64_t Imm) const {
3229   assert(MI.getOperand(OpNo).isReg() && "Operand must be a REG");
3230   // Replace the REG with the Immediate.
3231   Register InUseReg = MI.getOperand(OpNo).getReg();
3232   MI.getOperand(OpNo).ChangeToImmediate(Imm);
3233 
3234   // We need to make sure that the MI didn't have any implicit use
3235   // of this REG any more. We don't call MI.implicit_operands().empty() to
3236   // return early, since MI's MCID might be changed in calling context, as a
3237   // result its number of explicit operands may be changed, thus the begin of
3238   // implicit operand is changed.
3239   const TargetRegisterInfo *TRI = &getRegisterInfo();
3240   int UseOpIdx = MI.findRegisterUseOperandIdx(InUseReg, false, TRI);
3241   if (UseOpIdx >= 0) {
3242     MachineOperand &MO = MI.getOperand(UseOpIdx);
3243     if (MO.isImplicit())
3244       // The operands must always be in the following order:
3245       // - explicit reg defs,
3246       // - other explicit operands (reg uses, immediates, etc.),
3247       // - implicit reg defs
3248       // - implicit reg uses
3249       // Therefore, removing the implicit operand won't change the explicit
3250       // operands layout.
3251       MI.removeOperand(UseOpIdx);
3252   }
3253 }
3254 
3255 // Replace an instruction with one that materializes a constant (and sets
3256 // CR0 if the original instruction was a record-form instruction).
3257 void PPCInstrInfo::replaceInstrWithLI(MachineInstr &MI,
3258                                       const LoadImmediateInfo &LII) const {
3259   // Remove existing operands.
3260   int OperandToKeep = LII.SetCR ? 1 : 0;
3261   for (int i = MI.getNumOperands() - 1; i > OperandToKeep; i--)
3262     MI.removeOperand(i);
3263 
3264   // Replace the instruction.
3265   if (LII.SetCR) {
3266     MI.setDesc(get(LII.Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec));
3267     // Set the immediate.
3268     MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3269         .addImm(LII.Imm).addReg(PPC::CR0, RegState::ImplicitDefine);
3270     return;
3271   }
3272   else
3273     MI.setDesc(get(LII.Is64Bit ? PPC::LI8 : PPC::LI));
3274 
3275   // Set the immediate.
3276   MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3277       .addImm(LII.Imm);
3278 }
3279 
3280 MachineInstr *PPCInstrInfo::getDefMIPostRA(unsigned Reg, MachineInstr &MI,
3281                                            bool &SeenIntermediateUse) const {
3282   assert(!MI.getParent()->getParent()->getRegInfo().isSSA() &&
3283          "Should be called after register allocation.");
3284   const TargetRegisterInfo *TRI = &getRegisterInfo();
3285   MachineBasicBlock::reverse_iterator E = MI.getParent()->rend(), It = MI;
3286   It++;
3287   SeenIntermediateUse = false;
3288   for (; It != E; ++It) {
3289     if (It->modifiesRegister(Reg, TRI))
3290       return &*It;
3291     if (It->readsRegister(Reg, TRI))
3292       SeenIntermediateUse = true;
3293   }
3294   return nullptr;
3295 }
3296 
3297 void PPCInstrInfo::materializeImmPostRA(MachineBasicBlock &MBB,
3298                                         MachineBasicBlock::iterator MBBI,
3299                                         const DebugLoc &DL, Register Reg,
3300                                         int64_t Imm) const {
3301   assert(!MBB.getParent()->getRegInfo().isSSA() &&
3302          "Register should be in non-SSA form after RA");
3303   bool isPPC64 = Subtarget.isPPC64();
3304   // FIXME: Materialization here is not optimal.
3305   // For some special bit patterns we can use less instructions.
3306   // See `selectI64ImmDirect` in PPCISelDAGToDAG.cpp.
3307   if (isInt<16>(Imm)) {
3308     BuildMI(MBB, MBBI, DL, get(isPPC64 ? PPC::LI8 : PPC::LI), Reg).addImm(Imm);
3309   } else if (isInt<32>(Imm)) {
3310     BuildMI(MBB, MBBI, DL, get(isPPC64 ? PPC::LIS8 : PPC::LIS), Reg)
3311         .addImm(Imm >> 16);
3312     if (Imm & 0xFFFF)
3313       BuildMI(MBB, MBBI, DL, get(isPPC64 ? PPC::ORI8 : PPC::ORI), Reg)
3314           .addReg(Reg, RegState::Kill)
3315           .addImm(Imm & 0xFFFF);
3316   } else {
3317     assert(isPPC64 && "Materializing 64-bit immediate to single register is "
3318                       "only supported in PPC64");
3319     BuildMI(MBB, MBBI, DL, get(PPC::LIS8), Reg).addImm(Imm >> 48);
3320     if ((Imm >> 32) & 0xFFFF)
3321       BuildMI(MBB, MBBI, DL, get(PPC::ORI8), Reg)
3322           .addReg(Reg, RegState::Kill)
3323           .addImm((Imm >> 32) & 0xFFFF);
3324     BuildMI(MBB, MBBI, DL, get(PPC::RLDICR), Reg)
3325         .addReg(Reg, RegState::Kill)
3326         .addImm(32)
3327         .addImm(31);
3328     BuildMI(MBB, MBBI, DL, get(PPC::ORIS8), Reg)
3329         .addReg(Reg, RegState::Kill)
3330         .addImm((Imm >> 16) & 0xFFFF);
3331     if (Imm & 0xFFFF)
3332       BuildMI(MBB, MBBI, DL, get(PPC::ORI8), Reg)
3333           .addReg(Reg, RegState::Kill)
3334           .addImm(Imm & 0xFFFF);
3335   }
3336 }
3337 
3338 MachineInstr *PPCInstrInfo::getForwardingDefMI(
3339   MachineInstr &MI,
3340   unsigned &OpNoForForwarding,
3341   bool &SeenIntermediateUse) const {
3342   OpNoForForwarding = ~0U;
3343   MachineInstr *DefMI = nullptr;
3344   MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
3345   const TargetRegisterInfo *TRI = &getRegisterInfo();
3346   // If we're in SSA, get the defs through the MRI. Otherwise, only look
3347   // within the basic block to see if the register is defined using an
3348   // LI/LI8/ADDI/ADDI8.
3349   if (MRI->isSSA()) {
3350     for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
3351       if (!MI.getOperand(i).isReg())
3352         continue;
3353       Register Reg = MI.getOperand(i).getReg();
3354       if (!Reg.isVirtual())
3355         continue;
3356       Register TrueReg = TRI->lookThruCopyLike(Reg, MRI);
3357       if (TrueReg.isVirtual()) {
3358         MachineInstr *DefMIForTrueReg = MRI->getVRegDef(TrueReg);
3359         if (DefMIForTrueReg->getOpcode() == PPC::LI ||
3360             DefMIForTrueReg->getOpcode() == PPC::LI8 ||
3361             DefMIForTrueReg->getOpcode() == PPC::ADDI ||
3362             DefMIForTrueReg->getOpcode() == PPC::ADDI8) {
3363           OpNoForForwarding = i;
3364           DefMI = DefMIForTrueReg;
3365           // The ADDI and LI operand maybe exist in one instruction at same
3366           // time. we prefer to fold LI operand as LI only has one Imm operand
3367           // and is more possible to be converted. So if current DefMI is
3368           // ADDI/ADDI8, we continue to find possible LI/LI8.
3369           if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8)
3370             break;
3371         }
3372       }
3373     }
3374   } else {
3375     // Looking back through the definition for each operand could be expensive,
3376     // so exit early if this isn't an instruction that either has an immediate
3377     // form or is already an immediate form that we can handle.
3378     ImmInstrInfo III;
3379     unsigned Opc = MI.getOpcode();
3380     bool ConvertibleImmForm =
3381         Opc == PPC::CMPWI || Opc == PPC::CMPLWI || Opc == PPC::CMPDI ||
3382         Opc == PPC::CMPLDI || Opc == PPC::ADDI || Opc == PPC::ADDI8 ||
3383         Opc == PPC::ORI || Opc == PPC::ORI8 || Opc == PPC::XORI ||
3384         Opc == PPC::XORI8 || Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec ||
3385         Opc == PPC::RLDICL_32 || Opc == PPC::RLDICL_32_64 ||
3386         Opc == PPC::RLWINM || Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8 ||
3387         Opc == PPC::RLWINM8_rec;
3388     bool IsVFReg = (MI.getNumOperands() && MI.getOperand(0).isReg())
3389                        ? PPC::isVFRegister(MI.getOperand(0).getReg())
3390                        : false;
3391     if (!ConvertibleImmForm && !instrHasImmForm(Opc, IsVFReg, III, true))
3392       return nullptr;
3393 
3394     // Don't convert or %X, %Y, %Y since that's just a register move.
3395     if ((Opc == PPC::OR || Opc == PPC::OR8) &&
3396         MI.getOperand(1).getReg() == MI.getOperand(2).getReg())
3397       return nullptr;
3398     for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
3399       MachineOperand &MO = MI.getOperand(i);
3400       SeenIntermediateUse = false;
3401       if (MO.isReg() && MO.isUse() && !MO.isImplicit()) {
3402         Register Reg = MI.getOperand(i).getReg();
3403         // If we see another use of this reg between the def and the MI,
3404         // we want to flag it so the def isn't deleted.
3405         MachineInstr *DefMI = getDefMIPostRA(Reg, MI, SeenIntermediateUse);
3406         if (DefMI) {
3407           // Is this register defined by some form of add-immediate (including
3408           // load-immediate) within this basic block?
3409           switch (DefMI->getOpcode()) {
3410           default:
3411             break;
3412           case PPC::LI:
3413           case PPC::LI8:
3414           case PPC::ADDItocL:
3415           case PPC::ADDI:
3416           case PPC::ADDI8:
3417             OpNoForForwarding = i;
3418             return DefMI;
3419           }
3420         }
3421       }
3422     }
3423   }
3424   return OpNoForForwarding == ~0U ? nullptr : DefMI;
3425 }
3426 
3427 unsigned PPCInstrInfo::getSpillTarget() const {
3428   // With P10, we may need to spill paired vector registers or accumulator
3429   // registers. MMA implies paired vectors, so we can just check that.
3430   bool IsP10Variant = Subtarget.isISA3_1() || Subtarget.pairedVectorMemops();
3431   return Subtarget.isISAFuture() ? 3 : IsP10Variant ?
3432                                    2 : Subtarget.hasP9Vector() ?
3433                                    1 : 0;
3434 }
3435 
3436 ArrayRef<unsigned> PPCInstrInfo::getStoreOpcodesForSpillArray() const {
3437   return {StoreSpillOpcodesArray[getSpillTarget()], SOK_LastOpcodeSpill};
3438 }
3439 
3440 ArrayRef<unsigned> PPCInstrInfo::getLoadOpcodesForSpillArray() const {
3441   return {LoadSpillOpcodesArray[getSpillTarget()], SOK_LastOpcodeSpill};
3442 }
3443 
3444 // This opt tries to convert the following imm form to an index form to save an
3445 // add for stack variables.
3446 // Return false if no such pattern found.
3447 //
3448 // ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
3449 // ADD instr:  ToBeDeletedReg = ADD ToBeChangedReg(killed), ScaleReg
3450 // Imm instr:  Reg            = op OffsetImm, ToBeDeletedReg(killed)
3451 //
3452 // can be converted to:
3453 //
3454 // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, (OffsetAddi + OffsetImm)
3455 // Index instr:    Reg            = opx ScaleReg, ToBeChangedReg(killed)
3456 //
3457 // In order to eliminate ADD instr, make sure that:
3458 // 1: (OffsetAddi + OffsetImm) must be int16 since this offset will be used in
3459 //    new ADDI instr and ADDI can only take int16 Imm.
3460 // 2: ToBeChangedReg must be killed in ADD instr and there is no other use
3461 //    between ADDI and ADD instr since its original def in ADDI will be changed
3462 //    in new ADDI instr. And also there should be no new def for it between
3463 //    ADD and Imm instr as ToBeChangedReg will be used in Index instr.
3464 // 3: ToBeDeletedReg must be killed in Imm instr and there is no other use
3465 //    between ADD and Imm instr since ADD instr will be eliminated.
3466 // 4: ScaleReg must not be redefined between ADD and Imm instr since it will be
3467 //    moved to Index instr.
3468 bool PPCInstrInfo::foldFrameOffset(MachineInstr &MI) const {
3469   MachineFunction *MF = MI.getParent()->getParent();
3470   MachineRegisterInfo *MRI = &MF->getRegInfo();
3471   bool PostRA = !MRI->isSSA();
3472   // Do this opt after PEI which is after RA. The reason is stack slot expansion
3473   // in PEI may expose such opportunities since in PEI, stack slot offsets to
3474   // frame base(OffsetAddi) are determined.
3475   if (!PostRA)
3476     return false;
3477   unsigned ToBeDeletedReg = 0;
3478   int64_t OffsetImm = 0;
3479   unsigned XFormOpcode = 0;
3480   ImmInstrInfo III;
3481 
3482   // Check if Imm instr meets requirement.
3483   if (!isImmInstrEligibleForFolding(MI, ToBeDeletedReg, XFormOpcode, OffsetImm,
3484                                     III))
3485     return false;
3486 
3487   bool OtherIntermediateUse = false;
3488   MachineInstr *ADDMI = getDefMIPostRA(ToBeDeletedReg, MI, OtherIntermediateUse);
3489 
3490   // Exit if there is other use between ADD and Imm instr or no def found.
3491   if (OtherIntermediateUse || !ADDMI)
3492     return false;
3493 
3494   // Check if ADD instr meets requirement.
3495   if (!isADDInstrEligibleForFolding(*ADDMI))
3496     return false;
3497 
3498   unsigned ScaleRegIdx = 0;
3499   int64_t OffsetAddi = 0;
3500   MachineInstr *ADDIMI = nullptr;
3501 
3502   // Check if there is a valid ToBeChangedReg in ADDMI.
3503   // 1: It must be killed.
3504   // 2: Its definition must be a valid ADDIMI.
3505   // 3: It must satify int16 offset requirement.
3506   if (isValidToBeChangedReg(ADDMI, 1, ADDIMI, OffsetAddi, OffsetImm))
3507     ScaleRegIdx = 2;
3508   else if (isValidToBeChangedReg(ADDMI, 2, ADDIMI, OffsetAddi, OffsetImm))
3509     ScaleRegIdx = 1;
3510   else
3511     return false;
3512 
3513   assert(ADDIMI && "There should be ADDIMI for valid ToBeChangedReg.");
3514   Register ToBeChangedReg = ADDIMI->getOperand(0).getReg();
3515   Register ScaleReg = ADDMI->getOperand(ScaleRegIdx).getReg();
3516   auto NewDefFor = [&](unsigned Reg, MachineBasicBlock::iterator Start,
3517                        MachineBasicBlock::iterator End) {
3518     for (auto It = ++Start; It != End; It++)
3519       if (It->modifiesRegister(Reg, &getRegisterInfo()))
3520         return true;
3521     return false;
3522   };
3523 
3524   // We are trying to replace the ImmOpNo with ScaleReg. Give up if it is
3525   // treated as special zero when ScaleReg is R0/X0 register.
3526   if (III.ZeroIsSpecialOrig == III.ImmOpNo &&
3527       (ScaleReg == PPC::R0 || ScaleReg == PPC::X0))
3528     return false;
3529 
3530   // Make sure no other def for ToBeChangedReg and ScaleReg between ADD Instr
3531   // and Imm Instr.
3532   if (NewDefFor(ToBeChangedReg, *ADDMI, MI) || NewDefFor(ScaleReg, *ADDMI, MI))
3533     return false;
3534 
3535   // Now start to do the transformation.
3536   LLVM_DEBUG(dbgs() << "Replace instruction: "
3537                     << "\n");
3538   LLVM_DEBUG(ADDIMI->dump());
3539   LLVM_DEBUG(ADDMI->dump());
3540   LLVM_DEBUG(MI.dump());
3541   LLVM_DEBUG(dbgs() << "with: "
3542                     << "\n");
3543 
3544   // Update ADDI instr.
3545   ADDIMI->getOperand(2).setImm(OffsetAddi + OffsetImm);
3546 
3547   // Update Imm instr.
3548   MI.setDesc(get(XFormOpcode));
3549   MI.getOperand(III.ImmOpNo)
3550       .ChangeToRegister(ScaleReg, false, false,
3551                         ADDMI->getOperand(ScaleRegIdx).isKill());
3552 
3553   MI.getOperand(III.OpNoForForwarding)
3554       .ChangeToRegister(ToBeChangedReg, false, false, true);
3555 
3556   // Eliminate ADD instr.
3557   ADDMI->eraseFromParent();
3558 
3559   LLVM_DEBUG(ADDIMI->dump());
3560   LLVM_DEBUG(MI.dump());
3561 
3562   return true;
3563 }
3564 
3565 bool PPCInstrInfo::isADDIInstrEligibleForFolding(MachineInstr &ADDIMI,
3566                                                  int64_t &Imm) const {
3567   unsigned Opc = ADDIMI.getOpcode();
3568 
3569   // Exit if the instruction is not ADDI.
3570   if (Opc != PPC::ADDI && Opc != PPC::ADDI8)
3571     return false;
3572 
3573   // The operand may not necessarily be an immediate - it could be a relocation.
3574   if (!ADDIMI.getOperand(2).isImm())
3575     return false;
3576 
3577   Imm = ADDIMI.getOperand(2).getImm();
3578 
3579   return true;
3580 }
3581 
3582 bool PPCInstrInfo::isADDInstrEligibleForFolding(MachineInstr &ADDMI) const {
3583   unsigned Opc = ADDMI.getOpcode();
3584 
3585   // Exit if the instruction is not ADD.
3586   return Opc == PPC::ADD4 || Opc == PPC::ADD8;
3587 }
3588 
3589 bool PPCInstrInfo::isImmInstrEligibleForFolding(MachineInstr &MI,
3590                                                 unsigned &ToBeDeletedReg,
3591                                                 unsigned &XFormOpcode,
3592                                                 int64_t &OffsetImm,
3593                                                 ImmInstrInfo &III) const {
3594   // Only handle load/store.
3595   if (!MI.mayLoadOrStore())
3596     return false;
3597 
3598   unsigned Opc = MI.getOpcode();
3599 
3600   XFormOpcode = RI.getMappedIdxOpcForImmOpc(Opc);
3601 
3602   // Exit if instruction has no index form.
3603   if (XFormOpcode == PPC::INSTRUCTION_LIST_END)
3604     return false;
3605 
3606   // TODO: sync the logic between instrHasImmForm() and ImmToIdxMap.
3607   if (!instrHasImmForm(XFormOpcode,
3608                        PPC::isVFRegister(MI.getOperand(0).getReg()), III, true))
3609     return false;
3610 
3611   if (!III.IsSummingOperands)
3612     return false;
3613 
3614   MachineOperand ImmOperand = MI.getOperand(III.ImmOpNo);
3615   MachineOperand RegOperand = MI.getOperand(III.OpNoForForwarding);
3616   // Only support imm operands, not relocation slots or others.
3617   if (!ImmOperand.isImm())
3618     return false;
3619 
3620   assert(RegOperand.isReg() && "Instruction format is not right");
3621 
3622   // There are other use for ToBeDeletedReg after Imm instr, can not delete it.
3623   if (!RegOperand.isKill())
3624     return false;
3625 
3626   ToBeDeletedReg = RegOperand.getReg();
3627   OffsetImm = ImmOperand.getImm();
3628 
3629   return true;
3630 }
3631 
3632 bool PPCInstrInfo::isValidToBeChangedReg(MachineInstr *ADDMI, unsigned Index,
3633                                          MachineInstr *&ADDIMI,
3634                                          int64_t &OffsetAddi,
3635                                          int64_t OffsetImm) const {
3636   assert((Index == 1 || Index == 2) && "Invalid operand index for add.");
3637   MachineOperand &MO = ADDMI->getOperand(Index);
3638 
3639   if (!MO.isKill())
3640     return false;
3641 
3642   bool OtherIntermediateUse = false;
3643 
3644   ADDIMI = getDefMIPostRA(MO.getReg(), *ADDMI, OtherIntermediateUse);
3645   // Currently handle only one "add + Imminstr" pair case, exit if other
3646   // intermediate use for ToBeChangedReg found.
3647   // TODO: handle the cases where there are other "add + Imminstr" pairs
3648   // with same offset in Imminstr which is like:
3649   //
3650   // ADDI instr: ToBeChangedReg  = ADDI FrameBaseReg, OffsetAddi
3651   // ADD instr1: ToBeDeletedReg1 = ADD ToBeChangedReg, ScaleReg1
3652   // Imm instr1: Reg1            = op1 OffsetImm, ToBeDeletedReg1(killed)
3653   // ADD instr2: ToBeDeletedReg2 = ADD ToBeChangedReg(killed), ScaleReg2
3654   // Imm instr2: Reg2            = op2 OffsetImm, ToBeDeletedReg2(killed)
3655   //
3656   // can be converted to:
3657   //
3658   // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg,
3659   //                                       (OffsetAddi + OffsetImm)
3660   // Index instr1:   Reg1           = opx1 ScaleReg1, ToBeChangedReg
3661   // Index instr2:   Reg2           = opx2 ScaleReg2, ToBeChangedReg(killed)
3662 
3663   if (OtherIntermediateUse || !ADDIMI)
3664     return false;
3665   // Check if ADDI instr meets requirement.
3666   if (!isADDIInstrEligibleForFolding(*ADDIMI, OffsetAddi))
3667     return false;
3668 
3669   if (isInt<16>(OffsetAddi + OffsetImm))
3670     return true;
3671   return false;
3672 }
3673 
3674 // If this instruction has an immediate form and one of its operands is a
3675 // result of a load-immediate or an add-immediate, convert it to
3676 // the immediate form if the constant is in range.
3677 bool PPCInstrInfo::convertToImmediateForm(MachineInstr &MI,
3678                                           SmallSet<Register, 4> &RegsToUpdate,
3679                                           MachineInstr **KilledDef) const {
3680   MachineFunction *MF = MI.getParent()->getParent();
3681   MachineRegisterInfo *MRI = &MF->getRegInfo();
3682   bool PostRA = !MRI->isSSA();
3683   bool SeenIntermediateUse = true;
3684   unsigned ForwardingOperand = ~0U;
3685   MachineInstr *DefMI = getForwardingDefMI(MI, ForwardingOperand,
3686                                            SeenIntermediateUse);
3687   if (!DefMI)
3688     return false;
3689   assert(ForwardingOperand < MI.getNumOperands() &&
3690          "The forwarding operand needs to be valid at this point");
3691   bool IsForwardingOperandKilled = MI.getOperand(ForwardingOperand).isKill();
3692   bool KillFwdDefMI = !SeenIntermediateUse && IsForwardingOperandKilled;
3693   if (KilledDef && KillFwdDefMI)
3694     *KilledDef = DefMI;
3695 
3696   // Conservatively add defs from DefMI and defs/uses from MI to the set of
3697   // registers that need their kill flags updated.
3698   for (const MachineOperand &MO : DefMI->operands())
3699     if (MO.isReg() && MO.isDef())
3700       RegsToUpdate.insert(MO.getReg());
3701   for (const MachineOperand &MO : MI.operands())
3702     if (MO.isReg())
3703       RegsToUpdate.insert(MO.getReg());
3704 
3705   // If this is a imm instruction and its register operands is produced by ADDI,
3706   // put the imm into imm inst directly.
3707   if (RI.getMappedIdxOpcForImmOpc(MI.getOpcode()) !=
3708           PPC::INSTRUCTION_LIST_END &&
3709       transformToNewImmFormFedByAdd(MI, *DefMI, ForwardingOperand))
3710     return true;
3711 
3712   ImmInstrInfo III;
3713   bool IsVFReg = MI.getOperand(0).isReg()
3714                      ? PPC::isVFRegister(MI.getOperand(0).getReg())
3715                      : false;
3716   bool HasImmForm = instrHasImmForm(MI.getOpcode(), IsVFReg, III, PostRA);
3717   // If this is a reg+reg instruction that has a reg+imm form,
3718   // and one of the operands is produced by an add-immediate,
3719   // try to convert it.
3720   if (HasImmForm &&
3721       transformToImmFormFedByAdd(MI, III, ForwardingOperand, *DefMI,
3722                                  KillFwdDefMI))
3723     return true;
3724 
3725   // If this is a reg+reg instruction that has a reg+imm form,
3726   // and one of the operands is produced by LI, convert it now.
3727   if (HasImmForm &&
3728       transformToImmFormFedByLI(MI, III, ForwardingOperand, *DefMI))
3729     return true;
3730 
3731   // If this is not a reg+reg, but the DefMI is LI/LI8, check if its user MI
3732   // can be simpified to LI.
3733   if (!HasImmForm && simplifyToLI(MI, *DefMI, ForwardingOperand, KilledDef))
3734     return true;
3735 
3736   return false;
3737 }
3738 
3739 bool PPCInstrInfo::combineRLWINM(MachineInstr &MI,
3740                                  MachineInstr **ToErase) const {
3741   MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
3742   Register FoldingReg = MI.getOperand(1).getReg();
3743   if (!FoldingReg.isVirtual())
3744     return false;
3745   MachineInstr *SrcMI = MRI->getVRegDef(FoldingReg);
3746   if (SrcMI->getOpcode() != PPC::RLWINM &&
3747       SrcMI->getOpcode() != PPC::RLWINM_rec &&
3748       SrcMI->getOpcode() != PPC::RLWINM8 &&
3749       SrcMI->getOpcode() != PPC::RLWINM8_rec)
3750     return false;
3751   assert((MI.getOperand(2).isImm() && MI.getOperand(3).isImm() &&
3752           MI.getOperand(4).isImm() && SrcMI->getOperand(2).isImm() &&
3753           SrcMI->getOperand(3).isImm() && SrcMI->getOperand(4).isImm()) &&
3754          "Invalid PPC::RLWINM Instruction!");
3755   uint64_t SHSrc = SrcMI->getOperand(2).getImm();
3756   uint64_t SHMI = MI.getOperand(2).getImm();
3757   uint64_t MBSrc = SrcMI->getOperand(3).getImm();
3758   uint64_t MBMI = MI.getOperand(3).getImm();
3759   uint64_t MESrc = SrcMI->getOperand(4).getImm();
3760   uint64_t MEMI = MI.getOperand(4).getImm();
3761 
3762   assert((MEMI < 32 && MESrc < 32 && MBMI < 32 && MBSrc < 32) &&
3763          "Invalid PPC::RLWINM Instruction!");
3764   // If MBMI is bigger than MEMI, we always can not get run of ones.
3765   // RotatedSrcMask non-wrap:
3766   //                 0........31|32........63
3767   // RotatedSrcMask:   B---E        B---E
3768   // MaskMI:         -----------|--E  B------
3769   // Result:           -----          ---      (Bad candidate)
3770   //
3771   // RotatedSrcMask wrap:
3772   //                 0........31|32........63
3773   // RotatedSrcMask: --E   B----|--E    B----
3774   // MaskMI:         -----------|--E  B------
3775   // Result:         ---   -----|---    -----  (Bad candidate)
3776   //
3777   // One special case is RotatedSrcMask is a full set mask.
3778   // RotatedSrcMask full:
3779   //                 0........31|32........63
3780   // RotatedSrcMask: ------EB---|-------EB---
3781   // MaskMI:         -----------|--E  B------
3782   // Result:         -----------|---  -------  (Good candidate)
3783 
3784   // Mark special case.
3785   bool SrcMaskFull = (MBSrc - MESrc == 1) || (MBSrc == 0 && MESrc == 31);
3786 
3787   // For other MBMI > MEMI cases, just return.
3788   if ((MBMI > MEMI) && !SrcMaskFull)
3789     return false;
3790 
3791   // Handle MBMI <= MEMI cases.
3792   APInt MaskMI = APInt::getBitsSetWithWrap(32, 32 - MEMI - 1, 32 - MBMI);
3793   // In MI, we only need low 32 bits of SrcMI, just consider about low 32
3794   // bit of SrcMI mask. Note that in APInt, lowerest bit is at index 0,
3795   // while in PowerPC ISA, lowerest bit is at index 63.
3796   APInt MaskSrc = APInt::getBitsSetWithWrap(32, 32 - MESrc - 1, 32 - MBSrc);
3797 
3798   APInt RotatedSrcMask = MaskSrc.rotl(SHMI);
3799   APInt FinalMask = RotatedSrcMask & MaskMI;
3800   uint32_t NewMB, NewME;
3801   bool Simplified = false;
3802 
3803   // If final mask is 0, MI result should be 0 too.
3804   if (FinalMask.isZero()) {
3805     bool Is64Bit =
3806         (MI.getOpcode() == PPC::RLWINM8 || MI.getOpcode() == PPC::RLWINM8_rec);
3807     Simplified = true;
3808     LLVM_DEBUG(dbgs() << "Replace Instr: ");
3809     LLVM_DEBUG(MI.dump());
3810 
3811     if (MI.getOpcode() == PPC::RLWINM || MI.getOpcode() == PPC::RLWINM8) {
3812       // Replace MI with "LI 0"
3813       MI.removeOperand(4);
3814       MI.removeOperand(3);
3815       MI.removeOperand(2);
3816       MI.getOperand(1).ChangeToImmediate(0);
3817       MI.setDesc(get(Is64Bit ? PPC::LI8 : PPC::LI));
3818     } else {
3819       // Replace MI with "ANDI_rec reg, 0"
3820       MI.removeOperand(4);
3821       MI.removeOperand(3);
3822       MI.getOperand(2).setImm(0);
3823       MI.setDesc(get(Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec));
3824       MI.getOperand(1).setReg(SrcMI->getOperand(1).getReg());
3825       if (SrcMI->getOperand(1).isKill()) {
3826         MI.getOperand(1).setIsKill(true);
3827         SrcMI->getOperand(1).setIsKill(false);
3828       } else
3829         // About to replace MI.getOperand(1), clear its kill flag.
3830         MI.getOperand(1).setIsKill(false);
3831     }
3832 
3833     LLVM_DEBUG(dbgs() << "With: ");
3834     LLVM_DEBUG(MI.dump());
3835 
3836   } else if ((isRunOfOnes((unsigned)(FinalMask.getZExtValue()), NewMB, NewME) &&
3837               NewMB <= NewME) ||
3838              SrcMaskFull) {
3839     // Here we only handle MBMI <= MEMI case, so NewMB must be no bigger
3840     // than NewME. Otherwise we get a 64 bit value after folding, but MI
3841     // return a 32 bit value.
3842     Simplified = true;
3843     LLVM_DEBUG(dbgs() << "Converting Instr: ");
3844     LLVM_DEBUG(MI.dump());
3845 
3846     uint16_t NewSH = (SHSrc + SHMI) % 32;
3847     MI.getOperand(2).setImm(NewSH);
3848     // If SrcMI mask is full, no need to update MBMI and MEMI.
3849     if (!SrcMaskFull) {
3850       MI.getOperand(3).setImm(NewMB);
3851       MI.getOperand(4).setImm(NewME);
3852     }
3853     MI.getOperand(1).setReg(SrcMI->getOperand(1).getReg());
3854     if (SrcMI->getOperand(1).isKill()) {
3855       MI.getOperand(1).setIsKill(true);
3856       SrcMI->getOperand(1).setIsKill(false);
3857     } else
3858       // About to replace MI.getOperand(1), clear its kill flag.
3859       MI.getOperand(1).setIsKill(false);
3860 
3861     LLVM_DEBUG(dbgs() << "To: ");
3862     LLVM_DEBUG(MI.dump());
3863   }
3864   if (Simplified & MRI->use_nodbg_empty(FoldingReg) &&
3865       !SrcMI->hasImplicitDef()) {
3866     // If FoldingReg has no non-debug use and it has no implicit def (it
3867     // is not RLWINMO or RLWINM8o), it's safe to delete its def SrcMI.
3868     // Otherwise keep it.
3869     *ToErase = SrcMI;
3870     LLVM_DEBUG(dbgs() << "Delete dead instruction: ");
3871     LLVM_DEBUG(SrcMI->dump());
3872   }
3873   return Simplified;
3874 }
3875 
3876 bool PPCInstrInfo::instrHasImmForm(unsigned Opc, bool IsVFReg,
3877                                    ImmInstrInfo &III, bool PostRA) const {
3878   // The vast majority of the instructions would need their operand 2 replaced
3879   // with an immediate when switching to the reg+imm form. A marked exception
3880   // are the update form loads/stores for which a constant operand 2 would need
3881   // to turn into a displacement and move operand 1 to the operand 2 position.
3882   III.ImmOpNo = 2;
3883   III.OpNoForForwarding = 2;
3884   III.ImmWidth = 16;
3885   III.ImmMustBeMultipleOf = 1;
3886   III.TruncateImmTo = 0;
3887   III.IsSummingOperands = false;
3888   switch (Opc) {
3889   default: return false;
3890   case PPC::ADD4:
3891   case PPC::ADD8:
3892     III.SignedImm = true;
3893     III.ZeroIsSpecialOrig = 0;
3894     III.ZeroIsSpecialNew = 1;
3895     III.IsCommutative = true;
3896     III.IsSummingOperands = true;
3897     III.ImmOpcode = Opc == PPC::ADD4 ? PPC::ADDI : PPC::ADDI8;
3898     break;
3899   case PPC::ADDC:
3900   case PPC::ADDC8:
3901     III.SignedImm = true;
3902     III.ZeroIsSpecialOrig = 0;
3903     III.ZeroIsSpecialNew = 0;
3904     III.IsCommutative = true;
3905     III.IsSummingOperands = true;
3906     III.ImmOpcode = Opc == PPC::ADDC ? PPC::ADDIC : PPC::ADDIC8;
3907     break;
3908   case PPC::ADDC_rec:
3909     III.SignedImm = true;
3910     III.ZeroIsSpecialOrig = 0;
3911     III.ZeroIsSpecialNew = 0;
3912     III.IsCommutative = true;
3913     III.IsSummingOperands = true;
3914     III.ImmOpcode = PPC::ADDIC_rec;
3915     break;
3916   case PPC::SUBFC:
3917   case PPC::SUBFC8:
3918     III.SignedImm = true;
3919     III.ZeroIsSpecialOrig = 0;
3920     III.ZeroIsSpecialNew = 0;
3921     III.IsCommutative = false;
3922     III.ImmOpcode = Opc == PPC::SUBFC ? PPC::SUBFIC : PPC::SUBFIC8;
3923     break;
3924   case PPC::CMPW:
3925   case PPC::CMPD:
3926     III.SignedImm = true;
3927     III.ZeroIsSpecialOrig = 0;
3928     III.ZeroIsSpecialNew = 0;
3929     III.IsCommutative = false;
3930     III.ImmOpcode = Opc == PPC::CMPW ? PPC::CMPWI : PPC::CMPDI;
3931     break;
3932   case PPC::CMPLW:
3933   case PPC::CMPLD:
3934     III.SignedImm = false;
3935     III.ZeroIsSpecialOrig = 0;
3936     III.ZeroIsSpecialNew = 0;
3937     III.IsCommutative = false;
3938     III.ImmOpcode = Opc == PPC::CMPLW ? PPC::CMPLWI : PPC::CMPLDI;
3939     break;
3940   case PPC::AND_rec:
3941   case PPC::AND8_rec:
3942   case PPC::OR:
3943   case PPC::OR8:
3944   case PPC::XOR:
3945   case PPC::XOR8:
3946     III.SignedImm = false;
3947     III.ZeroIsSpecialOrig = 0;
3948     III.ZeroIsSpecialNew = 0;
3949     III.IsCommutative = true;
3950     switch(Opc) {
3951     default: llvm_unreachable("Unknown opcode");
3952     case PPC::AND_rec:
3953       III.ImmOpcode = PPC::ANDI_rec;
3954       break;
3955     case PPC::AND8_rec:
3956       III.ImmOpcode = PPC::ANDI8_rec;
3957       break;
3958     case PPC::OR: III.ImmOpcode = PPC::ORI; break;
3959     case PPC::OR8: III.ImmOpcode = PPC::ORI8; break;
3960     case PPC::XOR: III.ImmOpcode = PPC::XORI; break;
3961     case PPC::XOR8: III.ImmOpcode = PPC::XORI8; break;
3962     }
3963     break;
3964   case PPC::RLWNM:
3965   case PPC::RLWNM8:
3966   case PPC::RLWNM_rec:
3967   case PPC::RLWNM8_rec:
3968   case PPC::SLW:
3969   case PPC::SLW8:
3970   case PPC::SLW_rec:
3971   case PPC::SLW8_rec:
3972   case PPC::SRW:
3973   case PPC::SRW8:
3974   case PPC::SRW_rec:
3975   case PPC::SRW8_rec:
3976   case PPC::SRAW:
3977   case PPC::SRAW_rec:
3978     III.SignedImm = false;
3979     III.ZeroIsSpecialOrig = 0;
3980     III.ZeroIsSpecialNew = 0;
3981     III.IsCommutative = false;
3982     // This isn't actually true, but the instructions ignore any of the
3983     // upper bits, so any immediate loaded with an LI is acceptable.
3984     // This does not apply to shift right algebraic because a value
3985     // out of range will produce a -1/0.
3986     III.ImmWidth = 16;
3987     if (Opc == PPC::RLWNM || Opc == PPC::RLWNM8 || Opc == PPC::RLWNM_rec ||
3988         Opc == PPC::RLWNM8_rec)
3989       III.TruncateImmTo = 5;
3990     else
3991       III.TruncateImmTo = 6;
3992     switch(Opc) {
3993     default: llvm_unreachable("Unknown opcode");
3994     case PPC::RLWNM: III.ImmOpcode = PPC::RLWINM; break;
3995     case PPC::RLWNM8: III.ImmOpcode = PPC::RLWINM8; break;
3996     case PPC::RLWNM_rec:
3997       III.ImmOpcode = PPC::RLWINM_rec;
3998       break;
3999     case PPC::RLWNM8_rec:
4000       III.ImmOpcode = PPC::RLWINM8_rec;
4001       break;
4002     case PPC::SLW: III.ImmOpcode = PPC::RLWINM; break;
4003     case PPC::SLW8: III.ImmOpcode = PPC::RLWINM8; break;
4004     case PPC::SLW_rec:
4005       III.ImmOpcode = PPC::RLWINM_rec;
4006       break;
4007     case PPC::SLW8_rec:
4008       III.ImmOpcode = PPC::RLWINM8_rec;
4009       break;
4010     case PPC::SRW: III.ImmOpcode = PPC::RLWINM; break;
4011     case PPC::SRW8: III.ImmOpcode = PPC::RLWINM8; break;
4012     case PPC::SRW_rec:
4013       III.ImmOpcode = PPC::RLWINM_rec;
4014       break;
4015     case PPC::SRW8_rec:
4016       III.ImmOpcode = PPC::RLWINM8_rec;
4017       break;
4018     case PPC::SRAW:
4019       III.ImmWidth = 5;
4020       III.TruncateImmTo = 0;
4021       III.ImmOpcode = PPC::SRAWI;
4022       break;
4023     case PPC::SRAW_rec:
4024       III.ImmWidth = 5;
4025       III.TruncateImmTo = 0;
4026       III.ImmOpcode = PPC::SRAWI_rec;
4027       break;
4028     }
4029     break;
4030   case PPC::RLDCL:
4031   case PPC::RLDCL_rec:
4032   case PPC::RLDCR:
4033   case PPC::RLDCR_rec:
4034   case PPC::SLD:
4035   case PPC::SLD_rec:
4036   case PPC::SRD:
4037   case PPC::SRD_rec:
4038   case PPC::SRAD:
4039   case PPC::SRAD_rec:
4040     III.SignedImm = false;
4041     III.ZeroIsSpecialOrig = 0;
4042     III.ZeroIsSpecialNew = 0;
4043     III.IsCommutative = false;
4044     // This isn't actually true, but the instructions ignore any of the
4045     // upper bits, so any immediate loaded with an LI is acceptable.
4046     // This does not apply to shift right algebraic because a value
4047     // out of range will produce a -1/0.
4048     III.ImmWidth = 16;
4049     if (Opc == PPC::RLDCL || Opc == PPC::RLDCL_rec || Opc == PPC::RLDCR ||
4050         Opc == PPC::RLDCR_rec)
4051       III.TruncateImmTo = 6;
4052     else
4053       III.TruncateImmTo = 7;
4054     switch(Opc) {
4055     default: llvm_unreachable("Unknown opcode");
4056     case PPC::RLDCL: III.ImmOpcode = PPC::RLDICL; break;
4057     case PPC::RLDCL_rec:
4058       III.ImmOpcode = PPC::RLDICL_rec;
4059       break;
4060     case PPC::RLDCR: III.ImmOpcode = PPC::RLDICR; break;
4061     case PPC::RLDCR_rec:
4062       III.ImmOpcode = PPC::RLDICR_rec;
4063       break;
4064     case PPC::SLD: III.ImmOpcode = PPC::RLDICR; break;
4065     case PPC::SLD_rec:
4066       III.ImmOpcode = PPC::RLDICR_rec;
4067       break;
4068     case PPC::SRD: III.ImmOpcode = PPC::RLDICL; break;
4069     case PPC::SRD_rec:
4070       III.ImmOpcode = PPC::RLDICL_rec;
4071       break;
4072     case PPC::SRAD:
4073       III.ImmWidth = 6;
4074       III.TruncateImmTo = 0;
4075       III.ImmOpcode = PPC::SRADI;
4076        break;
4077     case PPC::SRAD_rec:
4078       III.ImmWidth = 6;
4079       III.TruncateImmTo = 0;
4080       III.ImmOpcode = PPC::SRADI_rec;
4081       break;
4082     }
4083     break;
4084   // Loads and stores:
4085   case PPC::LBZX:
4086   case PPC::LBZX8:
4087   case PPC::LHZX:
4088   case PPC::LHZX8:
4089   case PPC::LHAX:
4090   case PPC::LHAX8:
4091   case PPC::LWZX:
4092   case PPC::LWZX8:
4093   case PPC::LWAX:
4094   case PPC::LDX:
4095   case PPC::LFSX:
4096   case PPC::LFDX:
4097   case PPC::STBX:
4098   case PPC::STBX8:
4099   case PPC::STHX:
4100   case PPC::STHX8:
4101   case PPC::STWX:
4102   case PPC::STWX8:
4103   case PPC::STDX:
4104   case PPC::STFSX:
4105   case PPC::STFDX:
4106     III.SignedImm = true;
4107     III.ZeroIsSpecialOrig = 1;
4108     III.ZeroIsSpecialNew = 2;
4109     III.IsCommutative = true;
4110     III.IsSummingOperands = true;
4111     III.ImmOpNo = 1;
4112     III.OpNoForForwarding = 2;
4113     switch(Opc) {
4114     default: llvm_unreachable("Unknown opcode");
4115     case PPC::LBZX: III.ImmOpcode = PPC::LBZ; break;
4116     case PPC::LBZX8: III.ImmOpcode = PPC::LBZ8; break;
4117     case PPC::LHZX: III.ImmOpcode = PPC::LHZ; break;
4118     case PPC::LHZX8: III.ImmOpcode = PPC::LHZ8; break;
4119     case PPC::LHAX: III.ImmOpcode = PPC::LHA; break;
4120     case PPC::LHAX8: III.ImmOpcode = PPC::LHA8; break;
4121     case PPC::LWZX: III.ImmOpcode = PPC::LWZ; break;
4122     case PPC::LWZX8: III.ImmOpcode = PPC::LWZ8; break;
4123     case PPC::LWAX:
4124       III.ImmOpcode = PPC::LWA;
4125       III.ImmMustBeMultipleOf = 4;
4126       break;
4127     case PPC::LDX: III.ImmOpcode = PPC::LD; III.ImmMustBeMultipleOf = 4; break;
4128     case PPC::LFSX: III.ImmOpcode = PPC::LFS; break;
4129     case PPC::LFDX: III.ImmOpcode = PPC::LFD; break;
4130     case PPC::STBX: III.ImmOpcode = PPC::STB; break;
4131     case PPC::STBX8: III.ImmOpcode = PPC::STB8; break;
4132     case PPC::STHX: III.ImmOpcode = PPC::STH; break;
4133     case PPC::STHX8: III.ImmOpcode = PPC::STH8; break;
4134     case PPC::STWX: III.ImmOpcode = PPC::STW; break;
4135     case PPC::STWX8: III.ImmOpcode = PPC::STW8; break;
4136     case PPC::STDX:
4137       III.ImmOpcode = PPC::STD;
4138       III.ImmMustBeMultipleOf = 4;
4139       break;
4140     case PPC::STFSX: III.ImmOpcode = PPC::STFS; break;
4141     case PPC::STFDX: III.ImmOpcode = PPC::STFD; break;
4142     }
4143     break;
4144   case PPC::LBZUX:
4145   case PPC::LBZUX8:
4146   case PPC::LHZUX:
4147   case PPC::LHZUX8:
4148   case PPC::LHAUX:
4149   case PPC::LHAUX8:
4150   case PPC::LWZUX:
4151   case PPC::LWZUX8:
4152   case PPC::LDUX:
4153   case PPC::LFSUX:
4154   case PPC::LFDUX:
4155   case PPC::STBUX:
4156   case PPC::STBUX8:
4157   case PPC::STHUX:
4158   case PPC::STHUX8:
4159   case PPC::STWUX:
4160   case PPC::STWUX8:
4161   case PPC::STDUX:
4162   case PPC::STFSUX:
4163   case PPC::STFDUX:
4164     III.SignedImm = true;
4165     III.ZeroIsSpecialOrig = 2;
4166     III.ZeroIsSpecialNew = 3;
4167     III.IsCommutative = false;
4168     III.IsSummingOperands = true;
4169     III.ImmOpNo = 2;
4170     III.OpNoForForwarding = 3;
4171     switch(Opc) {
4172     default: llvm_unreachable("Unknown opcode");
4173     case PPC::LBZUX: III.ImmOpcode = PPC::LBZU; break;
4174     case PPC::LBZUX8: III.ImmOpcode = PPC::LBZU8; break;
4175     case PPC::LHZUX: III.ImmOpcode = PPC::LHZU; break;
4176     case PPC::LHZUX8: III.ImmOpcode = PPC::LHZU8; break;
4177     case PPC::LHAUX: III.ImmOpcode = PPC::LHAU; break;
4178     case PPC::LHAUX8: III.ImmOpcode = PPC::LHAU8; break;
4179     case PPC::LWZUX: III.ImmOpcode = PPC::LWZU; break;
4180     case PPC::LWZUX8: III.ImmOpcode = PPC::LWZU8; break;
4181     case PPC::LDUX:
4182       III.ImmOpcode = PPC::LDU;
4183       III.ImmMustBeMultipleOf = 4;
4184       break;
4185     case PPC::LFSUX: III.ImmOpcode = PPC::LFSU; break;
4186     case PPC::LFDUX: III.ImmOpcode = PPC::LFDU; break;
4187     case PPC::STBUX: III.ImmOpcode = PPC::STBU; break;
4188     case PPC::STBUX8: III.ImmOpcode = PPC::STBU8; break;
4189     case PPC::STHUX: III.ImmOpcode = PPC::STHU; break;
4190     case PPC::STHUX8: III.ImmOpcode = PPC::STHU8; break;
4191     case PPC::STWUX: III.ImmOpcode = PPC::STWU; break;
4192     case PPC::STWUX8: III.ImmOpcode = PPC::STWU8; break;
4193     case PPC::STDUX:
4194       III.ImmOpcode = PPC::STDU;
4195       III.ImmMustBeMultipleOf = 4;
4196       break;
4197     case PPC::STFSUX: III.ImmOpcode = PPC::STFSU; break;
4198     case PPC::STFDUX: III.ImmOpcode = PPC::STFDU; break;
4199     }
4200     break;
4201   // Power9 and up only. For some of these, the X-Form version has access to all
4202   // 64 VSR's whereas the D-Form only has access to the VR's. We replace those
4203   // with pseudo-ops pre-ra and for post-ra, we check that the register loaded
4204   // into or stored from is one of the VR registers.
4205   case PPC::LXVX:
4206   case PPC::LXSSPX:
4207   case PPC::LXSDX:
4208   case PPC::STXVX:
4209   case PPC::STXSSPX:
4210   case PPC::STXSDX:
4211   case PPC::XFLOADf32:
4212   case PPC::XFLOADf64:
4213   case PPC::XFSTOREf32:
4214   case PPC::XFSTOREf64:
4215     if (!Subtarget.hasP9Vector())
4216       return false;
4217     III.SignedImm = true;
4218     III.ZeroIsSpecialOrig = 1;
4219     III.ZeroIsSpecialNew = 2;
4220     III.IsCommutative = true;
4221     III.IsSummingOperands = true;
4222     III.ImmOpNo = 1;
4223     III.OpNoForForwarding = 2;
4224     III.ImmMustBeMultipleOf = 4;
4225     switch(Opc) {
4226     default: llvm_unreachable("Unknown opcode");
4227     case PPC::LXVX:
4228       III.ImmOpcode = PPC::LXV;
4229       III.ImmMustBeMultipleOf = 16;
4230       break;
4231     case PPC::LXSSPX:
4232       if (PostRA) {
4233         if (IsVFReg)
4234           III.ImmOpcode = PPC::LXSSP;
4235         else {
4236           III.ImmOpcode = PPC::LFS;
4237           III.ImmMustBeMultipleOf = 1;
4238         }
4239         break;
4240       }
4241       [[fallthrough]];
4242     case PPC::XFLOADf32:
4243       III.ImmOpcode = PPC::DFLOADf32;
4244       break;
4245     case PPC::LXSDX:
4246       if (PostRA) {
4247         if (IsVFReg)
4248           III.ImmOpcode = PPC::LXSD;
4249         else {
4250           III.ImmOpcode = PPC::LFD;
4251           III.ImmMustBeMultipleOf = 1;
4252         }
4253         break;
4254       }
4255       [[fallthrough]];
4256     case PPC::XFLOADf64:
4257       III.ImmOpcode = PPC::DFLOADf64;
4258       break;
4259     case PPC::STXVX:
4260       III.ImmOpcode = PPC::STXV;
4261       III.ImmMustBeMultipleOf = 16;
4262       break;
4263     case PPC::STXSSPX:
4264       if (PostRA) {
4265         if (IsVFReg)
4266           III.ImmOpcode = PPC::STXSSP;
4267         else {
4268           III.ImmOpcode = PPC::STFS;
4269           III.ImmMustBeMultipleOf = 1;
4270         }
4271         break;
4272       }
4273       [[fallthrough]];
4274     case PPC::XFSTOREf32:
4275       III.ImmOpcode = PPC::DFSTOREf32;
4276       break;
4277     case PPC::STXSDX:
4278       if (PostRA) {
4279         if (IsVFReg)
4280           III.ImmOpcode = PPC::STXSD;
4281         else {
4282           III.ImmOpcode = PPC::STFD;
4283           III.ImmMustBeMultipleOf = 1;
4284         }
4285         break;
4286       }
4287       [[fallthrough]];
4288     case PPC::XFSTOREf64:
4289       III.ImmOpcode = PPC::DFSTOREf64;
4290       break;
4291     }
4292     break;
4293   }
4294   return true;
4295 }
4296 
4297 // Utility function for swaping two arbitrary operands of an instruction.
4298 static void swapMIOperands(MachineInstr &MI, unsigned Op1, unsigned Op2) {
4299   assert(Op1 != Op2 && "Cannot swap operand with itself.");
4300 
4301   unsigned MaxOp = std::max(Op1, Op2);
4302   unsigned MinOp = std::min(Op1, Op2);
4303   MachineOperand MOp1 = MI.getOperand(MinOp);
4304   MachineOperand MOp2 = MI.getOperand(MaxOp);
4305   MI.removeOperand(std::max(Op1, Op2));
4306   MI.removeOperand(std::min(Op1, Op2));
4307 
4308   // If the operands we are swapping are the two at the end (the common case)
4309   // we can just remove both and add them in the opposite order.
4310   if (MaxOp - MinOp == 1 && MI.getNumOperands() == MinOp) {
4311     MI.addOperand(MOp2);
4312     MI.addOperand(MOp1);
4313   } else {
4314     // Store all operands in a temporary vector, remove them and re-add in the
4315     // right order.
4316     SmallVector<MachineOperand, 2> MOps;
4317     unsigned TotalOps = MI.getNumOperands() + 2; // We've already removed 2 ops.
4318     for (unsigned i = MI.getNumOperands() - 1; i >= MinOp; i--) {
4319       MOps.push_back(MI.getOperand(i));
4320       MI.removeOperand(i);
4321     }
4322     // MOp2 needs to be added next.
4323     MI.addOperand(MOp2);
4324     // Now add the rest.
4325     for (unsigned i = MI.getNumOperands(); i < TotalOps; i++) {
4326       if (i == MaxOp)
4327         MI.addOperand(MOp1);
4328       else {
4329         MI.addOperand(MOps.back());
4330         MOps.pop_back();
4331       }
4332     }
4333   }
4334 }
4335 
4336 // Check if the 'MI' that has the index OpNoForForwarding
4337 // meets the requirement described in the ImmInstrInfo.
4338 bool PPCInstrInfo::isUseMIElgibleForForwarding(MachineInstr &MI,
4339                                                const ImmInstrInfo &III,
4340                                                unsigned OpNoForForwarding
4341                                                ) const {
4342   // As the algorithm of checking for PPC::ZERO/PPC::ZERO8
4343   // would not work pre-RA, we can only do the check post RA.
4344   MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4345   if (MRI.isSSA())
4346     return false;
4347 
4348   // Cannot do the transform if MI isn't summing the operands.
4349   if (!III.IsSummingOperands)
4350     return false;
4351 
4352   // The instruction we are trying to replace must have the ZeroIsSpecialOrig set.
4353   if (!III.ZeroIsSpecialOrig)
4354     return false;
4355 
4356   // We cannot do the transform if the operand we are trying to replace
4357   // isn't the same as the operand the instruction allows.
4358   if (OpNoForForwarding != III.OpNoForForwarding)
4359     return false;
4360 
4361   // Check if the instruction we are trying to transform really has
4362   // the special zero register as its operand.
4363   if (MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO &&
4364       MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO8)
4365     return false;
4366 
4367   // This machine instruction is convertible if it is,
4368   // 1. summing the operands.
4369   // 2. one of the operands is special zero register.
4370   // 3. the operand we are trying to replace is allowed by the MI.
4371   return true;
4372 }
4373 
4374 // Check if the DefMI is the add inst and set the ImmMO and RegMO
4375 // accordingly.
4376 bool PPCInstrInfo::isDefMIElgibleForForwarding(MachineInstr &DefMI,
4377                                                const ImmInstrInfo &III,
4378                                                MachineOperand *&ImmMO,
4379                                                MachineOperand *&RegMO) const {
4380   unsigned Opc = DefMI.getOpcode();
4381   if (Opc != PPC::ADDItocL && Opc != PPC::ADDI && Opc != PPC::ADDI8)
4382     return false;
4383 
4384   assert(DefMI.getNumOperands() >= 3 &&
4385          "Add inst must have at least three operands");
4386   RegMO = &DefMI.getOperand(1);
4387   ImmMO = &DefMI.getOperand(2);
4388 
4389   // Before RA, ADDI first operand could be a frame index.
4390   if (!RegMO->isReg())
4391     return false;
4392 
4393   // This DefMI is elgible for forwarding if it is:
4394   // 1. add inst
4395   // 2. one of the operands is Imm/CPI/Global.
4396   return isAnImmediateOperand(*ImmMO);
4397 }
4398 
4399 bool PPCInstrInfo::isRegElgibleForForwarding(
4400     const MachineOperand &RegMO, const MachineInstr &DefMI,
4401     const MachineInstr &MI, bool KillDefMI,
4402     bool &IsFwdFeederRegKilled, bool &SeenIntermediateUse) const {
4403   // x = addi y, imm
4404   // ...
4405   // z = lfdx 0, x   -> z = lfd imm(y)
4406   // The Reg "y" can be forwarded to the MI(z) only when there is no DEF
4407   // of "y" between the DEF of "x" and "z".
4408   // The query is only valid post RA.
4409   const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4410   if (MRI.isSSA())
4411     return false;
4412 
4413   Register Reg = RegMO.getReg();
4414 
4415   // Walking the inst in reverse(MI-->DefMI) to get the last DEF of the Reg.
4416   MachineBasicBlock::const_reverse_iterator It = MI;
4417   MachineBasicBlock::const_reverse_iterator E = MI.getParent()->rend();
4418   It++;
4419   for (; It != E; ++It) {
4420     if (It->modifiesRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
4421       return false;
4422     else if (It->killsRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
4423       IsFwdFeederRegKilled = true;
4424     if (It->readsRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
4425       SeenIntermediateUse = true;
4426     // Made it to DefMI without encountering a clobber.
4427     if ((&*It) == &DefMI)
4428       break;
4429   }
4430   assert((&*It) == &DefMI && "DefMI is missing");
4431 
4432   // If DefMI also defines the register to be forwarded, we can only forward it
4433   // if DefMI is being erased.
4434   if (DefMI.modifiesRegister(Reg, &getRegisterInfo()))
4435     return KillDefMI;
4436 
4437   return true;
4438 }
4439 
4440 bool PPCInstrInfo::isImmElgibleForForwarding(const MachineOperand &ImmMO,
4441                                              const MachineInstr &DefMI,
4442                                              const ImmInstrInfo &III,
4443                                              int64_t &Imm,
4444                                              int64_t BaseImm) const {
4445   assert(isAnImmediateOperand(ImmMO) && "ImmMO is NOT an immediate");
4446   if (DefMI.getOpcode() == PPC::ADDItocL) {
4447     // The operand for ADDItocL is CPI, which isn't imm at compiling time,
4448     // However, we know that, it is 16-bit width, and has the alignment of 4.
4449     // Check if the instruction met the requirement.
4450     if (III.ImmMustBeMultipleOf > 4 ||
4451        III.TruncateImmTo || III.ImmWidth != 16)
4452       return false;
4453 
4454     // Going from XForm to DForm loads means that the displacement needs to be
4455     // not just an immediate but also a multiple of 4, or 16 depending on the
4456     // load. A DForm load cannot be represented if it is a multiple of say 2.
4457     // XForm loads do not have this restriction.
4458     if (ImmMO.isGlobal()) {
4459       const DataLayout &DL = ImmMO.getGlobal()->getParent()->getDataLayout();
4460       if (ImmMO.getGlobal()->getPointerAlignment(DL) < III.ImmMustBeMultipleOf)
4461         return false;
4462     }
4463 
4464     return true;
4465   }
4466 
4467   if (ImmMO.isImm()) {
4468     // It is Imm, we need to check if the Imm fit the range.
4469     // Sign-extend to 64-bits.
4470     // DefMI may be folded with another imm form instruction, the result Imm is
4471     // the sum of Imm of DefMI and BaseImm which is from imm form instruction.
4472     APInt ActualValue(64, ImmMO.getImm() + BaseImm, true);
4473     if (III.SignedImm && !ActualValue.isSignedIntN(III.ImmWidth))
4474       return false;
4475     if (!III.SignedImm && !ActualValue.isIntN(III.ImmWidth))
4476       return false;
4477     Imm = SignExtend64<16>(ImmMO.getImm() + BaseImm);
4478 
4479     if (Imm % III.ImmMustBeMultipleOf)
4480       return false;
4481     if (III.TruncateImmTo)
4482       Imm &= ((1 << III.TruncateImmTo) - 1);
4483   }
4484   else
4485     return false;
4486 
4487   // This ImmMO is forwarded if it meets the requriement describle
4488   // in ImmInstrInfo
4489   return true;
4490 }
4491 
4492 bool PPCInstrInfo::simplifyToLI(MachineInstr &MI, MachineInstr &DefMI,
4493                                 unsigned OpNoForForwarding,
4494                                 MachineInstr **KilledDef) const {
4495   if ((DefMI.getOpcode() != PPC::LI && DefMI.getOpcode() != PPC::LI8) ||
4496       !DefMI.getOperand(1).isImm())
4497     return false;
4498 
4499   MachineFunction *MF = MI.getParent()->getParent();
4500   MachineRegisterInfo *MRI = &MF->getRegInfo();
4501   bool PostRA = !MRI->isSSA();
4502 
4503   int64_t Immediate = DefMI.getOperand(1).getImm();
4504   // Sign-extend to 64-bits.
4505   int64_t SExtImm = SignExtend64<16>(Immediate);
4506 
4507   bool ReplaceWithLI = false;
4508   bool Is64BitLI = false;
4509   int64_t NewImm = 0;
4510   bool SetCR = false;
4511   unsigned Opc = MI.getOpcode();
4512   switch (Opc) {
4513   default:
4514     return false;
4515 
4516   // FIXME: Any branches conditional on such a comparison can be made
4517   // unconditional. At this time, this happens too infrequently to be worth
4518   // the implementation effort, but if that ever changes, we could convert
4519   // such a pattern here.
4520   case PPC::CMPWI:
4521   case PPC::CMPLWI:
4522   case PPC::CMPDI:
4523   case PPC::CMPLDI: {
4524     // Doing this post-RA would require dataflow analysis to reliably find uses
4525     // of the CR register set by the compare.
4526     // No need to fixup killed/dead flag since this transformation is only valid
4527     // before RA.
4528     if (PostRA)
4529       return false;
4530     // If a compare-immediate is fed by an immediate and is itself an input of
4531     // an ISEL (the most common case) into a COPY of the correct register.
4532     bool Changed = false;
4533     Register DefReg = MI.getOperand(0).getReg();
4534     int64_t Comparand = MI.getOperand(2).getImm();
4535     int64_t SExtComparand = ((uint64_t)Comparand & ~0x7FFFuLL) != 0
4536                                 ? (Comparand | 0xFFFFFFFFFFFF0000)
4537                                 : Comparand;
4538 
4539     for (auto &CompareUseMI : MRI->use_instructions(DefReg)) {
4540       unsigned UseOpc = CompareUseMI.getOpcode();
4541       if (UseOpc != PPC::ISEL && UseOpc != PPC::ISEL8)
4542         continue;
4543       unsigned CRSubReg = CompareUseMI.getOperand(3).getSubReg();
4544       Register TrueReg = CompareUseMI.getOperand(1).getReg();
4545       Register FalseReg = CompareUseMI.getOperand(2).getReg();
4546       unsigned RegToCopy =
4547           selectReg(SExtImm, SExtComparand, Opc, TrueReg, FalseReg, CRSubReg);
4548       if (RegToCopy == PPC::NoRegister)
4549         continue;
4550       // Can't use PPC::COPY to copy PPC::ZERO[8]. Convert it to LI[8] 0.
4551       if (RegToCopy == PPC::ZERO || RegToCopy == PPC::ZERO8) {
4552         CompareUseMI.setDesc(get(UseOpc == PPC::ISEL8 ? PPC::LI8 : PPC::LI));
4553         replaceInstrOperandWithImm(CompareUseMI, 1, 0);
4554         CompareUseMI.removeOperand(3);
4555         CompareUseMI.removeOperand(2);
4556         continue;
4557       }
4558       LLVM_DEBUG(
4559           dbgs() << "Found LI -> CMPI -> ISEL, replacing with a copy.\n");
4560       LLVM_DEBUG(DefMI.dump(); MI.dump(); CompareUseMI.dump());
4561       LLVM_DEBUG(dbgs() << "Is converted to:\n");
4562       // Convert to copy and remove unneeded operands.
4563       CompareUseMI.setDesc(get(PPC::COPY));
4564       CompareUseMI.removeOperand(3);
4565       CompareUseMI.removeOperand(RegToCopy == TrueReg ? 2 : 1);
4566       CmpIselsConverted++;
4567       Changed = true;
4568       LLVM_DEBUG(CompareUseMI.dump());
4569     }
4570     if (Changed)
4571       return true;
4572     // This may end up incremented multiple times since this function is called
4573     // during a fixed-point transformation, but it is only meant to indicate the
4574     // presence of this opportunity.
4575     MissedConvertibleImmediateInstrs++;
4576     return false;
4577   }
4578 
4579   // Immediate forms - may simply be convertable to an LI.
4580   case PPC::ADDI:
4581   case PPC::ADDI8: {
4582     // Does the sum fit in a 16-bit signed field?
4583     int64_t Addend = MI.getOperand(2).getImm();
4584     if (isInt<16>(Addend + SExtImm)) {
4585       ReplaceWithLI = true;
4586       Is64BitLI = Opc == PPC::ADDI8;
4587       NewImm = Addend + SExtImm;
4588       break;
4589     }
4590     return false;
4591   }
4592   case PPC::SUBFIC:
4593   case PPC::SUBFIC8: {
4594     // Only transform this if the CARRY implicit operand is dead.
4595     if (MI.getNumOperands() > 3 && !MI.getOperand(3).isDead())
4596       return false;
4597     int64_t Minuend = MI.getOperand(2).getImm();
4598     if (isInt<16>(Minuend - SExtImm)) {
4599       ReplaceWithLI = true;
4600       Is64BitLI = Opc == PPC::SUBFIC8;
4601       NewImm = Minuend - SExtImm;
4602       break;
4603     }
4604     return false;
4605   }
4606   case PPC::RLDICL:
4607   case PPC::RLDICL_rec:
4608   case PPC::RLDICL_32:
4609   case PPC::RLDICL_32_64: {
4610     // Use APInt's rotate function.
4611     int64_t SH = MI.getOperand(2).getImm();
4612     int64_t MB = MI.getOperand(3).getImm();
4613     APInt InVal((Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec) ? 64 : 32,
4614                 SExtImm, true);
4615     InVal = InVal.rotl(SH);
4616     uint64_t Mask = MB == 0 ? -1LLU : (1LLU << (63 - MB + 1)) - 1;
4617     InVal &= Mask;
4618     // Can't replace negative values with an LI as that will sign-extend
4619     // and not clear the left bits. If we're setting the CR bit, we will use
4620     // ANDI_rec which won't sign extend, so that's safe.
4621     if (isUInt<15>(InVal.getSExtValue()) ||
4622         (Opc == PPC::RLDICL_rec && isUInt<16>(InVal.getSExtValue()))) {
4623       ReplaceWithLI = true;
4624       Is64BitLI = Opc != PPC::RLDICL_32;
4625       NewImm = InVal.getSExtValue();
4626       SetCR = Opc == PPC::RLDICL_rec;
4627       break;
4628     }
4629     return false;
4630   }
4631   case PPC::RLWINM:
4632   case PPC::RLWINM8:
4633   case PPC::RLWINM_rec:
4634   case PPC::RLWINM8_rec: {
4635     int64_t SH = MI.getOperand(2).getImm();
4636     int64_t MB = MI.getOperand(3).getImm();
4637     int64_t ME = MI.getOperand(4).getImm();
4638     APInt InVal(32, SExtImm, true);
4639     InVal = InVal.rotl(SH);
4640     APInt Mask = APInt::getBitsSetWithWrap(32, 32 - ME - 1, 32 - MB);
4641     InVal &= Mask;
4642     // Can't replace negative values with an LI as that will sign-extend
4643     // and not clear the left bits. If we're setting the CR bit, we will use
4644     // ANDI_rec which won't sign extend, so that's safe.
4645     bool ValueFits = isUInt<15>(InVal.getSExtValue());
4646     ValueFits |= ((Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec) &&
4647                   isUInt<16>(InVal.getSExtValue()));
4648     if (ValueFits) {
4649       ReplaceWithLI = true;
4650       Is64BitLI = Opc == PPC::RLWINM8 || Opc == PPC::RLWINM8_rec;
4651       NewImm = InVal.getSExtValue();
4652       SetCR = Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec;
4653       break;
4654     }
4655     return false;
4656   }
4657   case PPC::ORI:
4658   case PPC::ORI8:
4659   case PPC::XORI:
4660   case PPC::XORI8: {
4661     int64_t LogicalImm = MI.getOperand(2).getImm();
4662     int64_t Result = 0;
4663     if (Opc == PPC::ORI || Opc == PPC::ORI8)
4664       Result = LogicalImm | SExtImm;
4665     else
4666       Result = LogicalImm ^ SExtImm;
4667     if (isInt<16>(Result)) {
4668       ReplaceWithLI = true;
4669       Is64BitLI = Opc == PPC::ORI8 || Opc == PPC::XORI8;
4670       NewImm = Result;
4671       break;
4672     }
4673     return false;
4674   }
4675   }
4676 
4677   if (ReplaceWithLI) {
4678     // We need to be careful with CR-setting instructions we're replacing.
4679     if (SetCR) {
4680       // We don't know anything about uses when we're out of SSA, so only
4681       // replace if the new immediate will be reproduced.
4682       bool ImmChanged = (SExtImm & NewImm) != NewImm;
4683       if (PostRA && ImmChanged)
4684         return false;
4685 
4686       if (!PostRA) {
4687         // If the defining load-immediate has no other uses, we can just replace
4688         // the immediate with the new immediate.
4689         if (MRI->hasOneUse(DefMI.getOperand(0).getReg()))
4690           DefMI.getOperand(1).setImm(NewImm);
4691 
4692         // If we're not using the GPR result of the CR-setting instruction, we
4693         // just need to and with zero/non-zero depending on the new immediate.
4694         else if (MRI->use_empty(MI.getOperand(0).getReg())) {
4695           if (NewImm) {
4696             assert(Immediate && "Transformation converted zero to non-zero?");
4697             NewImm = Immediate;
4698           }
4699         } else if (ImmChanged)
4700           return false;
4701       }
4702     }
4703 
4704     LLVM_DEBUG(dbgs() << "Replacing constant instruction:\n");
4705     LLVM_DEBUG(MI.dump());
4706     LLVM_DEBUG(dbgs() << "Fed by:\n");
4707     LLVM_DEBUG(DefMI.dump());
4708     LoadImmediateInfo LII;
4709     LII.Imm = NewImm;
4710     LII.Is64Bit = Is64BitLI;
4711     LII.SetCR = SetCR;
4712     // If we're setting the CR, the original load-immediate must be kept (as an
4713     // operand to ANDI_rec/ANDI8_rec).
4714     if (KilledDef && SetCR)
4715       *KilledDef = nullptr;
4716     replaceInstrWithLI(MI, LII);
4717 
4718     if (PostRA)
4719       recomputeLivenessFlags(*MI.getParent());
4720 
4721     LLVM_DEBUG(dbgs() << "With:\n");
4722     LLVM_DEBUG(MI.dump());
4723     return true;
4724   }
4725   return false;
4726 }
4727 
4728 bool PPCInstrInfo::transformToNewImmFormFedByAdd(
4729     MachineInstr &MI, MachineInstr &DefMI, unsigned OpNoForForwarding) const {
4730   MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
4731   bool PostRA = !MRI->isSSA();
4732   // FIXME: extend this to post-ra. Need to do some change in getForwardingDefMI
4733   // for post-ra.
4734   if (PostRA)
4735     return false;
4736 
4737   // Only handle load/store.
4738   if (!MI.mayLoadOrStore())
4739     return false;
4740 
4741   unsigned XFormOpcode = RI.getMappedIdxOpcForImmOpc(MI.getOpcode());
4742 
4743   assert((XFormOpcode != PPC::INSTRUCTION_LIST_END) &&
4744          "MI must have x-form opcode");
4745 
4746   // get Imm Form info.
4747   ImmInstrInfo III;
4748   bool IsVFReg = MI.getOperand(0).isReg()
4749                      ? PPC::isVFRegister(MI.getOperand(0).getReg())
4750                      : false;
4751 
4752   if (!instrHasImmForm(XFormOpcode, IsVFReg, III, PostRA))
4753     return false;
4754 
4755   if (!III.IsSummingOperands)
4756     return false;
4757 
4758   if (OpNoForForwarding != III.OpNoForForwarding)
4759     return false;
4760 
4761   MachineOperand ImmOperandMI = MI.getOperand(III.ImmOpNo);
4762   if (!ImmOperandMI.isImm())
4763     return false;
4764 
4765   // Check DefMI.
4766   MachineOperand *ImmMO = nullptr;
4767   MachineOperand *RegMO = nullptr;
4768   if (!isDefMIElgibleForForwarding(DefMI, III, ImmMO, RegMO))
4769     return false;
4770   assert(ImmMO && RegMO && "Imm and Reg operand must have been set");
4771 
4772   // Check Imm.
4773   // Set ImmBase from imm instruction as base and get new Imm inside
4774   // isImmElgibleForForwarding.
4775   int64_t ImmBase = ImmOperandMI.getImm();
4776   int64_t Imm = 0;
4777   if (!isImmElgibleForForwarding(*ImmMO, DefMI, III, Imm, ImmBase))
4778     return false;
4779 
4780   // Do the transform
4781   LLVM_DEBUG(dbgs() << "Replacing existing reg+imm instruction:\n");
4782   LLVM_DEBUG(MI.dump());
4783   LLVM_DEBUG(dbgs() << "Fed by:\n");
4784   LLVM_DEBUG(DefMI.dump());
4785 
4786   MI.getOperand(III.OpNoForForwarding).setReg(RegMO->getReg());
4787   MI.getOperand(III.ImmOpNo).setImm(Imm);
4788 
4789   LLVM_DEBUG(dbgs() << "With:\n");
4790   LLVM_DEBUG(MI.dump());
4791   return true;
4792 }
4793 
4794 // If an X-Form instruction is fed by an add-immediate and one of its operands
4795 // is the literal zero, attempt to forward the source of the add-immediate to
4796 // the corresponding D-Form instruction with the displacement coming from
4797 // the immediate being added.
4798 bool PPCInstrInfo::transformToImmFormFedByAdd(
4799     MachineInstr &MI, const ImmInstrInfo &III, unsigned OpNoForForwarding,
4800     MachineInstr &DefMI, bool KillDefMI) const {
4801   //         RegMO ImmMO
4802   //           |    |
4803   // x = addi reg, imm  <----- DefMI
4804   // y = op    0 ,  x   <----- MI
4805   //                |
4806   //         OpNoForForwarding
4807   // Check if the MI meet the requirement described in the III.
4808   if (!isUseMIElgibleForForwarding(MI, III, OpNoForForwarding))
4809     return false;
4810 
4811   // Check if the DefMI meet the requirement
4812   // described in the III. If yes, set the ImmMO and RegMO accordingly.
4813   MachineOperand *ImmMO = nullptr;
4814   MachineOperand *RegMO = nullptr;
4815   if (!isDefMIElgibleForForwarding(DefMI, III, ImmMO, RegMO))
4816     return false;
4817   assert(ImmMO && RegMO && "Imm and Reg operand must have been set");
4818 
4819   // As we get the Imm operand now, we need to check if the ImmMO meet
4820   // the requirement described in the III. If yes set the Imm.
4821   int64_t Imm = 0;
4822   if (!isImmElgibleForForwarding(*ImmMO, DefMI, III, Imm))
4823     return false;
4824 
4825   bool IsFwdFeederRegKilled = false;
4826   bool SeenIntermediateUse = false;
4827   // Check if the RegMO can be forwarded to MI.
4828   if (!isRegElgibleForForwarding(*RegMO, DefMI, MI, KillDefMI,
4829                                  IsFwdFeederRegKilled, SeenIntermediateUse))
4830     return false;
4831 
4832   MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4833   bool PostRA = !MRI.isSSA();
4834 
4835   // We know that, the MI and DefMI both meet the pattern, and
4836   // the Imm also meet the requirement with the new Imm-form.
4837   // It is safe to do the transformation now.
4838   LLVM_DEBUG(dbgs() << "Replacing indexed instruction:\n");
4839   LLVM_DEBUG(MI.dump());
4840   LLVM_DEBUG(dbgs() << "Fed by:\n");
4841   LLVM_DEBUG(DefMI.dump());
4842 
4843   // Update the base reg first.
4844   MI.getOperand(III.OpNoForForwarding).ChangeToRegister(RegMO->getReg(),
4845                                                         false, false,
4846                                                         RegMO->isKill());
4847 
4848   // Then, update the imm.
4849   if (ImmMO->isImm()) {
4850     // If the ImmMO is Imm, change the operand that has ZERO to that Imm
4851     // directly.
4852     replaceInstrOperandWithImm(MI, III.ZeroIsSpecialOrig, Imm);
4853   }
4854   else {
4855     // Otherwise, it is Constant Pool Index(CPI) or Global,
4856     // which is relocation in fact. We need to replace the special zero
4857     // register with ImmMO.
4858     // Before that, we need to fixup the target flags for imm.
4859     // For some reason, we miss to set the flag for the ImmMO if it is CPI.
4860     if (DefMI.getOpcode() == PPC::ADDItocL)
4861       ImmMO->setTargetFlags(PPCII::MO_TOC_LO);
4862 
4863     // MI didn't have the interface such as MI.setOperand(i) though
4864     // it has MI.getOperand(i). To repalce the ZERO MachineOperand with
4865     // ImmMO, we need to remove ZERO operand and all the operands behind it,
4866     // and, add the ImmMO, then, move back all the operands behind ZERO.
4867     SmallVector<MachineOperand, 2> MOps;
4868     for (unsigned i = MI.getNumOperands() - 1; i >= III.ZeroIsSpecialOrig; i--) {
4869       MOps.push_back(MI.getOperand(i));
4870       MI.removeOperand(i);
4871     }
4872 
4873     // Remove the last MO in the list, which is ZERO operand in fact.
4874     MOps.pop_back();
4875     // Add the imm operand.
4876     MI.addOperand(*ImmMO);
4877     // Now add the rest back.
4878     for (auto &MO : MOps)
4879       MI.addOperand(MO);
4880   }
4881 
4882   // Update the opcode.
4883   MI.setDesc(get(III.ImmOpcode));
4884 
4885   if (PostRA)
4886     recomputeLivenessFlags(*MI.getParent());
4887   LLVM_DEBUG(dbgs() << "With:\n");
4888   LLVM_DEBUG(MI.dump());
4889 
4890   return true;
4891 }
4892 
4893 bool PPCInstrInfo::transformToImmFormFedByLI(MachineInstr &MI,
4894                                              const ImmInstrInfo &III,
4895                                              unsigned ConstantOpNo,
4896                                              MachineInstr &DefMI) const {
4897   // DefMI must be LI or LI8.
4898   if ((DefMI.getOpcode() != PPC::LI && DefMI.getOpcode() != PPC::LI8) ||
4899       !DefMI.getOperand(1).isImm())
4900     return false;
4901 
4902   // Get Imm operand and Sign-extend to 64-bits.
4903   int64_t Imm = SignExtend64<16>(DefMI.getOperand(1).getImm());
4904 
4905   MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4906   bool PostRA = !MRI.isSSA();
4907   // Exit early if we can't convert this.
4908   if ((ConstantOpNo != III.OpNoForForwarding) && !III.IsCommutative)
4909     return false;
4910   if (Imm % III.ImmMustBeMultipleOf)
4911     return false;
4912   if (III.TruncateImmTo)
4913     Imm &= ((1 << III.TruncateImmTo) - 1);
4914   if (III.SignedImm) {
4915     APInt ActualValue(64, Imm, true);
4916     if (!ActualValue.isSignedIntN(III.ImmWidth))
4917       return false;
4918   } else {
4919     uint64_t UnsignedMax = (1 << III.ImmWidth) - 1;
4920     if ((uint64_t)Imm > UnsignedMax)
4921       return false;
4922   }
4923 
4924   // If we're post-RA, the instructions don't agree on whether register zero is
4925   // special, we can transform this as long as the register operand that will
4926   // end up in the location where zero is special isn't R0.
4927   if (PostRA && III.ZeroIsSpecialOrig != III.ZeroIsSpecialNew) {
4928     unsigned PosForOrigZero = III.ZeroIsSpecialOrig ? III.ZeroIsSpecialOrig :
4929       III.ZeroIsSpecialNew + 1;
4930     Register OrigZeroReg = MI.getOperand(PosForOrigZero).getReg();
4931     Register NewZeroReg = MI.getOperand(III.ZeroIsSpecialNew).getReg();
4932     // If R0 is in the operand where zero is special for the new instruction,
4933     // it is unsafe to transform if the constant operand isn't that operand.
4934     if ((NewZeroReg == PPC::R0 || NewZeroReg == PPC::X0) &&
4935         ConstantOpNo != III.ZeroIsSpecialNew)
4936       return false;
4937     if ((OrigZeroReg == PPC::R0 || OrigZeroReg == PPC::X0) &&
4938         ConstantOpNo != PosForOrigZero)
4939       return false;
4940   }
4941 
4942   unsigned Opc = MI.getOpcode();
4943   bool SpecialShift32 = Opc == PPC::SLW || Opc == PPC::SLW_rec ||
4944                         Opc == PPC::SRW || Opc == PPC::SRW_rec ||
4945                         Opc == PPC::SLW8 || Opc == PPC::SLW8_rec ||
4946                         Opc == PPC::SRW8 || Opc == PPC::SRW8_rec;
4947   bool SpecialShift64 = Opc == PPC::SLD || Opc == PPC::SLD_rec ||
4948                         Opc == PPC::SRD || Opc == PPC::SRD_rec;
4949   bool SetCR = Opc == PPC::SLW_rec || Opc == PPC::SRW_rec ||
4950                Opc == PPC::SLD_rec || Opc == PPC::SRD_rec;
4951   bool RightShift = Opc == PPC::SRW || Opc == PPC::SRW_rec || Opc == PPC::SRD ||
4952                     Opc == PPC::SRD_rec;
4953 
4954   LLVM_DEBUG(dbgs() << "Replacing reg+reg instruction: ");
4955   LLVM_DEBUG(MI.dump());
4956   LLVM_DEBUG(dbgs() << "Fed by load-immediate: ");
4957   LLVM_DEBUG(DefMI.dump());
4958   MI.setDesc(get(III.ImmOpcode));
4959   if (ConstantOpNo == III.OpNoForForwarding) {
4960     // Converting shifts to immediate form is a bit tricky since they may do
4961     // one of three things:
4962     // 1. If the shift amount is between OpSize and 2*OpSize, the result is zero
4963     // 2. If the shift amount is zero, the result is unchanged (save for maybe
4964     //    setting CR0)
4965     // 3. If the shift amount is in [1, OpSize), it's just a shift
4966     if (SpecialShift32 || SpecialShift64) {
4967       LoadImmediateInfo LII;
4968       LII.Imm = 0;
4969       LII.SetCR = SetCR;
4970       LII.Is64Bit = SpecialShift64;
4971       uint64_t ShAmt = Imm & (SpecialShift32 ? 0x1F : 0x3F);
4972       if (Imm & (SpecialShift32 ? 0x20 : 0x40))
4973         replaceInstrWithLI(MI, LII);
4974       // Shifts by zero don't change the value. If we don't need to set CR0,
4975       // just convert this to a COPY. Can't do this post-RA since we've already
4976       // cleaned up the copies.
4977       else if (!SetCR && ShAmt == 0 && !PostRA) {
4978         MI.removeOperand(2);
4979         MI.setDesc(get(PPC::COPY));
4980       } else {
4981         // The 32 bit and 64 bit instructions are quite different.
4982         if (SpecialShift32) {
4983           // Left shifts use (N, 0, 31-N).
4984           // Right shifts use (32-N, N, 31) if 0 < N < 32.
4985           //              use (0, 0, 31)    if N == 0.
4986           uint64_t SH = ShAmt == 0 ? 0 : RightShift ? 32 - ShAmt : ShAmt;
4987           uint64_t MB = RightShift ? ShAmt : 0;
4988           uint64_t ME = RightShift ? 31 : 31 - ShAmt;
4989           replaceInstrOperandWithImm(MI, III.OpNoForForwarding, SH);
4990           MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(MB)
4991             .addImm(ME);
4992         } else {
4993           // Left shifts use (N, 63-N).
4994           // Right shifts use (64-N, N) if 0 < N < 64.
4995           //              use (0, 0)    if N == 0.
4996           uint64_t SH = ShAmt == 0 ? 0 : RightShift ? 64 - ShAmt : ShAmt;
4997           uint64_t ME = RightShift ? ShAmt : 63 - ShAmt;
4998           replaceInstrOperandWithImm(MI, III.OpNoForForwarding, SH);
4999           MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(ME);
5000         }
5001       }
5002     } else
5003       replaceInstrOperandWithImm(MI, ConstantOpNo, Imm);
5004   }
5005   // Convert commutative instructions (switch the operands and convert the
5006   // desired one to an immediate.
5007   else if (III.IsCommutative) {
5008     replaceInstrOperandWithImm(MI, ConstantOpNo, Imm);
5009     swapMIOperands(MI, ConstantOpNo, III.OpNoForForwarding);
5010   } else
5011     llvm_unreachable("Should have exited early!");
5012 
5013   // For instructions for which the constant register replaces a different
5014   // operand than where the immediate goes, we need to swap them.
5015   if (III.OpNoForForwarding != III.ImmOpNo)
5016     swapMIOperands(MI, III.OpNoForForwarding, III.ImmOpNo);
5017 
5018   // If the special R0/X0 register index are different for original instruction
5019   // and new instruction, we need to fix up the register class in new
5020   // instruction.
5021   if (!PostRA && III.ZeroIsSpecialOrig != III.ZeroIsSpecialNew) {
5022     if (III.ZeroIsSpecialNew) {
5023       // If operand at III.ZeroIsSpecialNew is physical reg(eg: ZERO/ZERO8), no
5024       // need to fix up register class.
5025       Register RegToModify = MI.getOperand(III.ZeroIsSpecialNew).getReg();
5026       if (RegToModify.isVirtual()) {
5027         const TargetRegisterClass *NewRC =
5028           MRI.getRegClass(RegToModify)->hasSuperClassEq(&PPC::GPRCRegClass) ?
5029           &PPC::GPRC_and_GPRC_NOR0RegClass : &PPC::G8RC_and_G8RC_NOX0RegClass;
5030         MRI.setRegClass(RegToModify, NewRC);
5031       }
5032     }
5033   }
5034 
5035   if (PostRA)
5036     recomputeLivenessFlags(*MI.getParent());
5037 
5038   LLVM_DEBUG(dbgs() << "With: ");
5039   LLVM_DEBUG(MI.dump());
5040   LLVM_DEBUG(dbgs() << "\n");
5041   return true;
5042 }
5043 
5044 const TargetRegisterClass *
5045 PPCInstrInfo::updatedRC(const TargetRegisterClass *RC) const {
5046   if (Subtarget.hasVSX() && RC == &PPC::VRRCRegClass)
5047     return &PPC::VSRCRegClass;
5048   return RC;
5049 }
5050 
5051 int PPCInstrInfo::getRecordFormOpcode(unsigned Opcode) {
5052   return PPC::getRecordFormOpcode(Opcode);
5053 }
5054 
5055 static bool isOpZeroOfSubwordPreincLoad(int Opcode) {
5056   return (Opcode == PPC::LBZU || Opcode == PPC::LBZUX || Opcode == PPC::LBZU8 ||
5057           Opcode == PPC::LBZUX8 || Opcode == PPC::LHZU ||
5058           Opcode == PPC::LHZUX || Opcode == PPC::LHZU8 ||
5059           Opcode == PPC::LHZUX8);
5060 }
5061 
5062 // This function checks for sign extension from 32 bits to 64 bits.
5063 static bool definedBySignExtendingOp(const unsigned Reg,
5064                                      const MachineRegisterInfo *MRI) {
5065   if (!Register::isVirtualRegister(Reg))
5066     return false;
5067 
5068   MachineInstr *MI = MRI->getVRegDef(Reg);
5069   if (!MI)
5070     return false;
5071 
5072   int Opcode = MI->getOpcode();
5073   const PPCInstrInfo *TII =
5074       MI->getMF()->getSubtarget<PPCSubtarget>().getInstrInfo();
5075   if (TII->isSExt32To64(Opcode))
5076     return true;
5077 
5078   // The first def of LBZU/LHZU is sign extended.
5079   if (isOpZeroOfSubwordPreincLoad(Opcode) && MI->getOperand(0).getReg() == Reg)
5080     return true;
5081 
5082   // RLDICL generates sign-extended output if it clears at least
5083   // 33 bits from the left (MSB).
5084   if (Opcode == PPC::RLDICL && MI->getOperand(3).getImm() >= 33)
5085     return true;
5086 
5087   // If at least one bit from left in a lower word is masked out,
5088   // all of 0 to 32-th bits of the output are cleared.
5089   // Hence the output is already sign extended.
5090   if ((Opcode == PPC::RLWINM || Opcode == PPC::RLWINM_rec ||
5091        Opcode == PPC::RLWNM || Opcode == PPC::RLWNM_rec) &&
5092       MI->getOperand(3).getImm() > 0 &&
5093       MI->getOperand(3).getImm() <= MI->getOperand(4).getImm())
5094     return true;
5095 
5096   // If the most significant bit of immediate in ANDIS is zero,
5097   // all of 0 to 32-th bits are cleared.
5098   if (Opcode == PPC::ANDIS_rec || Opcode == PPC::ANDIS8_rec) {
5099     uint16_t Imm = MI->getOperand(2).getImm();
5100     if ((Imm & 0x8000) == 0)
5101       return true;
5102   }
5103 
5104   return false;
5105 }
5106 
5107 // This function checks the machine instruction that defines the input register
5108 // Reg. If that machine instruction always outputs a value that has only zeros
5109 // in the higher 32 bits then this function will return true.
5110 static bool definedByZeroExtendingOp(const unsigned Reg,
5111                                      const MachineRegisterInfo *MRI) {
5112   if (!Register::isVirtualRegister(Reg))
5113     return false;
5114 
5115   MachineInstr *MI = MRI->getVRegDef(Reg);
5116   if (!MI)
5117     return false;
5118 
5119   int Opcode = MI->getOpcode();
5120   const PPCInstrInfo *TII =
5121       MI->getMF()->getSubtarget<PPCSubtarget>().getInstrInfo();
5122   if (TII->isZExt32To64(Opcode))
5123     return true;
5124 
5125   // The first def of LBZU/LHZU/LWZU are zero extended.
5126   if ((isOpZeroOfSubwordPreincLoad(Opcode) || Opcode == PPC::LWZU ||
5127        Opcode == PPC::LWZUX || Opcode == PPC::LWZU8 || Opcode == PPC::LWZUX8) &&
5128       MI->getOperand(0).getReg() == Reg)
5129     return true;
5130 
5131   // The 16-bit immediate is sign-extended in li/lis.
5132   // If the most significant bit is zero, all higher bits are zero.
5133   if (Opcode == PPC::LI  || Opcode == PPC::LI8 ||
5134       Opcode == PPC::LIS || Opcode == PPC::LIS8) {
5135     int64_t Imm = MI->getOperand(1).getImm();
5136     if (((uint64_t)Imm & ~0x7FFFuLL) == 0)
5137       return true;
5138   }
5139 
5140   // We have some variations of rotate-and-mask instructions
5141   // that clear higher 32-bits.
5142   if ((Opcode == PPC::RLDICL || Opcode == PPC::RLDICL_rec ||
5143        Opcode == PPC::RLDCL || Opcode == PPC::RLDCL_rec ||
5144        Opcode == PPC::RLDICL_32_64) &&
5145       MI->getOperand(3).getImm() >= 32)
5146     return true;
5147 
5148   if ((Opcode == PPC::RLDIC || Opcode == PPC::RLDIC_rec) &&
5149       MI->getOperand(3).getImm() >= 32 &&
5150       MI->getOperand(3).getImm() <= 63 - MI->getOperand(2).getImm())
5151     return true;
5152 
5153   if ((Opcode == PPC::RLWINM || Opcode == PPC::RLWINM_rec ||
5154        Opcode == PPC::RLWNM || Opcode == PPC::RLWNM_rec ||
5155        Opcode == PPC::RLWINM8 || Opcode == PPC::RLWNM8) &&
5156       MI->getOperand(3).getImm() <= MI->getOperand(4).getImm())
5157     return true;
5158 
5159   return false;
5160 }
5161 
5162 // This function returns true if the input MachineInstr is a TOC save
5163 // instruction.
5164 bool PPCInstrInfo::isTOCSaveMI(const MachineInstr &MI) const {
5165   if (!MI.getOperand(1).isImm() || !MI.getOperand(2).isReg())
5166     return false;
5167   unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
5168   unsigned StackOffset = MI.getOperand(1).getImm();
5169   Register StackReg = MI.getOperand(2).getReg();
5170   Register SPReg = Subtarget.isPPC64() ? PPC::X1 : PPC::R1;
5171   if (StackReg == SPReg && StackOffset == TOCSaveOffset)
5172     return true;
5173 
5174   return false;
5175 }
5176 
5177 // We limit the max depth to track incoming values of PHIs or binary ops
5178 // (e.g. AND) to avoid excessive cost.
5179 const unsigned MAX_BINOP_DEPTH = 1;
5180 // The isSignOrZeroExtended function is recursive. The parameter BinOpDepth
5181 // does not count all of the recursions. The parameter BinOpDepth is incremented
5182 // only when isSignOrZeroExtended calls itself more than once. This is done to
5183 // prevent expontential recursion. There is no parameter to track linear
5184 // recursion.
5185 std::pair<bool, bool>
5186 PPCInstrInfo::isSignOrZeroExtended(const unsigned Reg,
5187                                    const unsigned BinOpDepth,
5188                                    const MachineRegisterInfo *MRI) const {
5189   if (!Register::isVirtualRegister(Reg))
5190     return std::pair<bool, bool>(false, false);
5191 
5192   MachineInstr *MI = MRI->getVRegDef(Reg);
5193   if (!MI)
5194     return std::pair<bool, bool>(false, false);
5195 
5196   bool IsSExt = definedBySignExtendingOp(Reg, MRI);
5197   bool IsZExt = definedByZeroExtendingOp(Reg, MRI);
5198 
5199   // If we know the instruction always returns sign- and zero-extended result,
5200   // return here.
5201   if (IsSExt && IsZExt)
5202     return std::pair<bool, bool>(IsSExt, IsZExt);
5203 
5204   switch (MI->getOpcode()) {
5205   case PPC::COPY: {
5206     Register SrcReg = MI->getOperand(1).getReg();
5207 
5208     // In both ELFv1 and v2 ABI, method parameters and the return value
5209     // are sign- or zero-extended.
5210     const MachineFunction *MF = MI->getMF();
5211 
5212     if (!MF->getSubtarget<PPCSubtarget>().isSVR4ABI()) {
5213       // If this is a copy from another register, we recursively check source.
5214       auto SrcExt = isSignOrZeroExtended(SrcReg, BinOpDepth, MRI);
5215       return std::pair<bool, bool>(SrcExt.first || IsSExt,
5216                                    SrcExt.second || IsZExt);
5217     }
5218 
5219     // From here on everything is SVR4ABI
5220     const PPCFunctionInfo *FuncInfo = MF->getInfo<PPCFunctionInfo>();
5221     // We check the ZExt/SExt flags for a method parameter.
5222     if (MI->getParent()->getBasicBlock() ==
5223         &MF->getFunction().getEntryBlock()) {
5224       Register VReg = MI->getOperand(0).getReg();
5225       if (MF->getRegInfo().isLiveIn(VReg)) {
5226         IsSExt |= FuncInfo->isLiveInSExt(VReg);
5227         IsZExt |= FuncInfo->isLiveInZExt(VReg);
5228         return std::pair<bool, bool>(IsSExt, IsZExt);
5229       }
5230     }
5231 
5232     if (SrcReg != PPC::X3) {
5233       // If this is a copy from another register, we recursively check source.
5234       auto SrcExt = isSignOrZeroExtended(SrcReg, BinOpDepth, MRI);
5235       return std::pair<bool, bool>(SrcExt.first || IsSExt,
5236                                    SrcExt.second || IsZExt);
5237     }
5238 
5239     // For a method return value, we check the ZExt/SExt flags in attribute.
5240     // We assume the following code sequence for method call.
5241     //   ADJCALLSTACKDOWN 32, implicit dead %r1, implicit %r1
5242     //   BL8_NOP @func,...
5243     //   ADJCALLSTACKUP 32, 0, implicit dead %r1, implicit %r1
5244     //   %5 = COPY %x3; G8RC:%5
5245     const MachineBasicBlock *MBB = MI->getParent();
5246     std::pair<bool, bool> IsExtendPair = std::pair<bool, bool>(IsSExt, IsZExt);
5247     MachineBasicBlock::const_instr_iterator II =
5248         MachineBasicBlock::const_instr_iterator(MI);
5249     if (II == MBB->instr_begin() || (--II)->getOpcode() != PPC::ADJCALLSTACKUP)
5250       return IsExtendPair;
5251 
5252     const MachineInstr &CallMI = *(--II);
5253     if (!CallMI.isCall() || !CallMI.getOperand(0).isGlobal())
5254       return IsExtendPair;
5255 
5256     const Function *CalleeFn =
5257         dyn_cast_if_present<Function>(CallMI.getOperand(0).getGlobal());
5258     if (!CalleeFn)
5259       return IsExtendPair;
5260     const IntegerType *IntTy = dyn_cast<IntegerType>(CalleeFn->getReturnType());
5261     if (IntTy && IntTy->getBitWidth() <= 32) {
5262       const AttributeSet &Attrs = CalleeFn->getAttributes().getRetAttrs();
5263       IsSExt |= Attrs.hasAttribute(Attribute::SExt);
5264       IsZExt |= Attrs.hasAttribute(Attribute::ZExt);
5265       return std::pair<bool, bool>(IsSExt, IsZExt);
5266     }
5267 
5268     return IsExtendPair;
5269   }
5270 
5271   // OR, XOR with 16-bit immediate does not change the upper 48 bits.
5272   // So, we track the operand register as we do for register copy.
5273   case PPC::ORI:
5274   case PPC::XORI:
5275   case PPC::ORI8:
5276   case PPC::XORI8: {
5277     Register SrcReg = MI->getOperand(1).getReg();
5278     auto SrcExt = isSignOrZeroExtended(SrcReg, BinOpDepth, MRI);
5279     return std::pair<bool, bool>(SrcExt.first || IsSExt,
5280                                  SrcExt.second || IsZExt);
5281   }
5282 
5283   // OR, XOR with shifted 16-bit immediate does not change the upper
5284   // 32 bits. So, we track the operand register for zero extension.
5285   // For sign extension when the MSB of the immediate is zero, we also
5286   // track the operand register since the upper 33 bits are unchanged.
5287   case PPC::ORIS:
5288   case PPC::XORIS:
5289   case PPC::ORIS8:
5290   case PPC::XORIS8: {
5291     Register SrcReg = MI->getOperand(1).getReg();
5292     auto SrcExt = isSignOrZeroExtended(SrcReg, BinOpDepth, MRI);
5293     uint16_t Imm = MI->getOperand(2).getImm();
5294     if (Imm & 0x8000)
5295       return std::pair<bool, bool>(false, SrcExt.second || IsZExt);
5296     else
5297       return std::pair<bool, bool>(SrcExt.first || IsSExt,
5298                                    SrcExt.second || IsZExt);
5299   }
5300 
5301   // If all incoming values are sign-/zero-extended,
5302   // the output of OR, ISEL or PHI is also sign-/zero-extended.
5303   case PPC::OR:
5304   case PPC::OR8:
5305   case PPC::ISEL:
5306   case PPC::PHI: {
5307     if (BinOpDepth >= MAX_BINOP_DEPTH)
5308       return std::pair<bool, bool>(false, false);
5309 
5310     // The input registers for PHI are operand 1, 3, ...
5311     // The input registers for others are operand 1 and 2.
5312     unsigned OperandEnd = 3, OperandStride = 1;
5313     if (MI->getOpcode() == PPC::PHI) {
5314       OperandEnd = MI->getNumOperands();
5315       OperandStride = 2;
5316     }
5317 
5318     IsSExt = true;
5319     IsZExt = true;
5320     for (unsigned I = 1; I != OperandEnd; I += OperandStride) {
5321       if (!MI->getOperand(I).isReg())
5322         return std::pair<bool, bool>(false, false);
5323 
5324       Register SrcReg = MI->getOperand(I).getReg();
5325       auto SrcExt = isSignOrZeroExtended(SrcReg, BinOpDepth + 1, MRI);
5326       IsSExt &= SrcExt.first;
5327       IsZExt &= SrcExt.second;
5328     }
5329     return std::pair<bool, bool>(IsSExt, IsZExt);
5330   }
5331 
5332   // If at least one of the incoming values of an AND is zero extended
5333   // then the output is also zero-extended. If both of the incoming values
5334   // are sign-extended then the output is also sign extended.
5335   case PPC::AND:
5336   case PPC::AND8: {
5337     if (BinOpDepth >= MAX_BINOP_DEPTH)
5338       return std::pair<bool, bool>(false, false);
5339 
5340     Register SrcReg1 = MI->getOperand(1).getReg();
5341     Register SrcReg2 = MI->getOperand(2).getReg();
5342     auto Src1Ext = isSignOrZeroExtended(SrcReg1, BinOpDepth + 1, MRI);
5343     auto Src2Ext = isSignOrZeroExtended(SrcReg2, BinOpDepth + 1, MRI);
5344     return std::pair<bool, bool>(Src1Ext.first && Src2Ext.first,
5345                                  Src1Ext.second || Src2Ext.second);
5346   }
5347 
5348   default:
5349     break;
5350   }
5351   return std::pair<bool, bool>(IsSExt, IsZExt);
5352 }
5353 
5354 bool PPCInstrInfo::isBDNZ(unsigned Opcode) const {
5355   return (Opcode == (Subtarget.isPPC64() ? PPC::BDNZ8 : PPC::BDNZ));
5356 }
5357 
5358 namespace {
5359 class PPCPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
5360   MachineInstr *Loop, *EndLoop, *LoopCount;
5361   MachineFunction *MF;
5362   const TargetInstrInfo *TII;
5363   int64_t TripCount;
5364 
5365 public:
5366   PPCPipelinerLoopInfo(MachineInstr *Loop, MachineInstr *EndLoop,
5367                        MachineInstr *LoopCount)
5368       : Loop(Loop), EndLoop(EndLoop), LoopCount(LoopCount),
5369         MF(Loop->getParent()->getParent()),
5370         TII(MF->getSubtarget().getInstrInfo()) {
5371     // Inspect the Loop instruction up-front, as it may be deleted when we call
5372     // createTripCountGreaterCondition.
5373     if (LoopCount->getOpcode() == PPC::LI8 || LoopCount->getOpcode() == PPC::LI)
5374       TripCount = LoopCount->getOperand(1).getImm();
5375     else
5376       TripCount = -1;
5377   }
5378 
5379   bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
5380     // Only ignore the terminator.
5381     return MI == EndLoop;
5382   }
5383 
5384   std::optional<bool> createTripCountGreaterCondition(
5385       int TC, MachineBasicBlock &MBB,
5386       SmallVectorImpl<MachineOperand> &Cond) override {
5387     if (TripCount == -1) {
5388       // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1,
5389       // so we don't need to generate any thing here.
5390       Cond.push_back(MachineOperand::CreateImm(0));
5391       Cond.push_back(MachineOperand::CreateReg(
5392           MF->getSubtarget<PPCSubtarget>().isPPC64() ? PPC::CTR8 : PPC::CTR,
5393           true));
5394       return {};
5395     }
5396 
5397     return TripCount > TC;
5398   }
5399 
5400   void setPreheader(MachineBasicBlock *NewPreheader) override {
5401     // Do nothing. We want the LOOP setup instruction to stay in the *old*
5402     // preheader, so we can use BDZ in the prologs to adapt the loop trip count.
5403   }
5404 
5405   void adjustTripCount(int TripCountAdjust) override {
5406     // If the loop trip count is a compile-time value, then just change the
5407     // value.
5408     if (LoopCount->getOpcode() == PPC::LI8 ||
5409         LoopCount->getOpcode() == PPC::LI) {
5410       int64_t TripCount = LoopCount->getOperand(1).getImm() + TripCountAdjust;
5411       LoopCount->getOperand(1).setImm(TripCount);
5412       return;
5413     }
5414 
5415     // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1,
5416     // so we don't need to generate any thing here.
5417   }
5418 
5419   void disposed() override {
5420     Loop->eraseFromParent();
5421     // Ensure the loop setup instruction is deleted too.
5422     LoopCount->eraseFromParent();
5423   }
5424 };
5425 } // namespace
5426 
5427 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
5428 PPCInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
5429   // We really "analyze" only hardware loops right now.
5430   MachineBasicBlock::iterator I = LoopBB->getFirstTerminator();
5431   MachineBasicBlock *Preheader = *LoopBB->pred_begin();
5432   if (Preheader == LoopBB)
5433     Preheader = *std::next(LoopBB->pred_begin());
5434   MachineFunction *MF = Preheader->getParent();
5435 
5436   if (I != LoopBB->end() && isBDNZ(I->getOpcode())) {
5437     SmallPtrSet<MachineBasicBlock *, 8> Visited;
5438     if (MachineInstr *LoopInst = findLoopInstr(*Preheader, Visited)) {
5439       Register LoopCountReg = LoopInst->getOperand(0).getReg();
5440       MachineRegisterInfo &MRI = MF->getRegInfo();
5441       MachineInstr *LoopCount = MRI.getUniqueVRegDef(LoopCountReg);
5442       return std::make_unique<PPCPipelinerLoopInfo>(LoopInst, &*I, LoopCount);
5443     }
5444   }
5445   return nullptr;
5446 }
5447 
5448 MachineInstr *PPCInstrInfo::findLoopInstr(
5449     MachineBasicBlock &PreHeader,
5450     SmallPtrSet<MachineBasicBlock *, 8> &Visited) const {
5451 
5452   unsigned LOOPi = (Subtarget.isPPC64() ? PPC::MTCTR8loop : PPC::MTCTRloop);
5453 
5454   // The loop set-up instruction should be in preheader
5455   for (auto &I : PreHeader.instrs())
5456     if (I.getOpcode() == LOOPi)
5457       return &I;
5458   return nullptr;
5459 }
5460 
5461 // Return true if get the base operand, byte offset of an instruction and the
5462 // memory width. Width is the size of memory that is being loaded/stored.
5463 bool PPCInstrInfo::getMemOperandWithOffsetWidth(
5464     const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset,
5465     unsigned &Width, const TargetRegisterInfo *TRI) const {
5466   if (!LdSt.mayLoadOrStore() || LdSt.getNumExplicitOperands() != 3)
5467     return false;
5468 
5469   // Handle only loads/stores with base register followed by immediate offset.
5470   if (!LdSt.getOperand(1).isImm() ||
5471       (!LdSt.getOperand(2).isReg() && !LdSt.getOperand(2).isFI()))
5472     return false;
5473   if (!LdSt.getOperand(1).isImm() ||
5474       (!LdSt.getOperand(2).isReg() && !LdSt.getOperand(2).isFI()))
5475     return false;
5476 
5477   if (!LdSt.hasOneMemOperand())
5478     return false;
5479 
5480   Width = (*LdSt.memoperands_begin())->getSize();
5481   Offset = LdSt.getOperand(1).getImm();
5482   BaseReg = &LdSt.getOperand(2);
5483   return true;
5484 }
5485 
5486 bool PPCInstrInfo::areMemAccessesTriviallyDisjoint(
5487     const MachineInstr &MIa, const MachineInstr &MIb) const {
5488   assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
5489   assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
5490 
5491   if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
5492       MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
5493     return false;
5494 
5495   // Retrieve the base register, offset from the base register and width. Width
5496   // is the size of memory that is being loaded/stored (e.g. 1, 2, 4).  If
5497   // base registers are identical, and the offset of a lower memory access +
5498   // the width doesn't overlap the offset of a higher memory access,
5499   // then the memory accesses are different.
5500   const TargetRegisterInfo *TRI = &getRegisterInfo();
5501   const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
5502   int64_t OffsetA = 0, OffsetB = 0;
5503   unsigned int WidthA = 0, WidthB = 0;
5504   if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) &&
5505       getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) {
5506     if (BaseOpA->isIdenticalTo(*BaseOpB)) {
5507       int LowOffset = std::min(OffsetA, OffsetB);
5508       int HighOffset = std::max(OffsetA, OffsetB);
5509       int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
5510       if (LowOffset + LowWidth <= HighOffset)
5511         return true;
5512     }
5513   }
5514   return false;
5515 }
5516