xref: /freebsd/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp (revision 31ba4ce8898f9dfa5e7f054fdbc26e50a599a6e3)
1 //===-- PPCISelDAGToDAG.cpp - PPC --pattern matching inst selector --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines a pattern matching instruction selector for PowerPC,
10 // converting from a legalized dag to a PPC dag.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "MCTargetDesc/PPCMCTargetDesc.h"
15 #include "MCTargetDesc/PPCPredicates.h"
16 #include "PPC.h"
17 #include "PPCISelLowering.h"
18 #include "PPCMachineFunctionInfo.h"
19 #include "PPCSubtarget.h"
20 #include "PPCTargetMachine.h"
21 #include "llvm/ADT/APInt.h"
22 #include "llvm/ADT/DenseMap.h"
23 #include "llvm/ADT/STLExtras.h"
24 #include "llvm/ADT/SmallPtrSet.h"
25 #include "llvm/ADT/SmallVector.h"
26 #include "llvm/ADT/Statistic.h"
27 #include "llvm/Analysis/BranchProbabilityInfo.h"
28 #include "llvm/CodeGen/FunctionLoweringInfo.h"
29 #include "llvm/CodeGen/ISDOpcodes.h"
30 #include "llvm/CodeGen/MachineBasicBlock.h"
31 #include "llvm/CodeGen/MachineFunction.h"
32 #include "llvm/CodeGen/MachineInstrBuilder.h"
33 #include "llvm/CodeGen/MachineRegisterInfo.h"
34 #include "llvm/CodeGen/SelectionDAG.h"
35 #include "llvm/CodeGen/SelectionDAGISel.h"
36 #include "llvm/CodeGen/SelectionDAGNodes.h"
37 #include "llvm/CodeGen/TargetInstrInfo.h"
38 #include "llvm/CodeGen/TargetRegisterInfo.h"
39 #include "llvm/CodeGen/ValueTypes.h"
40 #include "llvm/IR/BasicBlock.h"
41 #include "llvm/IR/DebugLoc.h"
42 #include "llvm/IR/Function.h"
43 #include "llvm/IR/GlobalValue.h"
44 #include "llvm/IR/InlineAsm.h"
45 #include "llvm/IR/InstrTypes.h"
46 #include "llvm/IR/IntrinsicsPowerPC.h"
47 #include "llvm/IR/Module.h"
48 #include "llvm/Support/Casting.h"
49 #include "llvm/Support/CodeGen.h"
50 #include "llvm/Support/CommandLine.h"
51 #include "llvm/Support/Compiler.h"
52 #include "llvm/Support/Debug.h"
53 #include "llvm/Support/ErrorHandling.h"
54 #include "llvm/Support/KnownBits.h"
55 #include "llvm/Support/MachineValueType.h"
56 #include "llvm/Support/MathExtras.h"
57 #include "llvm/Support/raw_ostream.h"
58 #include <algorithm>
59 #include <cassert>
60 #include <cstdint>
61 #include <iterator>
62 #include <limits>
63 #include <memory>
64 #include <new>
65 #include <tuple>
66 #include <utility>
67 
68 using namespace llvm;
69 
70 #define DEBUG_TYPE "ppc-codegen"
71 
72 STATISTIC(NumSextSetcc,
73           "Number of (sext(setcc)) nodes expanded into GPR sequence.");
74 STATISTIC(NumZextSetcc,
75           "Number of (zext(setcc)) nodes expanded into GPR sequence.");
76 STATISTIC(SignExtensionsAdded,
77           "Number of sign extensions for compare inputs added.");
78 STATISTIC(ZeroExtensionsAdded,
79           "Number of zero extensions for compare inputs added.");
80 STATISTIC(NumLogicOpsOnComparison,
81           "Number of logical ops on i1 values calculated in GPR.");
82 STATISTIC(OmittedForNonExtendUses,
83           "Number of compares not eliminated as they have non-extending uses.");
84 STATISTIC(NumP9Setb,
85           "Number of compares lowered to setb.");
86 
87 // FIXME: Remove this once the bug has been fixed!
88 cl::opt<bool> ANDIGlueBug("expose-ppc-andi-glue-bug",
89 cl::desc("expose the ANDI glue bug on PPC"), cl::Hidden);
90 
91 static cl::opt<bool>
92     UseBitPermRewriter("ppc-use-bit-perm-rewriter", cl::init(true),
93                        cl::desc("use aggressive ppc isel for bit permutations"),
94                        cl::Hidden);
95 static cl::opt<bool> BPermRewriterNoMasking(
96     "ppc-bit-perm-rewriter-stress-rotates",
97     cl::desc("stress rotate selection in aggressive ppc isel for "
98              "bit permutations"),
99     cl::Hidden);
100 
101 static cl::opt<bool> EnableBranchHint(
102   "ppc-use-branch-hint", cl::init(true),
103     cl::desc("Enable static hinting of branches on ppc"),
104     cl::Hidden);
105 
106 static cl::opt<bool> EnableTLSOpt(
107   "ppc-tls-opt", cl::init(true),
108     cl::desc("Enable tls optimization peephole"),
109     cl::Hidden);
110 
111 enum ICmpInGPRType { ICGPR_All, ICGPR_None, ICGPR_I32, ICGPR_I64,
112   ICGPR_NonExtIn, ICGPR_Zext, ICGPR_Sext, ICGPR_ZextI32,
113   ICGPR_SextI32, ICGPR_ZextI64, ICGPR_SextI64 };
114 
115 static cl::opt<ICmpInGPRType> CmpInGPR(
116   "ppc-gpr-icmps", cl::Hidden, cl::init(ICGPR_All),
117   cl::desc("Specify the types of comparisons to emit GPR-only code for."),
118   cl::values(clEnumValN(ICGPR_None, "none", "Do not modify integer comparisons."),
119              clEnumValN(ICGPR_All, "all", "All possible int comparisons in GPRs."),
120              clEnumValN(ICGPR_I32, "i32", "Only i32 comparisons in GPRs."),
121              clEnumValN(ICGPR_I64, "i64", "Only i64 comparisons in GPRs."),
122              clEnumValN(ICGPR_NonExtIn, "nonextin",
123                         "Only comparisons where inputs don't need [sz]ext."),
124              clEnumValN(ICGPR_Zext, "zext", "Only comparisons with zext result."),
125              clEnumValN(ICGPR_ZextI32, "zexti32",
126                         "Only i32 comparisons with zext result."),
127              clEnumValN(ICGPR_ZextI64, "zexti64",
128                         "Only i64 comparisons with zext result."),
129              clEnumValN(ICGPR_Sext, "sext", "Only comparisons with sext result."),
130              clEnumValN(ICGPR_SextI32, "sexti32",
131                         "Only i32 comparisons with sext result."),
132              clEnumValN(ICGPR_SextI64, "sexti64",
133                         "Only i64 comparisons with sext result.")));
134 namespace {
135 
136   //===--------------------------------------------------------------------===//
137   /// PPCDAGToDAGISel - PPC specific code to select PPC machine
138   /// instructions for SelectionDAG operations.
139   ///
140   class PPCDAGToDAGISel : public SelectionDAGISel {
141     const PPCTargetMachine &TM;
142     const PPCSubtarget *Subtarget = nullptr;
143     const PPCTargetLowering *PPCLowering = nullptr;
144     unsigned GlobalBaseReg = 0;
145 
146   public:
147     explicit PPCDAGToDAGISel(PPCTargetMachine &tm, CodeGenOpt::Level OptLevel)
148         : SelectionDAGISel(tm, OptLevel), TM(tm) {}
149 
150     bool runOnMachineFunction(MachineFunction &MF) override {
151       // Make sure we re-emit a set of the global base reg if necessary
152       GlobalBaseReg = 0;
153       Subtarget = &MF.getSubtarget<PPCSubtarget>();
154       PPCLowering = Subtarget->getTargetLowering();
155       SelectionDAGISel::runOnMachineFunction(MF);
156 
157       return true;
158     }
159 
160     void PreprocessISelDAG() override;
161     void PostprocessISelDAG() override;
162 
163     /// getI16Imm - Return a target constant with the specified value, of type
164     /// i16.
165     inline SDValue getI16Imm(unsigned Imm, const SDLoc &dl) {
166       return CurDAG->getTargetConstant(Imm, dl, MVT::i16);
167     }
168 
169     /// getI32Imm - Return a target constant with the specified value, of type
170     /// i32.
171     inline SDValue getI32Imm(unsigned Imm, const SDLoc &dl) {
172       return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
173     }
174 
175     /// getI64Imm - Return a target constant with the specified value, of type
176     /// i64.
177     inline SDValue getI64Imm(uint64_t Imm, const SDLoc &dl) {
178       return CurDAG->getTargetConstant(Imm, dl, MVT::i64);
179     }
180 
181     /// getSmallIPtrImm - Return a target constant of pointer type.
182     inline SDValue getSmallIPtrImm(unsigned Imm, const SDLoc &dl) {
183       return CurDAG->getTargetConstant(
184           Imm, dl, PPCLowering->getPointerTy(CurDAG->getDataLayout()));
185     }
186 
187     /// isRotateAndMask - Returns true if Mask and Shift can be folded into a
188     /// rotate and mask opcode and mask operation.
189     static bool isRotateAndMask(SDNode *N, unsigned Mask, bool isShiftMask,
190                                 unsigned &SH, unsigned &MB, unsigned &ME);
191 
192     /// getGlobalBaseReg - insert code into the entry mbb to materialize the PIC
193     /// base register.  Return the virtual register that holds this value.
194     SDNode *getGlobalBaseReg();
195 
196     void selectFrameIndex(SDNode *SN, SDNode *N, unsigned Offset = 0);
197 
198     // Select - Convert the specified operand from a target-independent to a
199     // target-specific node if it hasn't already been changed.
200     void Select(SDNode *N) override;
201 
202     bool tryBitfieldInsert(SDNode *N);
203     bool tryBitPermutation(SDNode *N);
204     bool tryIntCompareInGPR(SDNode *N);
205 
206     // tryTLSXFormLoad - Convert an ISD::LOAD fed by a PPCISD::ADD_TLS into
207     // an X-Form load instruction with the offset being a relocation coming from
208     // the PPCISD::ADD_TLS.
209     bool tryTLSXFormLoad(LoadSDNode *N);
210     // tryTLSXFormStore - Convert an ISD::STORE fed by a PPCISD::ADD_TLS into
211     // an X-Form store instruction with the offset being a relocation coming from
212     // the PPCISD::ADD_TLS.
213     bool tryTLSXFormStore(StoreSDNode *N);
214     /// SelectCC - Select a comparison of the specified values with the
215     /// specified condition code, returning the CR# of the expression.
216     SDValue SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC,
217                      const SDLoc &dl, SDValue Chain = SDValue());
218 
219     /// SelectAddrImmOffs - Return true if the operand is valid for a preinc
220     /// immediate field.  Note that the operand at this point is already the
221     /// result of a prior SelectAddressRegImm call.
222     bool SelectAddrImmOffs(SDValue N, SDValue &Out) const {
223       if (N.getOpcode() == ISD::TargetConstant ||
224           N.getOpcode() == ISD::TargetGlobalAddress) {
225         Out = N;
226         return true;
227       }
228 
229       return false;
230     }
231 
232     /// SelectAddrIdx - Given the specified address, check to see if it can be
233     /// represented as an indexed [r+r] operation.
234     /// This is for xform instructions whose associated displacement form is D.
235     /// The last parameter \p 0 means associated D form has no requirment for 16
236     /// bit signed displacement.
237     /// Returns false if it can be represented by [r+imm], which are preferred.
238     bool SelectAddrIdx(SDValue N, SDValue &Base, SDValue &Index) {
239       return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG, None);
240     }
241 
242     /// SelectAddrIdx4 - Given the specified address, check to see if it can be
243     /// represented as an indexed [r+r] operation.
244     /// This is for xform instructions whose associated displacement form is DS.
245     /// The last parameter \p 4 means associated DS form 16 bit signed
246     /// displacement must be a multiple of 4.
247     /// Returns false if it can be represented by [r+imm], which are preferred.
248     bool SelectAddrIdxX4(SDValue N, SDValue &Base, SDValue &Index) {
249       return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG,
250                                               Align(4));
251     }
252 
253     /// SelectAddrIdx16 - Given the specified address, check to see if it can be
254     /// represented as an indexed [r+r] operation.
255     /// This is for xform instructions whose associated displacement form is DQ.
256     /// The last parameter \p 16 means associated DQ form 16 bit signed
257     /// displacement must be a multiple of 16.
258     /// Returns false if it can be represented by [r+imm], which are preferred.
259     bool SelectAddrIdxX16(SDValue N, SDValue &Base, SDValue &Index) {
260       return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG,
261                                               Align(16));
262     }
263 
264     /// SelectAddrIdxOnly - Given the specified address, force it to be
265     /// represented as an indexed [r+r] operation.
266     bool SelectAddrIdxOnly(SDValue N, SDValue &Base, SDValue &Index) {
267       return PPCLowering->SelectAddressRegRegOnly(N, Base, Index, *CurDAG);
268     }
269 
270     /// SelectAddrImm - Returns true if the address N can be represented by
271     /// a base register plus a signed 16-bit displacement [r+imm].
272     /// The last parameter \p 0 means D form has no requirment for 16 bit signed
273     /// displacement.
274     bool SelectAddrImm(SDValue N, SDValue &Disp,
275                        SDValue &Base) {
276       return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, None);
277     }
278 
279     /// SelectAddrImmX4 - Returns true if the address N can be represented by
280     /// a base register plus a signed 16-bit displacement that is a multiple of
281     /// 4 (last parameter). Suitable for use by STD and friends.
282     bool SelectAddrImmX4(SDValue N, SDValue &Disp, SDValue &Base) {
283       return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, Align(4));
284     }
285 
286     /// SelectAddrImmX16 - Returns true if the address N can be represented by
287     /// a base register plus a signed 16-bit displacement that is a multiple of
288     /// 16(last parameter). Suitable for use by STXV and friends.
289     bool SelectAddrImmX16(SDValue N, SDValue &Disp, SDValue &Base) {
290       return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG,
291                                               Align(16));
292     }
293 
294     /// SelectAddrImmX34 - Returns true if the address N can be represented by
295     /// a base register plus a signed 34-bit displacement. Suitable for use by
296     /// PSTXVP and friends.
297     bool SelectAddrImmX34(SDValue N, SDValue &Disp, SDValue &Base) {
298       return PPCLowering->SelectAddressRegImm34(N, Disp, Base, *CurDAG);
299     }
300 
301     // Select an address into a single register.
302     bool SelectAddr(SDValue N, SDValue &Base) {
303       Base = N;
304       return true;
305     }
306 
307     bool SelectAddrPCRel(SDValue N, SDValue &Base) {
308       return PPCLowering->SelectAddressPCRel(N, Base);
309     }
310 
311     /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
312     /// inline asm expressions.  It is always correct to compute the value into
313     /// a register.  The case of adding a (possibly relocatable) constant to a
314     /// register can be improved, but it is wrong to substitute Reg+Reg for
315     /// Reg in an asm, because the load or store opcode would have to change.
316     bool SelectInlineAsmMemoryOperand(const SDValue &Op,
317                                       unsigned ConstraintID,
318                                       std::vector<SDValue> &OutOps) override {
319       switch(ConstraintID) {
320       default:
321         errs() << "ConstraintID: " << ConstraintID << "\n";
322         llvm_unreachable("Unexpected asm memory constraint");
323       case InlineAsm::Constraint_es:
324       case InlineAsm::Constraint_m:
325       case InlineAsm::Constraint_o:
326       case InlineAsm::Constraint_Q:
327       case InlineAsm::Constraint_Z:
328       case InlineAsm::Constraint_Zy:
329         // We need to make sure that this one operand does not end up in r0
330         // (because we might end up lowering this as 0(%op)).
331         const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
332         const TargetRegisterClass *TRC = TRI->getPointerRegClass(*MF, /*Kind=*/1);
333         SDLoc dl(Op);
334         SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i32);
335         SDValue NewOp =
336           SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
337                                          dl, Op.getValueType(),
338                                          Op, RC), 0);
339 
340         OutOps.push_back(NewOp);
341         return false;
342       }
343       return true;
344     }
345 
346     StringRef getPassName() const override {
347       return "PowerPC DAG->DAG Pattern Instruction Selection";
348     }
349 
350 // Include the pieces autogenerated from the target description.
351 #include "PPCGenDAGISel.inc"
352 
353 private:
354     bool trySETCC(SDNode *N);
355     bool tryFoldSWTestBRCC(SDNode *N);
356     bool tryAsSingleRLDICL(SDNode *N);
357     bool tryAsSingleRLDICR(SDNode *N);
358     bool tryAsSingleRLWINM(SDNode *N);
359     bool tryAsSingleRLWINM8(SDNode *N);
360     bool tryAsSingleRLWIMI(SDNode *N);
361     bool tryAsPairOfRLDICL(SDNode *N);
362     bool tryAsSingleRLDIMI(SDNode *N);
363 
364     void PeepholePPC64();
365     void PeepholePPC64ZExt();
366     void PeepholeCROps();
367 
368     SDValue combineToCMPB(SDNode *N);
369     void foldBoolExts(SDValue &Res, SDNode *&N);
370 
371     bool AllUsersSelectZero(SDNode *N);
372     void SwapAllSelectUsers(SDNode *N);
373 
374     bool isOffsetMultipleOf(SDNode *N, unsigned Val) const;
375     void transferMemOperands(SDNode *N, SDNode *Result);
376   };
377 
378 } // end anonymous namespace
379 
380 /// getGlobalBaseReg - Output the instructions required to put the
381 /// base address to use for accessing globals into a register.
382 ///
383 SDNode *PPCDAGToDAGISel::getGlobalBaseReg() {
384   if (!GlobalBaseReg) {
385     const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
386     // Insert the set of GlobalBaseReg into the first MBB of the function
387     MachineBasicBlock &FirstMBB = MF->front();
388     MachineBasicBlock::iterator MBBI = FirstMBB.begin();
389     const Module *M = MF->getFunction().getParent();
390     DebugLoc dl;
391 
392     if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) == MVT::i32) {
393       if (Subtarget->isTargetELF()) {
394         GlobalBaseReg = PPC::R30;
395         if (!Subtarget->isSecurePlt() &&
396             M->getPICLevel() == PICLevel::SmallPIC) {
397           BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MoveGOTtoLR));
398           BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg);
399           MF->getInfo<PPCFunctionInfo>()->setUsesPICBase(true);
400         } else {
401           BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR));
402           BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg);
403           Register TempReg = RegInfo->createVirtualRegister(&PPC::GPRCRegClass);
404           BuildMI(FirstMBB, MBBI, dl,
405                   TII.get(PPC::UpdateGBR), GlobalBaseReg)
406                   .addReg(TempReg, RegState::Define).addReg(GlobalBaseReg);
407           MF->getInfo<PPCFunctionInfo>()->setUsesPICBase(true);
408         }
409       } else {
410         GlobalBaseReg =
411           RegInfo->createVirtualRegister(&PPC::GPRC_and_GPRC_NOR0RegClass);
412         BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR));
413         BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg);
414       }
415     } else {
416       // We must ensure that this sequence is dominated by the prologue.
417       // FIXME: This is a bit of a big hammer since we don't get the benefits
418       // of shrink-wrapping whenever we emit this instruction. Considering
419       // this is used in any function where we emit a jump table, this may be
420       // a significant limitation. We should consider inserting this in the
421       // block where it is used and then commoning this sequence up if it
422       // appears in multiple places.
423       // Note: on ISA 3.0 cores, we can use lnia (addpcis) instead of
424       // MovePCtoLR8.
425       MF->getInfo<PPCFunctionInfo>()->setShrinkWrapDisabled(true);
426       GlobalBaseReg = RegInfo->createVirtualRegister(&PPC::G8RC_and_G8RC_NOX0RegClass);
427       BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR8));
428       BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR8), GlobalBaseReg);
429     }
430   }
431   return CurDAG->getRegister(GlobalBaseReg,
432                              PPCLowering->getPointerTy(CurDAG->getDataLayout()))
433       .getNode();
434 }
435 
436 /// isInt32Immediate - This method tests to see if the node is a 32-bit constant
437 /// operand. If so Imm will receive the 32-bit value.
438 static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
439   if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
440     Imm = cast<ConstantSDNode>(N)->getZExtValue();
441     return true;
442   }
443   return false;
444 }
445 
446 /// isInt64Immediate - This method tests to see if the node is a 64-bit constant
447 /// operand.  If so Imm will receive the 64-bit value.
448 static bool isInt64Immediate(SDNode *N, uint64_t &Imm) {
449   if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i64) {
450     Imm = cast<ConstantSDNode>(N)->getZExtValue();
451     return true;
452   }
453   return false;
454 }
455 
456 // isInt32Immediate - This method tests to see if a constant operand.
457 // If so Imm will receive the 32 bit value.
458 static bool isInt32Immediate(SDValue N, unsigned &Imm) {
459   return isInt32Immediate(N.getNode(), Imm);
460 }
461 
462 /// isInt64Immediate - This method tests to see if the value is a 64-bit
463 /// constant operand. If so Imm will receive the 64-bit value.
464 static bool isInt64Immediate(SDValue N, uint64_t &Imm) {
465   return isInt64Immediate(N.getNode(), Imm);
466 }
467 
468 static unsigned getBranchHint(unsigned PCC,
469                               const FunctionLoweringInfo &FuncInfo,
470                               const SDValue &DestMBB) {
471   assert(isa<BasicBlockSDNode>(DestMBB));
472 
473   if (!FuncInfo.BPI) return PPC::BR_NO_HINT;
474 
475   const BasicBlock *BB = FuncInfo.MBB->getBasicBlock();
476   const Instruction *BBTerm = BB->getTerminator();
477 
478   if (BBTerm->getNumSuccessors() != 2) return PPC::BR_NO_HINT;
479 
480   const BasicBlock *TBB = BBTerm->getSuccessor(0);
481   const BasicBlock *FBB = BBTerm->getSuccessor(1);
482 
483   auto TProb = FuncInfo.BPI->getEdgeProbability(BB, TBB);
484   auto FProb = FuncInfo.BPI->getEdgeProbability(BB, FBB);
485 
486   // We only want to handle cases which are easy to predict at static time, e.g.
487   // C++ throw statement, that is very likely not taken, or calling never
488   // returned function, e.g. stdlib exit(). So we set Threshold to filter
489   // unwanted cases.
490   //
491   // Below is LLVM branch weight table, we only want to handle case 1, 2
492   //
493   // Case                  Taken:Nontaken  Example
494   // 1. Unreachable        1048575:1       C++ throw, stdlib exit(),
495   // 2. Invoke-terminating 1:1048575
496   // 3. Coldblock          4:64            __builtin_expect
497   // 4. Loop Branch        124:4           For loop
498   // 5. PH/ZH/FPH          20:12
499   const uint32_t Threshold = 10000;
500 
501   if (std::max(TProb, FProb) / Threshold < std::min(TProb, FProb))
502     return PPC::BR_NO_HINT;
503 
504   LLVM_DEBUG(dbgs() << "Use branch hint for '" << FuncInfo.Fn->getName()
505                     << "::" << BB->getName() << "'\n"
506                     << " -> " << TBB->getName() << ": " << TProb << "\n"
507                     << " -> " << FBB->getName() << ": " << FProb << "\n");
508 
509   const BasicBlockSDNode *BBDN = cast<BasicBlockSDNode>(DestMBB);
510 
511   // If Dest BasicBlock is False-BasicBlock (FBB), swap branch probabilities,
512   // because we want 'TProb' stands for 'branch probability' to Dest BasicBlock
513   if (BBDN->getBasicBlock()->getBasicBlock() != TBB)
514     std::swap(TProb, FProb);
515 
516   return (TProb > FProb) ? PPC::BR_TAKEN_HINT : PPC::BR_NONTAKEN_HINT;
517 }
518 
519 // isOpcWithIntImmediate - This method tests to see if the node is a specific
520 // opcode and that it has a immediate integer right operand.
521 // If so Imm will receive the 32 bit value.
522 static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
523   return N->getOpcode() == Opc
524          && isInt32Immediate(N->getOperand(1).getNode(), Imm);
525 }
526 
527 void PPCDAGToDAGISel::selectFrameIndex(SDNode *SN, SDNode *N, unsigned Offset) {
528   SDLoc dl(SN);
529   int FI = cast<FrameIndexSDNode>(N)->getIndex();
530   SDValue TFI = CurDAG->getTargetFrameIndex(FI, N->getValueType(0));
531   unsigned Opc = N->getValueType(0) == MVT::i32 ? PPC::ADDI : PPC::ADDI8;
532   if (SN->hasOneUse())
533     CurDAG->SelectNodeTo(SN, Opc, N->getValueType(0), TFI,
534                          getSmallIPtrImm(Offset, dl));
535   else
536     ReplaceNode(SN, CurDAG->getMachineNode(Opc, dl, N->getValueType(0), TFI,
537                                            getSmallIPtrImm(Offset, dl)));
538 }
539 
540 bool PPCDAGToDAGISel::isRotateAndMask(SDNode *N, unsigned Mask,
541                                       bool isShiftMask, unsigned &SH,
542                                       unsigned &MB, unsigned &ME) {
543   // Don't even go down this path for i64, since different logic will be
544   // necessary for rldicl/rldicr/rldimi.
545   if (N->getValueType(0) != MVT::i32)
546     return false;
547 
548   unsigned Shift  = 32;
549   unsigned Indeterminant = ~0;  // bit mask marking indeterminant results
550   unsigned Opcode = N->getOpcode();
551   if (N->getNumOperands() != 2 ||
552       !isInt32Immediate(N->getOperand(1).getNode(), Shift) || (Shift > 31))
553     return false;
554 
555   if (Opcode == ISD::SHL) {
556     // apply shift left to mask if it comes first
557     if (isShiftMask) Mask = Mask << Shift;
558     // determine which bits are made indeterminant by shift
559     Indeterminant = ~(0xFFFFFFFFu << Shift);
560   } else if (Opcode == ISD::SRL) {
561     // apply shift right to mask if it comes first
562     if (isShiftMask) Mask = Mask >> Shift;
563     // determine which bits are made indeterminant by shift
564     Indeterminant = ~(0xFFFFFFFFu >> Shift);
565     // adjust for the left rotate
566     Shift = 32 - Shift;
567   } else if (Opcode == ISD::ROTL) {
568     Indeterminant = 0;
569   } else {
570     return false;
571   }
572 
573   // if the mask doesn't intersect any Indeterminant bits
574   if (Mask && !(Mask & Indeterminant)) {
575     SH = Shift & 31;
576     // make sure the mask is still a mask (wrap arounds may not be)
577     return isRunOfOnes(Mask, MB, ME);
578   }
579   return false;
580 }
581 
582 bool PPCDAGToDAGISel::tryTLSXFormStore(StoreSDNode *ST) {
583   SDValue Base = ST->getBasePtr();
584   if (Base.getOpcode() != PPCISD::ADD_TLS)
585     return false;
586   SDValue Offset = ST->getOffset();
587   if (!Offset.isUndef())
588     return false;
589   if (Base.getOperand(1).getOpcode() == PPCISD::TLS_LOCAL_EXEC_MAT_ADDR)
590     return false;
591 
592   SDLoc dl(ST);
593   EVT MemVT = ST->getMemoryVT();
594   EVT RegVT = ST->getValue().getValueType();
595 
596   unsigned Opcode;
597   switch (MemVT.getSimpleVT().SimpleTy) {
598     default:
599       return false;
600     case MVT::i8: {
601       Opcode = (RegVT == MVT::i32) ? PPC::STBXTLS_32 : PPC::STBXTLS;
602       break;
603     }
604     case MVT::i16: {
605       Opcode = (RegVT == MVT::i32) ? PPC::STHXTLS_32 : PPC::STHXTLS;
606       break;
607     }
608     case MVT::i32: {
609       Opcode = (RegVT == MVT::i32) ? PPC::STWXTLS_32 : PPC::STWXTLS;
610       break;
611     }
612     case MVT::i64: {
613       Opcode = PPC::STDXTLS;
614       break;
615     }
616   }
617   SDValue Chain = ST->getChain();
618   SDVTList VTs = ST->getVTList();
619   SDValue Ops[] = {ST->getValue(), Base.getOperand(0), Base.getOperand(1),
620                    Chain};
621   SDNode *MN = CurDAG->getMachineNode(Opcode, dl, VTs, Ops);
622   transferMemOperands(ST, MN);
623   ReplaceNode(ST, MN);
624   return true;
625 }
626 
627 bool PPCDAGToDAGISel::tryTLSXFormLoad(LoadSDNode *LD) {
628   SDValue Base = LD->getBasePtr();
629   if (Base.getOpcode() != PPCISD::ADD_TLS)
630     return false;
631   SDValue Offset = LD->getOffset();
632   if (!Offset.isUndef())
633     return false;
634   if (Base.getOperand(1).getOpcode() == PPCISD::TLS_LOCAL_EXEC_MAT_ADDR)
635     return false;
636 
637   SDLoc dl(LD);
638   EVT MemVT = LD->getMemoryVT();
639   EVT RegVT = LD->getValueType(0);
640   unsigned Opcode;
641   switch (MemVT.getSimpleVT().SimpleTy) {
642     default:
643       return false;
644     case MVT::i8: {
645       Opcode = (RegVT == MVT::i32) ? PPC::LBZXTLS_32 : PPC::LBZXTLS;
646       break;
647     }
648     case MVT::i16: {
649       Opcode = (RegVT == MVT::i32) ? PPC::LHZXTLS_32 : PPC::LHZXTLS;
650       break;
651     }
652     case MVT::i32: {
653       Opcode = (RegVT == MVT::i32) ? PPC::LWZXTLS_32 : PPC::LWZXTLS;
654       break;
655     }
656     case MVT::i64: {
657       Opcode = PPC::LDXTLS;
658       break;
659     }
660   }
661   SDValue Chain = LD->getChain();
662   SDVTList VTs = LD->getVTList();
663   SDValue Ops[] = {Base.getOperand(0), Base.getOperand(1), Chain};
664   SDNode *MN = CurDAG->getMachineNode(Opcode, dl, VTs, Ops);
665   transferMemOperands(LD, MN);
666   ReplaceNode(LD, MN);
667   return true;
668 }
669 
670 /// Turn an or of two masked values into the rotate left word immediate then
671 /// mask insert (rlwimi) instruction.
672 bool PPCDAGToDAGISel::tryBitfieldInsert(SDNode *N) {
673   SDValue Op0 = N->getOperand(0);
674   SDValue Op1 = N->getOperand(1);
675   SDLoc dl(N);
676 
677   KnownBits LKnown = CurDAG->computeKnownBits(Op0);
678   KnownBits RKnown = CurDAG->computeKnownBits(Op1);
679 
680   unsigned TargetMask = LKnown.Zero.getZExtValue();
681   unsigned InsertMask = RKnown.Zero.getZExtValue();
682 
683   if ((TargetMask | InsertMask) == 0xFFFFFFFF) {
684     unsigned Op0Opc = Op0.getOpcode();
685     unsigned Op1Opc = Op1.getOpcode();
686     unsigned Value, SH = 0;
687     TargetMask = ~TargetMask;
688     InsertMask = ~InsertMask;
689 
690     // If the LHS has a foldable shift and the RHS does not, then swap it to the
691     // RHS so that we can fold the shift into the insert.
692     if (Op0Opc == ISD::AND && Op1Opc == ISD::AND) {
693       if (Op0.getOperand(0).getOpcode() == ISD::SHL ||
694           Op0.getOperand(0).getOpcode() == ISD::SRL) {
695         if (Op1.getOperand(0).getOpcode() != ISD::SHL &&
696             Op1.getOperand(0).getOpcode() != ISD::SRL) {
697           std::swap(Op0, Op1);
698           std::swap(Op0Opc, Op1Opc);
699           std::swap(TargetMask, InsertMask);
700         }
701       }
702     } else if (Op0Opc == ISD::SHL || Op0Opc == ISD::SRL) {
703       if (Op1Opc == ISD::AND && Op1.getOperand(0).getOpcode() != ISD::SHL &&
704           Op1.getOperand(0).getOpcode() != ISD::SRL) {
705         std::swap(Op0, Op1);
706         std::swap(Op0Opc, Op1Opc);
707         std::swap(TargetMask, InsertMask);
708       }
709     }
710 
711     unsigned MB, ME;
712     if (isRunOfOnes(InsertMask, MB, ME)) {
713       if ((Op1Opc == ISD::SHL || Op1Opc == ISD::SRL) &&
714           isInt32Immediate(Op1.getOperand(1), Value)) {
715         Op1 = Op1.getOperand(0);
716         SH  = (Op1Opc == ISD::SHL) ? Value : 32 - Value;
717       }
718       if (Op1Opc == ISD::AND) {
719        // The AND mask might not be a constant, and we need to make sure that
720        // if we're going to fold the masking with the insert, all bits not
721        // know to be zero in the mask are known to be one.
722         KnownBits MKnown = CurDAG->computeKnownBits(Op1.getOperand(1));
723         bool CanFoldMask = InsertMask == MKnown.One.getZExtValue();
724 
725         unsigned SHOpc = Op1.getOperand(0).getOpcode();
726         if ((SHOpc == ISD::SHL || SHOpc == ISD::SRL) && CanFoldMask &&
727             isInt32Immediate(Op1.getOperand(0).getOperand(1), Value)) {
728           // Note that Value must be in range here (less than 32) because
729           // otherwise there would not be any bits set in InsertMask.
730           Op1 = Op1.getOperand(0).getOperand(0);
731           SH  = (SHOpc == ISD::SHL) ? Value : 32 - Value;
732         }
733       }
734 
735       SH &= 31;
736       SDValue Ops[] = { Op0, Op1, getI32Imm(SH, dl), getI32Imm(MB, dl),
737                           getI32Imm(ME, dl) };
738       ReplaceNode(N, CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops));
739       return true;
740     }
741   }
742   return false;
743 }
744 
745 static unsigned allUsesTruncate(SelectionDAG *CurDAG, SDNode *N) {
746   unsigned MaxTruncation = 0;
747   // Cannot use range-based for loop here as we need the actual use (i.e. we
748   // need the operand number corresponding to the use). A range-based for
749   // will unbox the use and provide an SDNode*.
750   for (SDNode::use_iterator Use = N->use_begin(), UseEnd = N->use_end();
751        Use != UseEnd; ++Use) {
752     unsigned Opc =
753       Use->isMachineOpcode() ? Use->getMachineOpcode() : Use->getOpcode();
754     switch (Opc) {
755     default: return 0;
756     case ISD::TRUNCATE:
757       if (Use->isMachineOpcode())
758         return 0;
759       MaxTruncation =
760         std::max(MaxTruncation, (unsigned)Use->getValueType(0).getSizeInBits());
761       continue;
762     case ISD::STORE: {
763       if (Use->isMachineOpcode())
764         return 0;
765       StoreSDNode *STN = cast<StoreSDNode>(*Use);
766       unsigned MemVTSize = STN->getMemoryVT().getSizeInBits();
767       if (MemVTSize == 64 || Use.getOperandNo() != 0)
768         return 0;
769       MaxTruncation = std::max(MaxTruncation, MemVTSize);
770       continue;
771     }
772     case PPC::STW8:
773     case PPC::STWX8:
774     case PPC::STWU8:
775     case PPC::STWUX8:
776       if (Use.getOperandNo() != 0)
777         return 0;
778       MaxTruncation = std::max(MaxTruncation, 32u);
779       continue;
780     case PPC::STH8:
781     case PPC::STHX8:
782     case PPC::STHU8:
783     case PPC::STHUX8:
784       if (Use.getOperandNo() != 0)
785         return 0;
786       MaxTruncation = std::max(MaxTruncation, 16u);
787       continue;
788     case PPC::STB8:
789     case PPC::STBX8:
790     case PPC::STBU8:
791     case PPC::STBUX8:
792       if (Use.getOperandNo() != 0)
793         return 0;
794       MaxTruncation = std::max(MaxTruncation, 8u);
795       continue;
796     }
797   }
798   return MaxTruncation;
799 }
800 
801 // For any 32 < Num < 64, check if the Imm contains at least Num consecutive
802 // zeros and return the number of bits by the left of these consecutive zeros.
803 static int findContiguousZerosAtLeast(uint64_t Imm, unsigned Num) {
804   unsigned HiTZ = countTrailingZeros<uint32_t>(Hi_32(Imm));
805   unsigned LoLZ = countLeadingZeros<uint32_t>(Lo_32(Imm));
806   if ((HiTZ + LoLZ) >= Num)
807     return (32 + HiTZ);
808   return 0;
809 }
810 
811 // Direct materialization of 64-bit constants by enumerated patterns.
812 static SDNode *selectI64ImmDirect(SelectionDAG *CurDAG, const SDLoc &dl,
813                                   uint64_t Imm, unsigned &InstCnt) {
814   unsigned TZ = countTrailingZeros<uint64_t>(Imm);
815   unsigned LZ = countLeadingZeros<uint64_t>(Imm);
816   unsigned TO = countTrailingOnes<uint64_t>(Imm);
817   unsigned LO = countLeadingOnes<uint64_t>(Imm);
818   unsigned Hi32 = Hi_32(Imm);
819   unsigned Lo32 = Lo_32(Imm);
820   SDNode *Result = nullptr;
821   unsigned Shift = 0;
822 
823   auto getI32Imm = [CurDAG, dl](unsigned Imm) {
824     return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
825   };
826 
827   // Following patterns use 1 instructions to materialize the Imm.
828   InstCnt = 1;
829   // 1-1) Patterns : {zeros}{15-bit valve}
830   //                 {ones}{15-bit valve}
831   if (isInt<16>(Imm)) {
832     SDValue SDImm = CurDAG->getTargetConstant(Imm, dl, MVT::i64);
833     return CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, SDImm);
834   }
835   // 1-2) Patterns : {zeros}{15-bit valve}{16 zeros}
836   //                 {ones}{15-bit valve}{16 zeros}
837   if (TZ > 15 && (LZ > 32 || LO > 32))
838     return CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64,
839                                   getI32Imm((Imm >> 16) & 0xffff));
840 
841   // Following patterns use 2 instructions to materialize the Imm.
842   InstCnt = 2;
843   assert(LZ < 64 && "Unexpected leading zeros here.");
844   // Count of ones follwing the leading zeros.
845   unsigned FO = countLeadingOnes<uint64_t>(Imm << LZ);
846   // 2-1) Patterns : {zeros}{31-bit value}
847   //                 {ones}{31-bit value}
848   if (isInt<32>(Imm)) {
849     uint64_t ImmHi16 = (Imm >> 16) & 0xffff;
850     unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8;
851     Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16));
852     return CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
853                                   getI32Imm(Imm & 0xffff));
854   }
855   // 2-2) Patterns : {zeros}{ones}{15-bit value}{zeros}
856   //                 {zeros}{15-bit value}{zeros}
857   //                 {zeros}{ones}{15-bit value}
858   //                 {ones}{15-bit value}{zeros}
859   // We can take advantage of LI's sign-extension semantics to generate leading
860   // ones, and then use RLDIC to mask off the ones in both sides after rotation.
861   if ((LZ + FO + TZ) > 48) {
862     Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64,
863                                     getI32Imm((Imm >> TZ) & 0xffff));
864     return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0),
865                                   getI32Imm(TZ), getI32Imm(LZ));
866   }
867   // 2-3) Pattern : {zeros}{15-bit value}{ones}
868   // Shift right the Imm by (48 - LZ) bits to construct a negtive 16 bits value,
869   // therefore we can take advantage of LI's sign-extension semantics, and then
870   // mask them off after rotation.
871   //
872   // +--LZ--||-15-bit-||--TO--+     +-------------|--16-bit--+
873   // |00000001bbbbbbbbb1111111| ->  |00000000000001bbbbbbbbb1|
874   // +------------------------+     +------------------------+
875   // 63                      0      63                      0
876   //          Imm                   (Imm >> (48 - LZ) & 0xffff)
877   // +----sext-----|--16-bit--+     +clear-|-----------------+
878   // |11111111111111bbbbbbbbb1| ->  |00000001bbbbbbbbb1111111|
879   // +------------------------+     +------------------------+
880   // 63                      0      63                      0
881   // LI8: sext many leading zeros   RLDICL: rotate left (48 - LZ), clear left LZ
882   if ((LZ + TO) > 48) {
883     // Since the immediates with (LZ > 32) have been handled by previous
884     // patterns, here we have (LZ <= 32) to make sure we will not shift right
885     // the Imm by a negative value.
886     assert(LZ <= 32 && "Unexpected shift value.");
887     Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64,
888                                     getI32Imm((Imm >> (48 - LZ) & 0xffff)));
889     return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
890                                   getI32Imm(48 - LZ), getI32Imm(LZ));
891   }
892   // 2-4) Patterns : {zeros}{ones}{15-bit value}{ones}
893   //                 {ones}{15-bit value}{ones}
894   // We can take advantage of LI's sign-extension semantics to generate leading
895   // ones, and then use RLDICL to mask off the ones in left sides (if required)
896   // after rotation.
897   //
898   // +-LZ-FO||-15-bit-||--TO--+     +-------------|--16-bit--+
899   // |00011110bbbbbbbbb1111111| ->  |000000000011110bbbbbbbbb|
900   // +------------------------+     +------------------------+
901   // 63                      0      63                      0
902   //            Imm                    (Imm >> TO) & 0xffff
903   // +----sext-----|--16-bit--+     +LZ|---------------------+
904   // |111111111111110bbbbbbbbb| ->  |00011110bbbbbbbbb1111111|
905   // +------------------------+     +------------------------+
906   // 63                      0      63                      0
907   // LI8: sext many leading zeros   RLDICL: rotate left TO, clear left LZ
908   if ((LZ + FO + TO) > 48) {
909     Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64,
910                                     getI32Imm((Imm >> TO) & 0xffff));
911     return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
912                                   getI32Imm(TO), getI32Imm(LZ));
913   }
914   // 2-5) Pattern : {32 zeros}{****}{0}{15-bit value}
915   // If Hi32 is zero and the Lo16(in Lo32) can be presented as a positive 16 bit
916   // value, we can use LI for Lo16 without generating leading ones then add the
917   // Hi16(in Lo32).
918   if (LZ == 32 && ((Lo32 & 0x8000) == 0)) {
919     Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64,
920                                     getI32Imm(Lo32 & 0xffff));
921     return CurDAG->getMachineNode(PPC::ORIS8, dl, MVT::i64, SDValue(Result, 0),
922                                   getI32Imm(Lo32 >> 16));
923   }
924   // 2-6) Patterns : {******}{49 zeros}{******}
925   //                 {******}{49 ones}{******}
926   // If the Imm contains 49 consecutive zeros/ones, it means that a total of 15
927   // bits remain on both sides. Rotate right the Imm to construct an int<16>
928   // value, use LI for int<16> value and then use RLDICL without mask to rotate
929   // it back.
930   //
931   // 1) findContiguousZerosAtLeast(Imm, 49)
932   // +------|--zeros-|------+     +---ones--||---15 bit--+
933   // |bbbbbb0000000000aaaaaa| ->  |0000000000aaaaaabbbbbb|
934   // +----------------------+     +----------------------+
935   // 63                    0      63                    0
936   //
937   // 2) findContiguousZerosAtLeast(~Imm, 49)
938   // +------|--ones--|------+     +---ones--||---15 bit--+
939   // |bbbbbb1111111111aaaaaa| ->  |1111111111aaaaaabbbbbb|
940   // +----------------------+     +----------------------+
941   // 63                    0      63                    0
942   if ((Shift = findContiguousZerosAtLeast(Imm, 49)) ||
943       (Shift = findContiguousZerosAtLeast(~Imm, 49))) {
944     uint64_t RotImm = (Imm >> Shift) | (Imm << (64 - Shift));
945     Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64,
946                                     getI32Imm(RotImm & 0xffff));
947     return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
948                                   getI32Imm(Shift), getI32Imm(0));
949   }
950 
951   // Following patterns use 3 instructions to materialize the Imm.
952   InstCnt = 3;
953   // 3-1) Patterns : {zeros}{ones}{31-bit value}{zeros}
954   //                 {zeros}{31-bit value}{zeros}
955   //                 {zeros}{ones}{31-bit value}
956   //                 {ones}{31-bit value}{zeros}
957   // We can take advantage of LIS's sign-extension semantics to generate leading
958   // ones, add the remaining bits with ORI, and then use RLDIC to mask off the
959   // ones in both sides after rotation.
960   if ((LZ + FO + TZ) > 32) {
961     uint64_t ImmHi16 = (Imm >> (TZ + 16)) & 0xffff;
962     unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8;
963     Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16));
964     Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
965                                     getI32Imm((Imm >> TZ) & 0xffff));
966     return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0),
967                                   getI32Imm(TZ), getI32Imm(LZ));
968   }
969   // 3-2) Pattern : {zeros}{31-bit value}{ones}
970   // Shift right the Imm by (32 - LZ) bits to construct a negtive 32 bits value,
971   // therefore we can take advantage of LIS's sign-extension semantics, add
972   // the remaining bits with ORI, and then mask them off after rotation.
973   // This is similar to Pattern 2-3, please refer to the diagram there.
974   if ((LZ + TO) > 32) {
975     // Since the immediates with (LZ > 32) have been handled by previous
976     // patterns, here we have (LZ <= 32) to make sure we will not shift right
977     // the Imm by a negative value.
978     assert(LZ <= 32 && "Unexpected shift value.");
979     Result = CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64,
980                                     getI32Imm((Imm >> (48 - LZ)) & 0xffff));
981     Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
982                                     getI32Imm((Imm >> (32 - LZ)) & 0xffff));
983     return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
984                                   getI32Imm(32 - LZ), getI32Imm(LZ));
985   }
986   // 3-3) Patterns : {zeros}{ones}{31-bit value}{ones}
987   //                 {ones}{31-bit value}{ones}
988   // We can take advantage of LIS's sign-extension semantics to generate leading
989   // ones, add the remaining bits with ORI, and then use RLDICL to mask off the
990   // ones in left sides (if required) after rotation.
991   // This is similar to Pattern 2-4, please refer to the diagram there.
992   if ((LZ + FO + TO) > 32) {
993     Result = CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64,
994                                     getI32Imm((Imm >> (TO + 16)) & 0xffff));
995     Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
996                                     getI32Imm((Imm >> TO) & 0xffff));
997     return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
998                                   getI32Imm(TO), getI32Imm(LZ));
999   }
1000   // 3-4) Patterns : High word == Low word
1001   if (Hi32 == Lo32) {
1002     // Handle the first 32 bits.
1003     uint64_t ImmHi16 = (Lo32 >> 16) & 0xffff;
1004     unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8;
1005     Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16));
1006     Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
1007                                     getI32Imm(Lo32 & 0xffff));
1008     // Use rldimi to insert the Low word into High word.
1009     SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(32),
1010                      getI32Imm(0)};
1011     return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops);
1012   }
1013   // 3-5) Patterns : {******}{33 zeros}{******}
1014   //                 {******}{33 ones}{******}
1015   // If the Imm contains 33 consecutive zeros/ones, it means that a total of 31
1016   // bits remain on both sides. Rotate right the Imm to construct an int<32>
1017   // value, use LIS + ORI for int<32> value and then use RLDICL without mask to
1018   // rotate it back.
1019   // This is similar to Pattern 2-6, please refer to the diagram there.
1020   if ((Shift = findContiguousZerosAtLeast(Imm, 33)) ||
1021       (Shift = findContiguousZerosAtLeast(~Imm, 33))) {
1022     uint64_t RotImm = (Imm >> Shift) | (Imm << (64 - Shift));
1023     uint64_t ImmHi16 = (RotImm >> 16) & 0xffff;
1024     unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8;
1025     Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16));
1026     Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
1027                                     getI32Imm(RotImm & 0xffff));
1028     return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
1029                                   getI32Imm(Shift), getI32Imm(0));
1030   }
1031 
1032   InstCnt = 0;
1033   return nullptr;
1034 }
1035 
1036 static SDNode *selectI64Imm(SelectionDAG *CurDAG, const SDLoc &dl, uint64_t Imm,
1037                             unsigned *InstCnt = nullptr) {
1038   unsigned InstCntDirect = 0;
1039   // No more than 3 instructions is used if we can select the i64 immediate
1040   // directly.
1041   SDNode *Result = selectI64ImmDirect(CurDAG, dl, Imm, InstCntDirect);
1042   if (Result) {
1043     if (InstCnt)
1044       *InstCnt = InstCntDirect;
1045     return Result;
1046   }
1047   auto getI32Imm = [CurDAG, dl](unsigned Imm) {
1048     return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
1049   };
1050   // Handle the upper 32 bit value.
1051   Result =
1052       selectI64ImmDirect(CurDAG, dl, Imm & 0xffffffff00000000, InstCntDirect);
1053   // Add in the last bits as required.
1054   if (uint32_t Hi16 = (Lo_32(Imm) >> 16) & 0xffff) {
1055     Result = CurDAG->getMachineNode(PPC::ORIS8, dl, MVT::i64,
1056                                     SDValue(Result, 0), getI32Imm(Hi16));
1057     ++InstCntDirect;
1058   }
1059   if (uint32_t Lo16 = Lo_32(Imm) & 0xffff) {
1060     Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
1061                                     getI32Imm(Lo16));
1062     ++InstCntDirect;
1063   }
1064   if (InstCnt)
1065     *InstCnt = InstCntDirect;
1066   return Result;
1067 }
1068 
1069 // Select a 64-bit constant.
1070 static SDNode *selectI64Imm(SelectionDAG *CurDAG, SDNode *N) {
1071   SDLoc dl(N);
1072 
1073   // Get 64 bit value.
1074   int64_t Imm = cast<ConstantSDNode>(N)->getZExtValue();
1075   if (unsigned MinSize = allUsesTruncate(CurDAG, N)) {
1076     uint64_t SextImm = SignExtend64(Imm, MinSize);
1077     SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i64);
1078     if (isInt<16>(SextImm))
1079       return CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, SDImm);
1080   }
1081   return selectI64Imm(CurDAG, dl, Imm);
1082 }
1083 
1084 namespace {
1085 
1086 class BitPermutationSelector {
1087   struct ValueBit {
1088     SDValue V;
1089 
1090     // The bit number in the value, using a convention where bit 0 is the
1091     // lowest-order bit.
1092     unsigned Idx;
1093 
1094     // ConstZero means a bit we need to mask off.
1095     // Variable is a bit comes from an input variable.
1096     // VariableKnownToBeZero is also a bit comes from an input variable,
1097     // but it is known to be already zero. So we do not need to mask them.
1098     enum Kind {
1099       ConstZero,
1100       Variable,
1101       VariableKnownToBeZero
1102     } K;
1103 
1104     ValueBit(SDValue V, unsigned I, Kind K = Variable)
1105       : V(V), Idx(I), K(K) {}
1106     ValueBit(Kind K = Variable)
1107       : V(SDValue(nullptr, 0)), Idx(UINT32_MAX), K(K) {}
1108 
1109     bool isZero() const {
1110       return K == ConstZero || K == VariableKnownToBeZero;
1111     }
1112 
1113     bool hasValue() const {
1114       return K == Variable || K == VariableKnownToBeZero;
1115     }
1116 
1117     SDValue getValue() const {
1118       assert(hasValue() && "Cannot get the value of a constant bit");
1119       return V;
1120     }
1121 
1122     unsigned getValueBitIndex() const {
1123       assert(hasValue() && "Cannot get the value bit index of a constant bit");
1124       return Idx;
1125     }
1126   };
1127 
1128   // A bit group has the same underlying value and the same rotate factor.
1129   struct BitGroup {
1130     SDValue V;
1131     unsigned RLAmt;
1132     unsigned StartIdx, EndIdx;
1133 
1134     // This rotation amount assumes that the lower 32 bits of the quantity are
1135     // replicated in the high 32 bits by the rotation operator (which is done
1136     // by rlwinm and friends in 64-bit mode).
1137     bool Repl32;
1138     // Did converting to Repl32 == true change the rotation factor? If it did,
1139     // it decreased it by 32.
1140     bool Repl32CR;
1141     // Was this group coalesced after setting Repl32 to true?
1142     bool Repl32Coalesced;
1143 
1144     BitGroup(SDValue V, unsigned R, unsigned S, unsigned E)
1145       : V(V), RLAmt(R), StartIdx(S), EndIdx(E), Repl32(false), Repl32CR(false),
1146         Repl32Coalesced(false) {
1147       LLVM_DEBUG(dbgs() << "\tbit group for " << V.getNode() << " RLAmt = " << R
1148                         << " [" << S << ", " << E << "]\n");
1149     }
1150   };
1151 
1152   // Information on each (Value, RLAmt) pair (like the number of groups
1153   // associated with each) used to choose the lowering method.
1154   struct ValueRotInfo {
1155     SDValue V;
1156     unsigned RLAmt = std::numeric_limits<unsigned>::max();
1157     unsigned NumGroups = 0;
1158     unsigned FirstGroupStartIdx = std::numeric_limits<unsigned>::max();
1159     bool Repl32 = false;
1160 
1161     ValueRotInfo() = default;
1162 
1163     // For sorting (in reverse order) by NumGroups, and then by
1164     // FirstGroupStartIdx.
1165     bool operator < (const ValueRotInfo &Other) const {
1166       // We need to sort so that the non-Repl32 come first because, when we're
1167       // doing masking, the Repl32 bit groups might be subsumed into the 64-bit
1168       // masking operation.
1169       if (Repl32 < Other.Repl32)
1170         return true;
1171       else if (Repl32 > Other.Repl32)
1172         return false;
1173       else if (NumGroups > Other.NumGroups)
1174         return true;
1175       else if (NumGroups < Other.NumGroups)
1176         return false;
1177       else if (RLAmt == 0 && Other.RLAmt != 0)
1178         return true;
1179       else if (RLAmt != 0 && Other.RLAmt == 0)
1180         return false;
1181       else if (FirstGroupStartIdx < Other.FirstGroupStartIdx)
1182         return true;
1183       return false;
1184     }
1185   };
1186 
1187   using ValueBitsMemoizedValue = std::pair<bool, SmallVector<ValueBit, 64>>;
1188   using ValueBitsMemoizer =
1189       DenseMap<SDValue, std::unique_ptr<ValueBitsMemoizedValue>>;
1190   ValueBitsMemoizer Memoizer;
1191 
1192   // Return a pair of bool and a SmallVector pointer to a memoization entry.
1193   // The bool is true if something interesting was deduced, otherwise if we're
1194   // providing only a generic representation of V (or something else likewise
1195   // uninteresting for instruction selection) through the SmallVector.
1196   std::pair<bool, SmallVector<ValueBit, 64> *> getValueBits(SDValue V,
1197                                                             unsigned NumBits) {
1198     auto &ValueEntry = Memoizer[V];
1199     if (ValueEntry)
1200       return std::make_pair(ValueEntry->first, &ValueEntry->second);
1201     ValueEntry.reset(new ValueBitsMemoizedValue());
1202     bool &Interesting = ValueEntry->first;
1203     SmallVector<ValueBit, 64> &Bits = ValueEntry->second;
1204     Bits.resize(NumBits);
1205 
1206     switch (V.getOpcode()) {
1207     default: break;
1208     case ISD::ROTL:
1209       if (isa<ConstantSDNode>(V.getOperand(1))) {
1210         unsigned RotAmt = V.getConstantOperandVal(1);
1211 
1212         const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second;
1213 
1214         for (unsigned i = 0; i < NumBits; ++i)
1215           Bits[i] = LHSBits[i < RotAmt ? i + (NumBits - RotAmt) : i - RotAmt];
1216 
1217         return std::make_pair(Interesting = true, &Bits);
1218       }
1219       break;
1220     case ISD::SHL:
1221     case PPCISD::SHL:
1222       if (isa<ConstantSDNode>(V.getOperand(1))) {
1223         unsigned ShiftAmt = V.getConstantOperandVal(1);
1224 
1225         const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second;
1226 
1227         for (unsigned i = ShiftAmt; i < NumBits; ++i)
1228           Bits[i] = LHSBits[i - ShiftAmt];
1229 
1230         for (unsigned i = 0; i < ShiftAmt; ++i)
1231           Bits[i] = ValueBit(ValueBit::ConstZero);
1232 
1233         return std::make_pair(Interesting = true, &Bits);
1234       }
1235       break;
1236     case ISD::SRL:
1237     case PPCISD::SRL:
1238       if (isa<ConstantSDNode>(V.getOperand(1))) {
1239         unsigned ShiftAmt = V.getConstantOperandVal(1);
1240 
1241         const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second;
1242 
1243         for (unsigned i = 0; i < NumBits - ShiftAmt; ++i)
1244           Bits[i] = LHSBits[i + ShiftAmt];
1245 
1246         for (unsigned i = NumBits - ShiftAmt; i < NumBits; ++i)
1247           Bits[i] = ValueBit(ValueBit::ConstZero);
1248 
1249         return std::make_pair(Interesting = true, &Bits);
1250       }
1251       break;
1252     case ISD::AND:
1253       if (isa<ConstantSDNode>(V.getOperand(1))) {
1254         uint64_t Mask = V.getConstantOperandVal(1);
1255 
1256         const SmallVector<ValueBit, 64> *LHSBits;
1257         // Mark this as interesting, only if the LHS was also interesting. This
1258         // prevents the overall procedure from matching a single immediate 'and'
1259         // (which is non-optimal because such an and might be folded with other
1260         // things if we don't select it here).
1261         std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0), NumBits);
1262 
1263         for (unsigned i = 0; i < NumBits; ++i)
1264           if (((Mask >> i) & 1) == 1)
1265             Bits[i] = (*LHSBits)[i];
1266           else {
1267             // AND instruction masks this bit. If the input is already zero,
1268             // we have nothing to do here. Otherwise, make the bit ConstZero.
1269             if ((*LHSBits)[i].isZero())
1270               Bits[i] = (*LHSBits)[i];
1271             else
1272               Bits[i] = ValueBit(ValueBit::ConstZero);
1273           }
1274 
1275         return std::make_pair(Interesting, &Bits);
1276       }
1277       break;
1278     case ISD::OR: {
1279       const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second;
1280       const auto &RHSBits = *getValueBits(V.getOperand(1), NumBits).second;
1281 
1282       bool AllDisjoint = true;
1283       SDValue LastVal = SDValue();
1284       unsigned LastIdx = 0;
1285       for (unsigned i = 0; i < NumBits; ++i) {
1286         if (LHSBits[i].isZero() && RHSBits[i].isZero()) {
1287           // If both inputs are known to be zero and one is ConstZero and
1288           // another is VariableKnownToBeZero, we can select whichever
1289           // we like. To minimize the number of bit groups, we select
1290           // VariableKnownToBeZero if this bit is the next bit of the same
1291           // input variable from the previous bit. Otherwise, we select
1292           // ConstZero.
1293           if (LHSBits[i].hasValue() && LHSBits[i].getValue() == LastVal &&
1294               LHSBits[i].getValueBitIndex() == LastIdx + 1)
1295             Bits[i] = LHSBits[i];
1296           else if (RHSBits[i].hasValue() && RHSBits[i].getValue() == LastVal &&
1297                    RHSBits[i].getValueBitIndex() == LastIdx + 1)
1298             Bits[i] = RHSBits[i];
1299           else
1300             Bits[i] = ValueBit(ValueBit::ConstZero);
1301         }
1302         else if (LHSBits[i].isZero())
1303           Bits[i] = RHSBits[i];
1304         else if (RHSBits[i].isZero())
1305           Bits[i] = LHSBits[i];
1306         else {
1307           AllDisjoint = false;
1308           break;
1309         }
1310         // We remember the value and bit index of this bit.
1311         if (Bits[i].hasValue()) {
1312           LastVal = Bits[i].getValue();
1313           LastIdx = Bits[i].getValueBitIndex();
1314         }
1315         else {
1316           if (LastVal) LastVal = SDValue();
1317           LastIdx = 0;
1318         }
1319       }
1320 
1321       if (!AllDisjoint)
1322         break;
1323 
1324       return std::make_pair(Interesting = true, &Bits);
1325     }
1326     case ISD::ZERO_EXTEND: {
1327       // We support only the case with zero extension from i32 to i64 so far.
1328       if (V.getValueType() != MVT::i64 ||
1329           V.getOperand(0).getValueType() != MVT::i32)
1330         break;
1331 
1332       const SmallVector<ValueBit, 64> *LHSBits;
1333       const unsigned NumOperandBits = 32;
1334       std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0),
1335                                                     NumOperandBits);
1336 
1337       for (unsigned i = 0; i < NumOperandBits; ++i)
1338         Bits[i] = (*LHSBits)[i];
1339 
1340       for (unsigned i = NumOperandBits; i < NumBits; ++i)
1341         Bits[i] = ValueBit(ValueBit::ConstZero);
1342 
1343       return std::make_pair(Interesting, &Bits);
1344     }
1345     case ISD::TRUNCATE: {
1346       EVT FromType = V.getOperand(0).getValueType();
1347       EVT ToType = V.getValueType();
1348       // We support only the case with truncate from i64 to i32.
1349       if (FromType != MVT::i64 || ToType != MVT::i32)
1350         break;
1351       const unsigned NumAllBits = FromType.getSizeInBits();
1352       SmallVector<ValueBit, 64> *InBits;
1353       std::tie(Interesting, InBits) = getValueBits(V.getOperand(0),
1354                                                     NumAllBits);
1355       const unsigned NumValidBits = ToType.getSizeInBits();
1356 
1357       // A 32-bit instruction cannot touch upper 32-bit part of 64-bit value.
1358       // So, we cannot include this truncate.
1359       bool UseUpper32bit = false;
1360       for (unsigned i = 0; i < NumValidBits; ++i)
1361         if ((*InBits)[i].hasValue() && (*InBits)[i].getValueBitIndex() >= 32) {
1362           UseUpper32bit = true;
1363           break;
1364         }
1365       if (UseUpper32bit)
1366         break;
1367 
1368       for (unsigned i = 0; i < NumValidBits; ++i)
1369         Bits[i] = (*InBits)[i];
1370 
1371       return std::make_pair(Interesting, &Bits);
1372     }
1373     case ISD::AssertZext: {
1374       // For AssertZext, we look through the operand and
1375       // mark the bits known to be zero.
1376       const SmallVector<ValueBit, 64> *LHSBits;
1377       std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0),
1378                                                     NumBits);
1379 
1380       EVT FromType = cast<VTSDNode>(V.getOperand(1))->getVT();
1381       const unsigned NumValidBits = FromType.getSizeInBits();
1382       for (unsigned i = 0; i < NumValidBits; ++i)
1383         Bits[i] = (*LHSBits)[i];
1384 
1385       // These bits are known to be zero but the AssertZext may be from a value
1386       // that already has some constant zero bits (i.e. from a masking and).
1387       for (unsigned i = NumValidBits; i < NumBits; ++i)
1388         Bits[i] = (*LHSBits)[i].hasValue()
1389                       ? ValueBit((*LHSBits)[i].getValue(),
1390                                  (*LHSBits)[i].getValueBitIndex(),
1391                                  ValueBit::VariableKnownToBeZero)
1392                       : ValueBit(ValueBit::ConstZero);
1393 
1394       return std::make_pair(Interesting, &Bits);
1395     }
1396     case ISD::LOAD:
1397       LoadSDNode *LD = cast<LoadSDNode>(V);
1398       if (ISD::isZEXTLoad(V.getNode()) && V.getResNo() == 0) {
1399         EVT VT = LD->getMemoryVT();
1400         const unsigned NumValidBits = VT.getSizeInBits();
1401 
1402         for (unsigned i = 0; i < NumValidBits; ++i)
1403           Bits[i] = ValueBit(V, i);
1404 
1405         // These bits are known to be zero.
1406         for (unsigned i = NumValidBits; i < NumBits; ++i)
1407           Bits[i] = ValueBit(V, i, ValueBit::VariableKnownToBeZero);
1408 
1409         // Zero-extending load itself cannot be optimized. So, it is not
1410         // interesting by itself though it gives useful information.
1411         return std::make_pair(Interesting = false, &Bits);
1412       }
1413       break;
1414     }
1415 
1416     for (unsigned i = 0; i < NumBits; ++i)
1417       Bits[i] = ValueBit(V, i);
1418 
1419     return std::make_pair(Interesting = false, &Bits);
1420   }
1421 
1422   // For each value (except the constant ones), compute the left-rotate amount
1423   // to get it from its original to final position.
1424   void computeRotationAmounts() {
1425     NeedMask = false;
1426     RLAmt.resize(Bits.size());
1427     for (unsigned i = 0; i < Bits.size(); ++i)
1428       if (Bits[i].hasValue()) {
1429         unsigned VBI = Bits[i].getValueBitIndex();
1430         if (i >= VBI)
1431           RLAmt[i] = i - VBI;
1432         else
1433           RLAmt[i] = Bits.size() - (VBI - i);
1434       } else if (Bits[i].isZero()) {
1435         NeedMask = true;
1436         RLAmt[i] = UINT32_MAX;
1437       } else {
1438         llvm_unreachable("Unknown value bit type");
1439       }
1440   }
1441 
1442   // Collect groups of consecutive bits with the same underlying value and
1443   // rotation factor. If we're doing late masking, we ignore zeros, otherwise
1444   // they break up groups.
1445   void collectBitGroups(bool LateMask) {
1446     BitGroups.clear();
1447 
1448     unsigned LastRLAmt = RLAmt[0];
1449     SDValue LastValue = Bits[0].hasValue() ? Bits[0].getValue() : SDValue();
1450     unsigned LastGroupStartIdx = 0;
1451     bool IsGroupOfZeros = !Bits[LastGroupStartIdx].hasValue();
1452     for (unsigned i = 1; i < Bits.size(); ++i) {
1453       unsigned ThisRLAmt = RLAmt[i];
1454       SDValue ThisValue = Bits[i].hasValue() ? Bits[i].getValue() : SDValue();
1455       if (LateMask && !ThisValue) {
1456         ThisValue = LastValue;
1457         ThisRLAmt = LastRLAmt;
1458         // If we're doing late masking, then the first bit group always starts
1459         // at zero (even if the first bits were zero).
1460         if (BitGroups.empty())
1461           LastGroupStartIdx = 0;
1462       }
1463 
1464       // If this bit is known to be zero and the current group is a bit group
1465       // of zeros, we do not need to terminate the current bit group even the
1466       // Value or RLAmt does not match here. Instead, we terminate this group
1467       // when the first non-zero bit appears later.
1468       if (IsGroupOfZeros && Bits[i].isZero())
1469         continue;
1470 
1471       // If this bit has the same underlying value and the same rotate factor as
1472       // the last one, then they're part of the same group.
1473       if (ThisRLAmt == LastRLAmt && ThisValue == LastValue)
1474         // We cannot continue the current group if this bits is not known to
1475         // be zero in a bit group of zeros.
1476         if (!(IsGroupOfZeros && ThisValue && !Bits[i].isZero()))
1477           continue;
1478 
1479       if (LastValue.getNode())
1480         BitGroups.push_back(BitGroup(LastValue, LastRLAmt, LastGroupStartIdx,
1481                                      i-1));
1482       LastRLAmt = ThisRLAmt;
1483       LastValue = ThisValue;
1484       LastGroupStartIdx = i;
1485       IsGroupOfZeros = !Bits[LastGroupStartIdx].hasValue();
1486     }
1487     if (LastValue.getNode())
1488       BitGroups.push_back(BitGroup(LastValue, LastRLAmt, LastGroupStartIdx,
1489                                    Bits.size()-1));
1490 
1491     if (BitGroups.empty())
1492       return;
1493 
1494     // We might be able to combine the first and last groups.
1495     if (BitGroups.size() > 1) {
1496       // If the first and last groups are the same, then remove the first group
1497       // in favor of the last group, making the ending index of the last group
1498       // equal to the ending index of the to-be-removed first group.
1499       if (BitGroups[0].StartIdx == 0 &&
1500           BitGroups[BitGroups.size()-1].EndIdx == Bits.size()-1 &&
1501           BitGroups[0].V == BitGroups[BitGroups.size()-1].V &&
1502           BitGroups[0].RLAmt == BitGroups[BitGroups.size()-1].RLAmt) {
1503         LLVM_DEBUG(dbgs() << "\tcombining final bit group with initial one\n");
1504         BitGroups[BitGroups.size()-1].EndIdx = BitGroups[0].EndIdx;
1505         BitGroups.erase(BitGroups.begin());
1506       }
1507     }
1508   }
1509 
1510   // Take all (SDValue, RLAmt) pairs and sort them by the number of groups
1511   // associated with each. If the number of groups are same, we prefer a group
1512   // which does not require rotate, i.e. RLAmt is 0, to avoid the first rotate
1513   // instruction. If there is a degeneracy, pick the one that occurs
1514   // first (in the final value).
1515   void collectValueRotInfo() {
1516     ValueRots.clear();
1517 
1518     for (auto &BG : BitGroups) {
1519       unsigned RLAmtKey = BG.RLAmt + (BG.Repl32 ? 64 : 0);
1520       ValueRotInfo &VRI = ValueRots[std::make_pair(BG.V, RLAmtKey)];
1521       VRI.V = BG.V;
1522       VRI.RLAmt = BG.RLAmt;
1523       VRI.Repl32 = BG.Repl32;
1524       VRI.NumGroups += 1;
1525       VRI.FirstGroupStartIdx = std::min(VRI.FirstGroupStartIdx, BG.StartIdx);
1526     }
1527 
1528     // Now that we've collected the various ValueRotInfo instances, we need to
1529     // sort them.
1530     ValueRotsVec.clear();
1531     for (auto &I : ValueRots) {
1532       ValueRotsVec.push_back(I.second);
1533     }
1534     llvm::sort(ValueRotsVec);
1535   }
1536 
1537   // In 64-bit mode, rlwinm and friends have a rotation operator that
1538   // replicates the low-order 32 bits into the high-order 32-bits. The mask
1539   // indices of these instructions can only be in the lower 32 bits, so they
1540   // can only represent some 64-bit bit groups. However, when they can be used,
1541   // the 32-bit replication can be used to represent, as a single bit group,
1542   // otherwise separate bit groups. We'll convert to replicated-32-bit bit
1543   // groups when possible. Returns true if any of the bit groups were
1544   // converted.
1545   void assignRepl32BitGroups() {
1546     // If we have bits like this:
1547     //
1548     // Indices:    15 14 13 12 11 10 9 8  7  6  5  4  3  2  1  0
1549     // V bits: ... 7  6  5  4  3  2  1 0 31 30 29 28 27 26 25 24
1550     // Groups:    |      RLAmt = 8      |      RLAmt = 40       |
1551     //
1552     // But, making use of a 32-bit operation that replicates the low-order 32
1553     // bits into the high-order 32 bits, this can be one bit group with a RLAmt
1554     // of 8.
1555 
1556     auto IsAllLow32 = [this](BitGroup & BG) {
1557       if (BG.StartIdx <= BG.EndIdx) {
1558         for (unsigned i = BG.StartIdx; i <= BG.EndIdx; ++i) {
1559           if (!Bits[i].hasValue())
1560             continue;
1561           if (Bits[i].getValueBitIndex() >= 32)
1562             return false;
1563         }
1564       } else {
1565         for (unsigned i = BG.StartIdx; i < Bits.size(); ++i) {
1566           if (!Bits[i].hasValue())
1567             continue;
1568           if (Bits[i].getValueBitIndex() >= 32)
1569             return false;
1570         }
1571         for (unsigned i = 0; i <= BG.EndIdx; ++i) {
1572           if (!Bits[i].hasValue())
1573             continue;
1574           if (Bits[i].getValueBitIndex() >= 32)
1575             return false;
1576         }
1577       }
1578 
1579       return true;
1580     };
1581 
1582     for (auto &BG : BitGroups) {
1583       // If this bit group has RLAmt of 0 and will not be merged with
1584       // another bit group, we don't benefit from Repl32. We don't mark
1585       // such group to give more freedom for later instruction selection.
1586       if (BG.RLAmt == 0) {
1587         auto PotentiallyMerged = [this](BitGroup & BG) {
1588           for (auto &BG2 : BitGroups)
1589             if (&BG != &BG2 && BG.V == BG2.V &&
1590                 (BG2.RLAmt == 0 || BG2.RLAmt == 32))
1591               return true;
1592           return false;
1593         };
1594         if (!PotentiallyMerged(BG))
1595           continue;
1596       }
1597       if (BG.StartIdx < 32 && BG.EndIdx < 32) {
1598         if (IsAllLow32(BG)) {
1599           if (BG.RLAmt >= 32) {
1600             BG.RLAmt -= 32;
1601             BG.Repl32CR = true;
1602           }
1603 
1604           BG.Repl32 = true;
1605 
1606           LLVM_DEBUG(dbgs() << "\t32-bit replicated bit group for "
1607                             << BG.V.getNode() << " RLAmt = " << BG.RLAmt << " ["
1608                             << BG.StartIdx << ", " << BG.EndIdx << "]\n");
1609         }
1610       }
1611     }
1612 
1613     // Now walk through the bit groups, consolidating where possible.
1614     for (auto I = BitGroups.begin(); I != BitGroups.end();) {
1615       // We might want to remove this bit group by merging it with the previous
1616       // group (which might be the ending group).
1617       auto IP = (I == BitGroups.begin()) ?
1618                 std::prev(BitGroups.end()) : std::prev(I);
1619       if (I->Repl32 && IP->Repl32 && I->V == IP->V && I->RLAmt == IP->RLAmt &&
1620           I->StartIdx == (IP->EndIdx + 1) % 64 && I != IP) {
1621 
1622         LLVM_DEBUG(dbgs() << "\tcombining 32-bit replicated bit group for "
1623                           << I->V.getNode() << " RLAmt = " << I->RLAmt << " ["
1624                           << I->StartIdx << ", " << I->EndIdx
1625                           << "] with group with range [" << IP->StartIdx << ", "
1626                           << IP->EndIdx << "]\n");
1627 
1628         IP->EndIdx = I->EndIdx;
1629         IP->Repl32CR = IP->Repl32CR || I->Repl32CR;
1630         IP->Repl32Coalesced = true;
1631         I = BitGroups.erase(I);
1632         continue;
1633       } else {
1634         // There is a special case worth handling: If there is a single group
1635         // covering the entire upper 32 bits, and it can be merged with both
1636         // the next and previous groups (which might be the same group), then
1637         // do so. If it is the same group (so there will be only one group in
1638         // total), then we need to reverse the order of the range so that it
1639         // covers the entire 64 bits.
1640         if (I->StartIdx == 32 && I->EndIdx == 63) {
1641           assert(std::next(I) == BitGroups.end() &&
1642                  "bit group ends at index 63 but there is another?");
1643           auto IN = BitGroups.begin();
1644 
1645           if (IP->Repl32 && IN->Repl32 && I->V == IP->V && I->V == IN->V &&
1646               (I->RLAmt % 32) == IP->RLAmt && (I->RLAmt % 32) == IN->RLAmt &&
1647               IP->EndIdx == 31 && IN->StartIdx == 0 && I != IP &&
1648               IsAllLow32(*I)) {
1649 
1650             LLVM_DEBUG(dbgs() << "\tcombining bit group for " << I->V.getNode()
1651                               << " RLAmt = " << I->RLAmt << " [" << I->StartIdx
1652                               << ", " << I->EndIdx
1653                               << "] with 32-bit replicated groups with ranges ["
1654                               << IP->StartIdx << ", " << IP->EndIdx << "] and ["
1655                               << IN->StartIdx << ", " << IN->EndIdx << "]\n");
1656 
1657             if (IP == IN) {
1658               // There is only one other group; change it to cover the whole
1659               // range (backward, so that it can still be Repl32 but cover the
1660               // whole 64-bit range).
1661               IP->StartIdx = 31;
1662               IP->EndIdx = 30;
1663               IP->Repl32CR = IP->Repl32CR || I->RLAmt >= 32;
1664               IP->Repl32Coalesced = true;
1665               I = BitGroups.erase(I);
1666             } else {
1667               // There are two separate groups, one before this group and one
1668               // after us (at the beginning). We're going to remove this group,
1669               // but also the group at the very beginning.
1670               IP->EndIdx = IN->EndIdx;
1671               IP->Repl32CR = IP->Repl32CR || IN->Repl32CR || I->RLAmt >= 32;
1672               IP->Repl32Coalesced = true;
1673               I = BitGroups.erase(I);
1674               BitGroups.erase(BitGroups.begin());
1675             }
1676 
1677             // This must be the last group in the vector (and we might have
1678             // just invalidated the iterator above), so break here.
1679             break;
1680           }
1681         }
1682       }
1683 
1684       ++I;
1685     }
1686   }
1687 
1688   SDValue getI32Imm(unsigned Imm, const SDLoc &dl) {
1689     return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
1690   }
1691 
1692   uint64_t getZerosMask() {
1693     uint64_t Mask = 0;
1694     for (unsigned i = 0; i < Bits.size(); ++i) {
1695       if (Bits[i].hasValue())
1696         continue;
1697       Mask |= (UINT64_C(1) << i);
1698     }
1699 
1700     return ~Mask;
1701   }
1702 
1703   // This method extends an input value to 64 bit if input is 32-bit integer.
1704   // While selecting instructions in BitPermutationSelector in 64-bit mode,
1705   // an input value can be a 32-bit integer if a ZERO_EXTEND node is included.
1706   // In such case, we extend it to 64 bit to be consistent with other values.
1707   SDValue ExtendToInt64(SDValue V, const SDLoc &dl) {
1708     if (V.getValueSizeInBits() == 64)
1709       return V;
1710 
1711     assert(V.getValueSizeInBits() == 32);
1712     SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32);
1713     SDValue ImDef = SDValue(CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl,
1714                                                    MVT::i64), 0);
1715     SDValue ExtVal = SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl,
1716                                                     MVT::i64, ImDef, V,
1717                                                     SubRegIdx), 0);
1718     return ExtVal;
1719   }
1720 
1721   SDValue TruncateToInt32(SDValue V, const SDLoc &dl) {
1722     if (V.getValueSizeInBits() == 32)
1723       return V;
1724 
1725     assert(V.getValueSizeInBits() == 64);
1726     SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32);
1727     SDValue SubVal = SDValue(CurDAG->getMachineNode(PPC::EXTRACT_SUBREG, dl,
1728                                                     MVT::i32, V, SubRegIdx), 0);
1729     return SubVal;
1730   }
1731 
1732   // Depending on the number of groups for a particular value, it might be
1733   // better to rotate, mask explicitly (using andi/andis), and then or the
1734   // result. Select this part of the result first.
1735   void SelectAndParts32(const SDLoc &dl, SDValue &Res, unsigned *InstCnt) {
1736     if (BPermRewriterNoMasking)
1737       return;
1738 
1739     for (ValueRotInfo &VRI : ValueRotsVec) {
1740       unsigned Mask = 0;
1741       for (unsigned i = 0; i < Bits.size(); ++i) {
1742         if (!Bits[i].hasValue() || Bits[i].getValue() != VRI.V)
1743           continue;
1744         if (RLAmt[i] != VRI.RLAmt)
1745           continue;
1746         Mask |= (1u << i);
1747       }
1748 
1749       // Compute the masks for andi/andis that would be necessary.
1750       unsigned ANDIMask = (Mask & UINT16_MAX), ANDISMask = Mask >> 16;
1751       assert((ANDIMask != 0 || ANDISMask != 0) &&
1752              "No set bits in mask for value bit groups");
1753       bool NeedsRotate = VRI.RLAmt != 0;
1754 
1755       // We're trying to minimize the number of instructions. If we have one
1756       // group, using one of andi/andis can break even.  If we have three
1757       // groups, we can use both andi and andis and break even (to use both
1758       // andi and andis we also need to or the results together). We need four
1759       // groups if we also need to rotate. To use andi/andis we need to do more
1760       // than break even because rotate-and-mask instructions tend to be easier
1761       // to schedule.
1762 
1763       // FIXME: We've biased here against using andi/andis, which is right for
1764       // POWER cores, but not optimal everywhere. For example, on the A2,
1765       // andi/andis have single-cycle latency whereas the rotate-and-mask
1766       // instructions take two cycles, and it would be better to bias toward
1767       // andi/andis in break-even cases.
1768 
1769       unsigned NumAndInsts = (unsigned) NeedsRotate +
1770                              (unsigned) (ANDIMask != 0) +
1771                              (unsigned) (ANDISMask != 0) +
1772                              (unsigned) (ANDIMask != 0 && ANDISMask != 0) +
1773                              (unsigned) (bool) Res;
1774 
1775       LLVM_DEBUG(dbgs() << "\t\trotation groups for " << VRI.V.getNode()
1776                         << " RL: " << VRI.RLAmt << ":"
1777                         << "\n\t\t\tisel using masking: " << NumAndInsts
1778                         << " using rotates: " << VRI.NumGroups << "\n");
1779 
1780       if (NumAndInsts >= VRI.NumGroups)
1781         continue;
1782 
1783       LLVM_DEBUG(dbgs() << "\t\t\t\tusing masking\n");
1784 
1785       if (InstCnt) *InstCnt += NumAndInsts;
1786 
1787       SDValue VRot;
1788       if (VRI.RLAmt) {
1789         SDValue Ops[] =
1790           { TruncateToInt32(VRI.V, dl), getI32Imm(VRI.RLAmt, dl),
1791             getI32Imm(0, dl), getI32Imm(31, dl) };
1792         VRot = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32,
1793                                               Ops), 0);
1794       } else {
1795         VRot = TruncateToInt32(VRI.V, dl);
1796       }
1797 
1798       SDValue ANDIVal, ANDISVal;
1799       if (ANDIMask != 0)
1800         ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI_rec, dl, MVT::i32,
1801                                                  VRot, getI32Imm(ANDIMask, dl)),
1802                           0);
1803       if (ANDISMask != 0)
1804         ANDISVal =
1805             SDValue(CurDAG->getMachineNode(PPC::ANDIS_rec, dl, MVT::i32, VRot,
1806                                            getI32Imm(ANDISMask, dl)),
1807                     0);
1808 
1809       SDValue TotalVal;
1810       if (!ANDIVal)
1811         TotalVal = ANDISVal;
1812       else if (!ANDISVal)
1813         TotalVal = ANDIVal;
1814       else
1815         TotalVal = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32,
1816                              ANDIVal, ANDISVal), 0);
1817 
1818       if (!Res)
1819         Res = TotalVal;
1820       else
1821         Res = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32,
1822                         Res, TotalVal), 0);
1823 
1824       // Now, remove all groups with this underlying value and rotation
1825       // factor.
1826       eraseMatchingBitGroups([VRI](const BitGroup &BG) {
1827         return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt;
1828       });
1829     }
1830   }
1831 
1832   // Instruction selection for the 32-bit case.
1833   SDNode *Select32(SDNode *N, bool LateMask, unsigned *InstCnt) {
1834     SDLoc dl(N);
1835     SDValue Res;
1836 
1837     if (InstCnt) *InstCnt = 0;
1838 
1839     // Take care of cases that should use andi/andis first.
1840     SelectAndParts32(dl, Res, InstCnt);
1841 
1842     // If we've not yet selected a 'starting' instruction, and we have no zeros
1843     // to fill in, select the (Value, RLAmt) with the highest priority (largest
1844     // number of groups), and start with this rotated value.
1845     if ((!NeedMask || LateMask) && !Res) {
1846       ValueRotInfo &VRI = ValueRotsVec[0];
1847       if (VRI.RLAmt) {
1848         if (InstCnt) *InstCnt += 1;
1849         SDValue Ops[] =
1850           { TruncateToInt32(VRI.V, dl), getI32Imm(VRI.RLAmt, dl),
1851             getI32Imm(0, dl), getI32Imm(31, dl) };
1852         Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops),
1853                       0);
1854       } else {
1855         Res = TruncateToInt32(VRI.V, dl);
1856       }
1857 
1858       // Now, remove all groups with this underlying value and rotation factor.
1859       eraseMatchingBitGroups([VRI](const BitGroup &BG) {
1860         return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt;
1861       });
1862     }
1863 
1864     if (InstCnt) *InstCnt += BitGroups.size();
1865 
1866     // Insert the other groups (one at a time).
1867     for (auto &BG : BitGroups) {
1868       if (!Res) {
1869         SDValue Ops[] =
1870           { TruncateToInt32(BG.V, dl), getI32Imm(BG.RLAmt, dl),
1871             getI32Imm(Bits.size() - BG.EndIdx - 1, dl),
1872             getI32Imm(Bits.size() - BG.StartIdx - 1, dl) };
1873         Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0);
1874       } else {
1875         SDValue Ops[] =
1876           { Res, TruncateToInt32(BG.V, dl), getI32Imm(BG.RLAmt, dl),
1877               getI32Imm(Bits.size() - BG.EndIdx - 1, dl),
1878             getI32Imm(Bits.size() - BG.StartIdx - 1, dl) };
1879         Res = SDValue(CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops), 0);
1880       }
1881     }
1882 
1883     if (LateMask) {
1884       unsigned Mask = (unsigned) getZerosMask();
1885 
1886       unsigned ANDIMask = (Mask & UINT16_MAX), ANDISMask = Mask >> 16;
1887       assert((ANDIMask != 0 || ANDISMask != 0) &&
1888              "No set bits in zeros mask?");
1889 
1890       if (InstCnt) *InstCnt += (unsigned) (ANDIMask != 0) +
1891                                (unsigned) (ANDISMask != 0) +
1892                                (unsigned) (ANDIMask != 0 && ANDISMask != 0);
1893 
1894       SDValue ANDIVal, ANDISVal;
1895       if (ANDIMask != 0)
1896         ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI_rec, dl, MVT::i32,
1897                                                  Res, getI32Imm(ANDIMask, dl)),
1898                           0);
1899       if (ANDISMask != 0)
1900         ANDISVal =
1901             SDValue(CurDAG->getMachineNode(PPC::ANDIS_rec, dl, MVT::i32, Res,
1902                                            getI32Imm(ANDISMask, dl)),
1903                     0);
1904 
1905       if (!ANDIVal)
1906         Res = ANDISVal;
1907       else if (!ANDISVal)
1908         Res = ANDIVal;
1909       else
1910         Res = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32,
1911                         ANDIVal, ANDISVal), 0);
1912     }
1913 
1914     return Res.getNode();
1915   }
1916 
1917   unsigned SelectRotMask64Count(unsigned RLAmt, bool Repl32,
1918                                 unsigned MaskStart, unsigned MaskEnd,
1919                                 bool IsIns) {
1920     // In the notation used by the instructions, 'start' and 'end' are reversed
1921     // because bits are counted from high to low order.
1922     unsigned InstMaskStart = 64 - MaskEnd - 1,
1923              InstMaskEnd   = 64 - MaskStart - 1;
1924 
1925     if (Repl32)
1926       return 1;
1927 
1928     if ((!IsIns && (InstMaskEnd == 63 || InstMaskStart == 0)) ||
1929         InstMaskEnd == 63 - RLAmt)
1930       return 1;
1931 
1932     return 2;
1933   }
1934 
1935   // For 64-bit values, not all combinations of rotates and masks are
1936   // available. Produce one if it is available.
1937   SDValue SelectRotMask64(SDValue V, const SDLoc &dl, unsigned RLAmt,
1938                           bool Repl32, unsigned MaskStart, unsigned MaskEnd,
1939                           unsigned *InstCnt = nullptr) {
1940     // In the notation used by the instructions, 'start' and 'end' are reversed
1941     // because bits are counted from high to low order.
1942     unsigned InstMaskStart = 64 - MaskEnd - 1,
1943              InstMaskEnd   = 64 - MaskStart - 1;
1944 
1945     if (InstCnt) *InstCnt += 1;
1946 
1947     if (Repl32) {
1948       // This rotation amount assumes that the lower 32 bits of the quantity
1949       // are replicated in the high 32 bits by the rotation operator (which is
1950       // done by rlwinm and friends).
1951       assert(InstMaskStart >= 32 && "Mask cannot start out of range");
1952       assert(InstMaskEnd   >= 32 && "Mask cannot end out of range");
1953       SDValue Ops[] =
1954         { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
1955           getI32Imm(InstMaskStart - 32, dl), getI32Imm(InstMaskEnd - 32, dl) };
1956       return SDValue(CurDAG->getMachineNode(PPC::RLWINM8, dl, MVT::i64,
1957                                             Ops), 0);
1958     }
1959 
1960     if (InstMaskEnd == 63) {
1961       SDValue Ops[] =
1962         { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
1963           getI32Imm(InstMaskStart, dl) };
1964       return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Ops), 0);
1965     }
1966 
1967     if (InstMaskStart == 0) {
1968       SDValue Ops[] =
1969         { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
1970           getI32Imm(InstMaskEnd, dl) };
1971       return SDValue(CurDAG->getMachineNode(PPC::RLDICR, dl, MVT::i64, Ops), 0);
1972     }
1973 
1974     if (InstMaskEnd == 63 - RLAmt) {
1975       SDValue Ops[] =
1976         { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
1977           getI32Imm(InstMaskStart, dl) };
1978       return SDValue(CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, Ops), 0);
1979     }
1980 
1981     // We cannot do this with a single instruction, so we'll use two. The
1982     // problem is that we're not free to choose both a rotation amount and mask
1983     // start and end independently. We can choose an arbitrary mask start and
1984     // end, but then the rotation amount is fixed. Rotation, however, can be
1985     // inverted, and so by applying an "inverse" rotation first, we can get the
1986     // desired result.
1987     if (InstCnt) *InstCnt += 1;
1988 
1989     // The rotation mask for the second instruction must be MaskStart.
1990     unsigned RLAmt2 = MaskStart;
1991     // The first instruction must rotate V so that the overall rotation amount
1992     // is RLAmt.
1993     unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64;
1994     if (RLAmt1)
1995       V = SelectRotMask64(V, dl, RLAmt1, false, 0, 63);
1996     return SelectRotMask64(V, dl, RLAmt2, false, MaskStart, MaskEnd);
1997   }
1998 
1999   // For 64-bit values, not all combinations of rotates and masks are
2000   // available. Produce a rotate-mask-and-insert if one is available.
2001   SDValue SelectRotMaskIns64(SDValue Base, SDValue V, const SDLoc &dl,
2002                              unsigned RLAmt, bool Repl32, unsigned MaskStart,
2003                              unsigned MaskEnd, unsigned *InstCnt = nullptr) {
2004     // In the notation used by the instructions, 'start' and 'end' are reversed
2005     // because bits are counted from high to low order.
2006     unsigned InstMaskStart = 64 - MaskEnd - 1,
2007              InstMaskEnd   = 64 - MaskStart - 1;
2008 
2009     if (InstCnt) *InstCnt += 1;
2010 
2011     if (Repl32) {
2012       // This rotation amount assumes that the lower 32 bits of the quantity
2013       // are replicated in the high 32 bits by the rotation operator (which is
2014       // done by rlwinm and friends).
2015       assert(InstMaskStart >= 32 && "Mask cannot start out of range");
2016       assert(InstMaskEnd   >= 32 && "Mask cannot end out of range");
2017       SDValue Ops[] =
2018         { ExtendToInt64(Base, dl), ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2019           getI32Imm(InstMaskStart - 32, dl), getI32Imm(InstMaskEnd - 32, dl) };
2020       return SDValue(CurDAG->getMachineNode(PPC::RLWIMI8, dl, MVT::i64,
2021                                             Ops), 0);
2022     }
2023 
2024     if (InstMaskEnd == 63 - RLAmt) {
2025       SDValue Ops[] =
2026         { ExtendToInt64(Base, dl), ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2027           getI32Imm(InstMaskStart, dl) };
2028       return SDValue(CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops), 0);
2029     }
2030 
2031     // We cannot do this with a single instruction, so we'll use two. The
2032     // problem is that we're not free to choose both a rotation amount and mask
2033     // start and end independently. We can choose an arbitrary mask start and
2034     // end, but then the rotation amount is fixed. Rotation, however, can be
2035     // inverted, and so by applying an "inverse" rotation first, we can get the
2036     // desired result.
2037     if (InstCnt) *InstCnt += 1;
2038 
2039     // The rotation mask for the second instruction must be MaskStart.
2040     unsigned RLAmt2 = MaskStart;
2041     // The first instruction must rotate V so that the overall rotation amount
2042     // is RLAmt.
2043     unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64;
2044     if (RLAmt1)
2045       V = SelectRotMask64(V, dl, RLAmt1, false, 0, 63);
2046     return SelectRotMaskIns64(Base, V, dl, RLAmt2, false, MaskStart, MaskEnd);
2047   }
2048 
2049   void SelectAndParts64(const SDLoc &dl, SDValue &Res, unsigned *InstCnt) {
2050     if (BPermRewriterNoMasking)
2051       return;
2052 
2053     // The idea here is the same as in the 32-bit version, but with additional
2054     // complications from the fact that Repl32 might be true. Because we
2055     // aggressively convert bit groups to Repl32 form (which, for small
2056     // rotation factors, involves no other change), and then coalesce, it might
2057     // be the case that a single 64-bit masking operation could handle both
2058     // some Repl32 groups and some non-Repl32 groups. If converting to Repl32
2059     // form allowed coalescing, then we must use a 32-bit rotaton in order to
2060     // completely capture the new combined bit group.
2061 
2062     for (ValueRotInfo &VRI : ValueRotsVec) {
2063       uint64_t Mask = 0;
2064 
2065       // We need to add to the mask all bits from the associated bit groups.
2066       // If Repl32 is false, we need to add bits from bit groups that have
2067       // Repl32 true, but are trivially convertable to Repl32 false. Such a
2068       // group is trivially convertable if it overlaps only with the lower 32
2069       // bits, and the group has not been coalesced.
2070       auto MatchingBG = [VRI](const BitGroup &BG) {
2071         if (VRI.V != BG.V)
2072           return false;
2073 
2074         unsigned EffRLAmt = BG.RLAmt;
2075         if (!VRI.Repl32 && BG.Repl32) {
2076           if (BG.StartIdx < 32 && BG.EndIdx < 32 && BG.StartIdx <= BG.EndIdx &&
2077               !BG.Repl32Coalesced) {
2078             if (BG.Repl32CR)
2079               EffRLAmt += 32;
2080           } else {
2081             return false;
2082           }
2083         } else if (VRI.Repl32 != BG.Repl32) {
2084           return false;
2085         }
2086 
2087         return VRI.RLAmt == EffRLAmt;
2088       };
2089 
2090       for (auto &BG : BitGroups) {
2091         if (!MatchingBG(BG))
2092           continue;
2093 
2094         if (BG.StartIdx <= BG.EndIdx) {
2095           for (unsigned i = BG.StartIdx; i <= BG.EndIdx; ++i)
2096             Mask |= (UINT64_C(1) << i);
2097         } else {
2098           for (unsigned i = BG.StartIdx; i < Bits.size(); ++i)
2099             Mask |= (UINT64_C(1) << i);
2100           for (unsigned i = 0; i <= BG.EndIdx; ++i)
2101             Mask |= (UINT64_C(1) << i);
2102         }
2103       }
2104 
2105       // We can use the 32-bit andi/andis technique if the mask does not
2106       // require any higher-order bits. This can save an instruction compared
2107       // to always using the general 64-bit technique.
2108       bool Use32BitInsts = isUInt<32>(Mask);
2109       // Compute the masks for andi/andis that would be necessary.
2110       unsigned ANDIMask = (Mask & UINT16_MAX),
2111                ANDISMask = (Mask >> 16) & UINT16_MAX;
2112 
2113       bool NeedsRotate = VRI.RLAmt || (VRI.Repl32 && !isUInt<32>(Mask));
2114 
2115       unsigned NumAndInsts = (unsigned) NeedsRotate +
2116                              (unsigned) (bool) Res;
2117       unsigned NumOfSelectInsts = 0;
2118       selectI64Imm(CurDAG, dl, Mask, &NumOfSelectInsts);
2119       assert(NumOfSelectInsts > 0 && "Failed to select an i64 constant.");
2120       if (Use32BitInsts)
2121         NumAndInsts += (unsigned) (ANDIMask != 0) + (unsigned) (ANDISMask != 0) +
2122                        (unsigned) (ANDIMask != 0 && ANDISMask != 0);
2123       else
2124         NumAndInsts += NumOfSelectInsts + /* and */ 1;
2125 
2126       unsigned NumRLInsts = 0;
2127       bool FirstBG = true;
2128       bool MoreBG = false;
2129       for (auto &BG : BitGroups) {
2130         if (!MatchingBG(BG)) {
2131           MoreBG = true;
2132           continue;
2133         }
2134         NumRLInsts +=
2135           SelectRotMask64Count(BG.RLAmt, BG.Repl32, BG.StartIdx, BG.EndIdx,
2136                                !FirstBG);
2137         FirstBG = false;
2138       }
2139 
2140       LLVM_DEBUG(dbgs() << "\t\trotation groups for " << VRI.V.getNode()
2141                         << " RL: " << VRI.RLAmt << (VRI.Repl32 ? " (32):" : ":")
2142                         << "\n\t\t\tisel using masking: " << NumAndInsts
2143                         << " using rotates: " << NumRLInsts << "\n");
2144 
2145       // When we'd use andi/andis, we bias toward using the rotates (andi only
2146       // has a record form, and is cracked on POWER cores). However, when using
2147       // general 64-bit constant formation, bias toward the constant form,
2148       // because that exposes more opportunities for CSE.
2149       if (NumAndInsts > NumRLInsts)
2150         continue;
2151       // When merging multiple bit groups, instruction or is used.
2152       // But when rotate is used, rldimi can inert the rotated value into any
2153       // register, so instruction or can be avoided.
2154       if ((Use32BitInsts || MoreBG) && NumAndInsts == NumRLInsts)
2155         continue;
2156 
2157       LLVM_DEBUG(dbgs() << "\t\t\t\tusing masking\n");
2158 
2159       if (InstCnt) *InstCnt += NumAndInsts;
2160 
2161       SDValue VRot;
2162       // We actually need to generate a rotation if we have a non-zero rotation
2163       // factor or, in the Repl32 case, if we care about any of the
2164       // higher-order replicated bits. In the latter case, we generate a mask
2165       // backward so that it actually includes the entire 64 bits.
2166       if (VRI.RLAmt || (VRI.Repl32 && !isUInt<32>(Mask)))
2167         VRot = SelectRotMask64(VRI.V, dl, VRI.RLAmt, VRI.Repl32,
2168                                VRI.Repl32 ? 31 : 0, VRI.Repl32 ? 30 : 63);
2169       else
2170         VRot = VRI.V;
2171 
2172       SDValue TotalVal;
2173       if (Use32BitInsts) {
2174         assert((ANDIMask != 0 || ANDISMask != 0) &&
2175                "No set bits in mask when using 32-bit ands for 64-bit value");
2176 
2177         SDValue ANDIVal, ANDISVal;
2178         if (ANDIMask != 0)
2179           ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI8_rec, dl, MVT::i64,
2180                                                    ExtendToInt64(VRot, dl),
2181                                                    getI32Imm(ANDIMask, dl)),
2182                             0);
2183         if (ANDISMask != 0)
2184           ANDISVal =
2185               SDValue(CurDAG->getMachineNode(PPC::ANDIS8_rec, dl, MVT::i64,
2186                                              ExtendToInt64(VRot, dl),
2187                                              getI32Imm(ANDISMask, dl)),
2188                       0);
2189 
2190         if (!ANDIVal)
2191           TotalVal = ANDISVal;
2192         else if (!ANDISVal)
2193           TotalVal = ANDIVal;
2194         else
2195           TotalVal = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64,
2196                                ExtendToInt64(ANDIVal, dl), ANDISVal), 0);
2197       } else {
2198         TotalVal = SDValue(selectI64Imm(CurDAG, dl, Mask), 0);
2199         TotalVal =
2200           SDValue(CurDAG->getMachineNode(PPC::AND8, dl, MVT::i64,
2201                                          ExtendToInt64(VRot, dl), TotalVal),
2202                   0);
2203      }
2204 
2205       if (!Res)
2206         Res = TotalVal;
2207       else
2208         Res = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64,
2209                                              ExtendToInt64(Res, dl), TotalVal),
2210                       0);
2211 
2212       // Now, remove all groups with this underlying value and rotation
2213       // factor.
2214       eraseMatchingBitGroups(MatchingBG);
2215     }
2216   }
2217 
2218   // Instruction selection for the 64-bit case.
2219   SDNode *Select64(SDNode *N, bool LateMask, unsigned *InstCnt) {
2220     SDLoc dl(N);
2221     SDValue Res;
2222 
2223     if (InstCnt) *InstCnt = 0;
2224 
2225     // Take care of cases that should use andi/andis first.
2226     SelectAndParts64(dl, Res, InstCnt);
2227 
2228     // If we've not yet selected a 'starting' instruction, and we have no zeros
2229     // to fill in, select the (Value, RLAmt) with the highest priority (largest
2230     // number of groups), and start with this rotated value.
2231     if ((!NeedMask || LateMask) && !Res) {
2232       // If we have both Repl32 groups and non-Repl32 groups, the non-Repl32
2233       // groups will come first, and so the VRI representing the largest number
2234       // of groups might not be first (it might be the first Repl32 groups).
2235       unsigned MaxGroupsIdx = 0;
2236       if (!ValueRotsVec[0].Repl32) {
2237         for (unsigned i = 0, ie = ValueRotsVec.size(); i < ie; ++i)
2238           if (ValueRotsVec[i].Repl32) {
2239             if (ValueRotsVec[i].NumGroups > ValueRotsVec[0].NumGroups)
2240               MaxGroupsIdx = i;
2241             break;
2242           }
2243       }
2244 
2245       ValueRotInfo &VRI = ValueRotsVec[MaxGroupsIdx];
2246       bool NeedsRotate = false;
2247       if (VRI.RLAmt) {
2248         NeedsRotate = true;
2249       } else if (VRI.Repl32) {
2250         for (auto &BG : BitGroups) {
2251           if (BG.V != VRI.V || BG.RLAmt != VRI.RLAmt ||
2252               BG.Repl32 != VRI.Repl32)
2253             continue;
2254 
2255           // We don't need a rotate if the bit group is confined to the lower
2256           // 32 bits.
2257           if (BG.StartIdx < 32 && BG.EndIdx < 32 && BG.StartIdx < BG.EndIdx)
2258             continue;
2259 
2260           NeedsRotate = true;
2261           break;
2262         }
2263       }
2264 
2265       if (NeedsRotate)
2266         Res = SelectRotMask64(VRI.V, dl, VRI.RLAmt, VRI.Repl32,
2267                               VRI.Repl32 ? 31 : 0, VRI.Repl32 ? 30 : 63,
2268                               InstCnt);
2269       else
2270         Res = VRI.V;
2271 
2272       // Now, remove all groups with this underlying value and rotation factor.
2273       if (Res)
2274         eraseMatchingBitGroups([VRI](const BitGroup &BG) {
2275           return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt &&
2276                  BG.Repl32 == VRI.Repl32;
2277         });
2278     }
2279 
2280     // Because 64-bit rotates are more flexible than inserts, we might have a
2281     // preference regarding which one we do first (to save one instruction).
2282     if (!Res)
2283       for (auto I = BitGroups.begin(), IE = BitGroups.end(); I != IE; ++I) {
2284         if (SelectRotMask64Count(I->RLAmt, I->Repl32, I->StartIdx, I->EndIdx,
2285                                 false) <
2286             SelectRotMask64Count(I->RLAmt, I->Repl32, I->StartIdx, I->EndIdx,
2287                                 true)) {
2288           if (I != BitGroups.begin()) {
2289             BitGroup BG = *I;
2290             BitGroups.erase(I);
2291             BitGroups.insert(BitGroups.begin(), BG);
2292           }
2293 
2294           break;
2295         }
2296       }
2297 
2298     // Insert the other groups (one at a time).
2299     for (auto &BG : BitGroups) {
2300       if (!Res)
2301         Res = SelectRotMask64(BG.V, dl, BG.RLAmt, BG.Repl32, BG.StartIdx,
2302                               BG.EndIdx, InstCnt);
2303       else
2304         Res = SelectRotMaskIns64(Res, BG.V, dl, BG.RLAmt, BG.Repl32,
2305                                  BG.StartIdx, BG.EndIdx, InstCnt);
2306     }
2307 
2308     if (LateMask) {
2309       uint64_t Mask = getZerosMask();
2310 
2311       // We can use the 32-bit andi/andis technique if the mask does not
2312       // require any higher-order bits. This can save an instruction compared
2313       // to always using the general 64-bit technique.
2314       bool Use32BitInsts = isUInt<32>(Mask);
2315       // Compute the masks for andi/andis that would be necessary.
2316       unsigned ANDIMask = (Mask & UINT16_MAX),
2317                ANDISMask = (Mask >> 16) & UINT16_MAX;
2318 
2319       if (Use32BitInsts) {
2320         assert((ANDIMask != 0 || ANDISMask != 0) &&
2321                "No set bits in mask when using 32-bit ands for 64-bit value");
2322 
2323         if (InstCnt) *InstCnt += (unsigned) (ANDIMask != 0) +
2324                                  (unsigned) (ANDISMask != 0) +
2325                                  (unsigned) (ANDIMask != 0 && ANDISMask != 0);
2326 
2327         SDValue ANDIVal, ANDISVal;
2328         if (ANDIMask != 0)
2329           ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI8_rec, dl, MVT::i64,
2330                                                    ExtendToInt64(Res, dl),
2331                                                    getI32Imm(ANDIMask, dl)),
2332                             0);
2333         if (ANDISMask != 0)
2334           ANDISVal =
2335               SDValue(CurDAG->getMachineNode(PPC::ANDIS8_rec, dl, MVT::i64,
2336                                              ExtendToInt64(Res, dl),
2337                                              getI32Imm(ANDISMask, dl)),
2338                       0);
2339 
2340         if (!ANDIVal)
2341           Res = ANDISVal;
2342         else if (!ANDISVal)
2343           Res = ANDIVal;
2344         else
2345           Res = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64,
2346                           ExtendToInt64(ANDIVal, dl), ANDISVal), 0);
2347       } else {
2348         unsigned NumOfSelectInsts = 0;
2349         SDValue MaskVal =
2350             SDValue(selectI64Imm(CurDAG, dl, Mask, &NumOfSelectInsts), 0);
2351         Res = SDValue(CurDAG->getMachineNode(PPC::AND8, dl, MVT::i64,
2352                                              ExtendToInt64(Res, dl), MaskVal),
2353                       0);
2354         if (InstCnt)
2355           *InstCnt += NumOfSelectInsts + /* and */ 1;
2356       }
2357     }
2358 
2359     return Res.getNode();
2360   }
2361 
2362   SDNode *Select(SDNode *N, bool LateMask, unsigned *InstCnt = nullptr) {
2363     // Fill in BitGroups.
2364     collectBitGroups(LateMask);
2365     if (BitGroups.empty())
2366       return nullptr;
2367 
2368     // For 64-bit values, figure out when we can use 32-bit instructions.
2369     if (Bits.size() == 64)
2370       assignRepl32BitGroups();
2371 
2372     // Fill in ValueRotsVec.
2373     collectValueRotInfo();
2374 
2375     if (Bits.size() == 32) {
2376       return Select32(N, LateMask, InstCnt);
2377     } else {
2378       assert(Bits.size() == 64 && "Not 64 bits here?");
2379       return Select64(N, LateMask, InstCnt);
2380     }
2381 
2382     return nullptr;
2383   }
2384 
2385   void eraseMatchingBitGroups(function_ref<bool(const BitGroup &)> F) {
2386     erase_if(BitGroups, F);
2387   }
2388 
2389   SmallVector<ValueBit, 64> Bits;
2390 
2391   bool NeedMask = false;
2392   SmallVector<unsigned, 64> RLAmt;
2393 
2394   SmallVector<BitGroup, 16> BitGroups;
2395 
2396   DenseMap<std::pair<SDValue, unsigned>, ValueRotInfo> ValueRots;
2397   SmallVector<ValueRotInfo, 16> ValueRotsVec;
2398 
2399   SelectionDAG *CurDAG = nullptr;
2400 
2401 public:
2402   BitPermutationSelector(SelectionDAG *DAG)
2403     : CurDAG(DAG) {}
2404 
2405   // Here we try to match complex bit permutations into a set of
2406   // rotate-and-shift/shift/and/or instructions, using a set of heuristics
2407   // known to produce optimal code for common cases (like i32 byte swapping).
2408   SDNode *Select(SDNode *N) {
2409     Memoizer.clear();
2410     auto Result =
2411         getValueBits(SDValue(N, 0), N->getValueType(0).getSizeInBits());
2412     if (!Result.first)
2413       return nullptr;
2414     Bits = std::move(*Result.second);
2415 
2416     LLVM_DEBUG(dbgs() << "Considering bit-permutation-based instruction"
2417                          " selection for:    ");
2418     LLVM_DEBUG(N->dump(CurDAG));
2419 
2420     // Fill it RLAmt and set NeedMask.
2421     computeRotationAmounts();
2422 
2423     if (!NeedMask)
2424       return Select(N, false);
2425 
2426     // We currently have two techniques for handling results with zeros: early
2427     // masking (the default) and late masking. Late masking is sometimes more
2428     // efficient, but because the structure of the bit groups is different, it
2429     // is hard to tell without generating both and comparing the results. With
2430     // late masking, we ignore zeros in the resulting value when inserting each
2431     // set of bit groups, and then mask in the zeros at the end. With early
2432     // masking, we only insert the non-zero parts of the result at every step.
2433 
2434     unsigned InstCnt = 0, InstCntLateMask = 0;
2435     LLVM_DEBUG(dbgs() << "\tEarly masking:\n");
2436     SDNode *RN = Select(N, false, &InstCnt);
2437     LLVM_DEBUG(dbgs() << "\t\tisel would use " << InstCnt << " instructions\n");
2438 
2439     LLVM_DEBUG(dbgs() << "\tLate masking:\n");
2440     SDNode *RNLM = Select(N, true, &InstCntLateMask);
2441     LLVM_DEBUG(dbgs() << "\t\tisel would use " << InstCntLateMask
2442                       << " instructions\n");
2443 
2444     if (InstCnt <= InstCntLateMask) {
2445       LLVM_DEBUG(dbgs() << "\tUsing early-masking for isel\n");
2446       return RN;
2447     }
2448 
2449     LLVM_DEBUG(dbgs() << "\tUsing late-masking for isel\n");
2450     return RNLM;
2451   }
2452 };
2453 
2454 class IntegerCompareEliminator {
2455   SelectionDAG *CurDAG;
2456   PPCDAGToDAGISel *S;
2457   // Conversion type for interpreting results of a 32-bit instruction as
2458   // a 64-bit value or vice versa.
2459   enum ExtOrTruncConversion { Ext, Trunc };
2460 
2461   // Modifiers to guide how an ISD::SETCC node's result is to be computed
2462   // in a GPR.
2463   // ZExtOrig - use the original condition code, zero-extend value
2464   // ZExtInvert - invert the condition code, zero-extend value
2465   // SExtOrig - use the original condition code, sign-extend value
2466   // SExtInvert - invert the condition code, sign-extend value
2467   enum SetccInGPROpts { ZExtOrig, ZExtInvert, SExtOrig, SExtInvert };
2468 
2469   // Comparisons against zero to emit GPR code sequences for. Each of these
2470   // sequences may need to be emitted for two or more equivalent patterns.
2471   // For example (a >= 0) == (a > -1). The direction of the comparison (</>)
2472   // matters as well as the extension type: sext (-1/0), zext (1/0).
2473   // GEZExt - (zext (LHS >= 0))
2474   // GESExt - (sext (LHS >= 0))
2475   // LEZExt - (zext (LHS <= 0))
2476   // LESExt - (sext (LHS <= 0))
2477   enum ZeroCompare { GEZExt, GESExt, LEZExt, LESExt };
2478 
2479   SDNode *tryEXTEND(SDNode *N);
2480   SDNode *tryLogicOpOfCompares(SDNode *N);
2481   SDValue computeLogicOpInGPR(SDValue LogicOp);
2482   SDValue signExtendInputIfNeeded(SDValue Input);
2483   SDValue zeroExtendInputIfNeeded(SDValue Input);
2484   SDValue addExtOrTrunc(SDValue NatWidthRes, ExtOrTruncConversion Conv);
2485   SDValue getCompoundZeroComparisonInGPR(SDValue LHS, SDLoc dl,
2486                                         ZeroCompare CmpTy);
2487   SDValue get32BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2488                               int64_t RHSValue, SDLoc dl);
2489  SDValue get32BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2490                               int64_t RHSValue, SDLoc dl);
2491   SDValue get64BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2492                               int64_t RHSValue, SDLoc dl);
2493   SDValue get64BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2494                               int64_t RHSValue, SDLoc dl);
2495   SDValue getSETCCInGPR(SDValue Compare, SetccInGPROpts ConvOpts);
2496 
2497 public:
2498   IntegerCompareEliminator(SelectionDAG *DAG,
2499                            PPCDAGToDAGISel *Sel) : CurDAG(DAG), S(Sel) {
2500     assert(CurDAG->getTargetLoweringInfo()
2501            .getPointerTy(CurDAG->getDataLayout()).getSizeInBits() == 64 &&
2502            "Only expecting to use this on 64 bit targets.");
2503   }
2504   SDNode *Select(SDNode *N) {
2505     if (CmpInGPR == ICGPR_None)
2506       return nullptr;
2507     switch (N->getOpcode()) {
2508     default: break;
2509     case ISD::ZERO_EXTEND:
2510       if (CmpInGPR == ICGPR_Sext || CmpInGPR == ICGPR_SextI32 ||
2511           CmpInGPR == ICGPR_SextI64)
2512         return nullptr;
2513       LLVM_FALLTHROUGH;
2514     case ISD::SIGN_EXTEND:
2515       if (CmpInGPR == ICGPR_Zext || CmpInGPR == ICGPR_ZextI32 ||
2516           CmpInGPR == ICGPR_ZextI64)
2517         return nullptr;
2518       return tryEXTEND(N);
2519     case ISD::AND:
2520     case ISD::OR:
2521     case ISD::XOR:
2522       return tryLogicOpOfCompares(N);
2523     }
2524     return nullptr;
2525   }
2526 };
2527 
2528 static bool isLogicOp(unsigned Opc) {
2529   return Opc == ISD::AND || Opc == ISD::OR || Opc == ISD::XOR;
2530 }
2531 // The obvious case for wanting to keep the value in a GPR. Namely, the
2532 // result of the comparison is actually needed in a GPR.
2533 SDNode *IntegerCompareEliminator::tryEXTEND(SDNode *N) {
2534   assert((N->getOpcode() == ISD::ZERO_EXTEND ||
2535           N->getOpcode() == ISD::SIGN_EXTEND) &&
2536          "Expecting a zero/sign extend node!");
2537   SDValue WideRes;
2538   // If we are zero-extending the result of a logical operation on i1
2539   // values, we can keep the values in GPRs.
2540   if (isLogicOp(N->getOperand(0).getOpcode()) &&
2541       N->getOperand(0).getValueType() == MVT::i1 &&
2542       N->getOpcode() == ISD::ZERO_EXTEND)
2543     WideRes = computeLogicOpInGPR(N->getOperand(0));
2544   else if (N->getOperand(0).getOpcode() != ISD::SETCC)
2545     return nullptr;
2546   else
2547     WideRes =
2548       getSETCCInGPR(N->getOperand(0),
2549                     N->getOpcode() == ISD::SIGN_EXTEND ?
2550                     SetccInGPROpts::SExtOrig : SetccInGPROpts::ZExtOrig);
2551 
2552   if (!WideRes)
2553     return nullptr;
2554 
2555   SDLoc dl(N);
2556   bool Input32Bit = WideRes.getValueType() == MVT::i32;
2557   bool Output32Bit = N->getValueType(0) == MVT::i32;
2558 
2559   NumSextSetcc += N->getOpcode() == ISD::SIGN_EXTEND ? 1 : 0;
2560   NumZextSetcc += N->getOpcode() == ISD::SIGN_EXTEND ? 0 : 1;
2561 
2562   SDValue ConvOp = WideRes;
2563   if (Input32Bit != Output32Bit)
2564     ConvOp = addExtOrTrunc(WideRes, Input32Bit ? ExtOrTruncConversion::Ext :
2565                            ExtOrTruncConversion::Trunc);
2566   return ConvOp.getNode();
2567 }
2568 
2569 // Attempt to perform logical operations on the results of comparisons while
2570 // keeping the values in GPRs. Without doing so, these would end up being
2571 // lowered to CR-logical operations which suffer from significant latency and
2572 // low ILP.
2573 SDNode *IntegerCompareEliminator::tryLogicOpOfCompares(SDNode *N) {
2574   if (N->getValueType(0) != MVT::i1)
2575     return nullptr;
2576   assert(isLogicOp(N->getOpcode()) &&
2577          "Expected a logic operation on setcc results.");
2578   SDValue LoweredLogical = computeLogicOpInGPR(SDValue(N, 0));
2579   if (!LoweredLogical)
2580     return nullptr;
2581 
2582   SDLoc dl(N);
2583   bool IsBitwiseNegate = LoweredLogical.getMachineOpcode() == PPC::XORI8;
2584   unsigned SubRegToExtract = IsBitwiseNegate ? PPC::sub_eq : PPC::sub_gt;
2585   SDValue CR0Reg = CurDAG->getRegister(PPC::CR0, MVT::i32);
2586   SDValue LHS = LoweredLogical.getOperand(0);
2587   SDValue RHS = LoweredLogical.getOperand(1);
2588   SDValue WideOp;
2589   SDValue OpToConvToRecForm;
2590 
2591   // Look through any 32-bit to 64-bit implicit extend nodes to find the
2592   // opcode that is input to the XORI.
2593   if (IsBitwiseNegate &&
2594       LoweredLogical.getOperand(0).getMachineOpcode() == PPC::INSERT_SUBREG)
2595     OpToConvToRecForm = LoweredLogical.getOperand(0).getOperand(1);
2596   else if (IsBitwiseNegate)
2597     // If the input to the XORI isn't an extension, that's what we're after.
2598     OpToConvToRecForm = LoweredLogical.getOperand(0);
2599   else
2600     // If this is not an XORI, it is a reg-reg logical op and we can convert
2601     // it to record-form.
2602     OpToConvToRecForm = LoweredLogical;
2603 
2604   // Get the record-form version of the node we're looking to use to get the
2605   // CR result from.
2606   uint16_t NonRecOpc = OpToConvToRecForm.getMachineOpcode();
2607   int NewOpc = PPCInstrInfo::getRecordFormOpcode(NonRecOpc);
2608 
2609   // Convert the right node to record-form. This is either the logical we're
2610   // looking at or it is the input node to the negation (if we're looking at
2611   // a bitwise negation).
2612   if (NewOpc != -1 && IsBitwiseNegate) {
2613     // The input to the XORI has a record-form. Use it.
2614     assert(LoweredLogical.getConstantOperandVal(1) == 1 &&
2615            "Expected a PPC::XORI8 only for bitwise negation.");
2616     // Emit the record-form instruction.
2617     std::vector<SDValue> Ops;
2618     for (int i = 0, e = OpToConvToRecForm.getNumOperands(); i < e; i++)
2619       Ops.push_back(OpToConvToRecForm.getOperand(i));
2620 
2621     WideOp =
2622       SDValue(CurDAG->getMachineNode(NewOpc, dl,
2623                                      OpToConvToRecForm.getValueType(),
2624                                      MVT::Glue, Ops), 0);
2625   } else {
2626     assert((NewOpc != -1 || !IsBitwiseNegate) &&
2627            "No record form available for AND8/OR8/XOR8?");
2628     WideOp =
2629         SDValue(CurDAG->getMachineNode(NewOpc == -1 ? PPC::ANDI8_rec : NewOpc,
2630                                        dl, MVT::i64, MVT::Glue, LHS, RHS),
2631                 0);
2632   }
2633 
2634   // Select this node to a single bit from CR0 set by the record-form node
2635   // just created. For bitwise negation, use the EQ bit which is the equivalent
2636   // of negating the result (i.e. it is a bit set when the result of the
2637   // operation is zero).
2638   SDValue SRIdxVal =
2639     CurDAG->getTargetConstant(SubRegToExtract, dl, MVT::i32);
2640   SDValue CRBit =
2641     SDValue(CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl,
2642                                    MVT::i1, CR0Reg, SRIdxVal,
2643                                    WideOp.getValue(1)), 0);
2644   return CRBit.getNode();
2645 }
2646 
2647 // Lower a logical operation on i1 values into a GPR sequence if possible.
2648 // The result can be kept in a GPR if requested.
2649 // Three types of inputs can be handled:
2650 // - SETCC
2651 // - TRUNCATE
2652 // - Logical operation (AND/OR/XOR)
2653 // There is also a special case that is handled (namely a complement operation
2654 // achieved with xor %a, -1).
2655 SDValue IntegerCompareEliminator::computeLogicOpInGPR(SDValue LogicOp) {
2656   assert(isLogicOp(LogicOp.getOpcode()) &&
2657         "Can only handle logic operations here.");
2658   assert(LogicOp.getValueType() == MVT::i1 &&
2659          "Can only handle logic operations on i1 values here.");
2660   SDLoc dl(LogicOp);
2661   SDValue LHS, RHS;
2662 
2663  // Special case: xor %a, -1
2664   bool IsBitwiseNegation = isBitwiseNot(LogicOp);
2665 
2666   // Produces a GPR sequence for each operand of the binary logic operation.
2667   // For SETCC, it produces the respective comparison, for TRUNCATE it truncates
2668   // the value in a GPR and for logic operations, it will recursively produce
2669   // a GPR sequence for the operation.
2670  auto getLogicOperand = [&] (SDValue Operand) -> SDValue {
2671     unsigned OperandOpcode = Operand.getOpcode();
2672     if (OperandOpcode == ISD::SETCC)
2673       return getSETCCInGPR(Operand, SetccInGPROpts::ZExtOrig);
2674     else if (OperandOpcode == ISD::TRUNCATE) {
2675       SDValue InputOp = Operand.getOperand(0);
2676      EVT InVT = InputOp.getValueType();
2677       return SDValue(CurDAG->getMachineNode(InVT == MVT::i32 ? PPC::RLDICL_32 :
2678                                             PPC::RLDICL, dl, InVT, InputOp,
2679                                             S->getI64Imm(0, dl),
2680                                             S->getI64Imm(63, dl)), 0);
2681     } else if (isLogicOp(OperandOpcode))
2682       return computeLogicOpInGPR(Operand);
2683     return SDValue();
2684   };
2685   LHS = getLogicOperand(LogicOp.getOperand(0));
2686   RHS = getLogicOperand(LogicOp.getOperand(1));
2687 
2688   // If a GPR sequence can't be produced for the LHS we can't proceed.
2689   // Not producing a GPR sequence for the RHS is only a problem if this isn't
2690   // a bitwise negation operation.
2691   if (!LHS || (!RHS && !IsBitwiseNegation))
2692     return SDValue();
2693 
2694   NumLogicOpsOnComparison++;
2695 
2696   // We will use the inputs as 64-bit values.
2697   if (LHS.getValueType() == MVT::i32)
2698     LHS = addExtOrTrunc(LHS, ExtOrTruncConversion::Ext);
2699   if (!IsBitwiseNegation && RHS.getValueType() == MVT::i32)
2700     RHS = addExtOrTrunc(RHS, ExtOrTruncConversion::Ext);
2701 
2702   unsigned NewOpc;
2703   switch (LogicOp.getOpcode()) {
2704   default: llvm_unreachable("Unknown logic operation.");
2705   case ISD::AND: NewOpc = PPC::AND8; break;
2706   case ISD::OR:  NewOpc = PPC::OR8;  break;
2707   case ISD::XOR: NewOpc = PPC::XOR8; break;
2708   }
2709 
2710   if (IsBitwiseNegation) {
2711     RHS = S->getI64Imm(1, dl);
2712     NewOpc = PPC::XORI8;
2713   }
2714 
2715   return SDValue(CurDAG->getMachineNode(NewOpc, dl, MVT::i64, LHS, RHS), 0);
2716 
2717 }
2718 
2719 /// If the value isn't guaranteed to be sign-extended to 64-bits, extend it.
2720 /// Otherwise just reinterpret it as a 64-bit value.
2721 /// Useful when emitting comparison code for 32-bit values without using
2722 /// the compare instruction (which only considers the lower 32-bits).
2723 SDValue IntegerCompareEliminator::signExtendInputIfNeeded(SDValue Input) {
2724   assert(Input.getValueType() == MVT::i32 &&
2725          "Can only sign-extend 32-bit values here.");
2726   unsigned Opc = Input.getOpcode();
2727 
2728   // The value was sign extended and then truncated to 32-bits. No need to
2729   // sign extend it again.
2730   if (Opc == ISD::TRUNCATE &&
2731       (Input.getOperand(0).getOpcode() == ISD::AssertSext ||
2732        Input.getOperand(0).getOpcode() == ISD::SIGN_EXTEND))
2733     return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
2734 
2735   LoadSDNode *InputLoad = dyn_cast<LoadSDNode>(Input);
2736   // The input is a sign-extending load. All ppc sign-extending loads
2737   // sign-extend to the full 64-bits.
2738   if (InputLoad && InputLoad->getExtensionType() == ISD::SEXTLOAD)
2739     return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
2740 
2741   ConstantSDNode *InputConst = dyn_cast<ConstantSDNode>(Input);
2742   // We don't sign-extend constants.
2743   if (InputConst)
2744     return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
2745 
2746   SDLoc dl(Input);
2747   SignExtensionsAdded++;
2748   return SDValue(CurDAG->getMachineNode(PPC::EXTSW_32_64, dl,
2749                                         MVT::i64, Input), 0);
2750 }
2751 
2752 /// If the value isn't guaranteed to be zero-extended to 64-bits, extend it.
2753 /// Otherwise just reinterpret it as a 64-bit value.
2754 /// Useful when emitting comparison code for 32-bit values without using
2755 /// the compare instruction (which only considers the lower 32-bits).
2756 SDValue IntegerCompareEliminator::zeroExtendInputIfNeeded(SDValue Input) {
2757   assert(Input.getValueType() == MVT::i32 &&
2758          "Can only zero-extend 32-bit values here.");
2759   unsigned Opc = Input.getOpcode();
2760 
2761   // The only condition under which we can omit the actual extend instruction:
2762   // - The value is a positive constant
2763   // - The value comes from a load that isn't a sign-extending load
2764   // An ISD::TRUNCATE needs to be zero-extended unless it is fed by a zext.
2765   bool IsTruncateOfZExt = Opc == ISD::TRUNCATE &&
2766     (Input.getOperand(0).getOpcode() == ISD::AssertZext ||
2767      Input.getOperand(0).getOpcode() == ISD::ZERO_EXTEND);
2768   if (IsTruncateOfZExt)
2769     return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
2770 
2771   ConstantSDNode *InputConst = dyn_cast<ConstantSDNode>(Input);
2772   if (InputConst && InputConst->getSExtValue() >= 0)
2773     return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
2774 
2775   LoadSDNode *InputLoad = dyn_cast<LoadSDNode>(Input);
2776   // The input is a load that doesn't sign-extend (it will be zero-extended).
2777   if (InputLoad && InputLoad->getExtensionType() != ISD::SEXTLOAD)
2778     return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
2779 
2780   // None of the above, need to zero-extend.
2781   SDLoc dl(Input);
2782   ZeroExtensionsAdded++;
2783   return SDValue(CurDAG->getMachineNode(PPC::RLDICL_32_64, dl, MVT::i64, Input,
2784                                         S->getI64Imm(0, dl),
2785                                         S->getI64Imm(32, dl)), 0);
2786 }
2787 
2788 // Handle a 32-bit value in a 64-bit register and vice-versa. These are of
2789 // course not actual zero/sign extensions that will generate machine code,
2790 // they're just a way to reinterpret a 32 bit value in a register as a
2791 // 64 bit value and vice-versa.
2792 SDValue IntegerCompareEliminator::addExtOrTrunc(SDValue NatWidthRes,
2793                                                 ExtOrTruncConversion Conv) {
2794   SDLoc dl(NatWidthRes);
2795 
2796   // For reinterpreting 32-bit values as 64 bit values, we generate
2797   // INSERT_SUBREG IMPLICIT_DEF:i64, <input>, TargetConstant:i32<1>
2798   if (Conv == ExtOrTruncConversion::Ext) {
2799     SDValue ImDef(CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, MVT::i64), 0);
2800     SDValue SubRegIdx =
2801       CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32);
2802     return SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl, MVT::i64,
2803                                           ImDef, NatWidthRes, SubRegIdx), 0);
2804   }
2805 
2806   assert(Conv == ExtOrTruncConversion::Trunc &&
2807          "Unknown convertion between 32 and 64 bit values.");
2808   // For reinterpreting 64-bit values as 32-bit values, we just need to
2809   // EXTRACT_SUBREG (i.e. extract the low word).
2810   SDValue SubRegIdx =
2811     CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32);
2812   return SDValue(CurDAG->getMachineNode(PPC::EXTRACT_SUBREG, dl, MVT::i32,
2813                                         NatWidthRes, SubRegIdx), 0);
2814 }
2815 
2816 // Produce a GPR sequence for compound comparisons (<=, >=) against zero.
2817 // Handle both zero-extensions and sign-extensions.
2818 SDValue
2819 IntegerCompareEliminator::getCompoundZeroComparisonInGPR(SDValue LHS, SDLoc dl,
2820                                                          ZeroCompare CmpTy) {
2821   EVT InVT = LHS.getValueType();
2822   bool Is32Bit = InVT == MVT::i32;
2823   SDValue ToExtend;
2824 
2825   // Produce the value that needs to be either zero or sign extended.
2826   switch (CmpTy) {
2827   case ZeroCompare::GEZExt:
2828   case ZeroCompare::GESExt:
2829     ToExtend = SDValue(CurDAG->getMachineNode(Is32Bit ? PPC::NOR : PPC::NOR8,
2830                                               dl, InVT, LHS, LHS), 0);
2831     break;
2832   case ZeroCompare::LEZExt:
2833   case ZeroCompare::LESExt: {
2834     if (Is32Bit) {
2835       // Upper 32 bits cannot be undefined for this sequence.
2836       LHS = signExtendInputIfNeeded(LHS);
2837       SDValue Neg =
2838         SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0);
2839       ToExtend =
2840         SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
2841                                        Neg, S->getI64Imm(1, dl),
2842                                        S->getI64Imm(63, dl)), 0);
2843     } else {
2844       SDValue Addi =
2845         SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS,
2846                                        S->getI64Imm(~0ULL, dl)), 0);
2847       ToExtend = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64,
2848                                                 Addi, LHS), 0);
2849     }
2850     break;
2851   }
2852   }
2853 
2854   // For 64-bit sequences, the extensions are the same for the GE/LE cases.
2855   if (!Is32Bit &&
2856       (CmpTy == ZeroCompare::GEZExt || CmpTy == ZeroCompare::LEZExt))
2857     return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
2858                                           ToExtend, S->getI64Imm(1, dl),
2859                                           S->getI64Imm(63, dl)), 0);
2860   if (!Is32Bit &&
2861       (CmpTy == ZeroCompare::GESExt || CmpTy == ZeroCompare::LESExt))
2862     return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, ToExtend,
2863                                           S->getI64Imm(63, dl)), 0);
2864 
2865   assert(Is32Bit && "Should have handled the 32-bit sequences above.");
2866   // For 32-bit sequences, the extensions differ between GE/LE cases.
2867   switch (CmpTy) {
2868   case ZeroCompare::GEZExt: {
2869     SDValue ShiftOps[] = { ToExtend, S->getI32Imm(1, dl), S->getI32Imm(31, dl),
2870                            S->getI32Imm(31, dl) };
2871     return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32,
2872                                           ShiftOps), 0);
2873   }
2874   case ZeroCompare::GESExt:
2875     return SDValue(CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, ToExtend,
2876                                           S->getI32Imm(31, dl)), 0);
2877   case ZeroCompare::LEZExt:
2878     return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, ToExtend,
2879                                           S->getI32Imm(1, dl)), 0);
2880   case ZeroCompare::LESExt:
2881     return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, ToExtend,
2882                                           S->getI32Imm(-1, dl)), 0);
2883   }
2884 
2885   // The above case covers all the enumerators so it can't have a default clause
2886   // to avoid compiler warnings.
2887   llvm_unreachable("Unknown zero-comparison type.");
2888 }
2889 
2890 /// Produces a zero-extended result of comparing two 32-bit values according to
2891 /// the passed condition code.
2892 SDValue
2893 IntegerCompareEliminator::get32BitZExtCompare(SDValue LHS, SDValue RHS,
2894                                               ISD::CondCode CC,
2895                                               int64_t RHSValue, SDLoc dl) {
2896   if (CmpInGPR == ICGPR_I64 || CmpInGPR == ICGPR_SextI64 ||
2897       CmpInGPR == ICGPR_ZextI64 || CmpInGPR == ICGPR_Sext)
2898     return SDValue();
2899   bool IsRHSZero = RHSValue == 0;
2900   bool IsRHSOne = RHSValue == 1;
2901   bool IsRHSNegOne = RHSValue == -1LL;
2902   switch (CC) {
2903   default: return SDValue();
2904   case ISD::SETEQ: {
2905     // (zext (setcc %a, %b, seteq)) -> (lshr (cntlzw (xor %a, %b)), 5)
2906     // (zext (setcc %a, 0, seteq))  -> (lshr (cntlzw %a), 5)
2907     SDValue Xor = IsRHSZero ? LHS :
2908       SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0);
2909     SDValue Clz =
2910       SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0);
2911     SDValue ShiftOps[] = { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl),
2912       S->getI32Imm(31, dl) };
2913     return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32,
2914                                           ShiftOps), 0);
2915   }
2916   case ISD::SETNE: {
2917     // (zext (setcc %a, %b, setne)) -> (xor (lshr (cntlzw (xor %a, %b)), 5), 1)
2918     // (zext (setcc %a, 0, setne))  -> (xor (lshr (cntlzw %a), 5), 1)
2919     SDValue Xor = IsRHSZero ? LHS :
2920       SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0);
2921     SDValue Clz =
2922       SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0);
2923     SDValue ShiftOps[] = { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl),
2924       S->getI32Imm(31, dl) };
2925     SDValue Shift =
2926       SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, ShiftOps), 0);
2927     return SDValue(CurDAG->getMachineNode(PPC::XORI, dl, MVT::i32, Shift,
2928                                           S->getI32Imm(1, dl)), 0);
2929   }
2930   case ISD::SETGE: {
2931     // (zext (setcc %a, %b, setge)) -> (xor (lshr (sub %a, %b), 63), 1)
2932     // (zext (setcc %a, 0, setge))  -> (lshr (~ %a), 31)
2933     if(IsRHSZero)
2934       return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt);
2935 
2936     // Not a special case (i.e. RHS == 0). Handle (%a >= %b) as (%b <= %a)
2937     // by swapping inputs and falling through.
2938     std::swap(LHS, RHS);
2939     ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
2940     IsRHSZero = RHSConst && RHSConst->isNullValue();
2941     LLVM_FALLTHROUGH;
2942   }
2943   case ISD::SETLE: {
2944     if (CmpInGPR == ICGPR_NonExtIn)
2945       return SDValue();
2946     // (zext (setcc %a, %b, setle)) -> (xor (lshr (sub %b, %a), 63), 1)
2947     // (zext (setcc %a, 0, setle))  -> (xor (lshr (- %a), 63), 1)
2948     if(IsRHSZero) {
2949       if (CmpInGPR == ICGPR_NonExtIn)
2950         return SDValue();
2951       return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt);
2952     }
2953 
2954     // The upper 32-bits of the register can't be undefined for this sequence.
2955     LHS = signExtendInputIfNeeded(LHS);
2956     RHS = signExtendInputIfNeeded(RHS);
2957     SDValue Sub =
2958       SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0);
2959     SDValue Shift =
2960       SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Sub,
2961                                      S->getI64Imm(1, dl), S->getI64Imm(63, dl)),
2962               0);
2963     return
2964       SDValue(CurDAG->getMachineNode(PPC::XORI8, dl,
2965                                      MVT::i64, Shift, S->getI32Imm(1, dl)), 0);
2966   }
2967   case ISD::SETGT: {
2968     // (zext (setcc %a, %b, setgt)) -> (lshr (sub %b, %a), 63)
2969     // (zext (setcc %a, -1, setgt)) -> (lshr (~ %a), 31)
2970     // (zext (setcc %a, 0, setgt))  -> (lshr (- %a), 63)
2971     // Handle SETLT -1 (which is equivalent to SETGE 0).
2972     if (IsRHSNegOne)
2973       return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt);
2974 
2975     if (IsRHSZero) {
2976       if (CmpInGPR == ICGPR_NonExtIn)
2977         return SDValue();
2978       // The upper 32-bits of the register can't be undefined for this sequence.
2979       LHS = signExtendInputIfNeeded(LHS);
2980       RHS = signExtendInputIfNeeded(RHS);
2981       SDValue Neg =
2982         SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0);
2983       return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
2984                      Neg, S->getI32Imm(1, dl), S->getI32Imm(63, dl)), 0);
2985     }
2986     // Not a special case (i.e. RHS == 0 or RHS == -1). Handle (%a > %b) as
2987     // (%b < %a) by swapping inputs and falling through.
2988     std::swap(LHS, RHS);
2989     ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
2990     IsRHSZero = RHSConst && RHSConst->isNullValue();
2991     IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1;
2992     LLVM_FALLTHROUGH;
2993   }
2994   case ISD::SETLT: {
2995     // (zext (setcc %a, %b, setlt)) -> (lshr (sub %a, %b), 63)
2996     // (zext (setcc %a, 1, setlt))  -> (xor (lshr (- %a), 63), 1)
2997     // (zext (setcc %a, 0, setlt))  -> (lshr %a, 31)
2998     // Handle SETLT 1 (which is equivalent to SETLE 0).
2999     if (IsRHSOne) {
3000       if (CmpInGPR == ICGPR_NonExtIn)
3001         return SDValue();
3002       return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt);
3003     }
3004 
3005     if (IsRHSZero) {
3006       SDValue ShiftOps[] = { LHS, S->getI32Imm(1, dl), S->getI32Imm(31, dl),
3007                              S->getI32Imm(31, dl) };
3008       return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32,
3009                                             ShiftOps), 0);
3010     }
3011 
3012     if (CmpInGPR == ICGPR_NonExtIn)
3013       return SDValue();
3014     // The upper 32-bits of the register can't be undefined for this sequence.
3015     LHS = signExtendInputIfNeeded(LHS);
3016     RHS = signExtendInputIfNeeded(RHS);
3017     SDValue SUBFNode =
3018       SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0);
3019     return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3020                                     SUBFNode, S->getI64Imm(1, dl),
3021                                     S->getI64Imm(63, dl)), 0);
3022   }
3023   case ISD::SETUGE:
3024     // (zext (setcc %a, %b, setuge)) -> (xor (lshr (sub %b, %a), 63), 1)
3025     // (zext (setcc %a, %b, setule)) -> (xor (lshr (sub %a, %b), 63), 1)
3026     std::swap(LHS, RHS);
3027     LLVM_FALLTHROUGH;
3028   case ISD::SETULE: {
3029     if (CmpInGPR == ICGPR_NonExtIn)
3030       return SDValue();
3031     // The upper 32-bits of the register can't be undefined for this sequence.
3032     LHS = zeroExtendInputIfNeeded(LHS);
3033     RHS = zeroExtendInputIfNeeded(RHS);
3034     SDValue Subtract =
3035       SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0);
3036     SDValue SrdiNode =
3037       SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3038                                           Subtract, S->getI64Imm(1, dl),
3039                                           S->getI64Imm(63, dl)), 0);
3040     return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, SrdiNode,
3041                                             S->getI32Imm(1, dl)), 0);
3042   }
3043   case ISD::SETUGT:
3044     // (zext (setcc %a, %b, setugt)) -> (lshr (sub %b, %a), 63)
3045     // (zext (setcc %a, %b, setult)) -> (lshr (sub %a, %b), 63)
3046     std::swap(LHS, RHS);
3047     LLVM_FALLTHROUGH;
3048   case ISD::SETULT: {
3049     if (CmpInGPR == ICGPR_NonExtIn)
3050       return SDValue();
3051     // The upper 32-bits of the register can't be undefined for this sequence.
3052     LHS = zeroExtendInputIfNeeded(LHS);
3053     RHS = zeroExtendInputIfNeeded(RHS);
3054     SDValue Subtract =
3055       SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0);
3056     return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3057                                           Subtract, S->getI64Imm(1, dl),
3058                                           S->getI64Imm(63, dl)), 0);
3059   }
3060   }
3061 }
3062 
3063 /// Produces a sign-extended result of comparing two 32-bit values according to
3064 /// the passed condition code.
3065 SDValue
3066 IntegerCompareEliminator::get32BitSExtCompare(SDValue LHS, SDValue RHS,
3067                                               ISD::CondCode CC,
3068                                               int64_t RHSValue, SDLoc dl) {
3069   if (CmpInGPR == ICGPR_I64 || CmpInGPR == ICGPR_SextI64 ||
3070       CmpInGPR == ICGPR_ZextI64 || CmpInGPR == ICGPR_Zext)
3071     return SDValue();
3072   bool IsRHSZero = RHSValue == 0;
3073   bool IsRHSOne = RHSValue == 1;
3074   bool IsRHSNegOne = RHSValue == -1LL;
3075 
3076   switch (CC) {
3077   default: return SDValue();
3078   case ISD::SETEQ: {
3079     // (sext (setcc %a, %b, seteq)) ->
3080     //   (ashr (shl (ctlz (xor %a, %b)), 58), 63)
3081     // (sext (setcc %a, 0, seteq)) ->
3082     //   (ashr (shl (ctlz %a), 58), 63)
3083     SDValue CountInput = IsRHSZero ? LHS :
3084       SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0);
3085     SDValue Cntlzw =
3086       SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, CountInput), 0);
3087     SDValue SHLOps[] = { Cntlzw, S->getI32Imm(27, dl),
3088                          S->getI32Imm(5, dl), S->getI32Imm(31, dl) };
3089     SDValue Slwi =
3090       SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, SHLOps), 0);
3091     return SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Slwi), 0);
3092   }
3093   case ISD::SETNE: {
3094     // Bitwise xor the operands, count leading zeros, shift right by 5 bits and
3095     // flip the bit, finally take 2's complement.
3096     // (sext (setcc %a, %b, setne)) ->
3097     //   (neg (xor (lshr (ctlz (xor %a, %b)), 5), 1))
3098     // Same as above, but the first xor is not needed.
3099     // (sext (setcc %a, 0, setne)) ->
3100     //   (neg (xor (lshr (ctlz %a), 5), 1))
3101     SDValue Xor = IsRHSZero ? LHS :
3102       SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0);
3103     SDValue Clz =
3104       SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0);
3105     SDValue ShiftOps[] =
3106       { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl), S->getI32Imm(31, dl) };
3107     SDValue Shift =
3108       SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, ShiftOps), 0);
3109     SDValue Xori =
3110       SDValue(CurDAG->getMachineNode(PPC::XORI, dl, MVT::i32, Shift,
3111                                      S->getI32Imm(1, dl)), 0);
3112     return SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Xori), 0);
3113   }
3114   case ISD::SETGE: {
3115     // (sext (setcc %a, %b, setge)) -> (add (lshr (sub %a, %b), 63), -1)
3116     // (sext (setcc %a, 0, setge))  -> (ashr (~ %a), 31)
3117     if (IsRHSZero)
3118       return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt);
3119 
3120     // Not a special case (i.e. RHS == 0). Handle (%a >= %b) as (%b <= %a)
3121     // by swapping inputs and falling through.
3122     std::swap(LHS, RHS);
3123     ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3124     IsRHSZero = RHSConst && RHSConst->isNullValue();
3125     LLVM_FALLTHROUGH;
3126   }
3127   case ISD::SETLE: {
3128     if (CmpInGPR == ICGPR_NonExtIn)
3129       return SDValue();
3130     // (sext (setcc %a, %b, setge)) -> (add (lshr (sub %b, %a), 63), -1)
3131     // (sext (setcc %a, 0, setle))  -> (add (lshr (- %a), 63), -1)
3132     if (IsRHSZero)
3133       return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt);
3134 
3135     // The upper 32-bits of the register can't be undefined for this sequence.
3136     LHS = signExtendInputIfNeeded(LHS);
3137     RHS = signExtendInputIfNeeded(RHS);
3138     SDValue SUBFNode =
3139       SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, MVT::Glue,
3140                                      LHS, RHS), 0);
3141     SDValue Srdi =
3142       SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3143                                      SUBFNode, S->getI64Imm(1, dl),
3144                                      S->getI64Imm(63, dl)), 0);
3145     return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, Srdi,
3146                                           S->getI32Imm(-1, dl)), 0);
3147   }
3148   case ISD::SETGT: {
3149     // (sext (setcc %a, %b, setgt)) -> (ashr (sub %b, %a), 63)
3150     // (sext (setcc %a, -1, setgt)) -> (ashr (~ %a), 31)
3151     // (sext (setcc %a, 0, setgt))  -> (ashr (- %a), 63)
3152     if (IsRHSNegOne)
3153       return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt);
3154     if (IsRHSZero) {
3155       if (CmpInGPR == ICGPR_NonExtIn)
3156         return SDValue();
3157       // The upper 32-bits of the register can't be undefined for this sequence.
3158       LHS = signExtendInputIfNeeded(LHS);
3159       RHS = signExtendInputIfNeeded(RHS);
3160       SDValue Neg =
3161         SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0);
3162         return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, Neg,
3163                                               S->getI64Imm(63, dl)), 0);
3164     }
3165     // Not a special case (i.e. RHS == 0 or RHS == -1). Handle (%a > %b) as
3166     // (%b < %a) by swapping inputs and falling through.
3167     std::swap(LHS, RHS);
3168     ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3169     IsRHSZero = RHSConst && RHSConst->isNullValue();
3170     IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1;
3171     LLVM_FALLTHROUGH;
3172   }
3173   case ISD::SETLT: {
3174     // (sext (setcc %a, %b, setgt)) -> (ashr (sub %a, %b), 63)
3175     // (sext (setcc %a, 1, setgt))  -> (add (lshr (- %a), 63), -1)
3176     // (sext (setcc %a, 0, setgt))  -> (ashr %a, 31)
3177     if (IsRHSOne) {
3178       if (CmpInGPR == ICGPR_NonExtIn)
3179         return SDValue();
3180       return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt);
3181     }
3182     if (IsRHSZero)
3183       return SDValue(CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, LHS,
3184                                             S->getI32Imm(31, dl)), 0);
3185 
3186     if (CmpInGPR == ICGPR_NonExtIn)
3187       return SDValue();
3188     // The upper 32-bits of the register can't be undefined for this sequence.
3189     LHS = signExtendInputIfNeeded(LHS);
3190     RHS = signExtendInputIfNeeded(RHS);
3191     SDValue SUBFNode =
3192       SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0);
3193     return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64,
3194                                           SUBFNode, S->getI64Imm(63, dl)), 0);
3195   }
3196   case ISD::SETUGE:
3197     // (sext (setcc %a, %b, setuge)) -> (add (lshr (sub %a, %b), 63), -1)
3198     // (sext (setcc %a, %b, setule)) -> (add (lshr (sub %b, %a), 63), -1)
3199     std::swap(LHS, RHS);
3200     LLVM_FALLTHROUGH;
3201   case ISD::SETULE: {
3202     if (CmpInGPR == ICGPR_NonExtIn)
3203       return SDValue();
3204     // The upper 32-bits of the register can't be undefined for this sequence.
3205     LHS = zeroExtendInputIfNeeded(LHS);
3206     RHS = zeroExtendInputIfNeeded(RHS);
3207     SDValue Subtract =
3208       SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0);
3209     SDValue Shift =
3210       SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Subtract,
3211                                      S->getI32Imm(1, dl), S->getI32Imm(63,dl)),
3212               0);
3213     return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, Shift,
3214                                           S->getI32Imm(-1, dl)), 0);
3215   }
3216   case ISD::SETUGT:
3217     // (sext (setcc %a, %b, setugt)) -> (ashr (sub %b, %a), 63)
3218     // (sext (setcc %a, %b, setugt)) -> (ashr (sub %a, %b), 63)
3219     std::swap(LHS, RHS);
3220     LLVM_FALLTHROUGH;
3221   case ISD::SETULT: {
3222     if (CmpInGPR == ICGPR_NonExtIn)
3223       return SDValue();
3224     // The upper 32-bits of the register can't be undefined for this sequence.
3225     LHS = zeroExtendInputIfNeeded(LHS);
3226     RHS = zeroExtendInputIfNeeded(RHS);
3227     SDValue Subtract =
3228       SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0);
3229     return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64,
3230                                           Subtract, S->getI64Imm(63, dl)), 0);
3231   }
3232   }
3233 }
3234 
3235 /// Produces a zero-extended result of comparing two 64-bit values according to
3236 /// the passed condition code.
3237 SDValue
3238 IntegerCompareEliminator::get64BitZExtCompare(SDValue LHS, SDValue RHS,
3239                                               ISD::CondCode CC,
3240                                               int64_t RHSValue, SDLoc dl) {
3241   if (CmpInGPR == ICGPR_I32 || CmpInGPR == ICGPR_SextI32 ||
3242       CmpInGPR == ICGPR_ZextI32 || CmpInGPR == ICGPR_Sext)
3243     return SDValue();
3244   bool IsRHSZero = RHSValue == 0;
3245   bool IsRHSOne = RHSValue == 1;
3246   bool IsRHSNegOne = RHSValue == -1LL;
3247   switch (CC) {
3248   default: return SDValue();
3249   case ISD::SETEQ: {
3250     // (zext (setcc %a, %b, seteq)) -> (lshr (ctlz (xor %a, %b)), 6)
3251     // (zext (setcc %a, 0, seteq)) ->  (lshr (ctlz %a), 6)
3252     SDValue Xor = IsRHSZero ? LHS :
3253       SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0);
3254     SDValue Clz =
3255       SDValue(CurDAG->getMachineNode(PPC::CNTLZD, dl, MVT::i64, Xor), 0);
3256     return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Clz,
3257                                           S->getI64Imm(58, dl),
3258                                           S->getI64Imm(63, dl)), 0);
3259   }
3260   case ISD::SETNE: {
3261     // {addc.reg, addc.CA} = (addcarry (xor %a, %b), -1)
3262     // (zext (setcc %a, %b, setne)) -> (sube addc.reg, addc.reg, addc.CA)
3263     // {addcz.reg, addcz.CA} = (addcarry %a, -1)
3264     // (zext (setcc %a, 0, setne)) -> (sube addcz.reg, addcz.reg, addcz.CA)
3265     SDValue Xor = IsRHSZero ? LHS :
3266       SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0);
3267     SDValue AC =
3268       SDValue(CurDAG->getMachineNode(PPC::ADDIC8, dl, MVT::i64, MVT::Glue,
3269                                      Xor, S->getI32Imm(~0U, dl)), 0);
3270     return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, AC,
3271                                           Xor, AC.getValue(1)), 0);
3272   }
3273   case ISD::SETGE: {
3274     // {subc.reg, subc.CA} = (subcarry %a, %b)
3275     // (zext (setcc %a, %b, setge)) ->
3276     //   (adde (lshr %b, 63), (ashr %a, 63), subc.CA)
3277     // (zext (setcc %a, 0, setge)) -> (lshr (~ %a), 63)
3278     if (IsRHSZero)
3279       return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt);
3280     std::swap(LHS, RHS);
3281     ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3282     IsRHSZero = RHSConst && RHSConst->isNullValue();
3283     LLVM_FALLTHROUGH;
3284   }
3285   case ISD::SETLE: {
3286     // {subc.reg, subc.CA} = (subcarry %b, %a)
3287     // (zext (setcc %a, %b, setge)) ->
3288     //   (adde (lshr %a, 63), (ashr %b, 63), subc.CA)
3289     // (zext (setcc %a, 0, setge)) -> (lshr (or %a, (add %a, -1)), 63)
3290     if (IsRHSZero)
3291       return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt);
3292     SDValue ShiftL =
3293       SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS,
3294                                      S->getI64Imm(1, dl),
3295                                      S->getI64Imm(63, dl)), 0);
3296     SDValue ShiftR =
3297       SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, RHS,
3298                                      S->getI64Imm(63, dl)), 0);
3299     SDValue SubtractCarry =
3300       SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3301                                      LHS, RHS), 1);
3302     return SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue,
3303                                           ShiftR, ShiftL, SubtractCarry), 0);
3304   }
3305   case ISD::SETGT: {
3306     // {subc.reg, subc.CA} = (subcarry %b, %a)
3307     // (zext (setcc %a, %b, setgt)) ->
3308     //   (xor (adde (lshr %a, 63), (ashr %b, 63), subc.CA), 1)
3309     // (zext (setcc %a, 0, setgt)) -> (lshr (nor (add %a, -1), %a), 63)
3310     if (IsRHSNegOne)
3311       return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt);
3312     if (IsRHSZero) {
3313       SDValue Addi =
3314         SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS,
3315                                        S->getI64Imm(~0ULL, dl)), 0);
3316       SDValue Nor =
3317         SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, Addi, LHS), 0);
3318       return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Nor,
3319                                             S->getI64Imm(1, dl),
3320                                             S->getI64Imm(63, dl)), 0);
3321     }
3322     std::swap(LHS, RHS);
3323     ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3324     IsRHSZero = RHSConst && RHSConst->isNullValue();
3325     IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1;
3326     LLVM_FALLTHROUGH;
3327   }
3328   case ISD::SETLT: {
3329     // {subc.reg, subc.CA} = (subcarry %a, %b)
3330     // (zext (setcc %a, %b, setlt)) ->
3331     //   (xor (adde (lshr %b, 63), (ashr %a, 63), subc.CA), 1)
3332     // (zext (setcc %a, 0, setlt)) -> (lshr %a, 63)
3333     if (IsRHSOne)
3334       return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt);
3335     if (IsRHSZero)
3336       return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS,
3337                                             S->getI64Imm(1, dl),
3338                                             S->getI64Imm(63, dl)), 0);
3339     SDValue SRADINode =
3340       SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64,
3341                                      LHS, S->getI64Imm(63, dl)), 0);
3342     SDValue SRDINode =
3343       SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3344                                      RHS, S->getI64Imm(1, dl),
3345                                      S->getI64Imm(63, dl)), 0);
3346     SDValue SUBFC8Carry =
3347       SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3348                                      RHS, LHS), 1);
3349     SDValue ADDE8Node =
3350       SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue,
3351                                      SRDINode, SRADINode, SUBFC8Carry), 0);
3352     return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64,
3353                                           ADDE8Node, S->getI64Imm(1, dl)), 0);
3354   }
3355   case ISD::SETUGE:
3356     // {subc.reg, subc.CA} = (subcarry %a, %b)
3357     // (zext (setcc %a, %b, setuge)) -> (add (sube %b, %b, subc.CA), 1)
3358     std::swap(LHS, RHS);
3359     LLVM_FALLTHROUGH;
3360   case ISD::SETULE: {
3361     // {subc.reg, subc.CA} = (subcarry %b, %a)
3362     // (zext (setcc %a, %b, setule)) -> (add (sube %a, %a, subc.CA), 1)
3363     SDValue SUBFC8Carry =
3364       SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3365                                      LHS, RHS), 1);
3366     SDValue SUBFE8Node =
3367       SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, MVT::Glue,
3368                                      LHS, LHS, SUBFC8Carry), 0);
3369     return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64,
3370                                           SUBFE8Node, S->getI64Imm(1, dl)), 0);
3371   }
3372   case ISD::SETUGT:
3373     // {subc.reg, subc.CA} = (subcarry %b, %a)
3374     // (zext (setcc %a, %b, setugt)) -> -(sube %b, %b, subc.CA)
3375     std::swap(LHS, RHS);
3376     LLVM_FALLTHROUGH;
3377   case ISD::SETULT: {
3378     // {subc.reg, subc.CA} = (subcarry %a, %b)
3379     // (zext (setcc %a, %b, setult)) -> -(sube %a, %a, subc.CA)
3380     SDValue SubtractCarry =
3381       SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3382                                      RHS, LHS), 1);
3383     SDValue ExtSub =
3384       SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64,
3385                                      LHS, LHS, SubtractCarry), 0);
3386     return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64,
3387                                           ExtSub), 0);
3388   }
3389   }
3390 }
3391 
3392 /// Produces a sign-extended result of comparing two 64-bit values according to
3393 /// the passed condition code.
3394 SDValue
3395 IntegerCompareEliminator::get64BitSExtCompare(SDValue LHS, SDValue RHS,
3396                                               ISD::CondCode CC,
3397                                               int64_t RHSValue, SDLoc dl) {
3398   if (CmpInGPR == ICGPR_I32 || CmpInGPR == ICGPR_SextI32 ||
3399       CmpInGPR == ICGPR_ZextI32 || CmpInGPR == ICGPR_Zext)
3400     return SDValue();
3401   bool IsRHSZero = RHSValue == 0;
3402   bool IsRHSOne = RHSValue == 1;
3403   bool IsRHSNegOne = RHSValue == -1LL;
3404   switch (CC) {
3405   default: return SDValue();
3406   case ISD::SETEQ: {
3407     // {addc.reg, addc.CA} = (addcarry (xor %a, %b), -1)
3408     // (sext (setcc %a, %b, seteq)) -> (sube addc.reg, addc.reg, addc.CA)
3409     // {addcz.reg, addcz.CA} = (addcarry %a, -1)
3410     // (sext (setcc %a, 0, seteq)) -> (sube addcz.reg, addcz.reg, addcz.CA)
3411     SDValue AddInput = IsRHSZero ? LHS :
3412       SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0);
3413     SDValue Addic =
3414       SDValue(CurDAG->getMachineNode(PPC::ADDIC8, dl, MVT::i64, MVT::Glue,
3415                                      AddInput, S->getI32Imm(~0U, dl)), 0);
3416     return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, Addic,
3417                                           Addic, Addic.getValue(1)), 0);
3418   }
3419   case ISD::SETNE: {
3420     // {subfc.reg, subfc.CA} = (subcarry 0, (xor %a, %b))
3421     // (sext (setcc %a, %b, setne)) -> (sube subfc.reg, subfc.reg, subfc.CA)
3422     // {subfcz.reg, subfcz.CA} = (subcarry 0, %a)
3423     // (sext (setcc %a, 0, setne)) -> (sube subfcz.reg, subfcz.reg, subfcz.CA)
3424     SDValue Xor = IsRHSZero ? LHS :
3425       SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0);
3426     SDValue SC =
3427       SDValue(CurDAG->getMachineNode(PPC::SUBFIC8, dl, MVT::i64, MVT::Glue,
3428                                      Xor, S->getI32Imm(0, dl)), 0);
3429     return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, SC,
3430                                           SC, SC.getValue(1)), 0);
3431   }
3432   case ISD::SETGE: {
3433     // {subc.reg, subc.CA} = (subcarry %a, %b)
3434     // (zext (setcc %a, %b, setge)) ->
3435     //   (- (adde (lshr %b, 63), (ashr %a, 63), subc.CA))
3436     // (zext (setcc %a, 0, setge)) -> (~ (ashr %a, 63))
3437     if (IsRHSZero)
3438       return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt);
3439     std::swap(LHS, RHS);
3440     ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3441     IsRHSZero = RHSConst && RHSConst->isNullValue();
3442     LLVM_FALLTHROUGH;
3443   }
3444   case ISD::SETLE: {
3445     // {subc.reg, subc.CA} = (subcarry %b, %a)
3446     // (zext (setcc %a, %b, setge)) ->
3447     //   (- (adde (lshr %a, 63), (ashr %b, 63), subc.CA))
3448     // (zext (setcc %a, 0, setge)) -> (ashr (or %a, (add %a, -1)), 63)
3449     if (IsRHSZero)
3450       return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt);
3451     SDValue ShiftR =
3452       SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, RHS,
3453                                      S->getI64Imm(63, dl)), 0);
3454     SDValue ShiftL =
3455       SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS,
3456                                      S->getI64Imm(1, dl),
3457                                      S->getI64Imm(63, dl)), 0);
3458     SDValue SubtractCarry =
3459       SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3460                                      LHS, RHS), 1);
3461     SDValue Adde =
3462       SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue,
3463                                      ShiftR, ShiftL, SubtractCarry), 0);
3464     return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, Adde), 0);
3465   }
3466   case ISD::SETGT: {
3467     // {subc.reg, subc.CA} = (subcarry %b, %a)
3468     // (zext (setcc %a, %b, setgt)) ->
3469     //   -(xor (adde (lshr %a, 63), (ashr %b, 63), subc.CA), 1)
3470     // (zext (setcc %a, 0, setgt)) -> (ashr (nor (add %a, -1), %a), 63)
3471     if (IsRHSNegOne)
3472       return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt);
3473     if (IsRHSZero) {
3474       SDValue Add =
3475         SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS,
3476                                        S->getI64Imm(-1, dl)), 0);
3477       SDValue Nor =
3478         SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, Add, LHS), 0);
3479       return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, Nor,
3480                                             S->getI64Imm(63, dl)), 0);
3481     }
3482     std::swap(LHS, RHS);
3483     ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3484     IsRHSZero = RHSConst && RHSConst->isNullValue();
3485     IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1;
3486     LLVM_FALLTHROUGH;
3487   }
3488   case ISD::SETLT: {
3489     // {subc.reg, subc.CA} = (subcarry %a, %b)
3490     // (zext (setcc %a, %b, setlt)) ->
3491     //   -(xor (adde (lshr %b, 63), (ashr %a, 63), subc.CA), 1)
3492     // (zext (setcc %a, 0, setlt)) -> (ashr %a, 63)
3493     if (IsRHSOne)
3494       return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt);
3495     if (IsRHSZero) {
3496       return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, LHS,
3497                                             S->getI64Imm(63, dl)), 0);
3498     }
3499     SDValue SRADINode =
3500       SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64,
3501                                      LHS, S->getI64Imm(63, dl)), 0);
3502     SDValue SRDINode =
3503       SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3504                                      RHS, S->getI64Imm(1, dl),
3505                                      S->getI64Imm(63, dl)), 0);
3506     SDValue SUBFC8Carry =
3507       SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3508                                      RHS, LHS), 1);
3509     SDValue ADDE8Node =
3510       SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64,
3511                                      SRDINode, SRADINode, SUBFC8Carry), 0);
3512     SDValue XORI8Node =
3513       SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64,
3514                                      ADDE8Node, S->getI64Imm(1, dl)), 0);
3515     return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64,
3516                                           XORI8Node), 0);
3517   }
3518   case ISD::SETUGE:
3519     // {subc.reg, subc.CA} = (subcarry %a, %b)
3520     // (sext (setcc %a, %b, setuge)) -> ~(sube %b, %b, subc.CA)
3521     std::swap(LHS, RHS);
3522     LLVM_FALLTHROUGH;
3523   case ISD::SETULE: {
3524     // {subc.reg, subc.CA} = (subcarry %b, %a)
3525     // (sext (setcc %a, %b, setule)) -> ~(sube %a, %a, subc.CA)
3526     SDValue SubtractCarry =
3527       SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3528                                      LHS, RHS), 1);
3529     SDValue ExtSub =
3530       SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, MVT::Glue, LHS,
3531                                      LHS, SubtractCarry), 0);
3532     return SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64,
3533                                           ExtSub, ExtSub), 0);
3534   }
3535   case ISD::SETUGT:
3536     // {subc.reg, subc.CA} = (subcarry %b, %a)
3537     // (sext (setcc %a, %b, setugt)) -> (sube %b, %b, subc.CA)
3538     std::swap(LHS, RHS);
3539     LLVM_FALLTHROUGH;
3540   case ISD::SETULT: {
3541     // {subc.reg, subc.CA} = (subcarry %a, %b)
3542     // (sext (setcc %a, %b, setult)) -> (sube %a, %a, subc.CA)
3543     SDValue SubCarry =
3544       SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3545                                      RHS, LHS), 1);
3546     return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64,
3547                                      LHS, LHS, SubCarry), 0);
3548   }
3549   }
3550 }
3551 
3552 /// Do all uses of this SDValue need the result in a GPR?
3553 /// This is meant to be used on values that have type i1 since
3554 /// it is somewhat meaningless to ask if values of other types
3555 /// should be kept in GPR's.
3556 static bool allUsesExtend(SDValue Compare, SelectionDAG *CurDAG) {
3557   assert(Compare.getOpcode() == ISD::SETCC &&
3558          "An ISD::SETCC node required here.");
3559 
3560   // For values that have a single use, the caller should obviously already have
3561   // checked if that use is an extending use. We check the other uses here.
3562   if (Compare.hasOneUse())
3563     return true;
3564   // We want the value in a GPR if it is being extended, used for a select, or
3565   // used in logical operations.
3566   for (auto CompareUse : Compare.getNode()->uses())
3567     if (CompareUse->getOpcode() != ISD::SIGN_EXTEND &&
3568         CompareUse->getOpcode() != ISD::ZERO_EXTEND &&
3569         CompareUse->getOpcode() != ISD::SELECT &&
3570         !isLogicOp(CompareUse->getOpcode())) {
3571       OmittedForNonExtendUses++;
3572       return false;
3573     }
3574   return true;
3575 }
3576 
3577 /// Returns an equivalent of a SETCC node but with the result the same width as
3578 /// the inputs. This can also be used for SELECT_CC if either the true or false
3579 /// values is a power of two while the other is zero.
3580 SDValue IntegerCompareEliminator::getSETCCInGPR(SDValue Compare,
3581                                                 SetccInGPROpts ConvOpts) {
3582   assert((Compare.getOpcode() == ISD::SETCC ||
3583           Compare.getOpcode() == ISD::SELECT_CC) &&
3584          "An ISD::SETCC node required here.");
3585 
3586   // Don't convert this comparison to a GPR sequence because there are uses
3587   // of the i1 result (i.e. uses that require the result in the CR).
3588   if ((Compare.getOpcode() == ISD::SETCC) && !allUsesExtend(Compare, CurDAG))
3589     return SDValue();
3590 
3591   SDValue LHS = Compare.getOperand(0);
3592   SDValue RHS = Compare.getOperand(1);
3593 
3594   // The condition code is operand 2 for SETCC and operand 4 for SELECT_CC.
3595   int CCOpNum = Compare.getOpcode() == ISD::SELECT_CC ? 4 : 2;
3596   ISD::CondCode CC =
3597     cast<CondCodeSDNode>(Compare.getOperand(CCOpNum))->get();
3598   EVT InputVT = LHS.getValueType();
3599   if (InputVT != MVT::i32 && InputVT != MVT::i64)
3600     return SDValue();
3601 
3602   if (ConvOpts == SetccInGPROpts::ZExtInvert ||
3603       ConvOpts == SetccInGPROpts::SExtInvert)
3604     CC = ISD::getSetCCInverse(CC, InputVT);
3605 
3606   bool Inputs32Bit = InputVT == MVT::i32;
3607 
3608   SDLoc dl(Compare);
3609   ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3610   int64_t RHSValue = RHSConst ? RHSConst->getSExtValue() : INT64_MAX;
3611   bool IsSext = ConvOpts == SetccInGPROpts::SExtOrig ||
3612     ConvOpts == SetccInGPROpts::SExtInvert;
3613 
3614   if (IsSext && Inputs32Bit)
3615     return get32BitSExtCompare(LHS, RHS, CC, RHSValue, dl);
3616   else if (Inputs32Bit)
3617     return get32BitZExtCompare(LHS, RHS, CC, RHSValue, dl);
3618   else if (IsSext)
3619     return get64BitSExtCompare(LHS, RHS, CC, RHSValue, dl);
3620   return get64BitZExtCompare(LHS, RHS, CC, RHSValue, dl);
3621 }
3622 
3623 } // end anonymous namespace
3624 
3625 bool PPCDAGToDAGISel::tryIntCompareInGPR(SDNode *N) {
3626   if (N->getValueType(0) != MVT::i32 &&
3627       N->getValueType(0) != MVT::i64)
3628     return false;
3629 
3630   // This optimization will emit code that assumes 64-bit registers
3631   // so we don't want to run it in 32-bit mode. Also don't run it
3632   // on functions that are not to be optimized.
3633   if (TM.getOptLevel() == CodeGenOpt::None || !TM.isPPC64())
3634     return false;
3635 
3636   // For POWER10, it is more profitable to use the set boolean extension
3637   // instructions rather than the integer compare elimination codegen.
3638   // Users can override this via the command line option, `--ppc-gpr-icmps`.
3639   if (!(CmpInGPR.getNumOccurrences() > 0) && Subtarget->isISA3_1())
3640     return false;
3641 
3642   switch (N->getOpcode()) {
3643   default: break;
3644   case ISD::ZERO_EXTEND:
3645   case ISD::SIGN_EXTEND:
3646   case ISD::AND:
3647   case ISD::OR:
3648   case ISD::XOR: {
3649     IntegerCompareEliminator ICmpElim(CurDAG, this);
3650     if (SDNode *New = ICmpElim.Select(N)) {
3651       ReplaceNode(N, New);
3652       return true;
3653     }
3654   }
3655   }
3656   return false;
3657 }
3658 
3659 bool PPCDAGToDAGISel::tryBitPermutation(SDNode *N) {
3660   if (N->getValueType(0) != MVT::i32 &&
3661       N->getValueType(0) != MVT::i64)
3662     return false;
3663 
3664   if (!UseBitPermRewriter)
3665     return false;
3666 
3667   switch (N->getOpcode()) {
3668   default: break;
3669   case ISD::ROTL:
3670   case ISD::SHL:
3671   case ISD::SRL:
3672   case ISD::AND:
3673   case ISD::OR: {
3674     BitPermutationSelector BPS(CurDAG);
3675     if (SDNode *New = BPS.Select(N)) {
3676       ReplaceNode(N, New);
3677       return true;
3678     }
3679     return false;
3680   }
3681   }
3682 
3683   return false;
3684 }
3685 
3686 /// SelectCC - Select a comparison of the specified values with the specified
3687 /// condition code, returning the CR# of the expression.
3688 SDValue PPCDAGToDAGISel::SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC,
3689                                   const SDLoc &dl, SDValue Chain) {
3690   // Always select the LHS.
3691   unsigned Opc;
3692 
3693   if (LHS.getValueType() == MVT::i32) {
3694     unsigned Imm;
3695     if (CC == ISD::SETEQ || CC == ISD::SETNE) {
3696       if (isInt32Immediate(RHS, Imm)) {
3697         // SETEQ/SETNE comparison with 16-bit immediate, fold it.
3698         if (isUInt<16>(Imm))
3699           return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS,
3700                                                 getI32Imm(Imm & 0xFFFF, dl)),
3701                          0);
3702         // If this is a 16-bit signed immediate, fold it.
3703         if (isInt<16>((int)Imm))
3704           return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS,
3705                                                 getI32Imm(Imm & 0xFFFF, dl)),
3706                          0);
3707 
3708         // For non-equality comparisons, the default code would materialize the
3709         // constant, then compare against it, like this:
3710         //   lis r2, 4660
3711         //   ori r2, r2, 22136
3712         //   cmpw cr0, r3, r2
3713         // Since we are just comparing for equality, we can emit this instead:
3714         //   xoris r0,r3,0x1234
3715         //   cmplwi cr0,r0,0x5678
3716         //   beq cr0,L6
3717         SDValue Xor(CurDAG->getMachineNode(PPC::XORIS, dl, MVT::i32, LHS,
3718                                            getI32Imm(Imm >> 16, dl)), 0);
3719         return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, Xor,
3720                                               getI32Imm(Imm & 0xFFFF, dl)), 0);
3721       }
3722       Opc = PPC::CMPLW;
3723     } else if (ISD::isUnsignedIntSetCC(CC)) {
3724       if (isInt32Immediate(RHS, Imm) && isUInt<16>(Imm))
3725         return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS,
3726                                               getI32Imm(Imm & 0xFFFF, dl)), 0);
3727       Opc = PPC::CMPLW;
3728     } else {
3729       int16_t SImm;
3730       if (isIntS16Immediate(RHS, SImm))
3731         return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS,
3732                                               getI32Imm((int)SImm & 0xFFFF,
3733                                                         dl)),
3734                          0);
3735       Opc = PPC::CMPW;
3736     }
3737   } else if (LHS.getValueType() == MVT::i64) {
3738     uint64_t Imm;
3739     if (CC == ISD::SETEQ || CC == ISD::SETNE) {
3740       if (isInt64Immediate(RHS.getNode(), Imm)) {
3741         // SETEQ/SETNE comparison with 16-bit immediate, fold it.
3742         if (isUInt<16>(Imm))
3743           return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS,
3744                                                 getI32Imm(Imm & 0xFFFF, dl)),
3745                          0);
3746         // If this is a 16-bit signed immediate, fold it.
3747         if (isInt<16>(Imm))
3748           return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS,
3749                                                 getI32Imm(Imm & 0xFFFF, dl)),
3750                          0);
3751 
3752         // For non-equality comparisons, the default code would materialize the
3753         // constant, then compare against it, like this:
3754         //   lis r2, 4660
3755         //   ori r2, r2, 22136
3756         //   cmpd cr0, r3, r2
3757         // Since we are just comparing for equality, we can emit this instead:
3758         //   xoris r0,r3,0x1234
3759         //   cmpldi cr0,r0,0x5678
3760         //   beq cr0,L6
3761         if (isUInt<32>(Imm)) {
3762           SDValue Xor(CurDAG->getMachineNode(PPC::XORIS8, dl, MVT::i64, LHS,
3763                                              getI64Imm(Imm >> 16, dl)), 0);
3764           return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, Xor,
3765                                                 getI64Imm(Imm & 0xFFFF, dl)),
3766                          0);
3767         }
3768       }
3769       Opc = PPC::CMPLD;
3770     } else if (ISD::isUnsignedIntSetCC(CC)) {
3771       if (isInt64Immediate(RHS.getNode(), Imm) && isUInt<16>(Imm))
3772         return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS,
3773                                               getI64Imm(Imm & 0xFFFF, dl)), 0);
3774       Opc = PPC::CMPLD;
3775     } else {
3776       int16_t SImm;
3777       if (isIntS16Immediate(RHS, SImm))
3778         return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS,
3779                                               getI64Imm(SImm & 0xFFFF, dl)),
3780                          0);
3781       Opc = PPC::CMPD;
3782     }
3783   } else if (LHS.getValueType() == MVT::f32) {
3784     if (Subtarget->hasSPE()) {
3785       switch (CC) {
3786         default:
3787         case ISD::SETEQ:
3788         case ISD::SETNE:
3789           Opc = PPC::EFSCMPEQ;
3790           break;
3791         case ISD::SETLT:
3792         case ISD::SETGE:
3793         case ISD::SETOLT:
3794         case ISD::SETOGE:
3795         case ISD::SETULT:
3796         case ISD::SETUGE:
3797           Opc = PPC::EFSCMPLT;
3798           break;
3799         case ISD::SETGT:
3800         case ISD::SETLE:
3801         case ISD::SETOGT:
3802         case ISD::SETOLE:
3803         case ISD::SETUGT:
3804         case ISD::SETULE:
3805           Opc = PPC::EFSCMPGT;
3806           break;
3807       }
3808     } else
3809       Opc = PPC::FCMPUS;
3810   } else if (LHS.getValueType() == MVT::f64) {
3811     if (Subtarget->hasSPE()) {
3812       switch (CC) {
3813         default:
3814         case ISD::SETEQ:
3815         case ISD::SETNE:
3816           Opc = PPC::EFDCMPEQ;
3817           break;
3818         case ISD::SETLT:
3819         case ISD::SETGE:
3820         case ISD::SETOLT:
3821         case ISD::SETOGE:
3822         case ISD::SETULT:
3823         case ISD::SETUGE:
3824           Opc = PPC::EFDCMPLT;
3825           break;
3826         case ISD::SETGT:
3827         case ISD::SETLE:
3828         case ISD::SETOGT:
3829         case ISD::SETOLE:
3830         case ISD::SETUGT:
3831         case ISD::SETULE:
3832           Opc = PPC::EFDCMPGT;
3833           break;
3834       }
3835     } else
3836       Opc = Subtarget->hasVSX() ? PPC::XSCMPUDP : PPC::FCMPUD;
3837   } else {
3838     assert(LHS.getValueType() == MVT::f128 && "Unknown vt!");
3839     assert(Subtarget->hasVSX() && "__float128 requires VSX");
3840     Opc = PPC::XSCMPUQP;
3841   }
3842   if (Chain)
3843     return SDValue(
3844         CurDAG->getMachineNode(Opc, dl, MVT::i32, MVT::Other, LHS, RHS, Chain),
3845         0);
3846   else
3847     return SDValue(CurDAG->getMachineNode(Opc, dl, MVT::i32, LHS, RHS), 0);
3848 }
3849 
3850 static PPC::Predicate getPredicateForSetCC(ISD::CondCode CC, const EVT &VT,
3851                                            const PPCSubtarget *Subtarget) {
3852   // For SPE instructions, the result is in GT bit of the CR
3853   bool UseSPE = Subtarget->hasSPE() && VT.isFloatingPoint();
3854 
3855   switch (CC) {
3856   case ISD::SETUEQ:
3857   case ISD::SETONE:
3858   case ISD::SETOLE:
3859   case ISD::SETOGE:
3860     llvm_unreachable("Should be lowered by legalize!");
3861   default: llvm_unreachable("Unknown condition!");
3862   case ISD::SETOEQ:
3863   case ISD::SETEQ:
3864     return UseSPE ? PPC::PRED_GT : PPC::PRED_EQ;
3865   case ISD::SETUNE:
3866   case ISD::SETNE:
3867     return UseSPE ? PPC::PRED_LE : PPC::PRED_NE;
3868   case ISD::SETOLT:
3869   case ISD::SETLT:
3870     return UseSPE ? PPC::PRED_GT : PPC::PRED_LT;
3871   case ISD::SETULE:
3872   case ISD::SETLE:
3873     return PPC::PRED_LE;
3874   case ISD::SETOGT:
3875   case ISD::SETGT:
3876     return PPC::PRED_GT;
3877   case ISD::SETUGE:
3878   case ISD::SETGE:
3879     return UseSPE ? PPC::PRED_LE : PPC::PRED_GE;
3880   case ISD::SETO:   return PPC::PRED_NU;
3881   case ISD::SETUO:  return PPC::PRED_UN;
3882     // These two are invalid for floating point.  Assume we have int.
3883   case ISD::SETULT: return PPC::PRED_LT;
3884   case ISD::SETUGT: return PPC::PRED_GT;
3885   }
3886 }
3887 
3888 /// getCRIdxForSetCC - Return the index of the condition register field
3889 /// associated with the SetCC condition, and whether or not the field is
3890 /// treated as inverted.  That is, lt = 0; ge = 0 inverted.
3891 static unsigned getCRIdxForSetCC(ISD::CondCode CC, bool &Invert) {
3892   Invert = false;
3893   switch (CC) {
3894   default: llvm_unreachable("Unknown condition!");
3895   case ISD::SETOLT:
3896   case ISD::SETLT:  return 0;                  // Bit #0 = SETOLT
3897   case ISD::SETOGT:
3898   case ISD::SETGT:  return 1;                  // Bit #1 = SETOGT
3899   case ISD::SETOEQ:
3900   case ISD::SETEQ:  return 2;                  // Bit #2 = SETOEQ
3901   case ISD::SETUO:  return 3;                  // Bit #3 = SETUO
3902   case ISD::SETUGE:
3903   case ISD::SETGE:  Invert = true; return 0;   // !Bit #0 = SETUGE
3904   case ISD::SETULE:
3905   case ISD::SETLE:  Invert = true; return 1;   // !Bit #1 = SETULE
3906   case ISD::SETUNE:
3907   case ISD::SETNE:  Invert = true; return 2;   // !Bit #2 = SETUNE
3908   case ISD::SETO:   Invert = true; return 3;   // !Bit #3 = SETO
3909   case ISD::SETUEQ:
3910   case ISD::SETOGE:
3911   case ISD::SETOLE:
3912   case ISD::SETONE:
3913     llvm_unreachable("Invalid branch code: should be expanded by legalize");
3914   // These are invalid for floating point.  Assume integer.
3915   case ISD::SETULT: return 0;
3916   case ISD::SETUGT: return 1;
3917   }
3918 }
3919 
3920 // getVCmpInst: return the vector compare instruction for the specified
3921 // vector type and condition code. Since this is for altivec specific code,
3922 // only support the altivec types (v16i8, v8i16, v4i32, v2i64, v1i128,
3923 // and v4f32).
3924 static unsigned int getVCmpInst(MVT VecVT, ISD::CondCode CC,
3925                                 bool HasVSX, bool &Swap, bool &Negate) {
3926   Swap = false;
3927   Negate = false;
3928 
3929   if (VecVT.isFloatingPoint()) {
3930     /* Handle some cases by swapping input operands.  */
3931     switch (CC) {
3932       case ISD::SETLE: CC = ISD::SETGE; Swap = true; break;
3933       case ISD::SETLT: CC = ISD::SETGT; Swap = true; break;
3934       case ISD::SETOLE: CC = ISD::SETOGE; Swap = true; break;
3935       case ISD::SETOLT: CC = ISD::SETOGT; Swap = true; break;
3936       case ISD::SETUGE: CC = ISD::SETULE; Swap = true; break;
3937       case ISD::SETUGT: CC = ISD::SETULT; Swap = true; break;
3938       default: break;
3939     }
3940     /* Handle some cases by negating the result.  */
3941     switch (CC) {
3942       case ISD::SETNE: CC = ISD::SETEQ; Negate = true; break;
3943       case ISD::SETUNE: CC = ISD::SETOEQ; Negate = true; break;
3944       case ISD::SETULE: CC = ISD::SETOGT; Negate = true; break;
3945       case ISD::SETULT: CC = ISD::SETOGE; Negate = true; break;
3946       default: break;
3947     }
3948     /* We have instructions implementing the remaining cases.  */
3949     switch (CC) {
3950       case ISD::SETEQ:
3951       case ISD::SETOEQ:
3952         if (VecVT == MVT::v4f32)
3953           return HasVSX ? PPC::XVCMPEQSP : PPC::VCMPEQFP;
3954         else if (VecVT == MVT::v2f64)
3955           return PPC::XVCMPEQDP;
3956         break;
3957       case ISD::SETGT:
3958       case ISD::SETOGT:
3959         if (VecVT == MVT::v4f32)
3960           return HasVSX ? PPC::XVCMPGTSP : PPC::VCMPGTFP;
3961         else if (VecVT == MVT::v2f64)
3962           return PPC::XVCMPGTDP;
3963         break;
3964       case ISD::SETGE:
3965       case ISD::SETOGE:
3966         if (VecVT == MVT::v4f32)
3967           return HasVSX ? PPC::XVCMPGESP : PPC::VCMPGEFP;
3968         else if (VecVT == MVT::v2f64)
3969           return PPC::XVCMPGEDP;
3970         break;
3971       default:
3972         break;
3973     }
3974     llvm_unreachable("Invalid floating-point vector compare condition");
3975   } else {
3976     /* Handle some cases by swapping input operands.  */
3977     switch (CC) {
3978       case ISD::SETGE: CC = ISD::SETLE; Swap = true; break;
3979       case ISD::SETLT: CC = ISD::SETGT; Swap = true; break;
3980       case ISD::SETUGE: CC = ISD::SETULE; Swap = true; break;
3981       case ISD::SETULT: CC = ISD::SETUGT; Swap = true; break;
3982       default: break;
3983     }
3984     /* Handle some cases by negating the result.  */
3985     switch (CC) {
3986       case ISD::SETNE: CC = ISD::SETEQ; Negate = true; break;
3987       case ISD::SETUNE: CC = ISD::SETUEQ; Negate = true; break;
3988       case ISD::SETLE: CC = ISD::SETGT; Negate = true; break;
3989       case ISD::SETULE: CC = ISD::SETUGT; Negate = true; break;
3990       default: break;
3991     }
3992     /* We have instructions implementing the remaining cases.  */
3993     switch (CC) {
3994       case ISD::SETEQ:
3995       case ISD::SETUEQ:
3996         if (VecVT == MVT::v16i8)
3997           return PPC::VCMPEQUB;
3998         else if (VecVT == MVT::v8i16)
3999           return PPC::VCMPEQUH;
4000         else if (VecVT == MVT::v4i32)
4001           return PPC::VCMPEQUW;
4002         else if (VecVT == MVT::v2i64)
4003           return PPC::VCMPEQUD;
4004         else if (VecVT == MVT::v1i128)
4005           return PPC::VCMPEQUQ;
4006         break;
4007       case ISD::SETGT:
4008         if (VecVT == MVT::v16i8)
4009           return PPC::VCMPGTSB;
4010         else if (VecVT == MVT::v8i16)
4011           return PPC::VCMPGTSH;
4012         else if (VecVT == MVT::v4i32)
4013           return PPC::VCMPGTSW;
4014         else if (VecVT == MVT::v2i64)
4015           return PPC::VCMPGTSD;
4016         else if (VecVT == MVT::v1i128)
4017            return PPC::VCMPGTSQ;
4018         break;
4019       case ISD::SETUGT:
4020         if (VecVT == MVT::v16i8)
4021           return PPC::VCMPGTUB;
4022         else if (VecVT == MVT::v8i16)
4023           return PPC::VCMPGTUH;
4024         else if (VecVT == MVT::v4i32)
4025           return PPC::VCMPGTUW;
4026         else if (VecVT == MVT::v2i64)
4027           return PPC::VCMPGTUD;
4028         else if (VecVT == MVT::v1i128)
4029            return PPC::VCMPGTUQ;
4030         break;
4031       default:
4032         break;
4033     }
4034     llvm_unreachable("Invalid integer vector compare condition");
4035   }
4036 }
4037 
4038 bool PPCDAGToDAGISel::trySETCC(SDNode *N) {
4039   SDLoc dl(N);
4040   unsigned Imm;
4041   bool IsStrict = N->isStrictFPOpcode();
4042   ISD::CondCode CC =
4043       cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
4044   EVT PtrVT =
4045       CurDAG->getTargetLoweringInfo().getPointerTy(CurDAG->getDataLayout());
4046   bool isPPC64 = (PtrVT == MVT::i64);
4047   SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
4048 
4049   SDValue LHS = N->getOperand(IsStrict ? 1 : 0);
4050   SDValue RHS = N->getOperand(IsStrict ? 2 : 1);
4051 
4052   if (!IsStrict && !Subtarget->useCRBits() && isInt32Immediate(RHS, Imm)) {
4053     // We can codegen setcc op, imm very efficiently compared to a brcond.
4054     // Check for those cases here.
4055     // setcc op, 0
4056     if (Imm == 0) {
4057       SDValue Op = LHS;
4058       switch (CC) {
4059       default: break;
4060       case ISD::SETEQ: {
4061         Op = SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Op), 0);
4062         SDValue Ops[] = { Op, getI32Imm(27, dl), getI32Imm(5, dl),
4063                           getI32Imm(31, dl) };
4064         CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4065         return true;
4066       }
4067       case ISD::SETNE: {
4068         if (isPPC64) break;
4069         SDValue AD =
4070           SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue,
4071                                          Op, getI32Imm(~0U, dl)), 0);
4072         CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, AD, Op, AD.getValue(1));
4073         return true;
4074       }
4075       case ISD::SETLT: {
4076         SDValue Ops[] = { Op, getI32Imm(1, dl), getI32Imm(31, dl),
4077                           getI32Imm(31, dl) };
4078         CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4079         return true;
4080       }
4081       case ISD::SETGT: {
4082         SDValue T =
4083           SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Op), 0);
4084         T = SDValue(CurDAG->getMachineNode(PPC::ANDC, dl, MVT::i32, T, Op), 0);
4085         SDValue Ops[] = { T, getI32Imm(1, dl), getI32Imm(31, dl),
4086                           getI32Imm(31, dl) };
4087         CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4088         return true;
4089       }
4090       }
4091     } else if (Imm == ~0U) {        // setcc op, -1
4092       SDValue Op = LHS;
4093       switch (CC) {
4094       default: break;
4095       case ISD::SETEQ:
4096         if (isPPC64) break;
4097         Op = SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue,
4098                                             Op, getI32Imm(1, dl)), 0);
4099         CurDAG->SelectNodeTo(N, PPC::ADDZE, MVT::i32,
4100                              SDValue(CurDAG->getMachineNode(PPC::LI, dl,
4101                                                             MVT::i32,
4102                                                             getI32Imm(0, dl)),
4103                                      0), Op.getValue(1));
4104         return true;
4105       case ISD::SETNE: {
4106         if (isPPC64) break;
4107         Op = SDValue(CurDAG->getMachineNode(PPC::NOR, dl, MVT::i32, Op, Op), 0);
4108         SDNode *AD = CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue,
4109                                             Op, getI32Imm(~0U, dl));
4110         CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, SDValue(AD, 0), Op,
4111                              SDValue(AD, 1));
4112         return true;
4113       }
4114       case ISD::SETLT: {
4115         SDValue AD = SDValue(CurDAG->getMachineNode(PPC::ADDI, dl, MVT::i32, Op,
4116                                                     getI32Imm(1, dl)), 0);
4117         SDValue AN = SDValue(CurDAG->getMachineNode(PPC::AND, dl, MVT::i32, AD,
4118                                                     Op), 0);
4119         SDValue Ops[] = { AN, getI32Imm(1, dl), getI32Imm(31, dl),
4120                           getI32Imm(31, dl) };
4121         CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4122         return true;
4123       }
4124       case ISD::SETGT: {
4125         SDValue Ops[] = { Op, getI32Imm(1, dl), getI32Imm(31, dl),
4126                           getI32Imm(31, dl) };
4127         Op = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0);
4128         CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Op, getI32Imm(1, dl));
4129         return true;
4130       }
4131       }
4132     }
4133   }
4134 
4135   // Altivec Vector compare instructions do not set any CR register by default and
4136   // vector compare operations return the same type as the operands.
4137   if (!IsStrict && LHS.getValueType().isVector()) {
4138     if (Subtarget->hasSPE())
4139       return false;
4140 
4141     EVT VecVT = LHS.getValueType();
4142     bool Swap, Negate;
4143     unsigned int VCmpInst =
4144         getVCmpInst(VecVT.getSimpleVT(), CC, Subtarget->hasVSX(), Swap, Negate);
4145     if (Swap)
4146       std::swap(LHS, RHS);
4147 
4148     EVT ResVT = VecVT.changeVectorElementTypeToInteger();
4149     if (Negate) {
4150       SDValue VCmp(CurDAG->getMachineNode(VCmpInst, dl, ResVT, LHS, RHS), 0);
4151       CurDAG->SelectNodeTo(N, Subtarget->hasVSX() ? PPC::XXLNOR : PPC::VNOR,
4152                            ResVT, VCmp, VCmp);
4153       return true;
4154     }
4155 
4156     CurDAG->SelectNodeTo(N, VCmpInst, ResVT, LHS, RHS);
4157     return true;
4158   }
4159 
4160   if (Subtarget->useCRBits())
4161     return false;
4162 
4163   bool Inv;
4164   unsigned Idx = getCRIdxForSetCC(CC, Inv);
4165   SDValue CCReg = SelectCC(LHS, RHS, CC, dl, Chain);
4166   if (IsStrict)
4167     CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 1), CCReg.getValue(1));
4168   SDValue IntCR;
4169 
4170   // SPE e*cmp* instructions only set the 'gt' bit, so hard-code that
4171   // The correct compare instruction is already set by SelectCC()
4172   if (Subtarget->hasSPE() && LHS.getValueType().isFloatingPoint()) {
4173     Idx = 1;
4174   }
4175 
4176   // Force the ccreg into CR7.
4177   SDValue CR7Reg = CurDAG->getRegister(PPC::CR7, MVT::i32);
4178 
4179   SDValue InFlag(nullptr, 0);  // Null incoming flag value.
4180   CCReg = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, CR7Reg, CCReg,
4181                                InFlag).getValue(1);
4182 
4183   IntCR = SDValue(CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, CR7Reg,
4184                                          CCReg), 0);
4185 
4186   SDValue Ops[] = { IntCR, getI32Imm((32 - (3 - Idx)) & 31, dl),
4187                       getI32Imm(31, dl), getI32Imm(31, dl) };
4188   if (!Inv) {
4189     CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4190     return true;
4191   }
4192 
4193   // Get the specified bit.
4194   SDValue Tmp =
4195     SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0);
4196   CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Tmp, getI32Imm(1, dl));
4197   return true;
4198 }
4199 
4200 /// Does this node represent a load/store node whose address can be represented
4201 /// with a register plus an immediate that's a multiple of \p Val:
4202 bool PPCDAGToDAGISel::isOffsetMultipleOf(SDNode *N, unsigned Val) const {
4203   LoadSDNode *LDN = dyn_cast<LoadSDNode>(N);
4204   StoreSDNode *STN = dyn_cast<StoreSDNode>(N);
4205   SDValue AddrOp;
4206   if (LDN)
4207     AddrOp = LDN->getOperand(1);
4208   else if (STN)
4209     AddrOp = STN->getOperand(2);
4210 
4211   // If the address points a frame object or a frame object with an offset,
4212   // we need to check the object alignment.
4213   short Imm = 0;
4214   if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(
4215           AddrOp.getOpcode() == ISD::ADD ? AddrOp.getOperand(0) :
4216                                            AddrOp)) {
4217     // If op0 is a frame index that is under aligned, we can't do it either,
4218     // because it is translated to r31 or r1 + slot + offset. We won't know the
4219     // slot number until the stack frame is finalized.
4220     const MachineFrameInfo &MFI = CurDAG->getMachineFunction().getFrameInfo();
4221     unsigned SlotAlign = MFI.getObjectAlign(FI->getIndex()).value();
4222     if ((SlotAlign % Val) != 0)
4223       return false;
4224 
4225     // If we have an offset, we need further check on the offset.
4226     if (AddrOp.getOpcode() != ISD::ADD)
4227       return true;
4228   }
4229 
4230   if (AddrOp.getOpcode() == ISD::ADD)
4231     return isIntS16Immediate(AddrOp.getOperand(1), Imm) && !(Imm % Val);
4232 
4233   // If the address comes from the outside, the offset will be zero.
4234   return AddrOp.getOpcode() == ISD::CopyFromReg;
4235 }
4236 
4237 void PPCDAGToDAGISel::transferMemOperands(SDNode *N, SDNode *Result) {
4238   // Transfer memoperands.
4239   MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
4240   CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {MemOp});
4241 }
4242 
4243 static bool mayUseP9Setb(SDNode *N, const ISD::CondCode &CC, SelectionDAG *DAG,
4244                          bool &NeedSwapOps, bool &IsUnCmp) {
4245 
4246   assert(N->getOpcode() == ISD::SELECT_CC && "Expecting a SELECT_CC here.");
4247 
4248   SDValue LHS = N->getOperand(0);
4249   SDValue RHS = N->getOperand(1);
4250   SDValue TrueRes = N->getOperand(2);
4251   SDValue FalseRes = N->getOperand(3);
4252   ConstantSDNode *TrueConst = dyn_cast<ConstantSDNode>(TrueRes);
4253   if (!TrueConst || (N->getSimpleValueType(0) != MVT::i64 &&
4254                      N->getSimpleValueType(0) != MVT::i32))
4255     return false;
4256 
4257   // We are looking for any of:
4258   // (select_cc lhs, rhs,  1, (sext (setcc [lr]hs, [lr]hs, cc2)), cc1)
4259   // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, cc2)), cc1)
4260   // (select_cc lhs, rhs,  0, (select_cc [lr]hs, [lr]hs,  1, -1, cc2), seteq)
4261   // (select_cc lhs, rhs,  0, (select_cc [lr]hs, [lr]hs, -1,  1, cc2), seteq)
4262   int64_t TrueResVal = TrueConst->getSExtValue();
4263   if ((TrueResVal < -1 || TrueResVal > 1) ||
4264       (TrueResVal == -1 && FalseRes.getOpcode() != ISD::ZERO_EXTEND) ||
4265       (TrueResVal == 1 && FalseRes.getOpcode() != ISD::SIGN_EXTEND) ||
4266       (TrueResVal == 0 &&
4267        (FalseRes.getOpcode() != ISD::SELECT_CC || CC != ISD::SETEQ)))
4268     return false;
4269 
4270   SDValue SetOrSelCC = FalseRes.getOpcode() == ISD::SELECT_CC
4271                            ? FalseRes
4272                            : FalseRes.getOperand(0);
4273   bool InnerIsSel = SetOrSelCC.getOpcode() == ISD::SELECT_CC;
4274   if (SetOrSelCC.getOpcode() != ISD::SETCC &&
4275       SetOrSelCC.getOpcode() != ISD::SELECT_CC)
4276     return false;
4277 
4278   // Without this setb optimization, the outer SELECT_CC will be manually
4279   // selected to SELECT_CC_I4/SELECT_CC_I8 Pseudo, then expand-isel-pseudos pass
4280   // transforms pseudo instruction to isel instruction. When there are more than
4281   // one use for result like zext/sext, with current optimization we only see
4282   // isel is replaced by setb but can't see any significant gain. Since
4283   // setb has longer latency than original isel, we should avoid this. Another
4284   // point is that setb requires comparison always kept, it can break the
4285   // opportunity to get the comparison away if we have in future.
4286   if (!SetOrSelCC.hasOneUse() || (!InnerIsSel && !FalseRes.hasOneUse()))
4287     return false;
4288 
4289   SDValue InnerLHS = SetOrSelCC.getOperand(0);
4290   SDValue InnerRHS = SetOrSelCC.getOperand(1);
4291   ISD::CondCode InnerCC =
4292       cast<CondCodeSDNode>(SetOrSelCC.getOperand(InnerIsSel ? 4 : 2))->get();
4293   // If the inner comparison is a select_cc, make sure the true/false values are
4294   // 1/-1 and canonicalize it if needed.
4295   if (InnerIsSel) {
4296     ConstantSDNode *SelCCTrueConst =
4297         dyn_cast<ConstantSDNode>(SetOrSelCC.getOperand(2));
4298     ConstantSDNode *SelCCFalseConst =
4299         dyn_cast<ConstantSDNode>(SetOrSelCC.getOperand(3));
4300     if (!SelCCTrueConst || !SelCCFalseConst)
4301       return false;
4302     int64_t SelCCTVal = SelCCTrueConst->getSExtValue();
4303     int64_t SelCCFVal = SelCCFalseConst->getSExtValue();
4304     // The values must be -1/1 (requiring a swap) or 1/-1.
4305     if (SelCCTVal == -1 && SelCCFVal == 1) {
4306       std::swap(InnerLHS, InnerRHS);
4307     } else if (SelCCTVal != 1 || SelCCFVal != -1)
4308       return false;
4309   }
4310 
4311   // Canonicalize unsigned case
4312   if (InnerCC == ISD::SETULT || InnerCC == ISD::SETUGT) {
4313     IsUnCmp = true;
4314     InnerCC = (InnerCC == ISD::SETULT) ? ISD::SETLT : ISD::SETGT;
4315   }
4316 
4317   bool InnerSwapped = false;
4318   if (LHS == InnerRHS && RHS == InnerLHS)
4319     InnerSwapped = true;
4320   else if (LHS != InnerLHS || RHS != InnerRHS)
4321     return false;
4322 
4323   switch (CC) {
4324   // (select_cc lhs, rhs,  0, \
4325   //     (select_cc [lr]hs, [lr]hs, 1, -1, setlt/setgt), seteq)
4326   case ISD::SETEQ:
4327     if (!InnerIsSel)
4328       return false;
4329     if (InnerCC != ISD::SETLT && InnerCC != ISD::SETGT)
4330       return false;
4331     NeedSwapOps = (InnerCC == ISD::SETGT) ? InnerSwapped : !InnerSwapped;
4332     break;
4333 
4334   // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, setne)), setu?lt)
4335   // (select_cc lhs, rhs, -1, (zext (setcc lhs, rhs, setgt)), setu?lt)
4336   // (select_cc lhs, rhs, -1, (zext (setcc rhs, lhs, setlt)), setu?lt)
4337   // (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, setne)), setu?lt)
4338   // (select_cc lhs, rhs, 1, (sext (setcc lhs, rhs, setgt)), setu?lt)
4339   // (select_cc lhs, rhs, 1, (sext (setcc rhs, lhs, setlt)), setu?lt)
4340   case ISD::SETULT:
4341     if (!IsUnCmp && InnerCC != ISD::SETNE)
4342       return false;
4343     IsUnCmp = true;
4344     LLVM_FALLTHROUGH;
4345   case ISD::SETLT:
4346     if (InnerCC == ISD::SETNE || (InnerCC == ISD::SETGT && !InnerSwapped) ||
4347         (InnerCC == ISD::SETLT && InnerSwapped))
4348       NeedSwapOps = (TrueResVal == 1);
4349     else
4350       return false;
4351     break;
4352 
4353   // (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, setne)), setu?gt)
4354   // (select_cc lhs, rhs, 1, (sext (setcc lhs, rhs, setlt)), setu?gt)
4355   // (select_cc lhs, rhs, 1, (sext (setcc rhs, lhs, setgt)), setu?gt)
4356   // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, setne)), setu?gt)
4357   // (select_cc lhs, rhs, -1, (zext (setcc lhs, rhs, setlt)), setu?gt)
4358   // (select_cc lhs, rhs, -1, (zext (setcc rhs, lhs, setgt)), setu?gt)
4359   case ISD::SETUGT:
4360     if (!IsUnCmp && InnerCC != ISD::SETNE)
4361       return false;
4362     IsUnCmp = true;
4363     LLVM_FALLTHROUGH;
4364   case ISD::SETGT:
4365     if (InnerCC == ISD::SETNE || (InnerCC == ISD::SETLT && !InnerSwapped) ||
4366         (InnerCC == ISD::SETGT && InnerSwapped))
4367       NeedSwapOps = (TrueResVal == -1);
4368     else
4369       return false;
4370     break;
4371 
4372   default:
4373     return false;
4374   }
4375 
4376   LLVM_DEBUG(dbgs() << "Found a node that can be lowered to a SETB: ");
4377   LLVM_DEBUG(N->dump());
4378 
4379   return true;
4380 }
4381 
4382 // Return true if it's a software square-root/divide operand.
4383 static bool isSWTestOp(SDValue N) {
4384   if (N.getOpcode() == PPCISD::FTSQRT)
4385     return true;
4386   if (N.getNumOperands() < 1 || !isa<ConstantSDNode>(N.getOperand(0)))
4387     return false;
4388   switch (N.getConstantOperandVal(0)) {
4389   case Intrinsic::ppc_vsx_xvtdivdp:
4390   case Intrinsic::ppc_vsx_xvtdivsp:
4391   case Intrinsic::ppc_vsx_xvtsqrtdp:
4392   case Intrinsic::ppc_vsx_xvtsqrtsp:
4393     return true;
4394   }
4395   return false;
4396 }
4397 
4398 bool PPCDAGToDAGISel::tryFoldSWTestBRCC(SDNode *N) {
4399   assert(N->getOpcode() == ISD::BR_CC && "ISD::BR_CC is expected.");
4400   // We are looking for following patterns, where `truncate to i1` actually has
4401   // the same semantic with `and 1`.
4402   // (br_cc seteq, (truncateToi1 SWTestOp), 0) -> (BCC PRED_NU, SWTestOp)
4403   // (br_cc seteq, (and SWTestOp, 2), 0) -> (BCC PRED_NE, SWTestOp)
4404   // (br_cc seteq, (and SWTestOp, 4), 0) -> (BCC PRED_LE, SWTestOp)
4405   // (br_cc seteq, (and SWTestOp, 8), 0) -> (BCC PRED_GE, SWTestOp)
4406   // (br_cc setne, (truncateToi1 SWTestOp), 0) -> (BCC PRED_UN, SWTestOp)
4407   // (br_cc setne, (and SWTestOp, 2), 0) -> (BCC PRED_EQ, SWTestOp)
4408   // (br_cc setne, (and SWTestOp, 4), 0) -> (BCC PRED_GT, SWTestOp)
4409   // (br_cc setne, (and SWTestOp, 8), 0) -> (BCC PRED_LT, SWTestOp)
4410   ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(1))->get();
4411   if (CC != ISD::SETEQ && CC != ISD::SETNE)
4412     return false;
4413 
4414   SDValue CmpRHS = N->getOperand(3);
4415   if (!isa<ConstantSDNode>(CmpRHS) ||
4416       cast<ConstantSDNode>(CmpRHS)->getSExtValue() != 0)
4417     return false;
4418 
4419   SDValue CmpLHS = N->getOperand(2);
4420   if (CmpLHS.getNumOperands() < 1 || !isSWTestOp(CmpLHS.getOperand(0)))
4421     return false;
4422 
4423   unsigned PCC = 0;
4424   bool IsCCNE = CC == ISD::SETNE;
4425   if (CmpLHS.getOpcode() == ISD::AND &&
4426       isa<ConstantSDNode>(CmpLHS.getOperand(1)))
4427     switch (CmpLHS.getConstantOperandVal(1)) {
4428     case 1:
4429       PCC = IsCCNE ? PPC::PRED_UN : PPC::PRED_NU;
4430       break;
4431     case 2:
4432       PCC = IsCCNE ? PPC::PRED_EQ : PPC::PRED_NE;
4433       break;
4434     case 4:
4435       PCC = IsCCNE ? PPC::PRED_GT : PPC::PRED_LE;
4436       break;
4437     case 8:
4438       PCC = IsCCNE ? PPC::PRED_LT : PPC::PRED_GE;
4439       break;
4440     default:
4441       return false;
4442     }
4443   else if (CmpLHS.getOpcode() == ISD::TRUNCATE &&
4444            CmpLHS.getValueType() == MVT::i1)
4445     PCC = IsCCNE ? PPC::PRED_UN : PPC::PRED_NU;
4446 
4447   if (PCC) {
4448     SDLoc dl(N);
4449     SDValue Ops[] = {getI32Imm(PCC, dl), CmpLHS.getOperand(0), N->getOperand(4),
4450                      N->getOperand(0)};
4451     CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops);
4452     return true;
4453   }
4454   return false;
4455 }
4456 
4457 bool PPCDAGToDAGISel::tryAsSingleRLWINM(SDNode *N) {
4458   assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected");
4459   unsigned Imm;
4460   if (!isInt32Immediate(N->getOperand(1), Imm))
4461     return false;
4462 
4463   SDLoc dl(N);
4464   SDValue Val = N->getOperand(0);
4465   unsigned SH, MB, ME;
4466   // If this is an and of a value rotated between 0 and 31 bits and then and'd
4467   // with a mask, emit rlwinm
4468   if (isRotateAndMask(Val.getNode(), Imm, false, SH, MB, ME)) {
4469     Val = Val.getOperand(0);
4470     SDValue Ops[] = {Val, getI32Imm(SH, dl), getI32Imm(MB, dl),
4471                      getI32Imm(ME, dl)};
4472     CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4473     return true;
4474   }
4475 
4476   // If this is just a masked value where the input is not handled, and
4477   // is not a rotate-left (handled by a pattern in the .td file), emit rlwinm
4478   if (isRunOfOnes(Imm, MB, ME) && Val.getOpcode() != ISD::ROTL) {
4479     SDValue Ops[] = {Val, getI32Imm(0, dl), getI32Imm(MB, dl),
4480                      getI32Imm(ME, dl)};
4481     CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4482     return true;
4483   }
4484 
4485   // AND X, 0 -> 0, not "rlwinm 32".
4486   if (Imm == 0) {
4487     ReplaceUses(SDValue(N, 0), N->getOperand(1));
4488     return true;
4489   }
4490 
4491   return false;
4492 }
4493 
4494 bool PPCDAGToDAGISel::tryAsSingleRLWINM8(SDNode *N) {
4495   assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected");
4496   uint64_t Imm64;
4497   if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64))
4498     return false;
4499 
4500   unsigned MB, ME;
4501   if (isRunOfOnes64(Imm64, MB, ME) && MB >= 32 && MB <= ME) {
4502     //                MB  ME
4503     // +----------------------+
4504     // |xxxxxxxxxxx00011111000|
4505     // +----------------------+
4506     //  0         32         64
4507     // We can only do it if the MB is larger than 32 and MB <= ME
4508     // as RLWINM will replace the contents of [0 - 32) with [32 - 64) even
4509     // we didn't rotate it.
4510     SDLoc dl(N);
4511     SDValue Ops[] = {N->getOperand(0), getI64Imm(0, dl), getI64Imm(MB - 32, dl),
4512                      getI64Imm(ME - 32, dl)};
4513     CurDAG->SelectNodeTo(N, PPC::RLWINM8, MVT::i64, Ops);
4514     return true;
4515   }
4516 
4517   return false;
4518 }
4519 
4520 bool PPCDAGToDAGISel::tryAsPairOfRLDICL(SDNode *N) {
4521   assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected");
4522   uint64_t Imm64;
4523   if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64))
4524     return false;
4525 
4526   // Do nothing if it is 16-bit imm as the pattern in the .td file handle
4527   // it well with "andi.".
4528   if (isUInt<16>(Imm64))
4529     return false;
4530 
4531   SDLoc Loc(N);
4532   SDValue Val = N->getOperand(0);
4533 
4534   // Optimized with two rldicl's as follows:
4535   // Add missing bits on left to the mask and check that the mask is a
4536   // wrapped run of ones, i.e.
4537   // Change pattern |0001111100000011111111|
4538   //             to |1111111100000011111111|.
4539   unsigned NumOfLeadingZeros = countLeadingZeros(Imm64);
4540   if (NumOfLeadingZeros != 0)
4541     Imm64 |= maskLeadingOnes<uint64_t>(NumOfLeadingZeros);
4542 
4543   unsigned MB, ME;
4544   if (!isRunOfOnes64(Imm64, MB, ME))
4545     return false;
4546 
4547   //         ME     MB                   MB-ME+63
4548   // +----------------------+     +----------------------+
4549   // |1111111100000011111111| ->  |0000001111111111111111|
4550   // +----------------------+     +----------------------+
4551   //  0                    63      0                    63
4552   // There are ME + 1 ones on the left and (MB - ME + 63) & 63 zeros in between.
4553   unsigned OnesOnLeft = ME + 1;
4554   unsigned ZerosInBetween = (MB - ME + 63) & 63;
4555   // Rotate left by OnesOnLeft (so leading ones are now trailing ones) and clear
4556   // on the left the bits that are already zeros in the mask.
4557   Val = SDValue(CurDAG->getMachineNode(PPC::RLDICL, Loc, MVT::i64, Val,
4558                                        getI64Imm(OnesOnLeft, Loc),
4559                                        getI64Imm(ZerosInBetween, Loc)),
4560                 0);
4561   //        MB-ME+63                      ME     MB
4562   // +----------------------+     +----------------------+
4563   // |0000001111111111111111| ->  |0001111100000011111111|
4564   // +----------------------+     +----------------------+
4565   //  0                    63      0                    63
4566   // Rotate back by 64 - OnesOnLeft to undo previous rotate. Then clear on the
4567   // left the number of ones we previously added.
4568   SDValue Ops[] = {Val, getI64Imm(64 - OnesOnLeft, Loc),
4569                    getI64Imm(NumOfLeadingZeros, Loc)};
4570   CurDAG->SelectNodeTo(N, PPC::RLDICL, MVT::i64, Ops);
4571   return true;
4572 }
4573 
4574 bool PPCDAGToDAGISel::tryAsSingleRLWIMI(SDNode *N) {
4575   assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected");
4576   unsigned Imm;
4577   if (!isInt32Immediate(N->getOperand(1), Imm))
4578     return false;
4579 
4580   SDValue Val = N->getOperand(0);
4581   unsigned Imm2;
4582   // ISD::OR doesn't get all the bitfield insertion fun.
4583   // (and (or x, c1), c2) where isRunOfOnes(~(c1^c2)) might be a
4584   // bitfield insert.
4585   if (Val.getOpcode() != ISD::OR || !isInt32Immediate(Val.getOperand(1), Imm2))
4586     return false;
4587 
4588   // The idea here is to check whether this is equivalent to:
4589   //   (c1 & m) | (x & ~m)
4590   // where m is a run-of-ones mask. The logic here is that, for each bit in
4591   // c1 and c2:
4592   //  - if both are 1, then the output will be 1.
4593   //  - if both are 0, then the output will be 0.
4594   //  - if the bit in c1 is 0, and the bit in c2 is 1, then the output will
4595   //    come from x.
4596   //  - if the bit in c1 is 1, and the bit in c2 is 0, then the output will
4597   //    be 0.
4598   //  If that last condition is never the case, then we can form m from the
4599   //  bits that are the same between c1 and c2.
4600   unsigned MB, ME;
4601   if (isRunOfOnes(~(Imm ^ Imm2), MB, ME) && !(~Imm & Imm2)) {
4602     SDLoc dl(N);
4603     SDValue Ops[] = {Val.getOperand(0), Val.getOperand(1), getI32Imm(0, dl),
4604                      getI32Imm(MB, dl), getI32Imm(ME, dl)};
4605     ReplaceNode(N, CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops));
4606     return true;
4607   }
4608 
4609   return false;
4610 }
4611 
4612 bool PPCDAGToDAGISel::tryAsSingleRLDICL(SDNode *N) {
4613   assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected");
4614   uint64_t Imm64;
4615   if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64) || !isMask_64(Imm64))
4616     return false;
4617 
4618   // If this is a 64-bit zero-extension mask, emit rldicl.
4619   unsigned MB = 64 - countTrailingOnes(Imm64);
4620   unsigned SH = 0;
4621   unsigned Imm;
4622   SDValue Val = N->getOperand(0);
4623   SDLoc dl(N);
4624 
4625   if (Val.getOpcode() == ISD::ANY_EXTEND) {
4626     auto Op0 = Val.getOperand(0);
4627     if (Op0.getOpcode() == ISD::SRL &&
4628         isInt32Immediate(Op0.getOperand(1).getNode(), Imm) && Imm <= MB) {
4629 
4630       auto ResultType = Val.getNode()->getValueType(0);
4631       auto ImDef = CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, ResultType);
4632       SDValue IDVal(ImDef, 0);
4633 
4634       Val = SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl, ResultType,
4635                                            IDVal, Op0.getOperand(0),
4636                                            getI32Imm(1, dl)),
4637                     0);
4638       SH = 64 - Imm;
4639     }
4640   }
4641 
4642   // If the operand is a logical right shift, we can fold it into this
4643   // instruction: rldicl(rldicl(x, 64-n, n), 0, mb) -> rldicl(x, 64-n, mb)
4644   // for n <= mb. The right shift is really a left rotate followed by a
4645   // mask, and this mask is a more-restrictive sub-mask of the mask implied
4646   // by the shift.
4647   if (Val.getOpcode() == ISD::SRL &&
4648       isInt32Immediate(Val.getOperand(1).getNode(), Imm) && Imm <= MB) {
4649     assert(Imm < 64 && "Illegal shift amount");
4650     Val = Val.getOperand(0);
4651     SH = 64 - Imm;
4652   }
4653 
4654   SDValue Ops[] = {Val, getI32Imm(SH, dl), getI32Imm(MB, dl)};
4655   CurDAG->SelectNodeTo(N, PPC::RLDICL, MVT::i64, Ops);
4656   return true;
4657 }
4658 
4659 bool PPCDAGToDAGISel::tryAsSingleRLDICR(SDNode *N) {
4660   assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected");
4661   uint64_t Imm64;
4662   if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64) ||
4663       !isMask_64(~Imm64))
4664     return false;
4665 
4666   // If this is a negated 64-bit zero-extension mask,
4667   // i.e. the immediate is a sequence of ones from most significant side
4668   // and all zero for reminder, we should use rldicr.
4669   unsigned MB = 63 - countTrailingOnes(~Imm64);
4670   unsigned SH = 0;
4671   SDLoc dl(N);
4672   SDValue Ops[] = {N->getOperand(0), getI32Imm(SH, dl), getI32Imm(MB, dl)};
4673   CurDAG->SelectNodeTo(N, PPC::RLDICR, MVT::i64, Ops);
4674   return true;
4675 }
4676 
4677 bool PPCDAGToDAGISel::tryAsSingleRLDIMI(SDNode *N) {
4678   assert(N->getOpcode() == ISD::OR && "ISD::OR SDNode expected");
4679   uint64_t Imm64;
4680   unsigned MB, ME;
4681   SDValue N0 = N->getOperand(0);
4682 
4683   // We won't get fewer instructions if the imm is 32-bit integer.
4684   // rldimi requires the imm to have consecutive ones with both sides zero.
4685   // Also, make sure the first Op has only one use, otherwise this may increase
4686   // register pressure since rldimi is destructive.
4687   if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64) ||
4688       isUInt<32>(Imm64) || !isRunOfOnes64(Imm64, MB, ME) || !N0.hasOneUse())
4689     return false;
4690 
4691   unsigned SH = 63 - ME;
4692   SDLoc Dl(N);
4693   // Use select64Imm for making LI instr instead of directly putting Imm64
4694   SDValue Ops[] = {
4695       N->getOperand(0),
4696       SDValue(selectI64Imm(CurDAG, getI64Imm(-1, Dl).getNode()), 0),
4697       getI32Imm(SH, Dl), getI32Imm(MB, Dl)};
4698   CurDAG->SelectNodeTo(N, PPC::RLDIMI, MVT::i64, Ops);
4699   return true;
4700 }
4701 
4702 // Select - Convert the specified operand from a target-independent to a
4703 // target-specific node if it hasn't already been changed.
4704 void PPCDAGToDAGISel::Select(SDNode *N) {
4705   SDLoc dl(N);
4706   if (N->isMachineOpcode()) {
4707     N->setNodeId(-1);
4708     return;   // Already selected.
4709   }
4710 
4711   // In case any misguided DAG-level optimizations form an ADD with a
4712   // TargetConstant operand, crash here instead of miscompiling (by selecting
4713   // an r+r add instead of some kind of r+i add).
4714   if (N->getOpcode() == ISD::ADD &&
4715       N->getOperand(1).getOpcode() == ISD::TargetConstant)
4716     llvm_unreachable("Invalid ADD with TargetConstant operand");
4717 
4718   // Try matching complex bit permutations before doing anything else.
4719   if (tryBitPermutation(N))
4720     return;
4721 
4722   // Try to emit integer compares as GPR-only sequences (i.e. no use of CR).
4723   if (tryIntCompareInGPR(N))
4724     return;
4725 
4726   switch (N->getOpcode()) {
4727   default: break;
4728 
4729   case ISD::Constant:
4730     if (N->getValueType(0) == MVT::i64) {
4731       ReplaceNode(N, selectI64Imm(CurDAG, N));
4732       return;
4733     }
4734     break;
4735 
4736   case ISD::INTRINSIC_WO_CHAIN: {
4737     if (!Subtarget->isISA3_1())
4738       break;
4739     unsigned Opcode = 0;
4740     switch (N->getConstantOperandVal(0)) {
4741     default:
4742       break;
4743     case Intrinsic::ppc_altivec_vstribr_p:
4744       Opcode = PPC::VSTRIBR_rec;
4745       break;
4746     case Intrinsic::ppc_altivec_vstribl_p:
4747       Opcode = PPC::VSTRIBL_rec;
4748       break;
4749     case Intrinsic::ppc_altivec_vstrihr_p:
4750       Opcode = PPC::VSTRIHR_rec;
4751       break;
4752     case Intrinsic::ppc_altivec_vstrihl_p:
4753       Opcode = PPC::VSTRIHL_rec;
4754       break;
4755     }
4756     if (!Opcode)
4757       break;
4758 
4759     // Generate the appropriate vector string isolate intrinsic to match.
4760     EVT VTs[] = {MVT::v16i8, MVT::Glue};
4761     SDValue VecStrOp =
4762         SDValue(CurDAG->getMachineNode(Opcode, dl, VTs, N->getOperand(2)), 0);
4763     // Vector string isolate instructions update the EQ bit of CR6.
4764     // Generate a SETBC instruction to extract the bit and place it in a GPR.
4765     SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_eq, dl, MVT::i32);
4766     SDValue CR6Reg = CurDAG->getRegister(PPC::CR6, MVT::i32);
4767     SDValue CRBit = SDValue(
4768         CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, MVT::i1,
4769                                CR6Reg, SubRegIdx, VecStrOp.getValue(1)),
4770         0);
4771     CurDAG->SelectNodeTo(N, PPC::SETBC, MVT::i32, CRBit);
4772     return;
4773   }
4774 
4775   case ISD::SETCC:
4776   case ISD::STRICT_FSETCC:
4777   case ISD::STRICT_FSETCCS:
4778     if (trySETCC(N))
4779       return;
4780     break;
4781   // These nodes will be transformed into GETtlsADDR32 node, which
4782   // later becomes BL_TLS __tls_get_addr(sym at tlsgd)@PLT
4783   case PPCISD::ADDI_TLSLD_L_ADDR:
4784   case PPCISD::ADDI_TLSGD_L_ADDR: {
4785     const Module *Mod = MF->getFunction().getParent();
4786     if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) != MVT::i32 ||
4787         !Subtarget->isSecurePlt() || !Subtarget->isTargetELF() ||
4788         Mod->getPICLevel() == PICLevel::SmallPIC)
4789       break;
4790     // Attach global base pointer on GETtlsADDR32 node in order to
4791     // generate secure plt code for TLS symbols.
4792     getGlobalBaseReg();
4793   } break;
4794   case PPCISD::CALL: {
4795     if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) != MVT::i32 ||
4796         !TM.isPositionIndependent() || !Subtarget->isSecurePlt() ||
4797         !Subtarget->isTargetELF())
4798       break;
4799 
4800     SDValue Op = N->getOperand(1);
4801 
4802     if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
4803       if (GA->getTargetFlags() == PPCII::MO_PLT)
4804         getGlobalBaseReg();
4805     }
4806     else if (ExternalSymbolSDNode *ES = dyn_cast<ExternalSymbolSDNode>(Op)) {
4807       if (ES->getTargetFlags() == PPCII::MO_PLT)
4808         getGlobalBaseReg();
4809     }
4810   }
4811     break;
4812 
4813   case PPCISD::GlobalBaseReg:
4814     ReplaceNode(N, getGlobalBaseReg());
4815     return;
4816 
4817   case ISD::FrameIndex:
4818     selectFrameIndex(N, N);
4819     return;
4820 
4821   case PPCISD::MFOCRF: {
4822     SDValue InFlag = N->getOperand(1);
4823     ReplaceNode(N, CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32,
4824                                           N->getOperand(0), InFlag));
4825     return;
4826   }
4827 
4828   case PPCISD::READ_TIME_BASE:
4829     ReplaceNode(N, CurDAG->getMachineNode(PPC::ReadTB, dl, MVT::i32, MVT::i32,
4830                                           MVT::Other, N->getOperand(0)));
4831     return;
4832 
4833   case PPCISD::SRA_ADDZE: {
4834     SDValue N0 = N->getOperand(0);
4835     SDValue ShiftAmt =
4836       CurDAG->getTargetConstant(*cast<ConstantSDNode>(N->getOperand(1))->
4837                                   getConstantIntValue(), dl,
4838                                   N->getValueType(0));
4839     if (N->getValueType(0) == MVT::i64) {
4840       SDNode *Op =
4841         CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, MVT::Glue,
4842                                N0, ShiftAmt);
4843       CurDAG->SelectNodeTo(N, PPC::ADDZE8, MVT::i64, SDValue(Op, 0),
4844                            SDValue(Op, 1));
4845       return;
4846     } else {
4847       assert(N->getValueType(0) == MVT::i32 &&
4848              "Expecting i64 or i32 in PPCISD::SRA_ADDZE");
4849       SDNode *Op =
4850         CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, MVT::Glue,
4851                                N0, ShiftAmt);
4852       CurDAG->SelectNodeTo(N, PPC::ADDZE, MVT::i32, SDValue(Op, 0),
4853                            SDValue(Op, 1));
4854       return;
4855     }
4856   }
4857 
4858   case ISD::STORE: {
4859     // Change TLS initial-exec D-form stores to X-form stores.
4860     StoreSDNode *ST = cast<StoreSDNode>(N);
4861     if (EnableTLSOpt && Subtarget->isELFv2ABI() &&
4862         ST->getAddressingMode() != ISD::PRE_INC)
4863       if (tryTLSXFormStore(ST))
4864         return;
4865     break;
4866   }
4867   case ISD::LOAD: {
4868     // Handle preincrement loads.
4869     LoadSDNode *LD = cast<LoadSDNode>(N);
4870     EVT LoadedVT = LD->getMemoryVT();
4871 
4872     // Normal loads are handled by code generated from the .td file.
4873     if (LD->getAddressingMode() != ISD::PRE_INC) {
4874       // Change TLS initial-exec D-form loads to X-form loads.
4875       if (EnableTLSOpt && Subtarget->isELFv2ABI())
4876         if (tryTLSXFormLoad(LD))
4877           return;
4878       break;
4879     }
4880 
4881     SDValue Offset = LD->getOffset();
4882     if (Offset.getOpcode() == ISD::TargetConstant ||
4883         Offset.getOpcode() == ISD::TargetGlobalAddress) {
4884 
4885       unsigned Opcode;
4886       bool isSExt = LD->getExtensionType() == ISD::SEXTLOAD;
4887       if (LD->getValueType(0) != MVT::i64) {
4888         // Handle PPC32 integer and normal FP loads.
4889         assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load");
4890         switch (LoadedVT.getSimpleVT().SimpleTy) {
4891           default: llvm_unreachable("Invalid PPC load type!");
4892           case MVT::f64: Opcode = PPC::LFDU; break;
4893           case MVT::f32: Opcode = PPC::LFSU; break;
4894           case MVT::i32: Opcode = PPC::LWZU; break;
4895           case MVT::i16: Opcode = isSExt ? PPC::LHAU : PPC::LHZU; break;
4896           case MVT::i1:
4897           case MVT::i8:  Opcode = PPC::LBZU; break;
4898         }
4899       } else {
4900         assert(LD->getValueType(0) == MVT::i64 && "Unknown load result type!");
4901         assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load");
4902         switch (LoadedVT.getSimpleVT().SimpleTy) {
4903           default: llvm_unreachable("Invalid PPC load type!");
4904           case MVT::i64: Opcode = PPC::LDU; break;
4905           case MVT::i32: Opcode = PPC::LWZU8; break;
4906           case MVT::i16: Opcode = isSExt ? PPC::LHAU8 : PPC::LHZU8; break;
4907           case MVT::i1:
4908           case MVT::i8:  Opcode = PPC::LBZU8; break;
4909         }
4910       }
4911 
4912       SDValue Chain = LD->getChain();
4913       SDValue Base = LD->getBasePtr();
4914       SDValue Ops[] = { Offset, Base, Chain };
4915       SDNode *MN = CurDAG->getMachineNode(
4916           Opcode, dl, LD->getValueType(0),
4917           PPCLowering->getPointerTy(CurDAG->getDataLayout()), MVT::Other, Ops);
4918       transferMemOperands(N, MN);
4919       ReplaceNode(N, MN);
4920       return;
4921     } else {
4922       unsigned Opcode;
4923       bool isSExt = LD->getExtensionType() == ISD::SEXTLOAD;
4924       if (LD->getValueType(0) != MVT::i64) {
4925         // Handle PPC32 integer and normal FP loads.
4926         assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load");
4927         switch (LoadedVT.getSimpleVT().SimpleTy) {
4928           default: llvm_unreachable("Invalid PPC load type!");
4929           case MVT::f64: Opcode = PPC::LFDUX; break;
4930           case MVT::f32: Opcode = PPC::LFSUX; break;
4931           case MVT::i32: Opcode = PPC::LWZUX; break;
4932           case MVT::i16: Opcode = isSExt ? PPC::LHAUX : PPC::LHZUX; break;
4933           case MVT::i1:
4934           case MVT::i8:  Opcode = PPC::LBZUX; break;
4935         }
4936       } else {
4937         assert(LD->getValueType(0) == MVT::i64 && "Unknown load result type!");
4938         assert((!isSExt || LoadedVT == MVT::i16 || LoadedVT == MVT::i32) &&
4939                "Invalid sext update load");
4940         switch (LoadedVT.getSimpleVT().SimpleTy) {
4941           default: llvm_unreachable("Invalid PPC load type!");
4942           case MVT::i64: Opcode = PPC::LDUX; break;
4943           case MVT::i32: Opcode = isSExt ? PPC::LWAUX  : PPC::LWZUX8; break;
4944           case MVT::i16: Opcode = isSExt ? PPC::LHAUX8 : PPC::LHZUX8; break;
4945           case MVT::i1:
4946           case MVT::i8:  Opcode = PPC::LBZUX8; break;
4947         }
4948       }
4949 
4950       SDValue Chain = LD->getChain();
4951       SDValue Base = LD->getBasePtr();
4952       SDValue Ops[] = { Base, Offset, Chain };
4953       SDNode *MN = CurDAG->getMachineNode(
4954           Opcode, dl, LD->getValueType(0),
4955           PPCLowering->getPointerTy(CurDAG->getDataLayout()), MVT::Other, Ops);
4956       transferMemOperands(N, MN);
4957       ReplaceNode(N, MN);
4958       return;
4959     }
4960   }
4961 
4962   case ISD::AND:
4963     // If this is an 'and' with a mask, try to emit rlwinm/rldicl/rldicr
4964     if (tryAsSingleRLWINM(N) || tryAsSingleRLWIMI(N) || tryAsSingleRLDICL(N) ||
4965         tryAsSingleRLDICR(N) || tryAsSingleRLWINM8(N) || tryAsPairOfRLDICL(N))
4966       return;
4967 
4968     // Other cases are autogenerated.
4969     break;
4970   case ISD::OR: {
4971     if (N->getValueType(0) == MVT::i32)
4972       if (tryBitfieldInsert(N))
4973         return;
4974 
4975     int16_t Imm;
4976     if (N->getOperand(0)->getOpcode() == ISD::FrameIndex &&
4977         isIntS16Immediate(N->getOperand(1), Imm)) {
4978       KnownBits LHSKnown = CurDAG->computeKnownBits(N->getOperand(0));
4979 
4980       // If this is equivalent to an add, then we can fold it with the
4981       // FrameIndex calculation.
4982       if ((LHSKnown.Zero.getZExtValue()|~(uint64_t)Imm) == ~0ULL) {
4983         selectFrameIndex(N, N->getOperand(0).getNode(), (int)Imm);
4984         return;
4985       }
4986     }
4987 
4988     // If this is 'or' against an imm with consecutive ones and both sides zero,
4989     // try to emit rldimi
4990     if (tryAsSingleRLDIMI(N))
4991       return;
4992 
4993     // OR with a 32-bit immediate can be handled by ori + oris
4994     // without creating an immediate in a GPR.
4995     uint64_t Imm64 = 0;
4996     bool IsPPC64 = Subtarget->isPPC64();
4997     if (IsPPC64 && isInt64Immediate(N->getOperand(1), Imm64) &&
4998         (Imm64 & ~0xFFFFFFFFuLL) == 0) {
4999       // If ImmHi (ImmHi) is zero, only one ori (oris) is generated later.
5000       uint64_t ImmHi = Imm64 >> 16;
5001       uint64_t ImmLo = Imm64 & 0xFFFF;
5002       if (ImmHi != 0 && ImmLo != 0) {
5003         SDNode *Lo = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64,
5004                                             N->getOperand(0),
5005                                             getI16Imm(ImmLo, dl));
5006         SDValue Ops1[] = { SDValue(Lo, 0), getI16Imm(ImmHi, dl)};
5007         CurDAG->SelectNodeTo(N, PPC::ORIS8, MVT::i64, Ops1);
5008         return;
5009       }
5010     }
5011 
5012     // Other cases are autogenerated.
5013     break;
5014   }
5015   case ISD::XOR: {
5016     // XOR with a 32-bit immediate can be handled by xori + xoris
5017     // without creating an immediate in a GPR.
5018     uint64_t Imm64 = 0;
5019     bool IsPPC64 = Subtarget->isPPC64();
5020     if (IsPPC64 && isInt64Immediate(N->getOperand(1), Imm64) &&
5021         (Imm64 & ~0xFFFFFFFFuLL) == 0) {
5022       // If ImmHi (ImmHi) is zero, only one xori (xoris) is generated later.
5023       uint64_t ImmHi = Imm64 >> 16;
5024       uint64_t ImmLo = Imm64 & 0xFFFF;
5025       if (ImmHi != 0 && ImmLo != 0) {
5026         SDNode *Lo = CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64,
5027                                             N->getOperand(0),
5028                                             getI16Imm(ImmLo, dl));
5029         SDValue Ops1[] = { SDValue(Lo, 0), getI16Imm(ImmHi, dl)};
5030         CurDAG->SelectNodeTo(N, PPC::XORIS8, MVT::i64, Ops1);
5031         return;
5032       }
5033     }
5034 
5035     break;
5036   }
5037   case ISD::ADD: {
5038     int16_t Imm;
5039     if (N->getOperand(0)->getOpcode() == ISD::FrameIndex &&
5040         isIntS16Immediate(N->getOperand(1), Imm)) {
5041       selectFrameIndex(N, N->getOperand(0).getNode(), (int)Imm);
5042       return;
5043     }
5044 
5045     break;
5046   }
5047   case ISD::SHL: {
5048     unsigned Imm, SH, MB, ME;
5049     if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, Imm) &&
5050         isRotateAndMask(N, Imm, true, SH, MB, ME)) {
5051       SDValue Ops[] = { N->getOperand(0).getOperand(0),
5052                           getI32Imm(SH, dl), getI32Imm(MB, dl),
5053                           getI32Imm(ME, dl) };
5054       CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
5055       return;
5056     }
5057 
5058     // Other cases are autogenerated.
5059     break;
5060   }
5061   case ISD::SRL: {
5062     unsigned Imm, SH, MB, ME;
5063     if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, Imm) &&
5064         isRotateAndMask(N, Imm, true, SH, MB, ME)) {
5065       SDValue Ops[] = { N->getOperand(0).getOperand(0),
5066                           getI32Imm(SH, dl), getI32Imm(MB, dl),
5067                           getI32Imm(ME, dl) };
5068       CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
5069       return;
5070     }
5071 
5072     // Other cases are autogenerated.
5073     break;
5074   }
5075   case ISD::MUL: {
5076     SDValue Op1 = N->getOperand(1);
5077     if (Op1.getOpcode() != ISD::Constant || Op1.getValueType() != MVT::i64)
5078       break;
5079 
5080     // If the multiplier fits int16, we can handle it with mulli.
5081     int64_t Imm = cast<ConstantSDNode>(Op1)->getZExtValue();
5082     unsigned Shift = countTrailingZeros<uint64_t>(Imm);
5083     if (isInt<16>(Imm) || !Shift)
5084       break;
5085 
5086     // If the shifted value fits int16, we can do this transformation:
5087     // (mul X, c1 << c2) -> (rldicr (mulli X, c1) c2). We do this in ISEL due to
5088     // DAGCombiner prefers (shl (mul X, c1), c2) -> (mul X, c1 << c2).
5089     uint64_t ImmSh = Imm >> Shift;
5090     if (isInt<16>(ImmSh)) {
5091       uint64_t SextImm = SignExtend64(ImmSh & 0xFFFF, 16);
5092       SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i64);
5093       SDNode *MulNode = CurDAG->getMachineNode(PPC::MULLI8, dl, MVT::i64,
5094                                                N->getOperand(0), SDImm);
5095       CurDAG->SelectNodeTo(N, PPC::RLDICR, MVT::i64, SDValue(MulNode, 0),
5096                            getI32Imm(Shift, dl), getI32Imm(63 - Shift, dl));
5097       return;
5098     }
5099     break;
5100   }
5101   // FIXME: Remove this once the ANDI glue bug is fixed:
5102   case PPCISD::ANDI_rec_1_EQ_BIT:
5103   case PPCISD::ANDI_rec_1_GT_BIT: {
5104     if (!ANDIGlueBug)
5105       break;
5106 
5107     EVT InVT = N->getOperand(0).getValueType();
5108     assert((InVT == MVT::i64 || InVT == MVT::i32) &&
5109            "Invalid input type for ANDI_rec_1_EQ_BIT");
5110 
5111     unsigned Opcode = (InVT == MVT::i64) ? PPC::ANDI8_rec : PPC::ANDI_rec;
5112     SDValue AndI(CurDAG->getMachineNode(Opcode, dl, InVT, MVT::Glue,
5113                                         N->getOperand(0),
5114                                         CurDAG->getTargetConstant(1, dl, InVT)),
5115                  0);
5116     SDValue CR0Reg = CurDAG->getRegister(PPC::CR0, MVT::i32);
5117     SDValue SRIdxVal = CurDAG->getTargetConstant(
5118         N->getOpcode() == PPCISD::ANDI_rec_1_EQ_BIT ? PPC::sub_eq : PPC::sub_gt,
5119         dl, MVT::i32);
5120 
5121     CurDAG->SelectNodeTo(N, TargetOpcode::EXTRACT_SUBREG, MVT::i1, CR0Reg,
5122                          SRIdxVal, SDValue(AndI.getNode(), 1) /* glue */);
5123     return;
5124   }
5125   case ISD::SELECT_CC: {
5126     ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(4))->get();
5127     EVT PtrVT =
5128         CurDAG->getTargetLoweringInfo().getPointerTy(CurDAG->getDataLayout());
5129     bool isPPC64 = (PtrVT == MVT::i64);
5130 
5131     // If this is a select of i1 operands, we'll pattern match it.
5132     if (Subtarget->useCRBits() && N->getOperand(0).getValueType() == MVT::i1)
5133       break;
5134 
5135     if (Subtarget->isISA3_0() && Subtarget->isPPC64()) {
5136       bool NeedSwapOps = false;
5137       bool IsUnCmp = false;
5138       if (mayUseP9Setb(N, CC, CurDAG, NeedSwapOps, IsUnCmp)) {
5139         SDValue LHS = N->getOperand(0);
5140         SDValue RHS = N->getOperand(1);
5141         if (NeedSwapOps)
5142           std::swap(LHS, RHS);
5143 
5144         // Make use of SelectCC to generate the comparison to set CR bits, for
5145         // equality comparisons having one literal operand, SelectCC probably
5146         // doesn't need to materialize the whole literal and just use xoris to
5147         // check it first, it leads the following comparison result can't
5148         // exactly represent GT/LT relationship. So to avoid this we specify
5149         // SETGT/SETUGT here instead of SETEQ.
5150         SDValue GenCC =
5151             SelectCC(LHS, RHS, IsUnCmp ? ISD::SETUGT : ISD::SETGT, dl);
5152         CurDAG->SelectNodeTo(
5153             N, N->getSimpleValueType(0) == MVT::i64 ? PPC::SETB8 : PPC::SETB,
5154             N->getValueType(0), GenCC);
5155         NumP9Setb++;
5156         return;
5157       }
5158     }
5159 
5160     // Handle the setcc cases here.  select_cc lhs, 0, 1, 0, cc
5161     if (!isPPC64)
5162       if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1)))
5163         if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N->getOperand(2)))
5164           if (ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N->getOperand(3)))
5165             if (N1C->isNullValue() && N3C->isNullValue() &&
5166                 N2C->getZExtValue() == 1ULL && CC == ISD::SETNE &&
5167                 // FIXME: Implement this optzn for PPC64.
5168                 N->getValueType(0) == MVT::i32) {
5169               SDNode *Tmp =
5170                 CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue,
5171                                        N->getOperand(0), getI32Imm(~0U, dl));
5172               CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, SDValue(Tmp, 0),
5173                                    N->getOperand(0), SDValue(Tmp, 1));
5174               return;
5175             }
5176 
5177     SDValue CCReg = SelectCC(N->getOperand(0), N->getOperand(1), CC, dl);
5178 
5179     if (N->getValueType(0) == MVT::i1) {
5180       // An i1 select is: (c & t) | (!c & f).
5181       bool Inv;
5182       unsigned Idx = getCRIdxForSetCC(CC, Inv);
5183 
5184       unsigned SRI;
5185       switch (Idx) {
5186       default: llvm_unreachable("Invalid CC index");
5187       case 0: SRI = PPC::sub_lt; break;
5188       case 1: SRI = PPC::sub_gt; break;
5189       case 2: SRI = PPC::sub_eq; break;
5190       case 3: SRI = PPC::sub_un; break;
5191       }
5192 
5193       SDValue CCBit = CurDAG->getTargetExtractSubreg(SRI, dl, MVT::i1, CCReg);
5194 
5195       SDValue NotCCBit(CurDAG->getMachineNode(PPC::CRNOR, dl, MVT::i1,
5196                                               CCBit, CCBit), 0);
5197       SDValue C =    Inv ? NotCCBit : CCBit,
5198               NotC = Inv ? CCBit    : NotCCBit;
5199 
5200       SDValue CAndT(CurDAG->getMachineNode(PPC::CRAND, dl, MVT::i1,
5201                                            C, N->getOperand(2)), 0);
5202       SDValue NotCAndF(CurDAG->getMachineNode(PPC::CRAND, dl, MVT::i1,
5203                                               NotC, N->getOperand(3)), 0);
5204 
5205       CurDAG->SelectNodeTo(N, PPC::CROR, MVT::i1, CAndT, NotCAndF);
5206       return;
5207     }
5208 
5209     unsigned BROpc =
5210         getPredicateForSetCC(CC, N->getOperand(0).getValueType(), Subtarget);
5211 
5212     unsigned SelectCCOp;
5213     if (N->getValueType(0) == MVT::i32)
5214       SelectCCOp = PPC::SELECT_CC_I4;
5215     else if (N->getValueType(0) == MVT::i64)
5216       SelectCCOp = PPC::SELECT_CC_I8;
5217     else if (N->getValueType(0) == MVT::f32) {
5218       if (Subtarget->hasP8Vector())
5219         SelectCCOp = PPC::SELECT_CC_VSSRC;
5220       else if (Subtarget->hasSPE())
5221         SelectCCOp = PPC::SELECT_CC_SPE4;
5222       else
5223         SelectCCOp = PPC::SELECT_CC_F4;
5224     } else if (N->getValueType(0) == MVT::f64) {
5225       if (Subtarget->hasVSX())
5226         SelectCCOp = PPC::SELECT_CC_VSFRC;
5227       else if (Subtarget->hasSPE())
5228         SelectCCOp = PPC::SELECT_CC_SPE;
5229       else
5230         SelectCCOp = PPC::SELECT_CC_F8;
5231     } else if (N->getValueType(0) == MVT::f128)
5232       SelectCCOp = PPC::SELECT_CC_F16;
5233     else if (Subtarget->hasSPE())
5234       SelectCCOp = PPC::SELECT_CC_SPE;
5235     else if (N->getValueType(0) == MVT::v2f64 ||
5236              N->getValueType(0) == MVT::v2i64)
5237       SelectCCOp = PPC::SELECT_CC_VSRC;
5238     else
5239       SelectCCOp = PPC::SELECT_CC_VRRC;
5240 
5241     SDValue Ops[] = { CCReg, N->getOperand(2), N->getOperand(3),
5242                         getI32Imm(BROpc, dl) };
5243     CurDAG->SelectNodeTo(N, SelectCCOp, N->getValueType(0), Ops);
5244     return;
5245   }
5246   case ISD::VECTOR_SHUFFLE:
5247     if (Subtarget->hasVSX() && (N->getValueType(0) == MVT::v2f64 ||
5248                                 N->getValueType(0) == MVT::v2i64)) {
5249       ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
5250 
5251       SDValue Op1 = N->getOperand(SVN->getMaskElt(0) < 2 ? 0 : 1),
5252               Op2 = N->getOperand(SVN->getMaskElt(1) < 2 ? 0 : 1);
5253       unsigned DM[2];
5254 
5255       for (int i = 0; i < 2; ++i)
5256         if (SVN->getMaskElt(i) <= 0 || SVN->getMaskElt(i) == 2)
5257           DM[i] = 0;
5258         else
5259           DM[i] = 1;
5260 
5261       if (Op1 == Op2 && DM[0] == 0 && DM[1] == 0 &&
5262           Op1.getOpcode() == ISD::SCALAR_TO_VECTOR &&
5263           isa<LoadSDNode>(Op1.getOperand(0))) {
5264         LoadSDNode *LD = cast<LoadSDNode>(Op1.getOperand(0));
5265         SDValue Base, Offset;
5266 
5267         if (LD->isUnindexed() && LD->hasOneUse() && Op1.hasOneUse() &&
5268             (LD->getMemoryVT() == MVT::f64 ||
5269              LD->getMemoryVT() == MVT::i64) &&
5270             SelectAddrIdxOnly(LD->getBasePtr(), Base, Offset)) {
5271           SDValue Chain = LD->getChain();
5272           SDValue Ops[] = { Base, Offset, Chain };
5273           MachineMemOperand *MemOp = LD->getMemOperand();
5274           SDNode *NewN = CurDAG->SelectNodeTo(N, PPC::LXVDSX,
5275                                               N->getValueType(0), Ops);
5276           CurDAG->setNodeMemRefs(cast<MachineSDNode>(NewN), {MemOp});
5277           return;
5278         }
5279       }
5280 
5281       // For little endian, we must swap the input operands and adjust
5282       // the mask elements (reverse and invert them).
5283       if (Subtarget->isLittleEndian()) {
5284         std::swap(Op1, Op2);
5285         unsigned tmp = DM[0];
5286         DM[0] = 1 - DM[1];
5287         DM[1] = 1 - tmp;
5288       }
5289 
5290       SDValue DMV = CurDAG->getTargetConstant(DM[1] | (DM[0] << 1), dl,
5291                                               MVT::i32);
5292       SDValue Ops[] = { Op1, Op2, DMV };
5293       CurDAG->SelectNodeTo(N, PPC::XXPERMDI, N->getValueType(0), Ops);
5294       return;
5295     }
5296 
5297     break;
5298   case PPCISD::BDNZ:
5299   case PPCISD::BDZ: {
5300     bool IsPPC64 = Subtarget->isPPC64();
5301     SDValue Ops[] = { N->getOperand(1), N->getOperand(0) };
5302     CurDAG->SelectNodeTo(N, N->getOpcode() == PPCISD::BDNZ
5303                                 ? (IsPPC64 ? PPC::BDNZ8 : PPC::BDNZ)
5304                                 : (IsPPC64 ? PPC::BDZ8 : PPC::BDZ),
5305                          MVT::Other, Ops);
5306     return;
5307   }
5308   case PPCISD::COND_BRANCH: {
5309     // Op #0 is the Chain.
5310     // Op #1 is the PPC::PRED_* number.
5311     // Op #2 is the CR#
5312     // Op #3 is the Dest MBB
5313     // Op #4 is the Flag.
5314     // Prevent PPC::PRED_* from being selected into LI.
5315     unsigned PCC = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
5316     if (EnableBranchHint)
5317       PCC |= getBranchHint(PCC, *FuncInfo, N->getOperand(3));
5318 
5319     SDValue Pred = getI32Imm(PCC, dl);
5320     SDValue Ops[] = { Pred, N->getOperand(2), N->getOperand(3),
5321       N->getOperand(0), N->getOperand(4) };
5322     CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops);
5323     return;
5324   }
5325   case ISD::BR_CC: {
5326     if (tryFoldSWTestBRCC(N))
5327       return;
5328     ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(1))->get();
5329     unsigned PCC =
5330         getPredicateForSetCC(CC, N->getOperand(2).getValueType(), Subtarget);
5331 
5332     if (N->getOperand(2).getValueType() == MVT::i1) {
5333       unsigned Opc;
5334       bool Swap;
5335       switch (PCC) {
5336       default: llvm_unreachable("Unexpected Boolean-operand predicate");
5337       case PPC::PRED_LT: Opc = PPC::CRANDC; Swap = true;  break;
5338       case PPC::PRED_LE: Opc = PPC::CRORC;  Swap = true;  break;
5339       case PPC::PRED_EQ: Opc = PPC::CREQV;  Swap = false; break;
5340       case PPC::PRED_GE: Opc = PPC::CRORC;  Swap = false; break;
5341       case PPC::PRED_GT: Opc = PPC::CRANDC; Swap = false; break;
5342       case PPC::PRED_NE: Opc = PPC::CRXOR;  Swap = false; break;
5343       }
5344 
5345       // A signed comparison of i1 values produces the opposite result to an
5346       // unsigned one if the condition code includes less-than or greater-than.
5347       // This is because 1 is the most negative signed i1 number and the most
5348       // positive unsigned i1 number. The CR-logical operations used for such
5349       // comparisons are non-commutative so for signed comparisons vs. unsigned
5350       // ones, the input operands just need to be swapped.
5351       if (ISD::isSignedIntSetCC(CC))
5352         Swap = !Swap;
5353 
5354       SDValue BitComp(CurDAG->getMachineNode(Opc, dl, MVT::i1,
5355                                              N->getOperand(Swap ? 3 : 2),
5356                                              N->getOperand(Swap ? 2 : 3)), 0);
5357       CurDAG->SelectNodeTo(N, PPC::BC, MVT::Other, BitComp, N->getOperand(4),
5358                            N->getOperand(0));
5359       return;
5360     }
5361 
5362     if (EnableBranchHint)
5363       PCC |= getBranchHint(PCC, *FuncInfo, N->getOperand(4));
5364 
5365     SDValue CondCode = SelectCC(N->getOperand(2), N->getOperand(3), CC, dl);
5366     SDValue Ops[] = { getI32Imm(PCC, dl), CondCode,
5367                         N->getOperand(4), N->getOperand(0) };
5368     CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops);
5369     return;
5370   }
5371   case ISD::BRIND: {
5372     // FIXME: Should custom lower this.
5373     SDValue Chain = N->getOperand(0);
5374     SDValue Target = N->getOperand(1);
5375     unsigned Opc = Target.getValueType() == MVT::i32 ? PPC::MTCTR : PPC::MTCTR8;
5376     unsigned Reg = Target.getValueType() == MVT::i32 ? PPC::BCTR : PPC::BCTR8;
5377     Chain = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, Target,
5378                                            Chain), 0);
5379     CurDAG->SelectNodeTo(N, Reg, MVT::Other, Chain);
5380     return;
5381   }
5382   case PPCISD::TOC_ENTRY: {
5383     const bool isPPC64 = Subtarget->isPPC64();
5384     const bool isELFABI = Subtarget->isSVR4ABI();
5385     const bool isAIXABI = Subtarget->isAIXABI();
5386 
5387     // PowerPC only support small, medium and large code model.
5388     const CodeModel::Model CModel = TM.getCodeModel();
5389     assert(!(CModel == CodeModel::Tiny || CModel == CodeModel::Kernel) &&
5390            "PowerPC doesn't support tiny or kernel code models.");
5391 
5392     if (isAIXABI && CModel == CodeModel::Medium)
5393       report_fatal_error("Medium code model is not supported on AIX.");
5394 
5395     // For 64-bit small code model, we allow SelectCodeCommon to handle this,
5396     // selecting one of LDtoc, LDtocJTI, LDtocCPT, and LDtocBA.
5397     if (isPPC64 && CModel == CodeModel::Small)
5398       break;
5399 
5400     // Handle 32-bit small code model.
5401     if (!isPPC64) {
5402       // Transforms the ISD::TOC_ENTRY node to a PPCISD::LWZtoc.
5403       auto replaceWithLWZtoc = [this, &dl](SDNode *TocEntry) {
5404         SDValue GA = TocEntry->getOperand(0);
5405         SDValue TocBase = TocEntry->getOperand(1);
5406         SDNode *MN = CurDAG->getMachineNode(PPC::LWZtoc, dl, MVT::i32, GA,
5407                                             TocBase);
5408         transferMemOperands(TocEntry, MN);
5409         ReplaceNode(TocEntry, MN);
5410       };
5411 
5412       if (isELFABI) {
5413         assert(TM.isPositionIndependent() &&
5414                "32-bit ELF can only have TOC entries in position independent"
5415                " code.");
5416         // 32-bit ELF always uses a small code model toc access.
5417         replaceWithLWZtoc(N);
5418         return;
5419       }
5420 
5421       if (isAIXABI && CModel == CodeModel::Small) {
5422         replaceWithLWZtoc(N);
5423         return;
5424       }
5425     }
5426 
5427     assert(CModel != CodeModel::Small && "All small code models handled.");
5428 
5429     assert((isPPC64 || (isAIXABI && !isPPC64)) && "We are dealing with 64-bit"
5430            " ELF/AIX or 32-bit AIX in the following.");
5431 
5432     // Transforms the ISD::TOC_ENTRY node for 32-bit AIX large code model mode
5433     // or 64-bit medium (ELF-only) or large (ELF and AIX) code model code. We
5434     // generate two instructions as described below. The first source operand
5435     // is a symbol reference. If it must be toc-referenced according to
5436     // Subtarget, we generate:
5437     // [32-bit AIX]
5438     //   LWZtocL(@sym, ADDIStocHA(%r2, @sym))
5439     // [64-bit ELF/AIX]
5440     //   LDtocL(@sym, ADDIStocHA8(%x2, @sym))
5441     // Otherwise we generate:
5442     //   ADDItocL(ADDIStocHA8(%x2, @sym), @sym)
5443     SDValue GA = N->getOperand(0);
5444     SDValue TOCbase = N->getOperand(1);
5445 
5446     EVT VT = isPPC64 ? MVT::i64 : MVT::i32;
5447     SDNode *Tmp = CurDAG->getMachineNode(
5448         isPPC64 ? PPC::ADDIStocHA8 : PPC::ADDIStocHA, dl, VT, TOCbase, GA);
5449 
5450     if (PPCLowering->isAccessedAsGotIndirect(GA)) {
5451       // If it is accessed as got-indirect, we need an extra LWZ/LD to load
5452       // the address.
5453       SDNode *MN = CurDAG->getMachineNode(
5454           isPPC64 ? PPC::LDtocL : PPC::LWZtocL, dl, VT, GA, SDValue(Tmp, 0));
5455 
5456       transferMemOperands(N, MN);
5457       ReplaceNode(N, MN);
5458       return;
5459     }
5460 
5461     // Build the address relative to the TOC-pointer.
5462     ReplaceNode(N, CurDAG->getMachineNode(PPC::ADDItocL, dl, MVT::i64,
5463                                           SDValue(Tmp, 0), GA));
5464     return;
5465   }
5466   case PPCISD::PPC32_PICGOT:
5467     // Generate a PIC-safe GOT reference.
5468     assert(Subtarget->is32BitELFABI() &&
5469            "PPCISD::PPC32_PICGOT is only supported for 32-bit SVR4");
5470     CurDAG->SelectNodeTo(N, PPC::PPC32PICGOT,
5471                          PPCLowering->getPointerTy(CurDAG->getDataLayout()),
5472                          MVT::i32);
5473     return;
5474 
5475   case PPCISD::VADD_SPLAT: {
5476     // This expands into one of three sequences, depending on whether
5477     // the first operand is odd or even, positive or negative.
5478     assert(isa<ConstantSDNode>(N->getOperand(0)) &&
5479            isa<ConstantSDNode>(N->getOperand(1)) &&
5480            "Invalid operand on VADD_SPLAT!");
5481 
5482     int Elt     = N->getConstantOperandVal(0);
5483     int EltSize = N->getConstantOperandVal(1);
5484     unsigned Opc1, Opc2, Opc3;
5485     EVT VT;
5486 
5487     if (EltSize == 1) {
5488       Opc1 = PPC::VSPLTISB;
5489       Opc2 = PPC::VADDUBM;
5490       Opc3 = PPC::VSUBUBM;
5491       VT = MVT::v16i8;
5492     } else if (EltSize == 2) {
5493       Opc1 = PPC::VSPLTISH;
5494       Opc2 = PPC::VADDUHM;
5495       Opc3 = PPC::VSUBUHM;
5496       VT = MVT::v8i16;
5497     } else {
5498       assert(EltSize == 4 && "Invalid element size on VADD_SPLAT!");
5499       Opc1 = PPC::VSPLTISW;
5500       Opc2 = PPC::VADDUWM;
5501       Opc3 = PPC::VSUBUWM;
5502       VT = MVT::v4i32;
5503     }
5504 
5505     if ((Elt & 1) == 0) {
5506       // Elt is even, in the range [-32,-18] + [16,30].
5507       //
5508       // Convert: VADD_SPLAT elt, size
5509       // Into:    tmp = VSPLTIS[BHW] elt
5510       //          VADDU[BHW]M tmp, tmp
5511       // Where:   [BHW] = B for size = 1, H for size = 2, W for size = 4
5512       SDValue EltVal = getI32Imm(Elt >> 1, dl);
5513       SDNode *Tmp = CurDAG->getMachineNode(Opc1, dl, VT, EltVal);
5514       SDValue TmpVal = SDValue(Tmp, 0);
5515       ReplaceNode(N, CurDAG->getMachineNode(Opc2, dl, VT, TmpVal, TmpVal));
5516       return;
5517     } else if (Elt > 0) {
5518       // Elt is odd and positive, in the range [17,31].
5519       //
5520       // Convert: VADD_SPLAT elt, size
5521       // Into:    tmp1 = VSPLTIS[BHW] elt-16
5522       //          tmp2 = VSPLTIS[BHW] -16
5523       //          VSUBU[BHW]M tmp1, tmp2
5524       SDValue EltVal = getI32Imm(Elt - 16, dl);
5525       SDNode *Tmp1 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal);
5526       EltVal = getI32Imm(-16, dl);
5527       SDNode *Tmp2 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal);
5528       ReplaceNode(N, CurDAG->getMachineNode(Opc3, dl, VT, SDValue(Tmp1, 0),
5529                                             SDValue(Tmp2, 0)));
5530       return;
5531     } else {
5532       // Elt is odd and negative, in the range [-31,-17].
5533       //
5534       // Convert: VADD_SPLAT elt, size
5535       // Into:    tmp1 = VSPLTIS[BHW] elt+16
5536       //          tmp2 = VSPLTIS[BHW] -16
5537       //          VADDU[BHW]M tmp1, tmp2
5538       SDValue EltVal = getI32Imm(Elt + 16, dl);
5539       SDNode *Tmp1 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal);
5540       EltVal = getI32Imm(-16, dl);
5541       SDNode *Tmp2 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal);
5542       ReplaceNode(N, CurDAG->getMachineNode(Opc2, dl, VT, SDValue(Tmp1, 0),
5543                                             SDValue(Tmp2, 0)));
5544       return;
5545     }
5546   }
5547   }
5548 
5549   SelectCode(N);
5550 }
5551 
5552 // If the target supports the cmpb instruction, do the idiom recognition here.
5553 // We don't do this as a DAG combine because we don't want to do it as nodes
5554 // are being combined (because we might miss part of the eventual idiom). We
5555 // don't want to do it during instruction selection because we want to reuse
5556 // the logic for lowering the masking operations already part of the
5557 // instruction selector.
5558 SDValue PPCDAGToDAGISel::combineToCMPB(SDNode *N) {
5559   SDLoc dl(N);
5560 
5561   assert(N->getOpcode() == ISD::OR &&
5562          "Only OR nodes are supported for CMPB");
5563 
5564   SDValue Res;
5565   if (!Subtarget->hasCMPB())
5566     return Res;
5567 
5568   if (N->getValueType(0) != MVT::i32 &&
5569       N->getValueType(0) != MVT::i64)
5570     return Res;
5571 
5572   EVT VT = N->getValueType(0);
5573 
5574   SDValue RHS, LHS;
5575   bool BytesFound[8] = {false, false, false, false, false, false, false, false};
5576   uint64_t Mask = 0, Alt = 0;
5577 
5578   auto IsByteSelectCC = [this](SDValue O, unsigned &b,
5579                                uint64_t &Mask, uint64_t &Alt,
5580                                SDValue &LHS, SDValue &RHS) {
5581     if (O.getOpcode() != ISD::SELECT_CC)
5582       return false;
5583     ISD::CondCode CC = cast<CondCodeSDNode>(O.getOperand(4))->get();
5584 
5585     if (!isa<ConstantSDNode>(O.getOperand(2)) ||
5586         !isa<ConstantSDNode>(O.getOperand(3)))
5587       return false;
5588 
5589     uint64_t PM = O.getConstantOperandVal(2);
5590     uint64_t PAlt = O.getConstantOperandVal(3);
5591     for (b = 0; b < 8; ++b) {
5592       uint64_t Mask = UINT64_C(0xFF) << (8*b);
5593       if (PM && (PM & Mask) == PM && (PAlt & Mask) == PAlt)
5594         break;
5595     }
5596 
5597     if (b == 8)
5598       return false;
5599     Mask |= PM;
5600     Alt  |= PAlt;
5601 
5602     if (!isa<ConstantSDNode>(O.getOperand(1)) ||
5603         O.getConstantOperandVal(1) != 0) {
5604       SDValue Op0 = O.getOperand(0), Op1 = O.getOperand(1);
5605       if (Op0.getOpcode() == ISD::TRUNCATE)
5606         Op0 = Op0.getOperand(0);
5607       if (Op1.getOpcode() == ISD::TRUNCATE)
5608         Op1 = Op1.getOperand(0);
5609 
5610       if (Op0.getOpcode() == ISD::SRL && Op1.getOpcode() == ISD::SRL &&
5611           Op0.getOperand(1) == Op1.getOperand(1) && CC == ISD::SETEQ &&
5612           isa<ConstantSDNode>(Op0.getOperand(1))) {
5613 
5614         unsigned Bits = Op0.getValueSizeInBits();
5615         if (b != Bits/8-1)
5616           return false;
5617         if (Op0.getConstantOperandVal(1) != Bits-8)
5618           return false;
5619 
5620         LHS = Op0.getOperand(0);
5621         RHS = Op1.getOperand(0);
5622         return true;
5623       }
5624 
5625       // When we have small integers (i16 to be specific), the form present
5626       // post-legalization uses SETULT in the SELECT_CC for the
5627       // higher-order byte, depending on the fact that the
5628       // even-higher-order bytes are known to all be zero, for example:
5629       //   select_cc (xor $lhs, $rhs), 256, 65280, 0, setult
5630       // (so when the second byte is the same, because all higher-order
5631       // bits from bytes 3 and 4 are known to be zero, the result of the
5632       // xor can be at most 255)
5633       if (Op0.getOpcode() == ISD::XOR && CC == ISD::SETULT &&
5634           isa<ConstantSDNode>(O.getOperand(1))) {
5635 
5636         uint64_t ULim = O.getConstantOperandVal(1);
5637         if (ULim != (UINT64_C(1) << b*8))
5638           return false;
5639 
5640         // Now we need to make sure that the upper bytes are known to be
5641         // zero.
5642         unsigned Bits = Op0.getValueSizeInBits();
5643         if (!CurDAG->MaskedValueIsZero(
5644                 Op0, APInt::getHighBitsSet(Bits, Bits - (b + 1) * 8)))
5645           return false;
5646 
5647         LHS = Op0.getOperand(0);
5648         RHS = Op0.getOperand(1);
5649         return true;
5650       }
5651 
5652       return false;
5653     }
5654 
5655     if (CC != ISD::SETEQ)
5656       return false;
5657 
5658     SDValue Op = O.getOperand(0);
5659     if (Op.getOpcode() == ISD::AND) {
5660       if (!isa<ConstantSDNode>(Op.getOperand(1)))
5661         return false;
5662       if (Op.getConstantOperandVal(1) != (UINT64_C(0xFF) << (8*b)))
5663         return false;
5664 
5665       SDValue XOR = Op.getOperand(0);
5666       if (XOR.getOpcode() == ISD::TRUNCATE)
5667         XOR = XOR.getOperand(0);
5668       if (XOR.getOpcode() != ISD::XOR)
5669         return false;
5670 
5671       LHS = XOR.getOperand(0);
5672       RHS = XOR.getOperand(1);
5673       return true;
5674     } else if (Op.getOpcode() == ISD::SRL) {
5675       if (!isa<ConstantSDNode>(Op.getOperand(1)))
5676         return false;
5677       unsigned Bits = Op.getValueSizeInBits();
5678       if (b != Bits/8-1)
5679         return false;
5680       if (Op.getConstantOperandVal(1) != Bits-8)
5681         return false;
5682 
5683       SDValue XOR = Op.getOperand(0);
5684       if (XOR.getOpcode() == ISD::TRUNCATE)
5685         XOR = XOR.getOperand(0);
5686       if (XOR.getOpcode() != ISD::XOR)
5687         return false;
5688 
5689       LHS = XOR.getOperand(0);
5690       RHS = XOR.getOperand(1);
5691       return true;
5692     }
5693 
5694     return false;
5695   };
5696 
5697   SmallVector<SDValue, 8> Queue(1, SDValue(N, 0));
5698   while (!Queue.empty()) {
5699     SDValue V = Queue.pop_back_val();
5700 
5701     for (const SDValue &O : V.getNode()->ops()) {
5702       unsigned b = 0;
5703       uint64_t M = 0, A = 0;
5704       SDValue OLHS, ORHS;
5705       if (O.getOpcode() == ISD::OR) {
5706         Queue.push_back(O);
5707       } else if (IsByteSelectCC(O, b, M, A, OLHS, ORHS)) {
5708         if (!LHS) {
5709           LHS = OLHS;
5710           RHS = ORHS;
5711           BytesFound[b] = true;
5712           Mask |= M;
5713           Alt  |= A;
5714         } else if ((LHS == ORHS && RHS == OLHS) ||
5715                    (RHS == ORHS && LHS == OLHS)) {
5716           BytesFound[b] = true;
5717           Mask |= M;
5718           Alt  |= A;
5719         } else {
5720           return Res;
5721         }
5722       } else {
5723         return Res;
5724       }
5725     }
5726   }
5727 
5728   unsigned LastB = 0, BCnt = 0;
5729   for (unsigned i = 0; i < 8; ++i)
5730     if (BytesFound[LastB]) {
5731       ++BCnt;
5732       LastB = i;
5733     }
5734 
5735   if (!LastB || BCnt < 2)
5736     return Res;
5737 
5738   // Because we'll be zero-extending the output anyway if don't have a specific
5739   // value for each input byte (via the Mask), we can 'anyext' the inputs.
5740   if (LHS.getValueType() != VT) {
5741     LHS = CurDAG->getAnyExtOrTrunc(LHS, dl, VT);
5742     RHS = CurDAG->getAnyExtOrTrunc(RHS, dl, VT);
5743   }
5744 
5745   Res = CurDAG->getNode(PPCISD::CMPB, dl, VT, LHS, RHS);
5746 
5747   bool NonTrivialMask = ((int64_t) Mask) != INT64_C(-1);
5748   if (NonTrivialMask && !Alt) {
5749     // Res = Mask & CMPB
5750     Res = CurDAG->getNode(ISD::AND, dl, VT, Res,
5751                           CurDAG->getConstant(Mask, dl, VT));
5752   } else if (Alt) {
5753     // Res = (CMPB & Mask) | (~CMPB & Alt)
5754     // Which, as suggested here:
5755     //   https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge
5756     // can be written as:
5757     // Res = Alt ^ ((Alt ^ Mask) & CMPB)
5758     // useful because the (Alt ^ Mask) can be pre-computed.
5759     Res = CurDAG->getNode(ISD::AND, dl, VT, Res,
5760                           CurDAG->getConstant(Mask ^ Alt, dl, VT));
5761     Res = CurDAG->getNode(ISD::XOR, dl, VT, Res,
5762                           CurDAG->getConstant(Alt, dl, VT));
5763   }
5764 
5765   return Res;
5766 }
5767 
5768 // When CR bit registers are enabled, an extension of an i1 variable to a i32
5769 // or i64 value is lowered in terms of a SELECT_I[48] operation, and thus
5770 // involves constant materialization of a 0 or a 1 or both. If the result of
5771 // the extension is then operated upon by some operator that can be constant
5772 // folded with a constant 0 or 1, and that constant can be materialized using
5773 // only one instruction (like a zero or one), then we should fold in those
5774 // operations with the select.
5775 void PPCDAGToDAGISel::foldBoolExts(SDValue &Res, SDNode *&N) {
5776   if (!Subtarget->useCRBits())
5777     return;
5778 
5779   if (N->getOpcode() != ISD::ZERO_EXTEND &&
5780       N->getOpcode() != ISD::SIGN_EXTEND &&
5781       N->getOpcode() != ISD::ANY_EXTEND)
5782     return;
5783 
5784   if (N->getOperand(0).getValueType() != MVT::i1)
5785     return;
5786 
5787   if (!N->hasOneUse())
5788     return;
5789 
5790   SDLoc dl(N);
5791   EVT VT = N->getValueType(0);
5792   SDValue Cond = N->getOperand(0);
5793   SDValue ConstTrue =
5794     CurDAG->getConstant(N->getOpcode() == ISD::SIGN_EXTEND ? -1 : 1, dl, VT);
5795   SDValue ConstFalse = CurDAG->getConstant(0, dl, VT);
5796 
5797   do {
5798     SDNode *User = *N->use_begin();
5799     if (User->getNumOperands() != 2)
5800       break;
5801 
5802     auto TryFold = [this, N, User, dl](SDValue Val) {
5803       SDValue UserO0 = User->getOperand(0), UserO1 = User->getOperand(1);
5804       SDValue O0 = UserO0.getNode() == N ? Val : UserO0;
5805       SDValue O1 = UserO1.getNode() == N ? Val : UserO1;
5806 
5807       return CurDAG->FoldConstantArithmetic(User->getOpcode(), dl,
5808                                             User->getValueType(0), {O0, O1});
5809     };
5810 
5811     // FIXME: When the semantics of the interaction between select and undef
5812     // are clearly defined, it may turn out to be unnecessary to break here.
5813     SDValue TrueRes = TryFold(ConstTrue);
5814     if (!TrueRes || TrueRes.isUndef())
5815       break;
5816     SDValue FalseRes = TryFold(ConstFalse);
5817     if (!FalseRes || FalseRes.isUndef())
5818       break;
5819 
5820     // For us to materialize these using one instruction, we must be able to
5821     // represent them as signed 16-bit integers.
5822     uint64_t True  = cast<ConstantSDNode>(TrueRes)->getZExtValue(),
5823              False = cast<ConstantSDNode>(FalseRes)->getZExtValue();
5824     if (!isInt<16>(True) || !isInt<16>(False))
5825       break;
5826 
5827     // We can replace User with a new SELECT node, and try again to see if we
5828     // can fold the select with its user.
5829     Res = CurDAG->getSelect(dl, User->getValueType(0), Cond, TrueRes, FalseRes);
5830     N = User;
5831     ConstTrue = TrueRes;
5832     ConstFalse = FalseRes;
5833   } while (N->hasOneUse());
5834 }
5835 
5836 void PPCDAGToDAGISel::PreprocessISelDAG() {
5837   SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
5838 
5839   bool MadeChange = false;
5840   while (Position != CurDAG->allnodes_begin()) {
5841     SDNode *N = &*--Position;
5842     if (N->use_empty())
5843       continue;
5844 
5845     SDValue Res;
5846     switch (N->getOpcode()) {
5847     default: break;
5848     case ISD::OR:
5849       Res = combineToCMPB(N);
5850       break;
5851     }
5852 
5853     if (!Res)
5854       foldBoolExts(Res, N);
5855 
5856     if (Res) {
5857       LLVM_DEBUG(dbgs() << "PPC DAG preprocessing replacing:\nOld:    ");
5858       LLVM_DEBUG(N->dump(CurDAG));
5859       LLVM_DEBUG(dbgs() << "\nNew: ");
5860       LLVM_DEBUG(Res.getNode()->dump(CurDAG));
5861       LLVM_DEBUG(dbgs() << "\n");
5862 
5863       CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res);
5864       MadeChange = true;
5865     }
5866   }
5867 
5868   if (MadeChange)
5869     CurDAG->RemoveDeadNodes();
5870 }
5871 
5872 /// PostprocessISelDAG - Perform some late peephole optimizations
5873 /// on the DAG representation.
5874 void PPCDAGToDAGISel::PostprocessISelDAG() {
5875   // Skip peepholes at -O0.
5876   if (TM.getOptLevel() == CodeGenOpt::None)
5877     return;
5878 
5879   PeepholePPC64();
5880   PeepholeCROps();
5881   PeepholePPC64ZExt();
5882 }
5883 
5884 // Check if all users of this node will become isel where the second operand
5885 // is the constant zero. If this is so, and if we can negate the condition,
5886 // then we can flip the true and false operands. This will allow the zero to
5887 // be folded with the isel so that we don't need to materialize a register
5888 // containing zero.
5889 bool PPCDAGToDAGISel::AllUsersSelectZero(SDNode *N) {
5890   for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
5891        UI != UE; ++UI) {
5892     SDNode *User = *UI;
5893     if (!User->isMachineOpcode())
5894       return false;
5895     if (User->getMachineOpcode() != PPC::SELECT_I4 &&
5896         User->getMachineOpcode() != PPC::SELECT_I8)
5897       return false;
5898 
5899     SDNode *Op1 = User->getOperand(1).getNode();
5900     SDNode *Op2 = User->getOperand(2).getNode();
5901     // If we have a degenerate select with two equal operands, swapping will
5902     // not do anything, and we may run into an infinite loop.
5903     if (Op1 == Op2)
5904       return false;
5905 
5906     if (!Op2->isMachineOpcode())
5907       return false;
5908 
5909     if (Op2->getMachineOpcode() != PPC::LI &&
5910         Op2->getMachineOpcode() != PPC::LI8)
5911       return false;
5912 
5913     ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op2->getOperand(0));
5914     if (!C)
5915       return false;
5916 
5917     if (!C->isNullValue())
5918       return false;
5919   }
5920 
5921   return true;
5922 }
5923 
5924 void PPCDAGToDAGISel::SwapAllSelectUsers(SDNode *N) {
5925   SmallVector<SDNode *, 4> ToReplace;
5926   for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
5927        UI != UE; ++UI) {
5928     SDNode *User = *UI;
5929     assert((User->getMachineOpcode() == PPC::SELECT_I4 ||
5930             User->getMachineOpcode() == PPC::SELECT_I8) &&
5931            "Must have all select users");
5932     ToReplace.push_back(User);
5933   }
5934 
5935   for (SmallVector<SDNode *, 4>::iterator UI = ToReplace.begin(),
5936        UE = ToReplace.end(); UI != UE; ++UI) {
5937     SDNode *User = *UI;
5938     SDNode *ResNode =
5939       CurDAG->getMachineNode(User->getMachineOpcode(), SDLoc(User),
5940                              User->getValueType(0), User->getOperand(0),
5941                              User->getOperand(2),
5942                              User->getOperand(1));
5943 
5944     LLVM_DEBUG(dbgs() << "CR Peephole replacing:\nOld:    ");
5945     LLVM_DEBUG(User->dump(CurDAG));
5946     LLVM_DEBUG(dbgs() << "\nNew: ");
5947     LLVM_DEBUG(ResNode->dump(CurDAG));
5948     LLVM_DEBUG(dbgs() << "\n");
5949 
5950     ReplaceUses(User, ResNode);
5951   }
5952 }
5953 
5954 void PPCDAGToDAGISel::PeepholeCROps() {
5955   bool IsModified;
5956   do {
5957     IsModified = false;
5958     for (SDNode &Node : CurDAG->allnodes()) {
5959       MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(&Node);
5960       if (!MachineNode || MachineNode->use_empty())
5961         continue;
5962       SDNode *ResNode = MachineNode;
5963 
5964       bool Op1Set   = false, Op1Unset = false,
5965            Op1Not   = false,
5966            Op2Set   = false, Op2Unset = false,
5967            Op2Not   = false;
5968 
5969       unsigned Opcode = MachineNode->getMachineOpcode();
5970       switch (Opcode) {
5971       default: break;
5972       case PPC::CRAND:
5973       case PPC::CRNAND:
5974       case PPC::CROR:
5975       case PPC::CRXOR:
5976       case PPC::CRNOR:
5977       case PPC::CREQV:
5978       case PPC::CRANDC:
5979       case PPC::CRORC: {
5980         SDValue Op = MachineNode->getOperand(1);
5981         if (Op.isMachineOpcode()) {
5982           if (Op.getMachineOpcode() == PPC::CRSET)
5983             Op2Set = true;
5984           else if (Op.getMachineOpcode() == PPC::CRUNSET)
5985             Op2Unset = true;
5986           else if (Op.getMachineOpcode() == PPC::CRNOR &&
5987                    Op.getOperand(0) == Op.getOperand(1))
5988             Op2Not = true;
5989         }
5990         LLVM_FALLTHROUGH;
5991       }
5992       case PPC::BC:
5993       case PPC::BCn:
5994       case PPC::SELECT_I4:
5995       case PPC::SELECT_I8:
5996       case PPC::SELECT_F4:
5997       case PPC::SELECT_F8:
5998       case PPC::SELECT_SPE:
5999       case PPC::SELECT_SPE4:
6000       case PPC::SELECT_VRRC:
6001       case PPC::SELECT_VSFRC:
6002       case PPC::SELECT_VSSRC:
6003       case PPC::SELECT_VSRC: {
6004         SDValue Op = MachineNode->getOperand(0);
6005         if (Op.isMachineOpcode()) {
6006           if (Op.getMachineOpcode() == PPC::CRSET)
6007             Op1Set = true;
6008           else if (Op.getMachineOpcode() == PPC::CRUNSET)
6009             Op1Unset = true;
6010           else if (Op.getMachineOpcode() == PPC::CRNOR &&
6011                    Op.getOperand(0) == Op.getOperand(1))
6012             Op1Not = true;
6013         }
6014         }
6015         break;
6016       }
6017 
6018       bool SelectSwap = false;
6019       switch (Opcode) {
6020       default: break;
6021       case PPC::CRAND:
6022         if (MachineNode->getOperand(0) == MachineNode->getOperand(1))
6023           // x & x = x
6024           ResNode = MachineNode->getOperand(0).getNode();
6025         else if (Op1Set)
6026           // 1 & y = y
6027           ResNode = MachineNode->getOperand(1).getNode();
6028         else if (Op2Set)
6029           // x & 1 = x
6030           ResNode = MachineNode->getOperand(0).getNode();
6031         else if (Op1Unset || Op2Unset)
6032           // x & 0 = 0 & y = 0
6033           ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode),
6034                                            MVT::i1);
6035         else if (Op1Not)
6036           // ~x & y = andc(y, x)
6037           ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode),
6038                                            MVT::i1, MachineNode->getOperand(1),
6039                                            MachineNode->getOperand(0).
6040                                              getOperand(0));
6041         else if (Op2Not)
6042           // x & ~y = andc(x, y)
6043           ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode),
6044                                            MVT::i1, MachineNode->getOperand(0),
6045                                            MachineNode->getOperand(1).
6046                                              getOperand(0));
6047         else if (AllUsersSelectZero(MachineNode)) {
6048           ResNode = CurDAG->getMachineNode(PPC::CRNAND, SDLoc(MachineNode),
6049                                            MVT::i1, MachineNode->getOperand(0),
6050                                            MachineNode->getOperand(1));
6051           SelectSwap = true;
6052         }
6053         break;
6054       case PPC::CRNAND:
6055         if (MachineNode->getOperand(0) == MachineNode->getOperand(1))
6056           // nand(x, x) -> nor(x, x)
6057           ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
6058                                            MVT::i1, MachineNode->getOperand(0),
6059                                            MachineNode->getOperand(0));
6060         else if (Op1Set)
6061           // nand(1, y) -> nor(y, y)
6062           ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
6063                                            MVT::i1, MachineNode->getOperand(1),
6064                                            MachineNode->getOperand(1));
6065         else if (Op2Set)
6066           // nand(x, 1) -> nor(x, x)
6067           ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
6068                                            MVT::i1, MachineNode->getOperand(0),
6069                                            MachineNode->getOperand(0));
6070         else if (Op1Unset || Op2Unset)
6071           // nand(x, 0) = nand(0, y) = 1
6072           ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode),
6073                                            MVT::i1);
6074         else if (Op1Not)
6075           // nand(~x, y) = ~(~x & y) = x | ~y = orc(x, y)
6076           ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode),
6077                                            MVT::i1, MachineNode->getOperand(0).
6078                                                       getOperand(0),
6079                                            MachineNode->getOperand(1));
6080         else if (Op2Not)
6081           // nand(x, ~y) = ~x | y = orc(y, x)
6082           ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode),
6083                                            MVT::i1, MachineNode->getOperand(1).
6084                                                       getOperand(0),
6085                                            MachineNode->getOperand(0));
6086         else if (AllUsersSelectZero(MachineNode)) {
6087           ResNode = CurDAG->getMachineNode(PPC::CRAND, SDLoc(MachineNode),
6088                                            MVT::i1, MachineNode->getOperand(0),
6089                                            MachineNode->getOperand(1));
6090           SelectSwap = true;
6091         }
6092         break;
6093       case PPC::CROR:
6094         if (MachineNode->getOperand(0) == MachineNode->getOperand(1))
6095           // x | x = x
6096           ResNode = MachineNode->getOperand(0).getNode();
6097         else if (Op1Set || Op2Set)
6098           // x | 1 = 1 | y = 1
6099           ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode),
6100                                            MVT::i1);
6101         else if (Op1Unset)
6102           // 0 | y = y
6103           ResNode = MachineNode->getOperand(1).getNode();
6104         else if (Op2Unset)
6105           // x | 0 = x
6106           ResNode = MachineNode->getOperand(0).getNode();
6107         else if (Op1Not)
6108           // ~x | y = orc(y, x)
6109           ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode),
6110                                            MVT::i1, MachineNode->getOperand(1),
6111                                            MachineNode->getOperand(0).
6112                                              getOperand(0));
6113         else if (Op2Not)
6114           // x | ~y = orc(x, y)
6115           ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode),
6116                                            MVT::i1, MachineNode->getOperand(0),
6117                                            MachineNode->getOperand(1).
6118                                              getOperand(0));
6119         else if (AllUsersSelectZero(MachineNode)) {
6120           ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
6121                                            MVT::i1, MachineNode->getOperand(0),
6122                                            MachineNode->getOperand(1));
6123           SelectSwap = true;
6124         }
6125         break;
6126       case PPC::CRXOR:
6127         if (MachineNode->getOperand(0) == MachineNode->getOperand(1))
6128           // xor(x, x) = 0
6129           ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode),
6130                                            MVT::i1);
6131         else if (Op1Set)
6132           // xor(1, y) -> nor(y, y)
6133           ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
6134                                            MVT::i1, MachineNode->getOperand(1),
6135                                            MachineNode->getOperand(1));
6136         else if (Op2Set)
6137           // xor(x, 1) -> nor(x, x)
6138           ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
6139                                            MVT::i1, MachineNode->getOperand(0),
6140                                            MachineNode->getOperand(0));
6141         else if (Op1Unset)
6142           // xor(0, y) = y
6143           ResNode = MachineNode->getOperand(1).getNode();
6144         else if (Op2Unset)
6145           // xor(x, 0) = x
6146           ResNode = MachineNode->getOperand(0).getNode();
6147         else if (Op1Not)
6148           // xor(~x, y) = eqv(x, y)
6149           ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode),
6150                                            MVT::i1, MachineNode->getOperand(0).
6151                                                       getOperand(0),
6152                                            MachineNode->getOperand(1));
6153         else if (Op2Not)
6154           // xor(x, ~y) = eqv(x, y)
6155           ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode),
6156                                            MVT::i1, MachineNode->getOperand(0),
6157                                            MachineNode->getOperand(1).
6158                                              getOperand(0));
6159         else if (AllUsersSelectZero(MachineNode)) {
6160           ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode),
6161                                            MVT::i1, MachineNode->getOperand(0),
6162                                            MachineNode->getOperand(1));
6163           SelectSwap = true;
6164         }
6165         break;
6166       case PPC::CRNOR:
6167         if (Op1Set || Op2Set)
6168           // nor(1, y) -> 0
6169           ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode),
6170                                            MVT::i1);
6171         else if (Op1Unset)
6172           // nor(0, y) = ~y -> nor(y, y)
6173           ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
6174                                            MVT::i1, MachineNode->getOperand(1),
6175                                            MachineNode->getOperand(1));
6176         else if (Op2Unset)
6177           // nor(x, 0) = ~x
6178           ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
6179                                            MVT::i1, MachineNode->getOperand(0),
6180                                            MachineNode->getOperand(0));
6181         else if (Op1Not)
6182           // nor(~x, y) = andc(x, y)
6183           ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode),
6184                                            MVT::i1, MachineNode->getOperand(0).
6185                                                       getOperand(0),
6186                                            MachineNode->getOperand(1));
6187         else if (Op2Not)
6188           // nor(x, ~y) = andc(y, x)
6189           ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode),
6190                                            MVT::i1, MachineNode->getOperand(1).
6191                                                       getOperand(0),
6192                                            MachineNode->getOperand(0));
6193         else if (AllUsersSelectZero(MachineNode)) {
6194           ResNode = CurDAG->getMachineNode(PPC::CROR, SDLoc(MachineNode),
6195                                            MVT::i1, MachineNode->getOperand(0),
6196                                            MachineNode->getOperand(1));
6197           SelectSwap = true;
6198         }
6199         break;
6200       case PPC::CREQV:
6201         if (MachineNode->getOperand(0) == MachineNode->getOperand(1))
6202           // eqv(x, x) = 1
6203           ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode),
6204                                            MVT::i1);
6205         else if (Op1Set)
6206           // eqv(1, y) = y
6207           ResNode = MachineNode->getOperand(1).getNode();
6208         else if (Op2Set)
6209           // eqv(x, 1) = x
6210           ResNode = MachineNode->getOperand(0).getNode();
6211         else if (Op1Unset)
6212           // eqv(0, y) = ~y -> nor(y, y)
6213           ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
6214                                            MVT::i1, MachineNode->getOperand(1),
6215                                            MachineNode->getOperand(1));
6216         else if (Op2Unset)
6217           // eqv(x, 0) = ~x
6218           ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
6219                                            MVT::i1, MachineNode->getOperand(0),
6220                                            MachineNode->getOperand(0));
6221         else if (Op1Not)
6222           // eqv(~x, y) = xor(x, y)
6223           ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode),
6224                                            MVT::i1, MachineNode->getOperand(0).
6225                                                       getOperand(0),
6226                                            MachineNode->getOperand(1));
6227         else if (Op2Not)
6228           // eqv(x, ~y) = xor(x, y)
6229           ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode),
6230                                            MVT::i1, MachineNode->getOperand(0),
6231                                            MachineNode->getOperand(1).
6232                                              getOperand(0));
6233         else if (AllUsersSelectZero(MachineNode)) {
6234           ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode),
6235                                            MVT::i1, MachineNode->getOperand(0),
6236                                            MachineNode->getOperand(1));
6237           SelectSwap = true;
6238         }
6239         break;
6240       case PPC::CRANDC:
6241         if (MachineNode->getOperand(0) == MachineNode->getOperand(1))
6242           // andc(x, x) = 0
6243           ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode),
6244                                            MVT::i1);
6245         else if (Op1Set)
6246           // andc(1, y) = ~y
6247           ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
6248                                            MVT::i1, MachineNode->getOperand(1),
6249                                            MachineNode->getOperand(1));
6250         else if (Op1Unset || Op2Set)
6251           // andc(0, y) = andc(x, 1) = 0
6252           ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode),
6253                                            MVT::i1);
6254         else if (Op2Unset)
6255           // andc(x, 0) = x
6256           ResNode = MachineNode->getOperand(0).getNode();
6257         else if (Op1Not)
6258           // andc(~x, y) = ~(x | y) = nor(x, y)
6259           ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
6260                                            MVT::i1, MachineNode->getOperand(0).
6261                                                       getOperand(0),
6262                                            MachineNode->getOperand(1));
6263         else if (Op2Not)
6264           // andc(x, ~y) = x & y
6265           ResNode = CurDAG->getMachineNode(PPC::CRAND, SDLoc(MachineNode),
6266                                            MVT::i1, MachineNode->getOperand(0),
6267                                            MachineNode->getOperand(1).
6268                                              getOperand(0));
6269         else if (AllUsersSelectZero(MachineNode)) {
6270           ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode),
6271                                            MVT::i1, MachineNode->getOperand(1),
6272                                            MachineNode->getOperand(0));
6273           SelectSwap = true;
6274         }
6275         break;
6276       case PPC::CRORC:
6277         if (MachineNode->getOperand(0) == MachineNode->getOperand(1))
6278           // orc(x, x) = 1
6279           ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode),
6280                                            MVT::i1);
6281         else if (Op1Set || Op2Unset)
6282           // orc(1, y) = orc(x, 0) = 1
6283           ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode),
6284                                            MVT::i1);
6285         else if (Op2Set)
6286           // orc(x, 1) = x
6287           ResNode = MachineNode->getOperand(0).getNode();
6288         else if (Op1Unset)
6289           // orc(0, y) = ~y
6290           ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode),
6291                                            MVT::i1, MachineNode->getOperand(1),
6292                                            MachineNode->getOperand(1));
6293         else if (Op1Not)
6294           // orc(~x, y) = ~(x & y) = nand(x, y)
6295           ResNode = CurDAG->getMachineNode(PPC::CRNAND, SDLoc(MachineNode),
6296                                            MVT::i1, MachineNode->getOperand(0).
6297                                                       getOperand(0),
6298                                            MachineNode->getOperand(1));
6299         else if (Op2Not)
6300           // orc(x, ~y) = x | y
6301           ResNode = CurDAG->getMachineNode(PPC::CROR, SDLoc(MachineNode),
6302                                            MVT::i1, MachineNode->getOperand(0),
6303                                            MachineNode->getOperand(1).
6304                                              getOperand(0));
6305         else if (AllUsersSelectZero(MachineNode)) {
6306           ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode),
6307                                            MVT::i1, MachineNode->getOperand(1),
6308                                            MachineNode->getOperand(0));
6309           SelectSwap = true;
6310         }
6311         break;
6312       case PPC::SELECT_I4:
6313       case PPC::SELECT_I8:
6314       case PPC::SELECT_F4:
6315       case PPC::SELECT_F8:
6316       case PPC::SELECT_SPE:
6317       case PPC::SELECT_SPE4:
6318       case PPC::SELECT_VRRC:
6319       case PPC::SELECT_VSFRC:
6320       case PPC::SELECT_VSSRC:
6321       case PPC::SELECT_VSRC:
6322         if (Op1Set)
6323           ResNode = MachineNode->getOperand(1).getNode();
6324         else if (Op1Unset)
6325           ResNode = MachineNode->getOperand(2).getNode();
6326         else if (Op1Not)
6327           ResNode = CurDAG->getMachineNode(MachineNode->getMachineOpcode(),
6328                                            SDLoc(MachineNode),
6329                                            MachineNode->getValueType(0),
6330                                            MachineNode->getOperand(0).
6331                                              getOperand(0),
6332                                            MachineNode->getOperand(2),
6333                                            MachineNode->getOperand(1));
6334         break;
6335       case PPC::BC:
6336       case PPC::BCn:
6337         if (Op1Not)
6338           ResNode = CurDAG->getMachineNode(Opcode == PPC::BC ? PPC::BCn :
6339                                                                PPC::BC,
6340                                            SDLoc(MachineNode),
6341                                            MVT::Other,
6342                                            MachineNode->getOperand(0).
6343                                              getOperand(0),
6344                                            MachineNode->getOperand(1),
6345                                            MachineNode->getOperand(2));
6346         // FIXME: Handle Op1Set, Op1Unset here too.
6347         break;
6348       }
6349 
6350       // If we're inverting this node because it is used only by selects that
6351       // we'd like to swap, then swap the selects before the node replacement.
6352       if (SelectSwap)
6353         SwapAllSelectUsers(MachineNode);
6354 
6355       if (ResNode != MachineNode) {
6356         LLVM_DEBUG(dbgs() << "CR Peephole replacing:\nOld:    ");
6357         LLVM_DEBUG(MachineNode->dump(CurDAG));
6358         LLVM_DEBUG(dbgs() << "\nNew: ");
6359         LLVM_DEBUG(ResNode->dump(CurDAG));
6360         LLVM_DEBUG(dbgs() << "\n");
6361 
6362         ReplaceUses(MachineNode, ResNode);
6363         IsModified = true;
6364       }
6365     }
6366     if (IsModified)
6367       CurDAG->RemoveDeadNodes();
6368   } while (IsModified);
6369 }
6370 
6371 // Gather the set of 32-bit operations that are known to have their
6372 // higher-order 32 bits zero, where ToPromote contains all such operations.
6373 static bool PeepholePPC64ZExtGather(SDValue Op32,
6374                                     SmallPtrSetImpl<SDNode *> &ToPromote) {
6375   if (!Op32.isMachineOpcode())
6376     return false;
6377 
6378   // First, check for the "frontier" instructions (those that will clear the
6379   // higher-order 32 bits.
6380 
6381   // For RLWINM and RLWNM, we need to make sure that the mask does not wrap
6382   // around. If it does not, then these instructions will clear the
6383   // higher-order bits.
6384   if ((Op32.getMachineOpcode() == PPC::RLWINM ||
6385        Op32.getMachineOpcode() == PPC::RLWNM) &&
6386       Op32.getConstantOperandVal(2) <= Op32.getConstantOperandVal(3)) {
6387     ToPromote.insert(Op32.getNode());
6388     return true;
6389   }
6390 
6391   // SLW and SRW always clear the higher-order bits.
6392   if (Op32.getMachineOpcode() == PPC::SLW ||
6393       Op32.getMachineOpcode() == PPC::SRW) {
6394     ToPromote.insert(Op32.getNode());
6395     return true;
6396   }
6397 
6398   // For LI and LIS, we need the immediate to be positive (so that it is not
6399   // sign extended).
6400   if (Op32.getMachineOpcode() == PPC::LI ||
6401       Op32.getMachineOpcode() == PPC::LIS) {
6402     if (!isUInt<15>(Op32.getConstantOperandVal(0)))
6403       return false;
6404 
6405     ToPromote.insert(Op32.getNode());
6406     return true;
6407   }
6408 
6409   // LHBRX and LWBRX always clear the higher-order bits.
6410   if (Op32.getMachineOpcode() == PPC::LHBRX ||
6411       Op32.getMachineOpcode() == PPC::LWBRX) {
6412     ToPromote.insert(Op32.getNode());
6413     return true;
6414   }
6415 
6416   // CNT[LT]ZW always produce a 64-bit value in [0,32], and so is zero extended.
6417   if (Op32.getMachineOpcode() == PPC::CNTLZW ||
6418       Op32.getMachineOpcode() == PPC::CNTTZW) {
6419     ToPromote.insert(Op32.getNode());
6420     return true;
6421   }
6422 
6423   // Next, check for those instructions we can look through.
6424 
6425   // Assuming the mask does not wrap around, then the higher-order bits are
6426   // taken directly from the first operand.
6427   if (Op32.getMachineOpcode() == PPC::RLWIMI &&
6428       Op32.getConstantOperandVal(3) <= Op32.getConstantOperandVal(4)) {
6429     SmallPtrSet<SDNode *, 16> ToPromote1;
6430     if (!PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1))
6431       return false;
6432 
6433     ToPromote.insert(Op32.getNode());
6434     ToPromote.insert(ToPromote1.begin(), ToPromote1.end());
6435     return true;
6436   }
6437 
6438   // For OR, the higher-order bits are zero if that is true for both operands.
6439   // For SELECT_I4, the same is true (but the relevant operand numbers are
6440   // shifted by 1).
6441   if (Op32.getMachineOpcode() == PPC::OR ||
6442       Op32.getMachineOpcode() == PPC::SELECT_I4) {
6443     unsigned B = Op32.getMachineOpcode() == PPC::SELECT_I4 ? 1 : 0;
6444     SmallPtrSet<SDNode *, 16> ToPromote1;
6445     if (!PeepholePPC64ZExtGather(Op32.getOperand(B+0), ToPromote1))
6446       return false;
6447     if (!PeepholePPC64ZExtGather(Op32.getOperand(B+1), ToPromote1))
6448       return false;
6449 
6450     ToPromote.insert(Op32.getNode());
6451     ToPromote.insert(ToPromote1.begin(), ToPromote1.end());
6452     return true;
6453   }
6454 
6455   // For ORI and ORIS, we need the higher-order bits of the first operand to be
6456   // zero, and also for the constant to be positive (so that it is not sign
6457   // extended).
6458   if (Op32.getMachineOpcode() == PPC::ORI ||
6459       Op32.getMachineOpcode() == PPC::ORIS) {
6460     SmallPtrSet<SDNode *, 16> ToPromote1;
6461     if (!PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1))
6462       return false;
6463     if (!isUInt<15>(Op32.getConstantOperandVal(1)))
6464       return false;
6465 
6466     ToPromote.insert(Op32.getNode());
6467     ToPromote.insert(ToPromote1.begin(), ToPromote1.end());
6468     return true;
6469   }
6470 
6471   // The higher-order bits of AND are zero if that is true for at least one of
6472   // the operands.
6473   if (Op32.getMachineOpcode() == PPC::AND) {
6474     SmallPtrSet<SDNode *, 16> ToPromote1, ToPromote2;
6475     bool Op0OK =
6476       PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1);
6477     bool Op1OK =
6478       PeepholePPC64ZExtGather(Op32.getOperand(1), ToPromote2);
6479     if (!Op0OK && !Op1OK)
6480       return false;
6481 
6482     ToPromote.insert(Op32.getNode());
6483 
6484     if (Op0OK)
6485       ToPromote.insert(ToPromote1.begin(), ToPromote1.end());
6486 
6487     if (Op1OK)
6488       ToPromote.insert(ToPromote2.begin(), ToPromote2.end());
6489 
6490     return true;
6491   }
6492 
6493   // For ANDI and ANDIS, the higher-order bits are zero if either that is true
6494   // of the first operand, or if the second operand is positive (so that it is
6495   // not sign extended).
6496   if (Op32.getMachineOpcode() == PPC::ANDI_rec ||
6497       Op32.getMachineOpcode() == PPC::ANDIS_rec) {
6498     SmallPtrSet<SDNode *, 16> ToPromote1;
6499     bool Op0OK =
6500       PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1);
6501     bool Op1OK = isUInt<15>(Op32.getConstantOperandVal(1));
6502     if (!Op0OK && !Op1OK)
6503       return false;
6504 
6505     ToPromote.insert(Op32.getNode());
6506 
6507     if (Op0OK)
6508       ToPromote.insert(ToPromote1.begin(), ToPromote1.end());
6509 
6510     return true;
6511   }
6512 
6513   return false;
6514 }
6515 
6516 void PPCDAGToDAGISel::PeepholePPC64ZExt() {
6517   if (!Subtarget->isPPC64())
6518     return;
6519 
6520   // When we zero-extend from i32 to i64, we use a pattern like this:
6521   // def : Pat<(i64 (zext i32:$in)),
6522   //           (RLDICL (INSERT_SUBREG (i64 (IMPLICIT_DEF)), $in, sub_32),
6523   //                   0, 32)>;
6524   // There are several 32-bit shift/rotate instructions, however, that will
6525   // clear the higher-order bits of their output, rendering the RLDICL
6526   // unnecessary. When that happens, we remove it here, and redefine the
6527   // relevant 32-bit operation to be a 64-bit operation.
6528 
6529   SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
6530 
6531   bool MadeChange = false;
6532   while (Position != CurDAG->allnodes_begin()) {
6533     SDNode *N = &*--Position;
6534     // Skip dead nodes and any non-machine opcodes.
6535     if (N->use_empty() || !N->isMachineOpcode())
6536       continue;
6537 
6538     if (N->getMachineOpcode() != PPC::RLDICL)
6539       continue;
6540 
6541     if (N->getConstantOperandVal(1) != 0 ||
6542         N->getConstantOperandVal(2) != 32)
6543       continue;
6544 
6545     SDValue ISR = N->getOperand(0);
6546     if (!ISR.isMachineOpcode() ||
6547         ISR.getMachineOpcode() != TargetOpcode::INSERT_SUBREG)
6548       continue;
6549 
6550     if (!ISR.hasOneUse())
6551       continue;
6552 
6553     if (ISR.getConstantOperandVal(2) != PPC::sub_32)
6554       continue;
6555 
6556     SDValue IDef = ISR.getOperand(0);
6557     if (!IDef.isMachineOpcode() ||
6558         IDef.getMachineOpcode() != TargetOpcode::IMPLICIT_DEF)
6559       continue;
6560 
6561     // We now know that we're looking at a canonical i32 -> i64 zext. See if we
6562     // can get rid of it.
6563 
6564     SDValue Op32 = ISR->getOperand(1);
6565     if (!Op32.isMachineOpcode())
6566       continue;
6567 
6568     // There are some 32-bit instructions that always clear the high-order 32
6569     // bits, there are also some instructions (like AND) that we can look
6570     // through.
6571     SmallPtrSet<SDNode *, 16> ToPromote;
6572     if (!PeepholePPC64ZExtGather(Op32, ToPromote))
6573       continue;
6574 
6575     // If the ToPromote set contains nodes that have uses outside of the set
6576     // (except for the original INSERT_SUBREG), then abort the transformation.
6577     bool OutsideUse = false;
6578     for (SDNode *PN : ToPromote) {
6579       for (SDNode *UN : PN->uses()) {
6580         if (!ToPromote.count(UN) && UN != ISR.getNode()) {
6581           OutsideUse = true;
6582           break;
6583         }
6584       }
6585 
6586       if (OutsideUse)
6587         break;
6588     }
6589     if (OutsideUse)
6590       continue;
6591 
6592     MadeChange = true;
6593 
6594     // We now know that this zero extension can be removed by promoting to
6595     // nodes in ToPromote to 64-bit operations, where for operations in the
6596     // frontier of the set, we need to insert INSERT_SUBREGs for their
6597     // operands.
6598     for (SDNode *PN : ToPromote) {
6599       unsigned NewOpcode;
6600       switch (PN->getMachineOpcode()) {
6601       default:
6602         llvm_unreachable("Don't know the 64-bit variant of this instruction");
6603       case PPC::RLWINM:    NewOpcode = PPC::RLWINM8; break;
6604       case PPC::RLWNM:     NewOpcode = PPC::RLWNM8; break;
6605       case PPC::SLW:       NewOpcode = PPC::SLW8; break;
6606       case PPC::SRW:       NewOpcode = PPC::SRW8; break;
6607       case PPC::LI:        NewOpcode = PPC::LI8; break;
6608       case PPC::LIS:       NewOpcode = PPC::LIS8; break;
6609       case PPC::LHBRX:     NewOpcode = PPC::LHBRX8; break;
6610       case PPC::LWBRX:     NewOpcode = PPC::LWBRX8; break;
6611       case PPC::CNTLZW:    NewOpcode = PPC::CNTLZW8; break;
6612       case PPC::CNTTZW:    NewOpcode = PPC::CNTTZW8; break;
6613       case PPC::RLWIMI:    NewOpcode = PPC::RLWIMI8; break;
6614       case PPC::OR:        NewOpcode = PPC::OR8; break;
6615       case PPC::SELECT_I4: NewOpcode = PPC::SELECT_I8; break;
6616       case PPC::ORI:       NewOpcode = PPC::ORI8; break;
6617       case PPC::ORIS:      NewOpcode = PPC::ORIS8; break;
6618       case PPC::AND:       NewOpcode = PPC::AND8; break;
6619       case PPC::ANDI_rec:
6620         NewOpcode = PPC::ANDI8_rec;
6621         break;
6622       case PPC::ANDIS_rec:
6623         NewOpcode = PPC::ANDIS8_rec;
6624         break;
6625       }
6626 
6627       // Note: During the replacement process, the nodes will be in an
6628       // inconsistent state (some instructions will have operands with values
6629       // of the wrong type). Once done, however, everything should be right
6630       // again.
6631 
6632       SmallVector<SDValue, 4> Ops;
6633       for (const SDValue &V : PN->ops()) {
6634         if (!ToPromote.count(V.getNode()) && V.getValueType() == MVT::i32 &&
6635             !isa<ConstantSDNode>(V)) {
6636           SDValue ReplOpOps[] = { ISR.getOperand(0), V, ISR.getOperand(2) };
6637           SDNode *ReplOp =
6638             CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG, SDLoc(V),
6639                                    ISR.getNode()->getVTList(), ReplOpOps);
6640           Ops.push_back(SDValue(ReplOp, 0));
6641         } else {
6642           Ops.push_back(V);
6643         }
6644       }
6645 
6646       // Because all to-be-promoted nodes only have users that are other
6647       // promoted nodes (or the original INSERT_SUBREG), we can safely replace
6648       // the i32 result value type with i64.
6649 
6650       SmallVector<EVT, 2> NewVTs;
6651       SDVTList VTs = PN->getVTList();
6652       for (unsigned i = 0, ie = VTs.NumVTs; i != ie; ++i)
6653         if (VTs.VTs[i] == MVT::i32)
6654           NewVTs.push_back(MVT::i64);
6655         else
6656           NewVTs.push_back(VTs.VTs[i]);
6657 
6658       LLVM_DEBUG(dbgs() << "PPC64 ZExt Peephole morphing:\nOld:    ");
6659       LLVM_DEBUG(PN->dump(CurDAG));
6660 
6661       CurDAG->SelectNodeTo(PN, NewOpcode, CurDAG->getVTList(NewVTs), Ops);
6662 
6663       LLVM_DEBUG(dbgs() << "\nNew: ");
6664       LLVM_DEBUG(PN->dump(CurDAG));
6665       LLVM_DEBUG(dbgs() << "\n");
6666     }
6667 
6668     // Now we replace the original zero extend and its associated INSERT_SUBREG
6669     // with the value feeding the INSERT_SUBREG (which has now been promoted to
6670     // return an i64).
6671 
6672     LLVM_DEBUG(dbgs() << "PPC64 ZExt Peephole replacing:\nOld:    ");
6673     LLVM_DEBUG(N->dump(CurDAG));
6674     LLVM_DEBUG(dbgs() << "\nNew: ");
6675     LLVM_DEBUG(Op32.getNode()->dump(CurDAG));
6676     LLVM_DEBUG(dbgs() << "\n");
6677 
6678     ReplaceUses(N, Op32.getNode());
6679   }
6680 
6681   if (MadeChange)
6682     CurDAG->RemoveDeadNodes();
6683 }
6684 
6685 void PPCDAGToDAGISel::PeepholePPC64() {
6686   SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
6687 
6688   while (Position != CurDAG->allnodes_begin()) {
6689     SDNode *N = &*--Position;
6690     // Skip dead nodes and any non-machine opcodes.
6691     if (N->use_empty() || !N->isMachineOpcode())
6692       continue;
6693 
6694     unsigned FirstOp;
6695     unsigned StorageOpcode = N->getMachineOpcode();
6696     bool RequiresMod4Offset = false;
6697 
6698     switch (StorageOpcode) {
6699     default: continue;
6700 
6701     case PPC::LWA:
6702     case PPC::LD:
6703     case PPC::DFLOADf64:
6704     case PPC::DFLOADf32:
6705       RequiresMod4Offset = true;
6706       LLVM_FALLTHROUGH;
6707     case PPC::LBZ:
6708     case PPC::LBZ8:
6709     case PPC::LFD:
6710     case PPC::LFS:
6711     case PPC::LHA:
6712     case PPC::LHA8:
6713     case PPC::LHZ:
6714     case PPC::LHZ8:
6715     case PPC::LWZ:
6716     case PPC::LWZ8:
6717       FirstOp = 0;
6718       break;
6719 
6720     case PPC::STD:
6721     case PPC::DFSTOREf64:
6722     case PPC::DFSTOREf32:
6723       RequiresMod4Offset = true;
6724       LLVM_FALLTHROUGH;
6725     case PPC::STB:
6726     case PPC::STB8:
6727     case PPC::STFD:
6728     case PPC::STFS:
6729     case PPC::STH:
6730     case PPC::STH8:
6731     case PPC::STW:
6732     case PPC::STW8:
6733       FirstOp = 1;
6734       break;
6735     }
6736 
6737     // If this is a load or store with a zero offset, or within the alignment,
6738     // we may be able to fold an add-immediate into the memory operation.
6739     // The check against alignment is below, as it can't occur until we check
6740     // the arguments to N
6741     if (!isa<ConstantSDNode>(N->getOperand(FirstOp)))
6742       continue;
6743 
6744     SDValue Base = N->getOperand(FirstOp + 1);
6745     if (!Base.isMachineOpcode())
6746       continue;
6747 
6748     unsigned Flags = 0;
6749     bool ReplaceFlags = true;
6750 
6751     // When the feeding operation is an add-immediate of some sort,
6752     // determine whether we need to add relocation information to the
6753     // target flags on the immediate operand when we fold it into the
6754     // load instruction.
6755     //
6756     // For something like ADDItocL, the relocation information is
6757     // inferred from the opcode; when we process it in the AsmPrinter,
6758     // we add the necessary relocation there.  A load, though, can receive
6759     // relocation from various flavors of ADDIxxx, so we need to carry
6760     // the relocation information in the target flags.
6761     switch (Base.getMachineOpcode()) {
6762     default: continue;
6763 
6764     case PPC::ADDI8:
6765     case PPC::ADDI:
6766       // In some cases (such as TLS) the relocation information
6767       // is already in place on the operand, so copying the operand
6768       // is sufficient.
6769       ReplaceFlags = false;
6770       // For these cases, the immediate may not be divisible by 4, in
6771       // which case the fold is illegal for DS-form instructions.  (The
6772       // other cases provide aligned addresses and are always safe.)
6773       if (RequiresMod4Offset &&
6774           (!isa<ConstantSDNode>(Base.getOperand(1)) ||
6775            Base.getConstantOperandVal(1) % 4 != 0))
6776         continue;
6777       break;
6778     case PPC::ADDIdtprelL:
6779       Flags = PPCII::MO_DTPREL_LO;
6780       break;
6781     case PPC::ADDItlsldL:
6782       Flags = PPCII::MO_TLSLD_LO;
6783       break;
6784     case PPC::ADDItocL:
6785       Flags = PPCII::MO_TOC_LO;
6786       break;
6787     }
6788 
6789     SDValue ImmOpnd = Base.getOperand(1);
6790 
6791     // On PPC64, the TOC base pointer is guaranteed by the ABI only to have
6792     // 8-byte alignment, and so we can only use offsets less than 8 (otherwise,
6793     // we might have needed different @ha relocation values for the offset
6794     // pointers).
6795     int MaxDisplacement = 7;
6796     if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(ImmOpnd)) {
6797       const GlobalValue *GV = GA->getGlobal();
6798       Align Alignment = GV->getPointerAlignment(CurDAG->getDataLayout());
6799       MaxDisplacement = std::min((int)Alignment.value() - 1, MaxDisplacement);
6800     }
6801 
6802     bool UpdateHBase = false;
6803     SDValue HBase = Base.getOperand(0);
6804 
6805     int Offset = N->getConstantOperandVal(FirstOp);
6806     if (ReplaceFlags) {
6807       if (Offset < 0 || Offset > MaxDisplacement) {
6808         // If we have a addi(toc@l)/addis(toc@ha) pair, and the addis has only
6809         // one use, then we can do this for any offset, we just need to also
6810         // update the offset (i.e. the symbol addend) on the addis also.
6811         if (Base.getMachineOpcode() != PPC::ADDItocL)
6812           continue;
6813 
6814         if (!HBase.isMachineOpcode() ||
6815             HBase.getMachineOpcode() != PPC::ADDIStocHA8)
6816           continue;
6817 
6818         if (!Base.hasOneUse() || !HBase.hasOneUse())
6819           continue;
6820 
6821         SDValue HImmOpnd = HBase.getOperand(1);
6822         if (HImmOpnd != ImmOpnd)
6823           continue;
6824 
6825         UpdateHBase = true;
6826       }
6827     } else {
6828       // If we're directly folding the addend from an addi instruction, then:
6829       //  1. In general, the offset on the memory access must be zero.
6830       //  2. If the addend is a constant, then it can be combined with a
6831       //     non-zero offset, but only if the result meets the encoding
6832       //     requirements.
6833       if (auto *C = dyn_cast<ConstantSDNode>(ImmOpnd)) {
6834         Offset += C->getSExtValue();
6835 
6836         if (RequiresMod4Offset && (Offset % 4) != 0)
6837           continue;
6838 
6839         if (!isInt<16>(Offset))
6840           continue;
6841 
6842         ImmOpnd = CurDAG->getTargetConstant(Offset, SDLoc(ImmOpnd),
6843                                             ImmOpnd.getValueType());
6844       } else if (Offset != 0) {
6845         continue;
6846       }
6847     }
6848 
6849     // We found an opportunity.  Reverse the operands from the add
6850     // immediate and substitute them into the load or store.  If
6851     // needed, update the target flags for the immediate operand to
6852     // reflect the necessary relocation information.
6853     LLVM_DEBUG(dbgs() << "Folding add-immediate into mem-op:\nBase:    ");
6854     LLVM_DEBUG(Base->dump(CurDAG));
6855     LLVM_DEBUG(dbgs() << "\nN: ");
6856     LLVM_DEBUG(N->dump(CurDAG));
6857     LLVM_DEBUG(dbgs() << "\n");
6858 
6859     // If the relocation information isn't already present on the
6860     // immediate operand, add it now.
6861     if (ReplaceFlags) {
6862       if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(ImmOpnd)) {
6863         SDLoc dl(GA);
6864         const GlobalValue *GV = GA->getGlobal();
6865         Align Alignment = GV->getPointerAlignment(CurDAG->getDataLayout());
6866         // We can't perform this optimization for data whose alignment
6867         // is insufficient for the instruction encoding.
6868         if (Alignment < 4 && (RequiresMod4Offset || (Offset % 4) != 0)) {
6869           LLVM_DEBUG(dbgs() << "Rejected this candidate for alignment.\n\n");
6870           continue;
6871         }
6872         ImmOpnd = CurDAG->getTargetGlobalAddress(GV, dl, MVT::i64, Offset, Flags);
6873       } else if (ConstantPoolSDNode *CP =
6874                  dyn_cast<ConstantPoolSDNode>(ImmOpnd)) {
6875         const Constant *C = CP->getConstVal();
6876         ImmOpnd = CurDAG->getTargetConstantPool(C, MVT::i64, CP->getAlign(),
6877                                                 Offset, Flags);
6878       }
6879     }
6880 
6881     if (FirstOp == 1) // Store
6882       (void)CurDAG->UpdateNodeOperands(N, N->getOperand(0), ImmOpnd,
6883                                        Base.getOperand(0), N->getOperand(3));
6884     else // Load
6885       (void)CurDAG->UpdateNodeOperands(N, ImmOpnd, Base.getOperand(0),
6886                                        N->getOperand(2));
6887 
6888     if (UpdateHBase)
6889       (void)CurDAG->UpdateNodeOperands(HBase.getNode(), HBase.getOperand(0),
6890                                        ImmOpnd);
6891 
6892     // The add-immediate may now be dead, in which case remove it.
6893     if (Base.getNode()->use_empty())
6894       CurDAG->RemoveDeadNode(Base.getNode());
6895   }
6896 }
6897 
6898 /// createPPCISelDag - This pass converts a legalized DAG into a
6899 /// PowerPC-specific DAG, ready for instruction scheduling.
6900 ///
6901 FunctionPass *llvm::createPPCISelDag(PPCTargetMachine &TM,
6902                                      CodeGenOpt::Level OptLevel) {
6903   return new PPCDAGToDAGISel(TM, OptLevel);
6904 }
6905