Lines Matching +full:un +full:- +full:masked
1 //===-- PPCISelDAGToDAG.cpp - PPC --pattern matching inst selector --------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
12 //===----------------------------------------------------------------------===//
72 #define DEBUG_TYPE "ppc-isel"
73 #define PASS_NAME "PowerPC DAG->DAG Pattern Instruction Selection"
86 "Number of compares not eliminated as they have non-extending uses.");
91 cl::opt<bool> ANDIGlueBug("expose-ppc-andi-glue-bug",
95 UseBitPermRewriter("ppc-use-bit-perm-rewriter", cl::init(true),
99 "ppc-bit-perm-rewriter-stress-rotates",
105 "ppc-use-branch-hint", cl::init(true),
110 "ppc-tls-opt", cl::init(true),
119 "ppc-gpr-icmps", cl::Hidden, cl::init(ICGPR_All),
120 cl::desc("Specify the types of comparisons to emit GPR-only code for."),
139 //===--------------------------------------------------------------------===//
140 /// PPCDAGToDAGISel - PPC specific code to select PPC machine
156 // Make sure we re-emit a set of the global base reg if necessary in runOnMachineFunction()
159 PPCLowering = Subtarget->getTargetLowering(); in runOnMachineFunction()
160 if (Subtarget->hasROPProtect()) { in runOnMachineFunction()
167 FI->setROPProtectionHashSaveIndex(Result); in runOnMachineFunction()
177 /// getI16Imm - Return a target constant with the specified value, of type
180 return CurDAG->getTargetConstant(Imm, dl, MVT::i16); in getI16Imm()
183 /// getI32Imm - Return a target constant with the specified value, of type
186 return CurDAG->getTargetConstant(Imm, dl, MVT::i32); in getI32Imm()
189 /// getI64Imm - Return a target constant with the specified value, of type
192 return CurDAG->getTargetConstant(Imm, dl, MVT::i64); in getI64Imm()
195 /// getSmallIPtrImm - Return a target constant of pointer type.
197 return CurDAG->getTargetConstant( in getSmallIPtrImm()
198 Imm, dl, PPCLowering->getPointerTy(CurDAG->getDataLayout())); in getSmallIPtrImm()
201 /// isRotateAndMask - Returns true if Mask and Shift can be folded into a
206 /// getGlobalBaseReg - insert code into the entry mbb to materialize the PIC
212 // Select - Convert the specified operand from a target-independent to a
213 // target-specific node if it hasn't already been changed.
220 // tryTLSXFormLoad - Convert an ISD::LOAD fed by a PPCISD::ADD_TLS into
221 // an X-Form load instruction with the offset being a relocation coming from
224 // tryTLSXFormStore - Convert an ISD::STORE fed by a PPCISD::ADD_TLS into
225 // an X-Form store instruction with the offset being a relocation coming from
228 /// SelectCC - Select a comparison of the specified values with the
233 /// SelectAddrImmOffs - Return true if the operand is valid for a preinc
246 /// SelectDSForm - Returns true if address N can be represented by the
248 /// 16-bit displacement that is a multiple of 4.
250 return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG, in SelectDSForm()
254 /// SelectDQForm - Returns true if address N can be represented by the
256 /// 16-bit displacement that is a multiple of 16.
258 return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG, in SelectDQForm()
262 /// SelectDForm - Returns true if address N can be represented by
264 /// signed 16-bit immediate.
266 return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG, in SelectDForm()
270 /// SelectPCRelForm - Returns true if address N can be represented by
271 /// PC-Relative addressing mode.
274 return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG, in SelectPCRelForm()
278 /// SelectPDForm - Returns true if address N can be represented by Prefixed
279 /// DForm addressing mode (a base register, plus a signed 34-bit immediate.
281 return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG, in SelectPDForm()
286 /// SelectXForm - Returns true if address N can be represented by the
289 return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG, in SelectXForm()
293 /// SelectForceXForm - Given the specified address, force it to be
297 return PPCLowering->SelectForceXFormMode(N, Disp, Base, *CurDAG) == in SelectForceXForm()
301 /// SelectAddrIdx - Given the specified address, check to see if it can be
308 return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG, in SelectAddrIdx()
312 /// SelectAddrIdx4 - Given the specified address, check to see if it can be
319 return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG, in SelectAddrIdxX4()
323 /// SelectAddrIdx16 - Given the specified address, check to see if it can be
330 return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG, in SelectAddrIdxX16()
334 /// SelectAddrIdxOnly - Given the specified address, force it to be
337 return PPCLowering->SelectAddressRegRegOnly(N, Base, Index, *CurDAG); in SelectAddrIdxOnly()
340 /// SelectAddrImm - Returns true if the address N can be represented by
341 /// a base register plus a signed 16-bit displacement [r+imm].
346 return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, in SelectAddrImm()
350 /// SelectAddrImmX4 - Returns true if the address N can be represented by
351 /// a base register plus a signed 16-bit displacement that is a multiple of
354 return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, Align(4)); in SelectAddrImmX4()
357 /// SelectAddrImmX16 - Returns true if the address N can be represented by
358 /// a base register plus a signed 16-bit displacement that is a multiple of
361 return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, in SelectAddrImmX16()
365 /// SelectAddrImmX34 - Returns true if the address N can be represented by
366 /// a base register plus a signed 34-bit displacement. Suitable for use by
369 return PPCLowering->SelectAddressRegImm34(N, Disp, Base, *CurDAG); in SelectAddrImmX34()
379 return PPCLowering->SelectAddressPCRel(N, Base); in SelectAddrPCRel()
382 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
403 const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo(); in SelectInlineAsmMemoryOperand()
404 const TargetRegisterClass *TRC = TRI->getPointerRegClass(*MF, /*Kind=*/1); in SelectInlineAsmMemoryOperand()
406 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i32); in SelectInlineAsmMemoryOperand()
408 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, in SelectInlineAsmMemoryOperand()
462 /// getGlobalBaseReg - Output the instructions required to put the in INITIALIZE_PASS()
467 const TargetInstrInfo &TII = *Subtarget->getInstrInfo(); in INITIALIZE_PASS()
469 MachineBasicBlock &FirstMBB = MF->front(); in INITIALIZE_PASS()
471 const Module *M = MF->getFunction().getParent(); in INITIALIZE_PASS()
474 if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) == MVT::i32) { in INITIALIZE_PASS()
475 if (Subtarget->isTargetELF()) { in INITIALIZE_PASS()
477 if (!Subtarget->isSecurePlt() && in INITIALIZE_PASS()
478 M->getPICLevel() == PICLevel::SmallPIC) { in INITIALIZE_PASS()
481 MF->getInfo<PPCFunctionInfo>()->setUsesPICBase(true); in INITIALIZE_PASS()
485 Register TempReg = RegInfo->createVirtualRegister(&PPC::GPRCRegClass); in INITIALIZE_PASS()
489 MF->getInfo<PPCFunctionInfo>()->setUsesPICBase(true); in INITIALIZE_PASS()
493 RegInfo->createVirtualRegister(&PPC::GPRC_and_GPRC_NOR0RegClass); in INITIALIZE_PASS()
500 // of shrink-wrapping whenever we emit this instruction. Considering in INITIALIZE_PASS()
507 MF->getInfo<PPCFunctionInfo>()->setShrinkWrapDisabled(true); in INITIALIZE_PASS()
508 GlobalBaseReg = RegInfo->createVirtualRegister(&PPC::G8RC_and_G8RC_NOX0RegClass); in INITIALIZE_PASS()
513 return CurDAG->getRegister(GlobalBaseReg, in INITIALIZE_PASS()
514 PPCLowering->getPointerTy(CurDAG->getDataLayout())) in INITIALIZE_PASS()
518 // Check if a SDValue has the toc-data attribute.
524 const GlobalVariable *GV = dyn_cast_or_null<GlobalVariable>(GA->getGlobal()); in hasTocDataAttr()
528 if (!GV->hasAttribute("toc-data")) in hasTocDataAttr()
540 GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Node->getOperand(0)); in getCodeModel()
544 const GlobalValue *GV = GA->getGlobal(); in getCodeModel()
551 /// isInt32Immediate - This method tests to see if the node is a 32-bit constant
552 /// operand. If so Imm will receive the 32-bit value.
554 if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) { in isInt32Immediate()
555 Imm = N->getAsZExtVal(); in isInt32Immediate()
561 /// isInt64Immediate - This method tests to see if the node is a 64-bit constant
562 /// operand. If so Imm will receive the 64-bit value.
564 if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i64) { in isInt64Immediate()
565 Imm = N->getAsZExtVal(); in isInt64Immediate()
571 // isInt32Immediate - This method tests to see if a constant operand.
577 /// isInt64Immediate - This method tests to see if the value is a 64-bit
578 /// constant operand. If so Imm will receive the 64-bit value.
590 const BasicBlock *BB = FuncInfo.MBB->getBasicBlock(); in getBranchHint()
591 const Instruction *BBTerm = BB->getTerminator(); in getBranchHint()
593 if (BBTerm->getNumSuccessors() != 2) return PPC::BR_NO_HINT; in getBranchHint()
595 const BasicBlock *TBB = BBTerm->getSuccessor(0); in getBranchHint()
596 const BasicBlock *FBB = BBTerm->getSuccessor(1); in getBranchHint()
598 auto TProb = FuncInfo.BPI->getEdgeProbability(BB, TBB); in getBranchHint()
599 auto FProb = FuncInfo.BPI->getEdgeProbability(BB, FBB); in getBranchHint()
610 // 2. Invoke-terminating 1:1048575 in getBranchHint()
619 LLVM_DEBUG(dbgs() << "Use branch hint for '" << FuncInfo.Fn->getName() in getBranchHint()
620 << "::" << BB->getName() << "'\n" in getBranchHint()
621 << " -> " << TBB->getName() << ": " << TProb << "\n" in getBranchHint()
622 << " -> " << FBB->getName() << ": " << FProb << "\n"); in getBranchHint()
626 // If Dest BasicBlock is False-BasicBlock (FBB), swap branch probabilities, in getBranchHint()
628 if (BBDN->getBasicBlock()->getBasicBlock() != TBB) in getBranchHint()
634 // isOpcWithIntImmediate - This method tests to see if the node is a specific
638 return N->getOpcode() == Opc in isOpcWithIntImmediate()
639 && isInt32Immediate(N->getOperand(1).getNode(), Imm); in isOpcWithIntImmediate()
644 int FI = cast<FrameIndexSDNode>(N)->getIndex(); in selectFrameIndex()
645 SDValue TFI = CurDAG->getTargetFrameIndex(FI, N->getValueType(0)); in selectFrameIndex()
646 unsigned Opc = N->getValueType(0) == MVT::i32 ? PPC::ADDI : PPC::ADDI8; in selectFrameIndex()
647 if (SN->hasOneUse()) in selectFrameIndex()
648 CurDAG->SelectNodeTo(SN, Opc, N->getValueType(0), TFI, in selectFrameIndex()
651 ReplaceNode(SN, CurDAG->getMachineNode(Opc, dl, N->getValueType(0), TFI, in selectFrameIndex()
660 if (N->getValueType(0) != MVT::i32) in isRotateAndMask()
665 unsigned Opcode = N->getOpcode(); in isRotateAndMask()
666 if (N->getNumOperands() != 2 || in isRotateAndMask()
667 !isInt32Immediate(N->getOperand(1).getNode(), Shift) || (Shift > 31)) in isRotateAndMask()
681 Shift = 32 - Shift; in isRotateAndMask()
697 // isThreadPointerAcquisitionNode - Check if the operands of an ADD_TLS
704 CurDAG->getMachineFunction().getSubtarget<PPCSubtarget>(); in isThreadPointerAcquisitionNode()
708 // Account for when ADD_TLS is used for the initial-exec TLS model on Linux. in isThreadPointerAcquisitionNode()
717 // When using PC-Relative instructions for initial-exec, a MAT_PCREL_ADDR in isThreadPointerAcquisitionNode()
720 if (LD && LD->getBasePtr().getOpcode() == PPCISD::MAT_PCREL_ADDR) in isThreadPointerAcquisitionNode()
723 // A GET_TPOINTER PPCISD node (only produced on AIX 32-bit mode) as an operand in isThreadPointerAcquisitionNode()
733 AddFirstOpReg->getReg() == Subtarget.getThreadPointerRegister()) in isThreadPointerAcquisitionNode()
739 // canOptimizeTLSDFormToXForm - Optimize TLS accesses when an ADD_TLS
740 // instruction is present. An ADD_TLS instruction, followed by a D-Form memory
741 // operation, can be optimized to use an X-Form load or store, allowing the
745 // Do not do this transformation at -O0. in canOptimizeTLSDFormToXForm()
746 if (CurDAG->getTarget().getOptLevel() == CodeGenOptLevel::None) in canOptimizeTLSDFormToXForm()
753 for (auto *ADDTLSUse : Base.getNode()->uses()) { in canOptimizeTLSDFormToXForm()
754 // The optimization to convert the D-Form load/store into its X-Form in canOptimizeTLSDFormToXForm()
758 if (LD->getSrcValueOffset() != 0 || !LD->getOffset().isUndef()) in canOptimizeTLSDFormToXForm()
761 if (ST->getSrcValueOffset() != 0 || !ST->getOffset().isUndef()) in canOptimizeTLSDFormToXForm()
777 SDValue Base = ST->getBasePtr(); in tryTLSXFormStore()
782 EVT MemVT = ST->getMemoryVT(); in tryTLSXFormStore()
783 EVT RegVT = ST->getValue().getValueType(); in tryTLSXFormStore()
814 SDValue Chain = ST->getChain(); in tryTLSXFormStore()
815 SDVTList VTs = ST->getVTList(); in tryTLSXFormStore()
816 SDValue Ops[] = {ST->getValue(), Base.getOperand(0), Base.getOperand(1), in tryTLSXFormStore()
818 SDNode *MN = CurDAG->getMachineNode(Opcode, dl, VTs, Ops); in tryTLSXFormStore()
825 SDValue Base = LD->getBasePtr(); in tryTLSXFormLoad()
830 EVT MemVT = LD->getMemoryVT(); in tryTLSXFormLoad()
831 EVT RegVT = LD->getValueType(0); in tryTLSXFormLoad()
832 bool isSExt = LD->getExtensionType() == ISD::SEXTLOAD; in tryTLSXFormLoad()
868 SDValue Chain = LD->getChain(); in tryTLSXFormLoad()
869 SDVTList VTs = LD->getVTList(); in tryTLSXFormLoad()
871 SDNode *MN = CurDAG->getMachineNode(Opcode, dl, VTs, Ops); in tryTLSXFormLoad()
877 /// Turn an or of two masked values into the rotate left word immediate then
880 SDValue Op0 = N->getOperand(0); in tryBitfieldInsert()
881 SDValue Op1 = N->getOperand(1); in tryBitfieldInsert()
884 KnownBits LKnown = CurDAG->computeKnownBits(Op0); in tryBitfieldInsert()
885 KnownBits RKnown = CurDAG->computeKnownBits(Op1); in tryBitfieldInsert()
923 SH = (Op1Opc == ISD::SHL) ? Value : 32 - Value; in tryBitfieldInsert()
929 KnownBits MKnown = CurDAG->computeKnownBits(Op1.getOperand(1)); in tryBitfieldInsert()
938 SH = (SHOpc == ISD::SHL) ? Value : 32 - Value; in tryBitfieldInsert()
945 ReplaceNode(N, CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops)); in tryBitfieldInsert()
954 // Cannot use range-based for loop here as we need the actual use (i.e. we in allUsesTruncate()
955 // need the operand number corresponding to the use). A range-based for in allUsesTruncate()
957 for (SDNode::use_iterator Use = N->use_begin(), UseEnd = N->use_end(); in allUsesTruncate()
960 Use->isMachineOpcode() ? Use->getMachineOpcode() : Use->getOpcode(); in allUsesTruncate()
964 if (Use->isMachineOpcode()) in allUsesTruncate()
967 std::max(MaxTruncation, (unsigned)Use->getValueType(0).getSizeInBits()); in allUsesTruncate()
970 if (Use->isMachineOpcode()) in allUsesTruncate()
973 unsigned MemVTSize = STN->getMemoryVT().getSizeInBits(); in allUsesTruncate()
1018 // Direct materialization of 64-bit constants by enumerated patterns.
1031 return CurDAG->getTargetConstant(Imm, dl, MVT::i32); in selectI64ImmDirect()
1036 // 1-1) Patterns : {zeros}{15-bit valve} in selectI64ImmDirect()
1037 // {ones}{15-bit valve} in selectI64ImmDirect()
1039 SDValue SDImm = CurDAG->getTargetConstant(Imm, dl, MVT::i64); in selectI64ImmDirect()
1040 return CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, SDImm); in selectI64ImmDirect()
1042 // 1-2) Patterns : {zeros}{15-bit valve}{16 zeros} in selectI64ImmDirect()
1043 // {ones}{15-bit valve}{16 zeros} in selectI64ImmDirect()
1045 return CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, in selectI64ImmDirect()
1053 // 2-1) Patterns : {zeros}{31-bit value} in selectI64ImmDirect()
1054 // {ones}{31-bit value} in selectI64ImmDirect()
1058 Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16)); in selectI64ImmDirect()
1059 return CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), in selectI64ImmDirect()
1062 // 2-2) Patterns : {zeros}{ones}{15-bit value}{zeros} in selectI64ImmDirect()
1063 // {zeros}{15-bit value}{zeros} in selectI64ImmDirect()
1064 // {zeros}{ones}{15-bit value} in selectI64ImmDirect()
1065 // {ones}{15-bit value}{zeros} in selectI64ImmDirect()
1066 // We can take advantage of LI's sign-extension semantics to generate leading in selectI64ImmDirect()
1069 Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, in selectI64ImmDirect()
1071 return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0), in selectI64ImmDirect()
1074 // 2-3) Pattern : {zeros}{15-bit value}{ones} in selectI64ImmDirect()
1075 // Shift right the Imm by (48 - LZ) bits to construct a negtive 16 bits value, in selectI64ImmDirect()
1076 // therefore we can take advantage of LI's sign-extension semantics, and then in selectI64ImmDirect()
1079 // +--LZ--||-15-bit-||--TO--+ +-------------|--16-bit--+ in selectI64ImmDirect()
1080 // |00000001bbbbbbbbb1111111| -> |00000000000001bbbbbbbbb1| in selectI64ImmDirect()
1081 // +------------------------+ +------------------------+ in selectI64ImmDirect()
1083 // Imm (Imm >> (48 - LZ) & 0xffff) in selectI64ImmDirect()
1084 // +----sext-----|--16-bit--+ +clear-|-----------------+ in selectI64ImmDirect()
1085 // |11111111111111bbbbbbbbb1| -> |00000001bbbbbbbbb1111111| in selectI64ImmDirect()
1086 // +------------------------+ +------------------------+ in selectI64ImmDirect()
1088 // LI8: sext many leading zeros RLDICL: rotate left (48 - LZ), clear left LZ in selectI64ImmDirect()
1094 Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, in selectI64ImmDirect()
1095 getI32Imm((Imm >> (48 - LZ) & 0xffff))); in selectI64ImmDirect()
1096 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), in selectI64ImmDirect()
1097 getI32Imm(48 - LZ), getI32Imm(LZ)); in selectI64ImmDirect()
1099 // 2-4) Patterns : {zeros}{ones}{15-bit value}{ones} in selectI64ImmDirect()
1100 // {ones}{15-bit value}{ones} in selectI64ImmDirect()
1101 // We can take advantage of LI's sign-extension semantics to generate leading in selectI64ImmDirect()
1105 // +-LZ-FO||-15-bit-||--TO--+ +-------------|--16-bit--+ in selectI64ImmDirect()
1106 // |00011110bbbbbbbbb1111111| -> |000000000011110bbbbbbbbb| in selectI64ImmDirect()
1107 // +------------------------+ +------------------------+ in selectI64ImmDirect()
1110 // +----sext-----|--16-bit--+ +LZ|---------------------+ in selectI64ImmDirect()
1111 // |111111111111110bbbbbbbbb| -> |00011110bbbbbbbbb1111111| in selectI64ImmDirect()
1112 // +------------------------+ +------------------------+ in selectI64ImmDirect()
1116 Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, in selectI64ImmDirect()
1118 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), in selectI64ImmDirect()
1121 // 2-5) Pattern : {32 zeros}{****}{0}{15-bit value} in selectI64ImmDirect()
1126 Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, in selectI64ImmDirect()
1128 return CurDAG->getMachineNode(PPC::ORIS8, dl, MVT::i64, SDValue(Result, 0), in selectI64ImmDirect()
1131 // 2-6) Patterns : {******}{49 zeros}{******} in selectI64ImmDirect()
1139 // +------|--zeros-|------+ +---ones--||---15 bit--+ in selectI64ImmDirect()
1140 // |bbbbbb0000000000aaaaaa| -> |0000000000aaaaaabbbbbb| in selectI64ImmDirect()
1141 // +----------------------+ +----------------------+ in selectI64ImmDirect()
1145 // +------|--ones--|------+ +---ones--||---15 bit--+ in selectI64ImmDirect()
1146 // |bbbbbb1111111111aaaaaa| -> |1111111111aaaaaabbbbbb| in selectI64ImmDirect()
1147 // +----------------------+ +----------------------+ in selectI64ImmDirect()
1152 Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, in selectI64ImmDirect()
1154 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), in selectI64ImmDirect()
1157 // 2-7) Patterns : High word == Low word in selectI64ImmDirect()
1166 CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, getI32Imm(ImmLo16)); in selectI64ImmDirect()
1169 CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, getI32Imm(ImmHi16)); in selectI64ImmDirect()
1173 CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, getI32Imm(ImmHi16)); in selectI64ImmDirect()
1174 Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, in selectI64ImmDirect()
1180 return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops); in selectI64ImmDirect()
1185 // 3-1) Patterns : {zeros}{ones}{31-bit value}{zeros} in selectI64ImmDirect()
1186 // {zeros}{31-bit value}{zeros} in selectI64ImmDirect()
1187 // {zeros}{ones}{31-bit value} in selectI64ImmDirect()
1188 // {ones}{31-bit value}{zeros} in selectI64ImmDirect()
1189 // We can take advantage of LIS's sign-extension semantics to generate leading in selectI64ImmDirect()
1195 Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16)); in selectI64ImmDirect()
1196 Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), in selectI64ImmDirect()
1198 return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0), in selectI64ImmDirect()
1201 // 3-2) Pattern : {zeros}{31-bit value}{ones} in selectI64ImmDirect()
1202 // Shift right the Imm by (32 - LZ) bits to construct a negative 32 bits in selectI64ImmDirect()
1203 // value, therefore we can take advantage of LIS's sign-extension semantics, in selectI64ImmDirect()
1205 // This is similar to Pattern 2-3, please refer to the diagram there. in selectI64ImmDirect()
1211 Result = CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, in selectI64ImmDirect()
1212 getI32Imm((Imm >> (48 - LZ)) & 0xffff)); in selectI64ImmDirect()
1213 Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), in selectI64ImmDirect()
1214 getI32Imm((Imm >> (32 - LZ)) & 0xffff)); in selectI64ImmDirect()
1215 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), in selectI64ImmDirect()
1216 getI32Imm(32 - LZ), getI32Imm(LZ)); in selectI64ImmDirect()
1218 // 3-3) Patterns : {zeros}{ones}{31-bit value}{ones} in selectI64ImmDirect()
1219 // {ones}{31-bit value}{ones} in selectI64ImmDirect()
1220 // We can take advantage of LIS's sign-extension semantics to generate leading in selectI64ImmDirect()
1223 // This is similar to Pattern 2-4, please refer to the diagram there. in selectI64ImmDirect()
1225 Result = CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, in selectI64ImmDirect()
1227 Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), in selectI64ImmDirect()
1229 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), in selectI64ImmDirect()
1232 // 3-4) Patterns : {******}{33 zeros}{******} in selectI64ImmDirect()
1238 // This is similar to Pattern 2-6, please refer to the diagram there. in selectI64ImmDirect()
1244 Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16)); in selectI64ImmDirect()
1245 Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), in selectI64ImmDirect()
1247 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), in selectI64ImmDirect()
1270 return CurDAG->getTargetConstant(Imm, dl, MVT::i32); in selectI64ImmDirectPrefix()
1274 return CurDAG->getTargetConstant(Imm, dl, MVT::i64); in selectI64ImmDirectPrefix()
1281 // If a constant fits within 34-bits, emit the pli instruction here directly. in selectI64ImmDirectPrefix()
1283 return CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, in selectI64ImmDirectPrefix()
1284 CurDAG->getTargetConstant(Imm, dl, MVT::i64)); in selectI64ImmDirectPrefix()
1289 // Patterns : {zeros}{ones}{33-bit value}{zeros} in selectI64ImmDirectPrefix()
1290 // {zeros}{33-bit value}{zeros} in selectI64ImmDirectPrefix()
1291 // {zeros}{ones}{33-bit value} in selectI64ImmDirectPrefix()
1292 // {ones}{33-bit value}{zeros} in selectI64ImmDirectPrefix()
1293 // We can take advantage of PLI's sign-extension semantics to generate leading in selectI64ImmDirectPrefix()
1298 Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, in selectI64ImmDirectPrefix()
1300 return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0), in selectI64ImmDirectPrefix()
1303 // Pattern : {zeros}{33-bit value}{ones} in selectI64ImmDirectPrefix()
1304 // Shift right the Imm by (30 - LZ) bits to construct a negative 34 bit value, in selectI64ImmDirectPrefix()
1305 // therefore we can take advantage of PLI's sign-extension semantics, and then in selectI64ImmDirectPrefix()
1308 // +--LZ--||-33-bit-||--TO--+ +-------------|--34-bit--+ in selectI64ImmDirectPrefix()
1309 // |00000001bbbbbbbbb1111111| -> |00000000000001bbbbbbbbb1| in selectI64ImmDirectPrefix()
1310 // +------------------------+ +------------------------+ in selectI64ImmDirectPrefix()
1313 // +----sext-----|--34-bit--+ +clear-|-----------------+ in selectI64ImmDirectPrefix()
1314 // |11111111111111bbbbbbbbb1| -> |00000001bbbbbbbbb1111111| in selectI64ImmDirectPrefix()
1315 // +------------------------+ +------------------------+ in selectI64ImmDirectPrefix()
1318 APInt SignedInt34 = APInt(34, (Imm >> (30 - LZ)) & 0x3ffffffff); in selectI64ImmDirectPrefix()
1320 Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, in selectI64ImmDirectPrefix()
1322 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), in selectI64ImmDirectPrefix()
1323 getI32Imm(30 - LZ), getI32Imm(LZ)); in selectI64ImmDirectPrefix()
1325 // Patterns : {zeros}{ones}{33-bit value}{ones} in selectI64ImmDirectPrefix()
1326 // {ones}{33-bit value}{ones} in selectI64ImmDirectPrefix()
1327 // Similar to LI we can take advantage of PLI's sign-extension semantics to in selectI64ImmDirectPrefix()
1333 Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, in selectI64ImmDirectPrefix()
1335 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), in selectI64ImmDirectPrefix()
1344 // +------|--ones--|------+ +---ones--||---33 bit--+ in selectI64ImmDirectPrefix()
1345 // |bbbbbb1111111111aaaaaa| -> |1111111111aaaaaabbbbbb| in selectI64ImmDirectPrefix()
1346 // +----------------------+ +----------------------+ in selectI64ImmDirectPrefix()
1352 CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(RotImm)); in selectI64ImmDirectPrefix()
1353 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, in selectI64ImmDirectPrefix()
1362 Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(Hi32)); in selectI64ImmDirectPrefix()
1365 return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops); in selectI64ImmDirectPrefix()
1369 // Catch-all in selectI64ImmDirectPrefix()
1372 CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(Hi32)); in selectI64ImmDirectPrefix()
1374 CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(Lo32)); in selectI64ImmDirectPrefix()
1377 return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops); in selectI64ImmDirectPrefix()
1388 CurDAG->getMachineFunction().getSubtarget<PPCSubtarget>(); in selectI64Imm()
1392 // non-prefixed, try that. in selectI64Imm()
1397 // 1) We have no result from the non-prefix case to use. in selectI64Imm()
1398 // 2) The non-prefix case uses more instructions than the prefix case. in selectI64Imm()
1399 // If the prefix and non-prefix cases use the same number of instructions in selectI64Imm()
1400 // we will prefer the non-prefix case. in selectI64Imm()
1414 return CurDAG->getTargetConstant(Imm, dl, MVT::i32); in selectI64Imm()
1429 CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, getI32Imm(Hi16)); in selectI64Imm()
1430 Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, in selectI64Imm()
1434 return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops); in selectI64Imm()
1443 Result = CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops); in selectI64Imm()
1450 Result = CurDAG->getMachineNode(PPC::RLWIMI8, dl, MVT::i64, Ops); in selectI64Imm()
1457 Result = CurDAG->getMachineNode(PPC::RLWIMI8, dl, MVT::i64, Ops); in selectI64Imm()
1471 Result = CurDAG->getMachineNode(PPC::ORIS8, dl, MVT::i64, in selectI64Imm()
1476 Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), in selectI64Imm()
1485 // Select a 64-bit constant.
1490 int64_t Imm = N->getAsZExtVal(); in selectI64Imm()
1493 SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i64); in selectI64Imm()
1495 return CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, SDImm); in selectI64Imm()
1507 // lowest-order bit.
1551 // by rlwinm and friends in 64-bit mode).
1581 // We need to sort so that the non-Repl32 come first because, when we're in operator <()
1582 // doing masking, the Repl32 bit groups might be subsumed into the 64-bit in operator <()
1615 return std::make_pair(ValueEntry->first, &ValueEntry->second); in getValueBits()
1617 bool &Interesting = ValueEntry->first; in getValueBits()
1618 SmallVector<ValueBit, 64> &Bits = ValueEntry->second; in getValueBits()
1626 unsigned RotAmt = V.getConstantOperandVal(1) & (NumBits - 1); in getValueBits()
1631 Bits[i] = LHSBits[i < RotAmt ? i + (NumBits - RotAmt) : i - RotAmt]; in getValueBits()
1640 unsigned ShiftAmt = V.getConstantOperandVal(1) & ((NumBits << 1) - 1); in getValueBits()
1649 Bits[i] = LHSBits[i - ShiftAmt]; in getValueBits()
1661 unsigned ShiftAmt = V.getConstantOperandVal(1) & ((NumBits << 1) - 1); in getValueBits()
1669 for (unsigned i = 0; i < NumBits - ShiftAmt; ++i) in getValueBits()
1671 for (unsigned i = NumBits - ShiftAmt; i < NumBits; ++i) in getValueBits()
1685 // (which is non-optimal because such an and might be folded with other in getValueBits()
1783 // A 32-bit instruction cannot touch upper 32-bit part of 64-bit value. in getValueBits()
1806 EVT FromType = cast<VTSDNode>(V.getOperand(1))->getVT(); in getValueBits()
1825 EVT VT = LD->getMemoryVT(); in getValueBits()
1835 // Zero-extending load itself cannot be optimized. So, it is not in getValueBits()
1848 // For each value (except the constant ones), compute the left-rotate amount
1857 RLAmt[i] = i - VBI; in computeRotationAmounts()
1859 RLAmt[i] = Bits.size() - (VBI - i); in computeRotationAmounts()
1893 // when the first non-zero bit appears later. in collectBitGroups()
1907 i-1)); in collectBitGroups()
1915 Bits.size()-1)); in collectBitGroups()
1924 // equal to the ending index of the to-be-removed first group. in collectBitGroups()
1926 BitGroups[BitGroups.size()-1].EndIdx == Bits.size()-1 && in collectBitGroups()
1927 BitGroups[0].V == BitGroups[BitGroups.size()-1].V && in collectBitGroups()
1928 BitGroups[0].RLAmt == BitGroups[BitGroups.size()-1].RLAmt) { in collectBitGroups()
1930 BitGroups[BitGroups.size()-1].EndIdx = BitGroups[0].EndIdx; in collectBitGroups()
1963 // In 64-bit mode, rlwinm and friends have a rotation operator that
1964 // replicates the low-order 32 bits into the high-order 32-bits. The mask
1966 // can only represent some 64-bit bit groups. However, when they can be used,
1967 // the 32-bit replication can be used to represent, as a single bit group,
1968 // otherwise separate bit groups. We'll convert to replicated-32-bit bit
1978 // But, making use of a 32-bit operation that replicates the low-order 32 in assignRepl32BitGroups()
1979 // bits into the high-order 32 bits, this can be one bit group with a RLAmt in assignRepl32BitGroups()
2026 BG.RLAmt -= 32; in assignRepl32BitGroups()
2032 LLVM_DEBUG(dbgs() << "\t32-bit replicated bit group for " in assignRepl32BitGroups()
2045 if (I->Repl32 && IP->Repl32 && I->V == IP->V && I->RLAmt == IP->RLAmt && in assignRepl32BitGroups()
2046 I->StartIdx == (IP->EndIdx + 1) % 64 && I != IP) { in assignRepl32BitGroups()
2048 LLVM_DEBUG(dbgs() << "\tcombining 32-bit replicated bit group for " in assignRepl32BitGroups()
2049 << I->V.getNode() << " RLAmt = " << I->RLAmt << " [" in assignRepl32BitGroups()
2050 << I->StartIdx << ", " << I->EndIdx in assignRepl32BitGroups()
2051 << "] with group with range [" << IP->StartIdx << ", " in assignRepl32BitGroups()
2052 << IP->EndIdx << "]\n"); in assignRepl32BitGroups()
2054 IP->EndIdx = I->EndIdx; in assignRepl32BitGroups()
2055 IP->Repl32CR = IP->Repl32CR || I->Repl32CR; in assignRepl32BitGroups()
2056 IP->Repl32Coalesced = true; in assignRepl32BitGroups()
2066 if (I->StartIdx == 32 && I->EndIdx == 63) { in assignRepl32BitGroups()
2071 if (IP->Repl32 && IN->Repl32 && I->V == IP->V && I->V == IN->V && in assignRepl32BitGroups()
2072 (I->RLAmt % 32) == IP->RLAmt && (I->RLAmt % 32) == IN->RLAmt && in assignRepl32BitGroups()
2073 IP->EndIdx == 31 && IN->StartIdx == 0 && I != IP && in assignRepl32BitGroups()
2076 LLVM_DEBUG(dbgs() << "\tcombining bit group for " << I->V.getNode() in assignRepl32BitGroups()
2077 << " RLAmt = " << I->RLAmt << " [" << I->StartIdx in assignRepl32BitGroups()
2078 << ", " << I->EndIdx in assignRepl32BitGroups()
2079 << "] with 32-bit replicated groups with ranges [" in assignRepl32BitGroups()
2080 << IP->StartIdx << ", " << IP->EndIdx << "] and [" in assignRepl32BitGroups()
2081 << IN->StartIdx << ", " << IN->EndIdx << "]\n"); in assignRepl32BitGroups()
2086 // whole 64-bit range). in assignRepl32BitGroups()
2087 IP->StartIdx = 31; in assignRepl32BitGroups()
2088 IP->EndIdx = 30; in assignRepl32BitGroups()
2089 IP->Repl32CR = IP->Repl32CR || I->RLAmt >= 32; in assignRepl32BitGroups()
2090 IP->Repl32Coalesced = true; in assignRepl32BitGroups()
2096 IP->EndIdx = IN->EndIdx; in assignRepl32BitGroups()
2097 IP->Repl32CR = IP->Repl32CR || IN->Repl32CR || I->RLAmt >= 32; in assignRepl32BitGroups()
2098 IP->Repl32Coalesced = true; in assignRepl32BitGroups()
2115 return CurDAG->getTargetConstant(Imm, dl, MVT::i32); in getI32Imm()
2129 // This method extends an input value to 64 bit if input is 32-bit integer.
2130 // While selecting instructions in BitPermutationSelector in 64-bit mode,
2131 // an input value can be a 32-bit integer if a ZERO_EXTEND node is included.
2138 SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32); in ExtendToInt64()
2139 SDValue ImDef = SDValue(CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, in ExtendToInt64()
2141 SDValue ExtVal = SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl, in ExtendToInt64()
2152 SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32); in TruncateToInt32()
2153 SDValue SubVal = SDValue(CurDAG->getMachineNode(PPC::EXTRACT_SUBREG, dl, in TruncateToInt32()
2186 // than break even because rotate-and-mask instructions tend to be easier in SelectAndParts32()
2191 // andi/andis have single-cycle latency whereas the rotate-and-mask in SelectAndParts32()
2193 // andi/andis in break-even cases. in SelectAndParts32()
2218 VRot = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, in SelectAndParts32()
2226 ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI_rec, dl, MVT::i32, in SelectAndParts32()
2231 SDValue(CurDAG->getMachineNode(PPC::ANDIS_rec, dl, MVT::i32, VRot, in SelectAndParts32()
2241 TotalVal = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32, in SelectAndParts32()
2247 Res = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32, in SelectAndParts32()
2258 // Instruction selection for the 32-bit case.
2278 Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), in Select32()
2297 getI32Imm(Bits.size() - BG.EndIdx - 1, dl), in Select32()
2298 getI32Imm(Bits.size() - BG.StartIdx - 1, dl) }; in Select32()
2299 Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); in Select32()
2303 getI32Imm(Bits.size() - BG.EndIdx - 1, dl), in Select32()
2304 getI32Imm(Bits.size() - BG.StartIdx - 1, dl) }; in Select32()
2305 Res = SDValue(CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops), 0); in Select32()
2322 ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI_rec, dl, MVT::i32, in Select32()
2327 SDValue(CurDAG->getMachineNode(PPC::ANDIS_rec, dl, MVT::i32, Res, in Select32()
2336 Res = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32, in Select32()
2348 unsigned InstMaskStart = 64 - MaskEnd - 1, in SelectRotMask64Count()
2349 InstMaskEnd = 64 - MaskStart - 1; in SelectRotMask64Count()
2355 InstMaskEnd == 63 - RLAmt) in SelectRotMask64Count()
2361 // For 64-bit values, not all combinations of rotates and masks are
2368 unsigned InstMaskStart = 64 - MaskEnd - 1, in SelectRotMask64()
2369 InstMaskEnd = 64 - MaskStart - 1; in SelectRotMask64()
2381 getI32Imm(InstMaskStart - 32, dl), getI32Imm(InstMaskEnd - 32, dl) }; in SelectRotMask64()
2382 return SDValue(CurDAG->getMachineNode(PPC::RLWINM8, dl, MVT::i64, in SelectRotMask64()
2390 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Ops), 0); in SelectRotMask64()
2397 return SDValue(CurDAG->getMachineNode(PPC::RLDICR, dl, MVT::i64, Ops), 0); in SelectRotMask64()
2400 if (InstMaskEnd == 63 - RLAmt) { in SelectRotMask64()
2404 return SDValue(CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, Ops), 0); in SelectRotMask64()
2419 unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64; in SelectRotMask64()
2425 // For 64-bit values, not all combinations of rotates and masks are
2426 // available. Produce a rotate-mask-and-insert if one is available.
2432 unsigned InstMaskStart = 64 - MaskEnd - 1, in SelectRotMaskIns64()
2433 InstMaskEnd = 64 - MaskStart - 1; in SelectRotMaskIns64()
2445 getI32Imm(InstMaskStart - 32, dl), getI32Imm(InstMaskEnd - 32, dl) }; in SelectRotMaskIns64()
2446 return SDValue(CurDAG->getMachineNode(PPC::RLWIMI8, dl, MVT::i64, in SelectRotMaskIns64()
2450 if (InstMaskEnd == 63 - RLAmt) { in SelectRotMaskIns64()
2454 return SDValue(CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops), 0); in SelectRotMaskIns64()
2469 unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64; in SelectRotMaskIns64()
2479 // The idea here is the same as in the 32-bit version, but with additional in SelectAndParts64()
2483 // be the case that a single 64-bit masking operation could handle both in SelectAndParts64()
2484 // some Repl32 groups and some non-Repl32 groups. If converting to Repl32 in SelectAndParts64()
2485 // form allowed coalescing, then we must use a 32-bit rotaton in order to in SelectAndParts64()
2531 // We can use the 32-bit andi/andis technique if the mask does not in SelectAndParts64()
2532 // require any higher-order bits. This can save an instruction compared in SelectAndParts64()
2533 // to always using the general 64-bit technique. in SelectAndParts64()
2573 // general 64-bit constant formation, bias toward the constant form, in SelectAndParts64()
2588 // We actually need to generate a rotation if we have a non-zero rotation in SelectAndParts64()
2590 // higher-order replicated bits. In the latter case, we generate a mask in SelectAndParts64()
2601 "No set bits in mask when using 32-bit ands for 64-bit value"); in SelectAndParts64()
2605 ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI8_rec, dl, MVT::i64, in SelectAndParts64()
2611 SDValue(CurDAG->getMachineNode(PPC::ANDIS8_rec, dl, MVT::i64, in SelectAndParts64()
2621 TotalVal = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, in SelectAndParts64()
2626 SDValue(CurDAG->getMachineNode(PPC::AND8, dl, MVT::i64, in SelectAndParts64()
2634 Res = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, in SelectAndParts64()
2644 // Instruction selection for the 64-bit case.
2658 // If we have both Repl32 groups and non-Repl32 groups, the non-Repl32 in Select64()
2706 // Because 64-bit rotates are more flexible than inserts, we might have a in Select64()
2710 if (SelectRotMask64Count(I->RLAmt, I->Repl32, I->StartIdx, I->EndIdx, in Select64()
2712 SelectRotMask64Count(I->RLAmt, I->Repl32, I->StartIdx, I->EndIdx, in Select64()
2737 // We can use the 32-bit andi/andis technique if the mask does not in Select64()
2738 // require any higher-order bits. This can save an instruction compared in Select64()
2739 // to always using the general 64-bit technique. in Select64()
2747 "No set bits in mask when using 32-bit ands for 64-bit value"); in Select64()
2755 ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI8_rec, dl, MVT::i64, in Select64()
2761 SDValue(CurDAG->getMachineNode(PPC::ANDIS8_rec, dl, MVT::i64, in Select64()
2771 Res = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, in Select64()
2777 Res = SDValue(CurDAG->getMachineNode(PPC::AND8, dl, MVT::i64, in Select64()
2794 // For 64-bit values, figure out when we can use 32-bit instructions. in Select()
2832 // rotate-and-shift/shift/and/or instructions, using a set of heuristics
2837 getValueBits(SDValue(N, 0), N->getValueType(0).getSizeInBits()); in Select()
2842 LLVM_DEBUG(dbgs() << "Considering bit-permutation-based instruction" in Select()
2844 LLVM_DEBUG(N->dump(CurDAG)); in Select()
2858 // masking, we only insert the non-zero parts of the result at every step. in Select()
2871 LLVM_DEBUG(dbgs() << "\tUsing early-masking for isel\n"); in Select()
2875 LLVM_DEBUG(dbgs() << "\tUsing late-masking for isel\n"); in Select()
2883 // Conversion type for interpreting results of a 32-bit instruction as
2884 // a 64-bit value or vice versa.
2889 // ZExtOrig - use the original condition code, zero-extend value
2890 // ZExtInvert - invert the condition code, zero-extend value
2891 // SExtOrig - use the original condition code, sign-extend value
2892 // SExtInvert - invert the condition code, sign-extend value
2897 // For example (a >= 0) == (a > -1). The direction of the comparison (</>)
2898 // matters as well as the extension type: sext (-1/0), zext (1/0).
2899 // GEZExt - (zext (LHS >= 0))
2900 // GESExt - (sext (LHS >= 0))
2901 // LEZExt - (zext (LHS <= 0))
2902 // LESExt - (sext (LHS <= 0))
2926 assert(CurDAG->getTargetLoweringInfo() in IntegerCompareEliminator()
2927 .getPointerTy(CurDAG->getDataLayout()).getSizeInBits() == 64 && in IntegerCompareEliminator()
2933 switch (N->getOpcode()) { in Select()
2957 assert((N->getOpcode() == ISD::ZERO_EXTEND || in tryEXTEND()
2958 N->getOpcode() == ISD::SIGN_EXTEND) && in tryEXTEND()
2961 // If we are zero-extending the result of a logical operation on i1 in tryEXTEND()
2963 if (ISD::isBitwiseLogicOp(N->getOperand(0).getOpcode()) && in tryEXTEND()
2964 N->getOperand(0).getValueType() == MVT::i1 && in tryEXTEND()
2965 N->getOpcode() == ISD::ZERO_EXTEND) in tryEXTEND()
2966 WideRes = computeLogicOpInGPR(N->getOperand(0)); in tryEXTEND()
2967 else if (N->getOperand(0).getOpcode() != ISD::SETCC) in tryEXTEND()
2971 getSETCCInGPR(N->getOperand(0), in tryEXTEND()
2972 N->getOpcode() == ISD::SIGN_EXTEND ? in tryEXTEND()
2980 bool Output32Bit = N->getValueType(0) == MVT::i32; in tryEXTEND()
2982 NumSextSetcc += N->getOpcode() == ISD::SIGN_EXTEND ? 1 : 0; in tryEXTEND()
2983 NumZextSetcc += N->getOpcode() == ISD::SIGN_EXTEND ? 0 : 1; in tryEXTEND()
2994 // lowered to CR-logical operations which suffer from significant latency and
2997 if (N->getValueType(0) != MVT::i1) in tryLogicOpOfCompares()
2999 assert(ISD::isBitwiseLogicOp(N->getOpcode()) && in tryLogicOpOfCompares()
3008 SDValue CR0Reg = CurDAG->getRegister(PPC::CR0, MVT::i32); in tryLogicOpOfCompares()
3014 // Look through any 32-bit to 64-bit implicit extend nodes to find the in tryLogicOpOfCompares()
3023 // If this is not an XORI, it is a reg-reg logical op and we can convert in tryLogicOpOfCompares()
3024 // it to record-form. in tryLogicOpOfCompares()
3027 // Get the record-form version of the node we're looking to use to get the in tryLogicOpOfCompares()
3032 // Convert the right node to record-form. This is either the logical we're in tryLogicOpOfCompares()
3035 if (NewOpc != -1 && IsBitwiseNegate) { in tryLogicOpOfCompares()
3036 // The input to the XORI has a record-form. Use it. in tryLogicOpOfCompares()
3039 // Emit the record-form instruction. in tryLogicOpOfCompares()
3045 SDValue(CurDAG->getMachineNode(NewOpc, dl, in tryLogicOpOfCompares()
3049 assert((NewOpc != -1 || !IsBitwiseNegate) && in tryLogicOpOfCompares()
3052 SDValue(CurDAG->getMachineNode(NewOpc == -1 ? PPC::ANDI8_rec : NewOpc, in tryLogicOpOfCompares()
3057 // Select this node to a single bit from CR0 set by the record-form node in tryLogicOpOfCompares()
3062 CurDAG->getTargetConstant(SubRegToExtract, dl, MVT::i32); in tryLogicOpOfCompares()
3064 SDValue(CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, in tryLogicOpOfCompares()
3073 // - SETCC
3074 // - TRUNCATE
3075 // - Logical operation (AND/OR/XOR)
3077 // achieved with xor %a, -1).
3086 // Special case: xor %a, -1 in computeLogicOpInGPR()
3093 auto getLogicOperand = [&] (SDValue Operand) -> SDValue { in computeLogicOpInGPR()
3100 return SDValue(CurDAG->getMachineNode(InVT == MVT::i32 ? PPC::RLDICL_32 : in computeLogicOpInGPR()
3102 S->getI64Imm(0, dl), in computeLogicOpInGPR()
3103 S->getI64Imm(63, dl)), 0); in computeLogicOpInGPR()
3119 // We will use the inputs as 64-bit values. in computeLogicOpInGPR()
3134 RHS = S->getI64Imm(1, dl); in computeLogicOpInGPR()
3138 return SDValue(CurDAG->getMachineNode(NewOpc, dl, MVT::i64, LHS, RHS), 0); in computeLogicOpInGPR()
3142 /// If the value isn't guaranteed to be sign-extended to 64-bits, extend it.
3143 /// Otherwise just reinterpret it as a 64-bit value.
3144 /// Useful when emitting comparison code for 32-bit values without using
3145 /// the compare instruction (which only considers the lower 32-bits).
3148 "Can only sign-extend 32-bit values here."); in signExtendInputIfNeeded()
3151 // The value was sign extended and then truncated to 32-bits. No need to in signExtendInputIfNeeded()
3159 // The input is a sign-extending load. All ppc sign-extending loads in signExtendInputIfNeeded()
3160 // sign-extend to the full 64-bits. in signExtendInputIfNeeded()
3161 if (InputLoad && InputLoad->getExtensionType() == ISD::SEXTLOAD) in signExtendInputIfNeeded()
3165 // We don't sign-extend constants. in signExtendInputIfNeeded()
3171 return SDValue(CurDAG->getMachineNode(PPC::EXTSW_32_64, dl, in signExtendInputIfNeeded()
3175 /// If the value isn't guaranteed to be zero-extended to 64-bits, extend it.
3176 /// Otherwise just reinterpret it as a 64-bit value.
3177 /// Useful when emitting comparison code for 32-bit values without using
3178 /// the compare instruction (which only considers the lower 32-bits).
3181 "Can only zero-extend 32-bit values here."); in zeroExtendInputIfNeeded()
3185 // - The value is a positive constant in zeroExtendInputIfNeeded()
3186 // - The value comes from a load that isn't a sign-extending load in zeroExtendInputIfNeeded()
3187 // An ISD::TRUNCATE needs to be zero-extended unless it is fed by a zext. in zeroExtendInputIfNeeded()
3195 if (InputConst && InputConst->getSExtValue() >= 0) in zeroExtendInputIfNeeded()
3199 // The input is a load that doesn't sign-extend (it will be zero-extended). in zeroExtendInputIfNeeded()
3200 if (InputLoad && InputLoad->getExtensionType() != ISD::SEXTLOAD) in zeroExtendInputIfNeeded()
3203 // None of the above, need to zero-extend. in zeroExtendInputIfNeeded()
3206 return SDValue(CurDAG->getMachineNode(PPC::RLDICL_32_64, dl, MVT::i64, Input, in zeroExtendInputIfNeeded()
3207 S->getI64Imm(0, dl), in zeroExtendInputIfNeeded()
3208 S->getI64Imm(32, dl)), 0); in zeroExtendInputIfNeeded()
3211 // Handle a 32-bit value in a 64-bit register and vice-versa. These are of
3214 // 64 bit value and vice-versa.
3219 // For reinterpreting 32-bit values as 64 bit values, we generate in addExtOrTrunc()
3222 SDValue ImDef(CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, MVT::i64), 0); in addExtOrTrunc()
3224 CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32); in addExtOrTrunc()
3225 return SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl, MVT::i64, in addExtOrTrunc()
3231 // For reinterpreting 64-bit values as 32-bit values, we just need to in addExtOrTrunc()
3234 CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32); in addExtOrTrunc()
3235 return SDValue(CurDAG->getMachineNode(PPC::EXTRACT_SUBREG, dl, MVT::i32, in addExtOrTrunc()
3240 // Handle both zero-extensions and sign-extensions.
3252 ToExtend = SDValue(CurDAG->getMachineNode(Is32Bit ? PPC::NOR : PPC::NOR8, in getCompoundZeroComparisonInGPR()
3261 SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0); in getCompoundZeroComparisonInGPR()
3263 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, in getCompoundZeroComparisonInGPR()
3264 Neg, S->getI64Imm(1, dl), in getCompoundZeroComparisonInGPR()
3265 S->getI64Imm(63, dl)), 0); in getCompoundZeroComparisonInGPR()
3268 SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS, in getCompoundZeroComparisonInGPR()
3269 S->getI64Imm(~0ULL, dl)), 0); in getCompoundZeroComparisonInGPR()
3270 ToExtend = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, in getCompoundZeroComparisonInGPR()
3277 // For 64-bit sequences, the extensions are the same for the GE/LE cases. in getCompoundZeroComparisonInGPR()
3280 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, in getCompoundZeroComparisonInGPR()
3281 ToExtend, S->getI64Imm(1, dl), in getCompoundZeroComparisonInGPR()
3282 S->getI64Imm(63, dl)), 0); in getCompoundZeroComparisonInGPR()
3285 return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, ToExtend, in getCompoundZeroComparisonInGPR()
3286 S->getI64Imm(63, dl)), 0); in getCompoundZeroComparisonInGPR()
3288 assert(Is32Bit && "Should have handled the 32-bit sequences above."); in getCompoundZeroComparisonInGPR()
3289 // For 32-bit sequences, the extensions differ between GE/LE cases. in getCompoundZeroComparisonInGPR()
3292 SDValue ShiftOps[] = { ToExtend, S->getI32Imm(1, dl), S->getI32Imm(31, dl), in getCompoundZeroComparisonInGPR()
3293 S->getI32Imm(31, dl) }; in getCompoundZeroComparisonInGPR()
3294 return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, in getCompoundZeroComparisonInGPR()
3298 return SDValue(CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, ToExtend, in getCompoundZeroComparisonInGPR()
3299 S->getI32Imm(31, dl)), 0); in getCompoundZeroComparisonInGPR()
3301 return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, ToExtend, in getCompoundZeroComparisonInGPR()
3302 S->getI32Imm(1, dl)), 0); in getCompoundZeroComparisonInGPR()
3304 return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, ToExtend, in getCompoundZeroComparisonInGPR()
3305 S->getI32Imm(-1, dl)), 0); in getCompoundZeroComparisonInGPR()
3310 llvm_unreachable("Unknown zero-comparison type."); in getCompoundZeroComparisonInGPR()
3313 /// Produces a zero-extended result of comparing two 32-bit values according to
3324 bool IsRHSNegOne = RHSValue == -1LL; in get32BitZExtCompare()
3328 // (zext (setcc %a, %b, seteq)) -> (lshr (cntlzw (xor %a, %b)), 5) in get32BitZExtCompare()
3329 // (zext (setcc %a, 0, seteq)) -> (lshr (cntlzw %a), 5) in get32BitZExtCompare()
3331 SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0); in get32BitZExtCompare()
3333 SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0); in get32BitZExtCompare()
3334 SDValue ShiftOps[] = { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl), in get32BitZExtCompare()
3335 S->getI32Imm(31, dl) }; in get32BitZExtCompare()
3336 return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, in get32BitZExtCompare()
3340 // (zext (setcc %a, %b, setne)) -> (xor (lshr (cntlzw (xor %a, %b)), 5), 1) in get32BitZExtCompare()
3341 // (zext (setcc %a, 0, setne)) -> (xor (lshr (cntlzw %a), 5), 1) in get32BitZExtCompare()
3343 SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0); in get32BitZExtCompare()
3345 SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0); in get32BitZExtCompare()
3346 SDValue ShiftOps[] = { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl), in get32BitZExtCompare()
3347 S->getI32Imm(31, dl) }; in get32BitZExtCompare()
3349 SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, ShiftOps), 0); in get32BitZExtCompare()
3350 return SDValue(CurDAG->getMachineNode(PPC::XORI, dl, MVT::i32, Shift, in get32BitZExtCompare()
3351 S->getI32Imm(1, dl)), 0); in get32BitZExtCompare()
3354 // (zext (setcc %a, %b, setge)) -> (xor (lshr (sub %a, %b), 63), 1) in get32BitZExtCompare()
3355 // (zext (setcc %a, 0, setge)) -> (lshr (~ %a), 31) in get32BitZExtCompare()
3363 IsRHSZero = RHSConst && RHSConst->isZero(); in get32BitZExtCompare()
3369 // (zext (setcc %a, %b, setle)) -> (xor (lshr (sub %b, %a), 63), 1) in get32BitZExtCompare()
3370 // (zext (setcc %a, 0, setle)) -> (xor (lshr (- %a), 63), 1) in get32BitZExtCompare()
3377 // The upper 32-bits of the register can't be undefined for this sequence. in get32BitZExtCompare()
3381 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0); in get32BitZExtCompare()
3383 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Sub, in get32BitZExtCompare()
3384 S->getI64Imm(1, dl), S->getI64Imm(63, dl)), in get32BitZExtCompare()
3387 SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, in get32BitZExtCompare()
3388 MVT::i64, Shift, S->getI32Imm(1, dl)), 0); in get32BitZExtCompare()
3391 // (zext (setcc %a, %b, setgt)) -> (lshr (sub %b, %a), 63) in get32BitZExtCompare()
3392 // (zext (setcc %a, -1, setgt)) -> (lshr (~ %a), 31) in get32BitZExtCompare()
3393 // (zext (setcc %a, 0, setgt)) -> (lshr (- %a), 63) in get32BitZExtCompare()
3394 // Handle SETLT -1 (which is equivalent to SETGE 0). in get32BitZExtCompare()
3401 // The upper 32-bits of the register can't be undefined for this sequence. in get32BitZExtCompare()
3405 SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0); in get32BitZExtCompare()
3406 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, in get32BitZExtCompare()
3407 Neg, S->getI32Imm(1, dl), S->getI32Imm(63, dl)), 0); in get32BitZExtCompare()
3409 // Not a special case (i.e. RHS == 0 or RHS == -1). Handle (%a > %b) as in get32BitZExtCompare()
3413 IsRHSZero = RHSConst && RHSConst->isZero(); in get32BitZExtCompare()
3414 IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1; in get32BitZExtCompare()
3418 // (zext (setcc %a, %b, setlt)) -> (lshr (sub %a, %b), 63) in get32BitZExtCompare()
3419 // (zext (setcc %a, 1, setlt)) -> (xor (lshr (- %a), 63), 1) in get32BitZExtCompare()
3420 // (zext (setcc %a, 0, setlt)) -> (lshr %a, 31) in get32BitZExtCompare()
3429 SDValue ShiftOps[] = { LHS, S->getI32Imm(1, dl), S->getI32Imm(31, dl), in get32BitZExtCompare()
3430 S->getI32Imm(31, dl) }; in get32BitZExtCompare()
3431 return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, in get32BitZExtCompare()
3437 // The upper 32-bits of the register can't be undefined for this sequence. in get32BitZExtCompare()
3441 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0); in get32BitZExtCompare()
3442 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, in get32BitZExtCompare()
3443 SUBFNode, S->getI64Imm(1, dl), in get32BitZExtCompare()
3444 S->getI64Imm(63, dl)), 0); in get32BitZExtCompare()
3447 // (zext (setcc %a, %b, setuge)) -> (xor (lshr (sub %b, %a), 63), 1) in get32BitZExtCompare()
3448 // (zext (setcc %a, %b, setule)) -> (xor (lshr (sub %a, %b), 63), 1) in get32BitZExtCompare()
3454 // The upper 32-bits of the register can't be undefined for this sequence. in get32BitZExtCompare()
3458 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0); in get32BitZExtCompare()
3460 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, in get32BitZExtCompare()
3461 Subtract, S->getI64Imm(1, dl), in get32BitZExtCompare()
3462 S->getI64Imm(63, dl)), 0); in get32BitZExtCompare()
3463 return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, SrdiNode, in get32BitZExtCompare()
3464 S->getI32Imm(1, dl)), 0); in get32BitZExtCompare()
3467 // (zext (setcc %a, %b, setugt)) -> (lshr (sub %b, %a), 63) in get32BitZExtCompare()
3468 // (zext (setcc %a, %b, setult)) -> (lshr (sub %a, %b), 63) in get32BitZExtCompare()
3474 // The upper 32-bits of the register can't be undefined for this sequence. in get32BitZExtCompare()
3478 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0); in get32BitZExtCompare()
3479 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, in get32BitZExtCompare()
3480 Subtract, S->getI64Imm(1, dl), in get32BitZExtCompare()
3481 S->getI64Imm(63, dl)), 0); in get32BitZExtCompare()
3486 /// Produces a sign-extended result of comparing two 32-bit values according to
3497 bool IsRHSNegOne = RHSValue == -1LL; in get32BitSExtCompare()
3502 // (sext (setcc %a, %b, seteq)) -> in get32BitSExtCompare()
3504 // (sext (setcc %a, 0, seteq)) -> in get32BitSExtCompare()
3507 SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0); in get32BitSExtCompare()
3509 SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, CountInput), 0); in get32BitSExtCompare()
3510 SDValue SHLOps[] = { Cntlzw, S->getI32Imm(27, dl), in get32BitSExtCompare()
3511 S->getI32Imm(5, dl), S->getI32Imm(31, dl) }; in get32BitSExtCompare()
3513 SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, SHLOps), 0); in get32BitSExtCompare()
3514 return SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Slwi), 0); in get32BitSExtCompare()
3519 // (sext (setcc %a, %b, setne)) -> in get32BitSExtCompare()
3522 // (sext (setcc %a, 0, setne)) -> in get32BitSExtCompare()
3525 SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0); in get32BitSExtCompare()
3527 SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0); in get32BitSExtCompare()
3529 { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl), S->getI32Imm(31, dl) }; in get32BitSExtCompare()
3531 SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, ShiftOps), 0); in get32BitSExtCompare()
3533 SDValue(CurDAG->getMachineNode(PPC::XORI, dl, MVT::i32, Shift, in get32BitSExtCompare()
3534 S->getI32Imm(1, dl)), 0); in get32BitSExtCompare()
3535 return SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Xori), 0); in get32BitSExtCompare()
3538 // (sext (setcc %a, %b, setge)) -> (add (lshr (sub %a, %b), 63), -1) in get32BitSExtCompare()
3539 // (sext (setcc %a, 0, setge)) -> (ashr (~ %a), 31) in get32BitSExtCompare()
3547 IsRHSZero = RHSConst && RHSConst->isZero(); in get32BitSExtCompare()
3553 // (sext (setcc %a, %b, setge)) -> (add (lshr (sub %b, %a), 63), -1) in get32BitSExtCompare()
3554 // (sext (setcc %a, 0, setle)) -> (add (lshr (- %a), 63), -1) in get32BitSExtCompare()
3558 // The upper 32-bits of the register can't be undefined for this sequence. in get32BitSExtCompare()
3562 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, MVT::Glue, in get32BitSExtCompare()
3565 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, in get32BitSExtCompare()
3566 SUBFNode, S->getI64Imm(1, dl), in get32BitSExtCompare()
3567 S->getI64Imm(63, dl)), 0); in get32BitSExtCompare()
3568 return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, Srdi, in get32BitSExtCompare()
3569 S->getI32Imm(-1, dl)), 0); in get32BitSExtCompare()
3572 // (sext (setcc %a, %b, setgt)) -> (ashr (sub %b, %a), 63) in get32BitSExtCompare()
3573 // (sext (setcc %a, -1, setgt)) -> (ashr (~ %a), 31) in get32BitSExtCompare()
3574 // (sext (setcc %a, 0, setgt)) -> (ashr (- %a), 63) in get32BitSExtCompare()
3580 // The upper 32-bits of the register can't be undefined for this sequence. in get32BitSExtCompare()
3584 SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0); in get32BitSExtCompare()
3585 return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, Neg, in get32BitSExtCompare()
3586 S->getI64Imm(63, dl)), 0); in get32BitSExtCompare()
3588 // Not a special case (i.e. RHS == 0 or RHS == -1). Handle (%a > %b) as in get32BitSExtCompare()
3592 IsRHSZero = RHSConst && RHSConst->isZero(); in get32BitSExtCompare()
3593 IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1; in get32BitSExtCompare()
3597 // (sext (setcc %a, %b, setgt)) -> (ashr (sub %a, %b), 63) in get32BitSExtCompare()
3598 // (sext (setcc %a, 1, setgt)) -> (add (lshr (- %a), 63), -1) in get32BitSExtCompare()
3599 // (sext (setcc %a, 0, setgt)) -> (ashr %a, 31) in get32BitSExtCompare()
3606 return SDValue(CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, LHS, in get32BitSExtCompare()
3607 S->getI32Imm(31, dl)), 0); in get32BitSExtCompare()
3611 // The upper 32-bits of the register can't be undefined for this sequence. in get32BitSExtCompare()
3615 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0); in get32BitSExtCompare()
3616 return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, in get32BitSExtCompare()
3617 SUBFNode, S->getI64Imm(63, dl)), 0); in get32BitSExtCompare()
3620 // (sext (setcc %a, %b, setuge)) -> (add (lshr (sub %a, %b), 63), -1) in get32BitSExtCompare()
3621 // (sext (setcc %a, %b, setule)) -> (add (lshr (sub %b, %a), 63), -1) in get32BitSExtCompare()
3627 // The upper 32-bits of the register can't be undefined for this sequence. in get32BitSExtCompare()
3631 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0); in get32BitSExtCompare()
3633 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Subtract, in get32BitSExtCompare()
3634 S->getI32Imm(1, dl), S->getI32Imm(63,dl)), in get32BitSExtCompare()
3636 return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, Shift, in get32BitSExtCompare()
3637 S->getI32Imm(-1, dl)), 0); in get32BitSExtCompare()
3640 // (sext (setcc %a, %b, setugt)) -> (ashr (sub %b, %a), 63) in get32BitSExtCompare()
3641 // (sext (setcc %a, %b, setugt)) -> (ashr (sub %a, %b), 63) in get32BitSExtCompare()
3647 // The upper 32-bits of the register can't be undefined for this sequence. in get32BitSExtCompare()
3651 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0); in get32BitSExtCompare()
3652 return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, in get32BitSExtCompare()
3653 Subtract, S->getI64Imm(63, dl)), 0); in get32BitSExtCompare()
3658 /// Produces a zero-extended result of comparing two 64-bit values according to
3669 bool IsRHSNegOne = RHSValue == -1LL; in get64BitZExtCompare()
3673 // (zext (setcc %a, %b, seteq)) -> (lshr (ctlz (xor %a, %b)), 6) in get64BitZExtCompare()
3674 // (zext (setcc %a, 0, seteq)) -> (lshr (ctlz %a), 6) in get64BitZExtCompare()
3676 SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0); in get64BitZExtCompare()
3678 SDValue(CurDAG->getMachineNode(PPC::CNTLZD, dl, MVT::i64, Xor), 0); in get64BitZExtCompare()
3679 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Clz, in get64BitZExtCompare()
3680 S->getI64Imm(58, dl), in get64BitZExtCompare()
3681 S->getI64Imm(63, dl)), 0); in get64BitZExtCompare()
3684 // {addc.reg, addc.CA} = (addcarry (xor %a, %b), -1) in get64BitZExtCompare()
3685 // (zext (setcc %a, %b, setne)) -> (sube addc.reg, addc.reg, addc.CA) in get64BitZExtCompare()
3686 // {addcz.reg, addcz.CA} = (addcarry %a, -1) in get64BitZExtCompare()
3687 // (zext (setcc %a, 0, setne)) -> (sube addcz.reg, addcz.reg, addcz.CA) in get64BitZExtCompare()
3689 SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0); in get64BitZExtCompare()
3691 SDValue(CurDAG->getMachineNode(PPC::ADDIC8, dl, MVT::i64, MVT::Glue, in get64BitZExtCompare()
3692 Xor, S->getI32Imm(~0U, dl)), 0); in get64BitZExtCompare()
3693 return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, AC, in get64BitZExtCompare()
3698 // (zext (setcc %a, %b, setge)) -> in get64BitZExtCompare()
3700 // (zext (setcc %a, 0, setge)) -> (lshr (~ %a), 63) in get64BitZExtCompare()
3705 IsRHSZero = RHSConst && RHSConst->isZero(); in get64BitZExtCompare()
3710 // (zext (setcc %a, %b, setge)) -> in get64BitZExtCompare()
3712 // (zext (setcc %a, 0, setge)) -> (lshr (or %a, (add %a, -1)), 63) in get64BitZExtCompare()
3716 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS, in get64BitZExtCompare()
3717 S->getI64Imm(1, dl), in get64BitZExtCompare()
3718 S->getI64Imm(63, dl)), 0); in get64BitZExtCompare()
3720 SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, RHS, in get64BitZExtCompare()
3721 S->getI64Imm(63, dl)), 0); in get64BitZExtCompare()
3723 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, in get64BitZExtCompare()
3725 return SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue, in get64BitZExtCompare()
3730 // (zext (setcc %a, %b, setgt)) -> in get64BitZExtCompare()
3732 // (zext (setcc %a, 0, setgt)) -> (lshr (nor (add %a, -1), %a), 63) in get64BitZExtCompare()
3737 SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS, in get64BitZExtCompare()
3738 S->getI64Imm(~0ULL, dl)), 0); in get64BitZExtCompare()
3740 SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, Addi, LHS), 0); in get64BitZExtCompare()
3741 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Nor, in get64BitZExtCompare()
3742 S->getI64Imm(1, dl), in get64BitZExtCompare()
3743 S->getI64Imm(63, dl)), 0); in get64BitZExtCompare()
3747 IsRHSZero = RHSConst && RHSConst->isZero(); in get64BitZExtCompare()
3748 IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1; in get64BitZExtCompare()
3753 // (zext (setcc %a, %b, setlt)) -> in get64BitZExtCompare()
3755 // (zext (setcc %a, 0, setlt)) -> (lshr %a, 63) in get64BitZExtCompare()
3759 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS, in get64BitZExtCompare()
3760 S->getI64Imm(1, dl), in get64BitZExtCompare()
3761 S->getI64Imm(63, dl)), 0); in get64BitZExtCompare()
3763 SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, in get64BitZExtCompare()
3764 LHS, S->getI64Imm(63, dl)), 0); in get64BitZExtCompare()
3766 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, in get64BitZExtCompare()
3767 RHS, S->getI64Imm(1, dl), in get64BitZExtCompare()
3768 S->getI64Imm(63, dl)), 0); in get64BitZExtCompare()
3770 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, in get64BitZExtCompare()
3773 SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue, in get64BitZExtCompare()
3775 return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, in get64BitZExtCompare()
3776 ADDE8Node, S->getI64Imm(1, dl)), 0); in get64BitZExtCompare()
3780 // (zext (setcc %a, %b, setuge)) -> (add (sube %b, %b, subc.CA), 1) in get64BitZExtCompare()
3785 // (zext (setcc %a, %b, setule)) -> (add (sube %a, %a, subc.CA), 1) in get64BitZExtCompare()
3787 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, in get64BitZExtCompare()
3790 SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, MVT::Glue, in get64BitZExtCompare()
3792 return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, in get64BitZExtCompare()
3793 SUBFE8Node, S->getI64Imm(1, dl)), 0); in get64BitZExtCompare()
3797 // (zext (setcc %a, %b, setugt)) -> -(sube %b, %b, subc.CA) in get64BitZExtCompare()
3802 // (zext (setcc %a, %b, setult)) -> -(sube %a, %a, subc.CA) in get64BitZExtCompare()
3804 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, in get64BitZExtCompare()
3807 SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, in get64BitZExtCompare()
3809 return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, in get64BitZExtCompare()
3815 /// Produces a sign-extended result of comparing two 64-bit values according to
3826 bool IsRHSNegOne = RHSValue == -1LL; in get64BitSExtCompare()
3830 // {addc.reg, addc.CA} = (addcarry (xor %a, %b), -1) in get64BitSExtCompare()
3831 // (sext (setcc %a, %b, seteq)) -> (sube addc.reg, addc.reg, addc.CA) in get64BitSExtCompare()
3832 // {addcz.reg, addcz.CA} = (addcarry %a, -1) in get64BitSExtCompare()
3833 // (sext (setcc %a, 0, seteq)) -> (sube addcz.reg, addcz.reg, addcz.CA) in get64BitSExtCompare()
3835 SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0); in get64BitSExtCompare()
3837 SDValue(CurDAG->getMachineNode(PPC::ADDIC8, dl, MVT::i64, MVT::Glue, in get64BitSExtCompare()
3838 AddInput, S->getI32Imm(~0U, dl)), 0); in get64BitSExtCompare()
3839 return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, Addic, in get64BitSExtCompare()
3844 // (sext (setcc %a, %b, setne)) -> (sube subfc.reg, subfc.reg, subfc.CA) in get64BitSExtCompare()
3846 // (sext (setcc %a, 0, setne)) -> (sube subfcz.reg, subfcz.reg, subfcz.CA) in get64BitSExtCompare()
3848 SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0); in get64BitSExtCompare()
3850 SDValue(CurDAG->getMachineNode(PPC::SUBFIC8, dl, MVT::i64, MVT::Glue, in get64BitSExtCompare()
3851 Xor, S->getI32Imm(0, dl)), 0); in get64BitSExtCompare()
3852 return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, SC, in get64BitSExtCompare()
3857 // (zext (setcc %a, %b, setge)) -> in get64BitSExtCompare()
3858 // (- (adde (lshr %b, 63), (ashr %a, 63), subc.CA)) in get64BitSExtCompare()
3859 // (zext (setcc %a, 0, setge)) -> (~ (ashr %a, 63)) in get64BitSExtCompare()
3864 IsRHSZero = RHSConst && RHSConst->isZero(); in get64BitSExtCompare()
3869 // (zext (setcc %a, %b, setge)) -> in get64BitSExtCompare()
3870 // (- (adde (lshr %a, 63), (ashr %b, 63), subc.CA)) in get64BitSExtCompare()
3871 // (zext (setcc %a, 0, setge)) -> (ashr (or %a, (add %a, -1)), 63) in get64BitSExtCompare()
3875 SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, RHS, in get64BitSExtCompare()
3876 S->getI64Imm(63, dl)), 0); in get64BitSExtCompare()
3878 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS, in get64BitSExtCompare()
3879 S->getI64Imm(1, dl), in get64BitSExtCompare()
3880 S->getI64Imm(63, dl)), 0); in get64BitSExtCompare()
3882 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, in get64BitSExtCompare()
3885 SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue, in get64BitSExtCompare()
3887 return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, Adde), 0); in get64BitSExtCompare()
3891 // (zext (setcc %a, %b, setgt)) -> in get64BitSExtCompare()
3892 // -(xor (adde (lshr %a, 63), (ashr %b, 63), subc.CA), 1) in get64BitSExtCompare()
3893 // (zext (setcc %a, 0, setgt)) -> (ashr (nor (add %a, -1), %a), 63) in get64BitSExtCompare()
3898 SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS, in get64BitSExtCompare()
3899 S->getI64Imm(-1, dl)), 0); in get64BitSExtCompare()
3901 SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, Add, LHS), 0); in get64BitSExtCompare()
3902 return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, Nor, in get64BitSExtCompare()
3903 S->getI64Imm(63, dl)), 0); in get64BitSExtCompare()
3907 IsRHSZero = RHSConst && RHSConst->isZero(); in get64BitSExtCompare()
3908 IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1; in get64BitSExtCompare()
3913 // (zext (setcc %a, %b, setlt)) -> in get64BitSExtCompare()
3914 // -(xor (adde (lshr %b, 63), (ashr %a, 63), subc.CA), 1) in get64BitSExtCompare()
3915 // (zext (setcc %a, 0, setlt)) -> (ashr %a, 63) in get64BitSExtCompare()
3919 return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, LHS, in get64BitSExtCompare()
3920 S->getI64Imm(63, dl)), 0); in get64BitSExtCompare()
3923 SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, in get64BitSExtCompare()
3924 LHS, S->getI64Imm(63, dl)), 0); in get64BitSExtCompare()
3926 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, in get64BitSExtCompare()
3927 RHS, S->getI64Imm(1, dl), in get64BitSExtCompare()
3928 S->getI64Imm(63, dl)), 0); in get64BitSExtCompare()
3930 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, in get64BitSExtCompare()
3933 SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, in get64BitSExtCompare()
3936 SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, in get64BitSExtCompare()
3937 ADDE8Node, S->getI64Imm(1, dl)), 0); in get64BitSExtCompare()
3938 return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, in get64BitSExtCompare()
3943 // (sext (setcc %a, %b, setuge)) -> ~(sube %b, %b, subc.CA) in get64BitSExtCompare()
3948 // (sext (setcc %a, %b, setule)) -> ~(sube %a, %a, subc.CA) in get64BitSExtCompare()
3950 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, in get64BitSExtCompare()
3953 SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, MVT::Glue, LHS, in get64BitSExtCompare()
3955 return SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, in get64BitSExtCompare()
3960 // (sext (setcc %a, %b, setugt)) -> (sube %b, %b, subc.CA) in get64BitSExtCompare()
3965 // (sext (setcc %a, %b, setult)) -> (sube %a, %a, subc.CA) in get64BitSExtCompare()
3967 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, in get64BitSExtCompare()
3969 return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, in get64BitSExtCompare()
3989 for (auto *CompareUse : Compare.getNode()->uses()) in allUsesExtend()
3990 if (CompareUse->getOpcode() != ISD::SIGN_EXTEND && in allUsesExtend()
3991 CompareUse->getOpcode() != ISD::ZERO_EXTEND && in allUsesExtend()
3992 CompareUse->getOpcode() != ISD::SELECT && in allUsesExtend()
3993 !ISD::isBitwiseLogicOp(CompareUse->getOpcode())) { in allUsesExtend()
4020 cast<CondCodeSDNode>(Compare.getOperand(CCOpNum))->get(); in getSETCCInGPR()
4033 int64_t RHSValue = RHSConst ? RHSConst->getSExtValue() : INT64_MAX; in getSETCCInGPR()
4049 if (N->getValueType(0) != MVT::i32 && in tryIntCompareInGPR()
4050 N->getValueType(0) != MVT::i64) in tryIntCompareInGPR()
4053 // This optimization will emit code that assumes 64-bit registers in tryIntCompareInGPR()
4054 // so we don't want to run it in 32-bit mode. Also don't run it in tryIntCompareInGPR()
4061 // Users can override this via the command line option, `--ppc-gpr-icmps`. in tryIntCompareInGPR()
4062 if (!(CmpInGPR.getNumOccurrences() > 0) && Subtarget->isISA3_1()) in tryIntCompareInGPR()
4065 switch (N->getOpcode()) { in tryIntCompareInGPR()
4083 if (N->getValueType(0) != MVT::i32 && in tryBitPermutation()
4084 N->getValueType(0) != MVT::i64) in tryBitPermutation()
4090 switch (N->getOpcode()) { in tryBitPermutation()
4093 // If we are on P10, we have a pattern for 32-bit (srl (bswap r), 16) that in tryBitPermutation()
4095 if (Subtarget->isISA3_1() && N->getValueType(0) == MVT::i32 && in tryBitPermutation()
4096 N->getOperand(0).getOpcode() == ISD::BSWAP) { in tryBitPermutation()
4097 auto &OpRight = N->getOperand(1); in tryBitPermutation()
4099 if (SRLConst && SRLConst->getSExtValue() == 16) in tryBitPermutation()
4119 /// SelectCC - Select a comparison of the specified values with the specified
4130 // SETEQ/SETNE comparison with 16-bit immediate, fold it. in SelectCC()
4132 return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS, in SelectCC()
4135 // If this is a 16-bit signed immediate, fold it. in SelectCC()
4137 return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS, in SelectCC()
4141 // For non-equality comparisons, the default code would materialize the in SelectCC()
4150 SDValue Xor(CurDAG->getMachineNode(PPC::XORIS, dl, MVT::i32, LHS, in SelectCC()
4152 return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, Xor, in SelectCC()
4158 return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS, in SelectCC()
4164 return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS, in SelectCC()
4174 // SETEQ/SETNE comparison with 16-bit immediate, fold it. in SelectCC()
4176 return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS, in SelectCC()
4179 // If this is a 16-bit signed immediate, fold it. in SelectCC()
4181 return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS, in SelectCC()
4185 // For non-equality comparisons, the default code would materialize the in SelectCC()
4195 SDValue Xor(CurDAG->getMachineNode(PPC::XORIS8, dl, MVT::i64, LHS, in SelectCC()
4197 return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, Xor, in SelectCC()
4205 return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS, in SelectCC()
4211 return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS, in SelectCC()
4217 if (Subtarget->hasSPE()) { in SelectCC()
4244 if (Subtarget->hasSPE()) { in SelectCC()
4269 Opc = Subtarget->hasVSX() ? PPC::XSCMPUDP : PPC::FCMPUD; in SelectCC()
4272 assert(Subtarget->hasP9Vector() && "XSCMPUQP requires Power9 Vector"); in SelectCC()
4277 CurDAG->getMachineNode(Opc, dl, MVT::i32, MVT::Other, LHS, RHS, Chain), in SelectCC()
4280 return SDValue(CurDAG->getMachineNode(Opc, dl, MVT::i32, LHS, RHS), 0); in SelectCC()
4286 bool UseSPE = Subtarget->hasSPE() && VT.isFloatingPoint(); in getPredicateForSetCC()
4321 /// getCRIdxForSetCC - Return the index of the condition register field
4407 llvm_unreachable("Invalid floating-point vector compare condition"); in getVCmpInst()
4474 bool IsStrict = N->isStrictFPOpcode(); in trySETCC()
4476 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get(); in trySETCC()
4478 CurDAG->getTargetLoweringInfo().getPointerTy(CurDAG->getDataLayout()); in trySETCC()
4480 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); in trySETCC()
4482 SDValue LHS = N->getOperand(IsStrict ? 1 : 0); in trySETCC()
4483 SDValue RHS = N->getOperand(IsStrict ? 2 : 1); in trySETCC()
4485 if (!IsStrict && !Subtarget->useCRBits() && isInt32Immediate(RHS, Imm)) { in trySETCC()
4494 Op = SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Op), 0); in trySETCC()
4497 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); in trySETCC()
4503 SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, in trySETCC()
4505 CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, AD, Op, AD.getValue(1)); in trySETCC()
4511 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); in trySETCC()
4516 SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Op), 0); in trySETCC()
4517 T = SDValue(CurDAG->getMachineNode(PPC::ANDC, dl, MVT::i32, T, Op), 0); in trySETCC()
4520 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); in trySETCC()
4524 } else if (Imm == ~0U) { // setcc op, -1 in trySETCC()
4530 Op = SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, in trySETCC()
4532 CurDAG->SelectNodeTo(N, PPC::ADDZE, MVT::i32, in trySETCC()
4533 SDValue(CurDAG->getMachineNode(PPC::LI, dl, in trySETCC()
4540 Op = SDValue(CurDAG->getMachineNode(PPC::NOR, dl, MVT::i32, Op, Op), 0); in trySETCC()
4541 SDNode *AD = CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, in trySETCC()
4543 CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, SDValue(AD, 0), Op, in trySETCC()
4548 SDValue AD = SDValue(CurDAG->getMachineNode(PPC::ADDI, dl, MVT::i32, Op, in trySETCC()
4550 SDValue AN = SDValue(CurDAG->getMachineNode(PPC::AND, dl, MVT::i32, AD, in trySETCC()
4554 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); in trySETCC()
4560 Op = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); in trySETCC()
4561 CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Op, getI32Imm(1, dl)); in trySETCC()
4571 if (Subtarget->hasSPE()) in trySETCC()
4577 getVCmpInst(VecVT.getSimpleVT(), CC, Subtarget->hasVSX(), Swap, Negate); in trySETCC()
4583 SDValue VCmp(CurDAG->getMachineNode(VCmpInst, dl, ResVT, LHS, RHS), 0); in trySETCC()
4584 CurDAG->SelectNodeTo(N, Subtarget->hasVSX() ? PPC::XXLNOR : PPC::VNOR, in trySETCC()
4589 CurDAG->SelectNodeTo(N, VCmpInst, ResVT, LHS, RHS); in trySETCC()
4593 if (Subtarget->useCRBits()) in trySETCC()
4600 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 1), CCReg.getValue(1)); in trySETCC()
4603 // SPE e*cmp* instructions only set the 'gt' bit, so hard-code that in trySETCC()
4605 if (Subtarget->hasSPE() && LHS.getValueType().isFloatingPoint()) { in trySETCC()
4610 SDValue CR7Reg = CurDAG->getRegister(PPC::CR7, MVT::i32); in trySETCC()
4613 CCReg = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, CR7Reg, CCReg, in trySETCC()
4616 IntCR = SDValue(CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, CR7Reg, in trySETCC()
4619 SDValue Ops[] = { IntCR, getI32Imm((32 - (3 - Idx)) & 31, dl), in trySETCC()
4622 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); in trySETCC()
4628 SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); in trySETCC()
4629 CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Tmp, getI32Imm(1, dl)); in trySETCC()
4640 if (LDN || (MIN && MIN->getOpcode() == PPCISD::LD_SPLAT)) in isOffsetMultipleOf()
4641 AddrOp = N->getOperand(1); in isOffsetMultipleOf()
4643 AddrOp = STN->getOperand(2); in isOffsetMultipleOf()
4654 const MachineFrameInfo &MFI = CurDAG->getMachineFunction().getFrameInfo(); in isOffsetMultipleOf()
4655 unsigned SlotAlign = MFI.getObjectAlign(FI->getIndex()).value(); in isOffsetMultipleOf()
4673 MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand(); in transferMemOperands()
4674 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {MemOp}); in transferMemOperands()
4680 assert(N->getOpcode() == ISD::SELECT_CC && "Expecting a SELECT_CC here."); in mayUseP9Setb()
4682 SDValue LHS = N->getOperand(0); in mayUseP9Setb()
4683 SDValue RHS = N->getOperand(1); in mayUseP9Setb()
4684 SDValue TrueRes = N->getOperand(2); in mayUseP9Setb()
4685 SDValue FalseRes = N->getOperand(3); in mayUseP9Setb()
4687 if (!TrueConst || (N->getSimpleValueType(0) != MVT::i64 && in mayUseP9Setb()
4688 N->getSimpleValueType(0) != MVT::i32)) in mayUseP9Setb()
4693 // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, cc2)), cc1) in mayUseP9Setb()
4694 // (select_cc lhs, rhs, 0, (select_cc [lr]hs, [lr]hs, 1, -1, cc2), seteq) in mayUseP9Setb()
4695 // (select_cc lhs, rhs, 0, (select_cc [lr]hs, [lr]hs, -1, 1, cc2), seteq) in mayUseP9Setb()
4696 int64_t TrueResVal = TrueConst->getSExtValue(); in mayUseP9Setb()
4697 if ((TrueResVal < -1 || TrueResVal > 1) || in mayUseP9Setb()
4698 (TrueResVal == -1 && FalseRes.getOpcode() != ISD::ZERO_EXTEND) || in mayUseP9Setb()
4713 // selected to SELECT_CC_I4/SELECT_CC_I8 Pseudo, then expand-isel-pseudos pass in mayUseP9Setb()
4726 cast<CondCodeSDNode>(SetOrSelCC.getOperand(InnerIsSel ? 4 : 2))->get(); in mayUseP9Setb()
4728 // 1/-1 and canonicalize it if needed. in mayUseP9Setb()
4736 int64_t SelCCTVal = SelCCTrueConst->getSExtValue(); in mayUseP9Setb()
4737 int64_t SelCCFVal = SelCCFalseConst->getSExtValue(); in mayUseP9Setb()
4738 // The values must be -1/1 (requiring a swap) or 1/-1. in mayUseP9Setb()
4739 if (SelCCTVal == -1 && SelCCFVal == 1) { in mayUseP9Setb()
4741 } else if (SelCCTVal != 1 || SelCCFVal != -1) in mayUseP9Setb()
4759 // (select_cc [lr]hs, [lr]hs, 1, -1, setlt/setgt), seteq) in mayUseP9Setb()
4768 // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, setne)), setu?lt) in mayUseP9Setb()
4769 // (select_cc lhs, rhs, -1, (zext (setcc lhs, rhs, setgt)), setu?lt) in mayUseP9Setb()
4770 // (select_cc lhs, rhs, -1, (zext (setcc rhs, lhs, setlt)), setu?lt) in mayUseP9Setb()
4790 // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, setne)), setu?gt) in mayUseP9Setb()
4791 // (select_cc lhs, rhs, -1, (zext (setcc lhs, rhs, setlt)), setu?gt) in mayUseP9Setb()
4792 // (select_cc lhs, rhs, -1, (zext (setcc rhs, lhs, setgt)), setu?gt) in mayUseP9Setb()
4801 NeedSwapOps = (TrueResVal == -1); in mayUseP9Setb()
4811 LLVM_DEBUG(N->dump()); in mayUseP9Setb()
4816 // Return true if it's a software square-root/divide operand.
4834 assert(N->getOpcode() == ISD::BR_CC && "ISD::BR_CC is expected."); in tryFoldSWTestBRCC()
4837 // (br_cc seteq, (truncateToi1 SWTestOp), 0) -> (BCC PRED_NU, SWTestOp) in tryFoldSWTestBRCC()
4838 // (br_cc seteq, (and SWTestOp, 2), 0) -> (BCC PRED_NE, SWTestOp) in tryFoldSWTestBRCC()
4839 // (br_cc seteq, (and SWTestOp, 4), 0) -> (BCC PRED_LE, SWTestOp) in tryFoldSWTestBRCC()
4840 // (br_cc seteq, (and SWTestOp, 8), 0) -> (BCC PRED_GE, SWTestOp) in tryFoldSWTestBRCC()
4841 // (br_cc setne, (truncateToi1 SWTestOp), 0) -> (BCC PRED_UN, SWTestOp) in tryFoldSWTestBRCC()
4842 // (br_cc setne, (and SWTestOp, 2), 0) -> (BCC PRED_EQ, SWTestOp) in tryFoldSWTestBRCC()
4843 // (br_cc setne, (and SWTestOp, 4), 0) -> (BCC PRED_GT, SWTestOp) in tryFoldSWTestBRCC()
4844 // (br_cc setne, (and SWTestOp, 8), 0) -> (BCC PRED_LT, SWTestOp) in tryFoldSWTestBRCC()
4845 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(1))->get(); in tryFoldSWTestBRCC()
4849 SDValue CmpRHS = N->getOperand(3); in tryFoldSWTestBRCC()
4853 SDValue CmpLHS = N->getOperand(2); in tryFoldSWTestBRCC()
4883 SDValue Ops[] = {getI32Imm(PCC, dl), CmpLHS.getOperand(0), N->getOperand(4), in tryFoldSWTestBRCC()
4884 N->getOperand(0)}; in tryFoldSWTestBRCC()
4885 CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops); in tryFoldSWTestBRCC()
4892 // Sometimes the promoted value of the intrinsic is ANDed by some non-zero in trySelectLoopCountIntrinsic()
4895 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(1))->get(); in trySelectLoopCountIntrinsic()
4896 SDValue LHS = N->getOperand(2), RHS = N->getOperand(3); in trySelectLoopCountIntrinsic()
4917 SDValue DecrementOps[] = {Subtarget->isPPC64() ? getI64Imm(1, DecrementLoc) in trySelectLoopCountIntrinsic()
4920 Subtarget->isPPC64() ? PPC::DecreaseCTR8loop : PPC::DecreaseCTRloop; in trySelectLoopCountIntrinsic()
4921 SDNode *NewDecrement = CurDAG->getMachineNode(DecrementOpcode, DecrementLoc, in trySelectLoopCountIntrinsic()
4924 unsigned Val = RHS->getAsZExtVal(); in trySelectLoopCountIntrinsic()
4929 CurDAG->RemoveDeadNode(LHS.getNode()); in trySelectLoopCountIntrinsic()
4933 CurDAG->RemoveDeadNode(OldDecrement.getNode()); in trySelectLoopCountIntrinsic()
4935 SDValue Chain = CurDAG->getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, in trySelectLoopCountIntrinsic()
4936 ChainInput, N->getOperand(0)); in trySelectLoopCountIntrinsic()
4938 CurDAG->SelectNodeTo(N, Opcode, MVT::Other, SDValue(NewDecrement, 0), in trySelectLoopCountIntrinsic()
4939 N->getOperand(4), Chain); in trySelectLoopCountIntrinsic()
4944 assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected"); in tryAsSingleRLWINM()
4946 if (!isInt32Immediate(N->getOperand(1), Imm)) in tryAsSingleRLWINM()
4950 SDValue Val = N->getOperand(0); in tryAsSingleRLWINM()
4958 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); in tryAsSingleRLWINM()
4962 // If this is just a masked value where the input is not handled, and in tryAsSingleRLWINM()
4963 // is not a rotate-left (handled by a pattern in the .td file), emit rlwinm in tryAsSingleRLWINM()
4967 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); in tryAsSingleRLWINM()
4971 // AND X, 0 -> 0, not "rlwinm 32". in tryAsSingleRLWINM()
4973 ReplaceUses(SDValue(N, 0), N->getOperand(1)); in tryAsSingleRLWINM()
4981 assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected"); in tryAsSingleRLWINM8()
4983 if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64)) in tryAsSingleRLWINM8()
4989 // +----------------------+ in tryAsSingleRLWINM8()
4991 // +----------------------+ in tryAsSingleRLWINM8()
4994 // as RLWINM will replace the contents of [0 - 32) with [32 - 64) even in tryAsSingleRLWINM8()
4997 SDValue Ops[] = {N->getOperand(0), getI64Imm(0, dl), getI64Imm(MB - 32, dl), in tryAsSingleRLWINM8()
4998 getI64Imm(ME - 32, dl)}; in tryAsSingleRLWINM8()
4999 CurDAG->SelectNodeTo(N, PPC::RLWINM8, MVT::i64, Ops); in tryAsSingleRLWINM8()
5007 assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected"); in tryAsPairOfRLDICL()
5009 if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64)) in tryAsPairOfRLDICL()
5012 // Do nothing if it is 16-bit imm as the pattern in the .td file handle in tryAsPairOfRLDICL()
5018 SDValue Val = N->getOperand(0); in tryAsPairOfRLDICL()
5033 // ME MB MB-ME+63 in tryAsPairOfRLDICL()
5034 // +----------------------+ +----------------------+ in tryAsPairOfRLDICL()
5035 // |1111111100000011111111| -> |0000001111111111111111| in tryAsPairOfRLDICL()
5036 // +----------------------+ +----------------------+ in tryAsPairOfRLDICL()
5038 // There are ME + 1 ones on the left and (MB - ME + 63) & 63 zeros in between. in tryAsPairOfRLDICL()
5040 unsigned ZerosInBetween = (MB - ME + 63) & 63; in tryAsPairOfRLDICL()
5043 Val = SDValue(CurDAG->getMachineNode(PPC::RLDICL, Loc, MVT::i64, Val, in tryAsPairOfRLDICL()
5047 // MB-ME+63 ME MB in tryAsPairOfRLDICL()
5048 // +----------------------+ +----------------------+ in tryAsPairOfRLDICL()
5049 // |0000001111111111111111| -> |0001111100000011111111| in tryAsPairOfRLDICL()
5050 // +----------------------+ +----------------------+ in tryAsPairOfRLDICL()
5052 // Rotate back by 64 - OnesOnLeft to undo previous rotate. Then clear on the in tryAsPairOfRLDICL()
5054 SDValue Ops[] = {Val, getI64Imm(64 - OnesOnLeft, Loc), in tryAsPairOfRLDICL()
5056 CurDAG->SelectNodeTo(N, PPC::RLDICL, MVT::i64, Ops); in tryAsPairOfRLDICL()
5061 assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected"); in tryAsSingleRLWIMI()
5063 if (!isInt32Immediate(N->getOperand(1), Imm)) in tryAsSingleRLWIMI()
5066 SDValue Val = N->getOperand(0); in tryAsSingleRLWIMI()
5076 // where m is a run-of-ones mask. The logic here is that, for each bit in in tryAsSingleRLWIMI()
5078 // - if both are 1, then the output will be 1. in tryAsSingleRLWIMI()
5079 // - if both are 0, then the output will be 0. in tryAsSingleRLWIMI()
5080 // - if the bit in c1 is 0, and the bit in c2 is 1, then the output will in tryAsSingleRLWIMI()
5082 // - if the bit in c1 is 1, and the bit in c2 is 0, then the output will in tryAsSingleRLWIMI()
5091 ReplaceNode(N, CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops)); in tryAsSingleRLWIMI()
5099 assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected"); in tryAsSingleRLDCL()
5102 if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64) || !isMask_64(Imm64)) in tryAsSingleRLDCL()
5105 SDValue Val = N->getOperand(0); in tryAsSingleRLDCL()
5120 unsigned MB = 64 - llvm::countr_one(Imm64); in tryAsSingleRLDCL()
5123 CurDAG->SelectNodeTo(N, PPC::RLDCL, MVT::i64, Ops); in tryAsSingleRLDCL()
5128 assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected"); in tryAsSingleRLDICL()
5130 if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64) || !isMask_64(Imm64)) in tryAsSingleRLDICL()
5133 // If this is a 64-bit zero-extension mask, emit rldicl. in tryAsSingleRLDICL()
5134 unsigned MB = 64 - llvm::countr_one(Imm64); in tryAsSingleRLDICL()
5137 SDValue Val = N->getOperand(0); in tryAsSingleRLDICL()
5145 auto ResultType = Val.getNode()->getValueType(0); in tryAsSingleRLDICL()
5146 auto ImDef = CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, ResultType); in tryAsSingleRLDICL()
5149 Val = SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl, ResultType, in tryAsSingleRLDICL()
5153 SH = 64 - Imm; in tryAsSingleRLDICL()
5158 // instruction: rldicl(rldicl(x, 64-n, n), 0, mb) -> rldicl(x, 64-n, mb) in tryAsSingleRLDICL()
5160 // mask, and this mask is a more-restrictive sub-mask of the mask implied in tryAsSingleRLDICL()
5166 SH = 64 - Imm; in tryAsSingleRLDICL()
5170 CurDAG->SelectNodeTo(N, PPC::RLDICL, MVT::i64, Ops); in tryAsSingleRLDICL()
5175 assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected"); in tryAsSingleRLDICR()
5177 if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64) || in tryAsSingleRLDICR()
5181 // If this is a negated 64-bit zero-extension mask, in tryAsSingleRLDICR()
5184 unsigned MB = 63 - llvm::countr_one(~Imm64); in tryAsSingleRLDICR()
5187 SDValue Ops[] = {N->getOperand(0), getI32Imm(SH, dl), getI32Imm(MB, dl)}; in tryAsSingleRLDICR()
5188 CurDAG->SelectNodeTo(N, PPC::RLDICR, MVT::i64, Ops); in tryAsSingleRLDICR()
5193 assert(N->getOpcode() == ISD::OR && "ISD::OR SDNode expected"); in tryAsSingleRLDIMI()
5196 SDValue N0 = N->getOperand(0); in tryAsSingleRLDIMI()
5198 // We won't get fewer instructions if the imm is 32-bit integer. in tryAsSingleRLDIMI()
5202 if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64) || in tryAsSingleRLDIMI()
5206 unsigned SH = 63 - ME; in tryAsSingleRLDIMI()
5210 N->getOperand(0), in tryAsSingleRLDIMI()
5211 SDValue(selectI64Imm(CurDAG, getI64Imm(-1, Dl).getNode()), 0), in tryAsSingleRLDIMI()
5213 CurDAG->SelectNodeTo(N, PPC::RLDIMI, MVT::i64, Ops); in tryAsSingleRLDIMI()
5217 // Select - Convert the specified operand from a target-independent to a
5218 // target-specific node if it hasn't already been changed.
5221 if (N->isMachineOpcode()) { in Select()
5222 N->setNodeId(-1); in Select()
5226 // In case any misguided DAG-level optimizations form an ADD with a in Select()
5229 if (N->getOpcode() == ISD::ADD && in Select()
5230 N->getOperand(1).getOpcode() == ISD::TargetConstant) in Select()
5237 // Try to emit integer compares as GPR-only sequences (i.e. no use of CR). in Select()
5241 switch (N->getOpcode()) { in Select()
5245 if (N->getValueType(0) == MVT::i64) { in Select()
5252 auto IntrinsicID = N->getConstantOperandVal(1); in Select()
5265 SDValue Ops[] = {N->getOperand(4), N->getOperand(2), N->getOperand(3)}; in Select()
5270 isIntS16Immediate(N->getOperand(2), SImmOperand2); in Select()
5272 isIntS16Immediate(N->getOperand(3), SImmOperand3); in Select()
5284 isIntS16Immediate(N->getOperand(4), SImmOperand4); in Select()
5292 TO = (TO & 0x1) ? TO + 1 : TO - 1; in Select()
5295 TO = (TO & 0x8) ? TO + 8 : TO - 8; in Select()
5297 Ops[1] = N->getOperand(3); in Select()
5303 OpsWithMD = {getI32Imm(24, dl), N->getOperand(2), getI32Imm(0, dl)}; in Select()
5307 if (N->getNumOperands() > MDIndex) { in Select()
5308 SDValue MDV = N->getOperand(MDIndex); in Select()
5309 const MDNode *MD = cast<MDNodeSDNode>(MDV)->getMD(); in Select()
5310 assert(MD->getNumOperands() != 0 && "Empty MDNode in operands!"); in Select()
5311 assert((isa<MDString>(MD->getOperand(0)) && in Select()
5312 cast<MDString>(MD->getOperand(0))->getString() == in Select()
5313 "ppc-trap-reason") && in Select()
5315 for (unsigned i = 1; i < MD->getNumOperands(); i++) { in Select()
5316 assert(isa<MDString>(MD->getOperand(i)) && in Select()
5317 "Invalid data type for annotation ppc-trap-reason!"); in Select()
5320 MD->getOperand(i))->getString().str()), dl)); in Select()
5323 OpsWithMD.push_back(N->getOperand(0)); // chain in Select()
5324 CurDAG->SelectNodeTo(N, Opcode, MVT::Other, OpsWithMD); in Select()
5330 // the comparison operand. The FSELS instruction is defined to use an 8-byte in Select()
5331 // comparison like the FSELD version. The fsels intrinsic takes a 4-byte in Select()
5334 // !Subtarget->isISA3_1() check. in Select()
5335 auto IntID = N->getConstantOperandVal(0); in Select()
5337 SDValue Ops[] = {N->getOperand(1), N->getOperand(2), N->getOperand(3)}; in Select()
5338 CurDAG->SelectNodeTo(N, PPC::FSELS, MVT::f32, Ops); in Select()
5343 auto Pred = N->getConstantOperandVal(1); in Select()
5387 SDValue Ops[] = {N->getOperand(2), N->getOperand(3), in Select()
5388 CurDAG->getTargetConstant(0, dl, MVT::i32)}; in Select()
5389 SDValue BCDOp = SDValue(CurDAG->getMachineNode(Opcode, dl, VTs, Ops), 0); in Select()
5390 SDValue CR6Reg = CurDAG->getRegister(PPC::CR6, MVT::i32); in Select()
5393 if (Subtarget->isISA3_1()) { in Select()
5394 SDValue SubRegIdx = CurDAG->getTargetConstant(SubReg, dl, MVT::i32); in Select()
5396 CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, MVT::i1, in Select()
5399 CurDAG->SelectNodeTo(N, Reverse ? PPC::SETBCR : PPC::SETBC, MVT::i32, in Select()
5403 SDValue(CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, CR6Reg, in Select()
5406 SDValue Ops[] = {Move, getI32Imm((32 - (4 + ShiftVal)) & 31, dl), in Select()
5409 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); in Select()
5412 CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); in Select()
5413 CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Shift, getI32Imm(1, dl)); in Select()
5419 if (!Subtarget->isISA3_1()) in Select()
5444 SDValue(CurDAG->getMachineNode(Opcode, dl, VTs, N->getOperand(2)), 0); in Select()
5447 SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_eq, dl, MVT::i32); in Select()
5448 SDValue CR6Reg = CurDAG->getRegister(PPC::CR6, MVT::i32); in Select()
5450 CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, MVT::i1, in Select()
5453 CurDAG->SelectNodeTo(N, PPC::SETBC, MVT::i32, CRBit); in Select()
5467 const Module *Mod = MF->getFunction().getParent(); in Select()
5468 if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) != MVT::i32 || in Select()
5469 !Subtarget->isSecurePlt() || !Subtarget->isTargetELF() || in Select()
5470 Mod->getPICLevel() == PICLevel::SmallPIC) in Select()
5477 if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) != MVT::i32 || in Select()
5478 !TM.isPositionIndependent() || !Subtarget->isSecurePlt() || in Select()
5479 !Subtarget->isTargetELF()) in Select()
5482 SDValue Op = N->getOperand(1); in Select()
5485 if (GA->getTargetFlags() == PPCII::MO_PLT) in Select()
5489 if (ES->getTargetFlags() == PPCII::MO_PLT) in Select()
5504 SDValue InGlue = N->getOperand(1); in Select()
5505 ReplaceNode(N, CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, in Select()
5506 N->getOperand(0), InGlue)); in Select()
5511 ReplaceNode(N, CurDAG->getMachineNode(PPC::ReadTB, dl, MVT::i32, MVT::i32, in Select()
5512 MVT::Other, N->getOperand(0))); in Select()
5516 SDValue N0 = N->getOperand(0); in Select()
5518 CurDAG->getTargetConstant(*cast<ConstantSDNode>(N->getOperand(1))-> in Select()
5520 N->getValueType(0)); in Select()
5521 if (N->getValueType(0) == MVT::i64) { in Select()
5523 CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, MVT::Glue, in Select()
5525 CurDAG->SelectNodeTo(N, PPC::ADDZE8, MVT::i64, SDValue(Op, 0), in Select()
5529 assert(N->getValueType(0) == MVT::i32 && in Select()
5532 CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, MVT::Glue, in Select()
5534 CurDAG->SelectNodeTo(N, PPC::ADDZE, MVT::i32, SDValue(Op, 0), in Select()
5541 // Change TLS initial-exec (or TLS local-exec on AIX) D-form stores to in Select()
5542 // X-form stores. in Select()
5544 if (EnableTLSOpt && (Subtarget->isELFv2ABI() || Subtarget->isAIXABI()) && in Select()
5545 ST->getAddressingMode() != ISD::PRE_INC) in Select()
5553 EVT LoadedVT = LD->getMemoryVT(); in Select()
5556 if (LD->getAddressingMode() != ISD::PRE_INC) { in Select()
5557 // Change TLS initial-exec (or TLS local-exec on AIX) D-form loads to in Select()
5558 // X-form loads. in Select()
5559 if (EnableTLSOpt && (Subtarget->isELFv2ABI() || Subtarget->isAIXABI())) in Select()
5565 SDValue Offset = LD->getOffset(); in Select()
5570 bool isSExt = LD->getExtensionType() == ISD::SEXTLOAD; in Select()
5571 if (LD->getValueType(0) != MVT::i64) { in Select()
5584 assert(LD->getValueType(0) == MVT::i64 && "Unknown load result type!"); in Select()
5596 SDValue Chain = LD->getChain(); in Select()
5597 SDValue Base = LD->getBasePtr(); in Select()
5599 SDNode *MN = CurDAG->getMachineNode( in Select()
5600 Opcode, dl, LD->getValueType(0), in Select()
5601 PPCLowering->getPointerTy(CurDAG->getDataLayout()), MVT::Other, Ops); in Select()
5607 bool isSExt = LD->getExtensionType() == ISD::SEXTLOAD; in Select()
5608 if (LD->getValueType(0) != MVT::i64) { in Select()
5621 assert(LD->getValueType(0) == MVT::i64 && "Unknown load result type!"); in Select()
5634 SDValue Chain = LD->getChain(); in Select()
5635 SDValue Base = LD->getBasePtr(); in Select()
5637 SDNode *MN = CurDAG->getMachineNode( in Select()
5638 Opcode, dl, LD->getValueType(0), in Select()
5639 PPCLowering->getPointerTy(CurDAG->getDataLayout()), MVT::Other, Ops); in Select()
5656 if (N->getValueType(0) == MVT::i32) in Select()
5661 if (N->getOperand(0)->getOpcode() == ISD::FrameIndex && in Select()
5662 isIntS16Immediate(N->getOperand(1), Imm)) { in Select()
5663 KnownBits LHSKnown = CurDAG->computeKnownBits(N->getOperand(0)); in Select()
5668 selectFrameIndex(N, N->getOperand(0).getNode(), (int64_t)Imm); in Select()
5678 // OR with a 32-bit immediate can be handled by ori + oris in Select()
5681 bool IsPPC64 = Subtarget->isPPC64(); in Select()
5682 if (IsPPC64 && isInt64Immediate(N->getOperand(1), Imm64) && in Select()
5688 SDNode *Lo = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, in Select()
5689 N->getOperand(0), in Select()
5692 CurDAG->SelectNodeTo(N, PPC::ORIS8, MVT::i64, Ops1); in Select()
5701 // XOR with a 32-bit immediate can be handled by xori + xoris in Select()
5704 bool IsPPC64 = Subtarget->isPPC64(); in Select()
5705 if (IsPPC64 && isInt64Immediate(N->getOperand(1), Imm64) && in Select()
5711 SDNode *Lo = CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, in Select()
5712 N->getOperand(0), in Select()
5715 CurDAG->SelectNodeTo(N, PPC::XORIS8, MVT::i64, Ops1); in Select()
5724 if (N->getOperand(0)->getOpcode() == ISD::FrameIndex && in Select()
5725 isIntS16Immediate(N->getOperand(1), Imm)) { in Select()
5726 selectFrameIndex(N, N->getOperand(0).getNode(), (int64_t)Imm); in Select()
5734 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, Imm) && in Select()
5736 SDValue Ops[] = { N->getOperand(0).getOperand(0), in Select()
5739 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); in Select()
5748 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, Imm) && in Select()
5750 SDValue Ops[] = { N->getOperand(0).getOperand(0), in Select()
5753 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); in Select()
5761 SDValue Op1 = N->getOperand(1); in Select()
5767 int64_t Imm = Op1->getAsZExtVal(); in Select()
5773 // (mul X, c1 << c2) -> (rldicr (mulli X, c1) c2). We do this in ISEL due to in Select()
5774 // DAGCombiner prefers (shl (mul X, c1), c2) -> (mul X, c1 << c2). in Select()
5781 SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i64); in Select()
5782 SDNode *MulNode = CurDAG->getMachineNode(PPC::MULLI8, dl, MVT::i64, in Select()
5783 N->getOperand(0), SDImm); in Select()
5786 getI32Imm(63 - Shift, dl)}; in Select()
5787 CurDAG->SelectNodeTo(N, PPC::RLDICR, MVT::i64, Ops); in Select()
5790 SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i32); in Select()
5791 SDNode *MulNode = CurDAG->getMachineNode(PPC::MULLI, dl, MVT::i32, in Select()
5792 N->getOperand(0), SDImm); in Select()
5795 getI32Imm(0, dl), getI32Imm(31 - Shift, dl)}; in Select()
5796 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); in Select()
5807 EVT InVT = N->getOperand(0).getValueType(); in Select()
5812 SDValue AndI(CurDAG->getMachineNode(Opcode, dl, InVT, MVT::Glue, in Select()
5813 N->getOperand(0), in Select()
5814 CurDAG->getTargetConstant(1, dl, InVT)), in Select()
5816 SDValue CR0Reg = CurDAG->getRegister(PPC::CR0, MVT::i32); in Select()
5817 SDValue SRIdxVal = CurDAG->getTargetConstant( in Select()
5818 N->getOpcode() == PPCISD::ANDI_rec_1_EQ_BIT ? PPC::sub_eq : PPC::sub_gt, in Select()
5821 CurDAG->SelectNodeTo(N, TargetOpcode::EXTRACT_SUBREG, MVT::i1, CR0Reg, in Select()
5826 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(4))->get(); in Select()
5828 CurDAG->getTargetLoweringInfo().getPointerTy(CurDAG->getDataLayout()); in Select()
5832 if (Subtarget->useCRBits() && N->getOperand(0).getValueType() == MVT::i1) in Select()
5835 if (Subtarget->isISA3_0() && Subtarget->isPPC64()) { in Select()
5839 SDValue LHS = N->getOperand(0); in Select()
5840 SDValue RHS = N->getOperand(1); in Select()
5852 CurDAG->SelectNodeTo( in Select()
5853 N, N->getSimpleValueType(0) == MVT::i64 ? PPC::SETB8 : PPC::SETB, in Select()
5854 N->getValueType(0), GenCC); in Select()
5861 if (!isPPC64 && isNullConstant(N->getOperand(1)) && in Select()
5862 isOneConstant(N->getOperand(2)) && isNullConstant(N->getOperand(3)) && in Select()
5865 N->getValueType(0) == MVT::i32) { in Select()
5867 CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, in Select()
5868 N->getOperand(0), getI32Imm(~0U, dl)); in Select()
5869 CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, SDValue(Tmp, 0), in Select()
5870 N->getOperand(0), SDValue(Tmp, 1)); in Select()
5874 SDValue CCReg = SelectCC(N->getOperand(0), N->getOperand(1), CC, dl); in Select()
5876 if (N->getValueType(0) == MVT::i1) { in Select()
5890 SDValue CCBit = CurDAG->getTargetExtractSubreg(SRI, dl, MVT::i1, CCReg); in Select()
5892 SDValue NotCCBit(CurDAG->getMachineNode(PPC::CRNOR, dl, MVT::i1, in Select()
5897 SDValue CAndT(CurDAG->getMachineNode(PPC::CRAND, dl, MVT::i1, in Select()
5898 C, N->getOperand(2)), 0); in Select()
5899 SDValue NotCAndF(CurDAG->getMachineNode(PPC::CRAND, dl, MVT::i1, in Select()
5900 NotC, N->getOperand(3)), 0); in Select()
5902 CurDAG->SelectNodeTo(N, PPC::CROR, MVT::i1, CAndT, NotCAndF); in Select()
5907 getPredicateForSetCC(CC, N->getOperand(0).getValueType(), Subtarget); in Select()
5910 if (N->getValueType(0) == MVT::i32) in Select()
5912 else if (N->getValueType(0) == MVT::i64) in Select()
5914 else if (N->getValueType(0) == MVT::f32) { in Select()
5915 if (Subtarget->hasP8Vector()) in Select()
5917 else if (Subtarget->hasSPE()) in Select()
5921 } else if (N->getValueType(0) == MVT::f64) { in Select()
5922 if (Subtarget->hasVSX()) in Select()
5924 else if (Subtarget->hasSPE()) in Select()
5928 } else if (N->getValueType(0) == MVT::f128) in Select()
5930 else if (Subtarget->hasSPE()) in Select()
5932 else if (N->getValueType(0) == MVT::v2f64 || in Select()
5933 N->getValueType(0) == MVT::v2i64) in Select()
5938 SDValue Ops[] = { CCReg, N->getOperand(2), N->getOperand(3), in Select()
5940 CurDAG->SelectNodeTo(N, SelectCCOp, N->getValueType(0), Ops); in Select()
5944 if (Subtarget->hasVSX() && (N->getValueType(0) == MVT::v2f64 || in Select()
5945 N->getValueType(0) == MVT::v2i64)) { in Select()
5948 SDValue Op1 = N->getOperand(SVN->getMaskElt(0) < 2 ? 0 : 1), in Select()
5949 Op2 = N->getOperand(SVN->getMaskElt(1) < 2 ? 0 : 1); in Select()
5953 if (SVN->getMaskElt(i) <= 0 || SVN->getMaskElt(i) == 2) in Select()
5964 if (LD->isUnindexed() && LD->hasOneUse() && Op1.hasOneUse() && in Select()
5965 (LD->getMemoryVT() == MVT::f64 || in Select()
5966 LD->getMemoryVT() == MVT::i64) && in Select()
5967 SelectAddrIdxOnly(LD->getBasePtr(), Base, Offset)) { in Select()
5968 SDValue Chain = LD->getChain(); in Select()
5970 MachineMemOperand *MemOp = LD->getMemOperand(); in Select()
5971 SDNode *NewN = CurDAG->SelectNodeTo(N, PPC::LXVDSX, in Select()
5972 N->getValueType(0), Ops); in Select()
5973 CurDAG->setNodeMemRefs(cast<MachineSDNode>(NewN), {MemOp}); in Select()
5980 if (Subtarget->isLittleEndian()) { in Select()
5983 DM[0] = 1 - DM[1]; in Select()
5984 DM[1] = 1 - tmp; in Select()
5987 SDValue DMV = CurDAG->getTargetConstant(DM[1] | (DM[0] << 1), dl, in Select()
5990 CurDAG->SelectNodeTo(N, PPC::XXPERMDI, N->getValueType(0), Ops); in Select()
5997 bool IsPPC64 = Subtarget->isPPC64(); in Select()
5998 SDValue Ops[] = { N->getOperand(1), N->getOperand(0) }; in Select()
5999 CurDAG->SelectNodeTo(N, N->getOpcode() == PPCISD::BDNZ in Select()
6012 unsigned PCC = N->getConstantOperandVal(1); in Select()
6014 PCC |= getBranchHint(PCC, *FuncInfo, N->getOperand(3)); in Select()
6017 SDValue Ops[] = { Pred, N->getOperand(2), N->getOperand(3), in Select()
6018 N->getOperand(0), N->getOperand(4) }; in Select()
6019 CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops); in Select()
6027 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(1))->get(); in Select()
6029 getPredicateForSetCC(CC, N->getOperand(2).getValueType(), Subtarget); in Select()
6031 if (N->getOperand(2).getValueType() == MVT::i1) { in Select()
6035 default: llvm_unreachable("Unexpected Boolean-operand predicate"); in Select()
6045 // unsigned one if the condition code includes less-than or greater-than. in Select()
6047 // positive unsigned i1 number. The CR-logical operations used for such in Select()
6048 // comparisons are non-commutative so for signed comparisons vs. unsigned in Select()
6053 SDValue BitComp(CurDAG->getMachineNode(Opc, dl, MVT::i1, in Select()
6054 N->getOperand(Swap ? 3 : 2), in Select()
6055 N->getOperand(Swap ? 2 : 3)), 0); in Select()
6056 CurDAG->SelectNodeTo(N, PPC::BC, MVT::Other, BitComp, N->getOperand(4), in Select()
6057 N->getOperand(0)); in Select()
6062 PCC |= getBranchHint(PCC, *FuncInfo, N->getOperand(4)); in Select()
6064 SDValue CondCode = SelectCC(N->getOperand(2), N->getOperand(3), CC, dl); in Select()
6066 N->getOperand(4), N->getOperand(0) }; in Select()
6067 CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops); in Select()
6072 SDValue Chain = N->getOperand(0); in Select()
6073 SDValue Target = N->getOperand(1); in Select()
6076 Chain = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, Target, in Select()
6078 CurDAG->SelectNodeTo(N, Reg, MVT::Other, Chain); in Select()
6082 const bool isPPC64 = Subtarget->isPPC64(); in Select()
6083 const bool isELFABI = Subtarget->isSVR4ABI(); in Select()
6084 const bool isAIXABI = Subtarget->isAIXABI(); in Select()
6095 // For 64-bit ELF small code model, we allow SelectCodeCommon to handle in Select()
6097 // small code model, we need to check for a toc-data attribute. in Select()
6103 SDValue GA = TocEntry->getOperand(0); in Select()
6104 SDValue TocBase = TocEntry->getOperand(1); in Select()
6107 // toc-data access doesn't involve in loading from got, no need to in Select()
6109 MN = CurDAG->getMachineNode(OpCode, dl, OperandTy, TocBase, GA); in Select()
6111 MN = CurDAG->getMachineNode(OpCode, dl, OperandTy, GA, TocBase); in Select()
6117 // Handle 32-bit small code model. in Select()
6123 "32-bit ELF can only have TOC entries in position independent" in Select()
6125 // 32-bit ELF always uses a small code model toc access. in Select()
6132 if (hasTocDataAttr(N->getOperand(0))) { in Select()
6144 if (hasTocDataAttr(N->getOperand(0))) { in Select()
6155 assert((isPPC64 || (isAIXABI && !isPPC64)) && "We are dealing with 64-bit" in Select()
6156 " ELF/AIX or 32-bit AIX in the following."); in Select()
6158 // Transforms the ISD::TOC_ENTRY node for 32-bit AIX large code model mode, in Select()
6159 // 64-bit medium (ELF-only), or 64-bit large (ELF and AIX) code model code in Select()
6163 // [32-bit AIX] in Select()
6165 // [64-bit ELF/AIX] in Select()
6170 // And finally for AIX with toc-data we generate: in Select()
6171 // [32-bit AIX] in Select()
6173 // [64-bit AIX] in Select()
6176 SDValue GA = N->getOperand(0); in Select()
6177 SDValue TOCbase = N->getOperand(1); in Select()
6180 SDNode *Tmp = CurDAG->getMachineNode( in Select()
6183 // On AIX, if the symbol has the toc-data attribute it will be defined in Select()
6187 N, CurDAG->getMachineNode(isPPC64 ? PPC::ADDItocL8 : PPC::ADDItocL, in Select()
6192 if (PPCLowering->isAccessedAsGotIndirect(GA)) { in Select()
6193 // If it is accessed as got-indirect, we need an extra LWZ/LD to load in Select()
6195 SDNode *MN = CurDAG->getMachineNode( in Select()
6203 assert(isPPC64 && "TOC_ENTRY already handled for 32-bit."); in Select()
6204 // Build the address relative to the TOC-pointer. in Select()
6205 ReplaceNode(N, CurDAG->getMachineNode(PPC::ADDItocL8, dl, MVT::i64, in Select()
6210 // Generate a PIC-safe GOT reference. in Select()
6211 assert(Subtarget->is32BitELFABI() && in Select()
6212 "PPCISD::PPC32_PICGOT is only supported for 32-bit SVR4"); in Select()
6213 CurDAG->SelectNodeTo(N, PPC::PPC32PICGOT, in Select()
6214 PPCLowering->getPointerTy(CurDAG->getDataLayout()), in Select()
6221 assert(isa<ConstantSDNode>(N->getOperand(0)) && in Select()
6222 isa<ConstantSDNode>(N->getOperand(1)) && in Select()
6225 int Elt = N->getConstantOperandVal(0); in Select()
6226 int EltSize = N->getConstantOperandVal(1); in Select()
6249 // Elt is even, in the range [-32,-18] + [16,30]. in Select()
6256 SDNode *Tmp = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); in Select()
6258 ReplaceNode(N, CurDAG->getMachineNode(Opc2, dl, VT, TmpVal, TmpVal)); in Select()
6264 // Into: tmp1 = VSPLTIS[BHW] elt-16 in Select()
6265 // tmp2 = VSPLTIS[BHW] -16 in Select()
6267 SDValue EltVal = getI32Imm(Elt - 16, dl); in Select()
6268 SDNode *Tmp1 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); in Select()
6269 EltVal = getI32Imm(-16, dl); in Select()
6270 SDNode *Tmp2 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); in Select()
6271 ReplaceNode(N, CurDAG->getMachineNode(Opc3, dl, VT, SDValue(Tmp1, 0), in Select()
6275 // Elt is odd and negative, in the range [-31,-17]. in Select()
6279 // tmp2 = VSPLTIS[BHW] -16 in Select()
6282 SDNode *Tmp1 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); in Select()
6283 EltVal = getI32Imm(-16, dl); in Select()
6284 SDNode *Tmp2 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); in Select()
6285 ReplaceNode(N, CurDAG->getMachineNode(Opc2, dl, VT, SDValue(Tmp1, 0), in Select()
6294 if (!Subtarget->hasAltivec() || Subtarget->hasDirectMove()) in Select()
6297 EVT Type = N->getValueType(0); in Select()
6306 if (cast<MemIntrinsicSDNode>(N)->getAlign() >= Align(16) && in Select()
6311 CurDAG->getRegister(Subtarget->isPPC64() ? PPC::ZERO8 : PPC::ZERO, in Select()
6312 Subtarget->isPPC64() ? MVT::i64 : MVT::i32); in Select()
6313 unsigned LIOpcode = Subtarget->isPPC64() ? PPC::LI8 : PPC::LI; in Select()
6330 Subtarget->isLittleEndian() ? ((Type == MVT::v16i8) ? 15 : 7) : 0; in Select()
6332 SDNode *Mask = CurDAG->getMachineNode( in Select()
6333 Subtarget->isLittleEndian() ? PPC::LVSR : PPC::LVSL, dl, Type, ZeroReg, in Select()
6334 N->getOperand(1)); in Select()
6337 CurDAG->getMachineNode(PPC::LVX, dl, MVT::v16i8, MVT::Other, in Select()
6338 {ZeroReg, N->getOperand(1), N->getOperand(0)}); in Select()
6342 LoadHigh = CurDAG->getMachineNode( in Select()
6344 {SDValue(CurDAG->getMachineNode( in Select()
6346 CurDAG->getTargetConstant(1, dl, MVT::i8)), in Select()
6348 N->getOperand(1), SDValue(LoadLow, 1)}); in Select()
6351 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 1), SDValue(LoadHigh, 1)); in Select()
6355 CurDAG->getMachineNode(PPC::VPERM, dl, Type, SDValue(LoadLow, 0), in Select()
6357 CurDAG->SelectNodeTo(N, SplatOp, Type, in Select()
6358 CurDAG->getTargetConstant(SplatElemIndex, dl, MVT::i8), in Select()
6376 assert(N->getOpcode() == ISD::OR && in combineToCMPB()
6380 if (!Subtarget->hasCMPB()) in combineToCMPB()
6383 if (N->getValueType(0) != MVT::i32 && in combineToCMPB()
6384 N->getValueType(0) != MVT::i64) in combineToCMPB()
6387 EVT VT = N->getValueType(0); in combineToCMPB()
6398 ISD::CondCode CC = cast<CondCodeSDNode>(O.getOperand(4))->get(); in combineToCMPB()
6430 if (b != Bits/8-1) in combineToCMPB()
6432 if (Op0.getConstantOperandVal(1) != Bits-8) in combineToCMPB()
6441 // post-legalization uses SETULT in the SELECT_CC for the in combineToCMPB()
6442 // higher-order byte, depending on the fact that the in combineToCMPB()
6443 // even-higher-order bytes are known to all be zero, for example: in combineToCMPB()
6445 // (so when the second byte is the same, because all higher-order in combineToCMPB()
6458 if (!CurDAG->MaskedValueIsZero( in combineToCMPB()
6459 Op0, APInt::getHighBitsSet(Bits, Bits - (b + 1) * 8))) in combineToCMPB()
6493 if (b != Bits/8-1) in combineToCMPB()
6495 if (Op.getConstantOperandVal(1) != Bits-8) in combineToCMPB()
6516 for (const SDValue &O : V.getNode()->ops()) { in combineToCMPB()
6553 // Because we'll be zero-extending the output anyway if don't have a specific in combineToCMPB()
6556 LHS = CurDAG->getAnyExtOrTrunc(LHS, dl, VT); in combineToCMPB()
6557 RHS = CurDAG->getAnyExtOrTrunc(RHS, dl, VT); in combineToCMPB()
6560 Res = CurDAG->getNode(PPCISD::CMPB, dl, VT, LHS, RHS); in combineToCMPB()
6562 bool NonTrivialMask = ((int64_t) Mask) != INT64_C(-1); in combineToCMPB()
6565 Res = CurDAG->getNode(ISD::AND, dl, VT, Res, in combineToCMPB()
6566 CurDAG->getConstant(Mask, dl, VT)); in combineToCMPB()
6573 // useful because the (Alt ^ Mask) can be pre-computed. in combineToCMPB()
6574 Res = CurDAG->getNode(ISD::AND, dl, VT, Res, in combineToCMPB()
6575 CurDAG->getConstant(Mask ^ Alt, dl, VT)); in combineToCMPB()
6576 Res = CurDAG->getNode(ISD::XOR, dl, VT, Res, in combineToCMPB()
6577 CurDAG->getConstant(Alt, dl, VT)); in combineToCMPB()
6591 if (!Subtarget->useCRBits()) in foldBoolExts()
6594 if (N->getOpcode() != ISD::ZERO_EXTEND && in foldBoolExts()
6595 N->getOpcode() != ISD::SIGN_EXTEND && in foldBoolExts()
6596 N->getOpcode() != ISD::ANY_EXTEND) in foldBoolExts()
6599 if (N->getOperand(0).getValueType() != MVT::i1) in foldBoolExts()
6602 if (!N->hasOneUse()) in foldBoolExts()
6606 EVT VT = N->getValueType(0); in foldBoolExts()
6607 SDValue Cond = N->getOperand(0); in foldBoolExts()
6609 CurDAG->getConstant(N->getOpcode() == ISD::SIGN_EXTEND ? -1 : 1, dl, VT); in foldBoolExts()
6610 SDValue ConstFalse = CurDAG->getConstant(0, dl, VT); in foldBoolExts()
6613 SDNode *User = *N->use_begin(); in foldBoolExts()
6614 if (User->getNumOperands() != 2) in foldBoolExts()
6618 SDValue UserO0 = User->getOperand(0), UserO1 = User->getOperand(1); in foldBoolExts()
6622 return CurDAG->FoldConstantArithmetic(User->getOpcode(), dl, in foldBoolExts()
6623 User->getValueType(0), {O0, O1}); in foldBoolExts()
6636 // represent them as signed 16-bit integers. in foldBoolExts()
6637 uint64_t True = TrueRes->getAsZExtVal(), False = FalseRes->getAsZExtVal(); in foldBoolExts()
6643 Res = CurDAG->getSelect(dl, User->getValueType(0), Cond, TrueRes, FalseRes); in foldBoolExts()
6647 } while (N->hasOneUse()); in foldBoolExts()
6651 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); in PreprocessISelDAG()
6654 while (Position != CurDAG->allnodes_begin()) { in PreprocessISelDAG()
6655 SDNode *N = &*--Position; in PreprocessISelDAG()
6656 if (N->use_empty()) in PreprocessISelDAG()
6660 switch (N->getOpcode()) { in PreprocessISelDAG()
6672 LLVM_DEBUG(N->dump(CurDAG)); in PreprocessISelDAG()
6674 LLVM_DEBUG(Res.getNode()->dump(CurDAG)); in PreprocessISelDAG()
6677 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res); in PreprocessISelDAG()
6683 CurDAG->RemoveDeadNodes(); in PreprocessISelDAG()
6686 /// PostprocessISelDAG - Perform some late peephole optimizations
6689 // Skip peepholes at -O0. in PostprocessISelDAG()
6704 for (const SDNode *User : N->uses()) { in AllUsersSelectZero()
6705 if (!User->isMachineOpcode()) in AllUsersSelectZero()
6707 if (User->getMachineOpcode() != PPC::SELECT_I4 && in AllUsersSelectZero()
6708 User->getMachineOpcode() != PPC::SELECT_I8) in AllUsersSelectZero()
6711 SDNode *Op1 = User->getOperand(1).getNode(); in AllUsersSelectZero()
6712 SDNode *Op2 = User->getOperand(2).getNode(); in AllUsersSelectZero()
6718 if (!Op2->isMachineOpcode()) in AllUsersSelectZero()
6721 if (Op2->getMachineOpcode() != PPC::LI && in AllUsersSelectZero()
6722 Op2->getMachineOpcode() != PPC::LI8) in AllUsersSelectZero()
6725 if (!isNullConstant(Op2->getOperand(0))) in AllUsersSelectZero()
6734 for (SDNode *User : N->uses()) { in SwapAllSelectUsers()
6735 assert((User->getMachineOpcode() == PPC::SELECT_I4 || in SwapAllSelectUsers()
6736 User->getMachineOpcode() == PPC::SELECT_I8) && in SwapAllSelectUsers()
6743 CurDAG->getMachineNode(User->getMachineOpcode(), SDLoc(User), in SwapAllSelectUsers()
6744 User->getValueType(0), User->getOperand(0), in SwapAllSelectUsers()
6745 User->getOperand(2), in SwapAllSelectUsers()
6746 User->getOperand(1)); in SwapAllSelectUsers()
6749 LLVM_DEBUG(User->dump(CurDAG)); in SwapAllSelectUsers()
6751 LLVM_DEBUG(ResNode->dump(CurDAG)); in SwapAllSelectUsers()
6762 for (SDNode &Node : CurDAG->allnodes()) { in PeepholeCROps()
6764 if (!MachineNode || MachineNode->use_empty()) in PeepholeCROps()
6773 unsigned Opcode = MachineNode->getMachineOpcode(); in PeepholeCROps()
6784 SDValue Op = MachineNode->getOperand(1); in PeepholeCROps()
6809 SDValue Op = MachineNode->getOperand(0); in PeepholeCROps()
6828 if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) in PeepholeCROps()
6830 ResNode = MachineNode->getOperand(0).getNode(); in PeepholeCROps()
6833 ResNode = MachineNode->getOperand(1).getNode(); in PeepholeCROps()
6836 ResNode = MachineNode->getOperand(0).getNode(); in PeepholeCROps()
6839 ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), in PeepholeCROps()
6843 ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), in PeepholeCROps()
6844 MVT::i1, MachineNode->getOperand(1), in PeepholeCROps()
6845 MachineNode->getOperand(0). in PeepholeCROps()
6849 ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), in PeepholeCROps()
6850 MVT::i1, MachineNode->getOperand(0), in PeepholeCROps()
6851 MachineNode->getOperand(1). in PeepholeCROps()
6854 ResNode = CurDAG->getMachineNode(PPC::CRNAND, SDLoc(MachineNode), in PeepholeCROps()
6855 MVT::i1, MachineNode->getOperand(0), in PeepholeCROps()
6856 MachineNode->getOperand(1)); in PeepholeCROps()
6861 if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) in PeepholeCROps()
6862 // nand(x, x) -> nor(x, x) in PeepholeCROps()
6863 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), in PeepholeCROps()
6864 MVT::i1, MachineNode->getOperand(0), in PeepholeCROps()
6865 MachineNode->getOperand(0)); in PeepholeCROps()
6867 // nand(1, y) -> nor(y, y) in PeepholeCROps()
6868 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), in PeepholeCROps()
6869 MVT::i1, MachineNode->getOperand(1), in PeepholeCROps()
6870 MachineNode->getOperand(1)); in PeepholeCROps()
6872 // nand(x, 1) -> nor(x, x) in PeepholeCROps()
6873 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), in PeepholeCROps()
6874 MVT::i1, MachineNode->getOperand(0), in PeepholeCROps()
6875 MachineNode->getOperand(0)); in PeepholeCROps()
6878 ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), in PeepholeCROps()
6882 ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), in PeepholeCROps()
6883 MVT::i1, MachineNode->getOperand(0). in PeepholeCROps()
6885 MachineNode->getOperand(1)); in PeepholeCROps()
6888 ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), in PeepholeCROps()
6889 MVT::i1, MachineNode->getOperand(1). in PeepholeCROps()
6891 MachineNode->getOperand(0)); in PeepholeCROps()
6893 ResNode = CurDAG->getMachineNode(PPC::CRAND, SDLoc(MachineNode), in PeepholeCROps()
6894 MVT::i1, MachineNode->getOperand(0), in PeepholeCROps()
6895 MachineNode->getOperand(1)); in PeepholeCROps()
6900 if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) in PeepholeCROps()
6902 ResNode = MachineNode->getOperand(0).getNode(); in PeepholeCROps()
6905 ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), in PeepholeCROps()
6909 ResNode = MachineNode->getOperand(1).getNode(); in PeepholeCROps()
6912 ResNode = MachineNode->getOperand(0).getNode(); in PeepholeCROps()
6915 ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), in PeepholeCROps()
6916 MVT::i1, MachineNode->getOperand(1), in PeepholeCROps()
6917 MachineNode->getOperand(0). in PeepholeCROps()
6921 ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), in PeepholeCROps()
6922 MVT::i1, MachineNode->getOperand(0), in PeepholeCROps()
6923 MachineNode->getOperand(1). in PeepholeCROps()
6926 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), in PeepholeCROps()
6927 MVT::i1, MachineNode->getOperand(0), in PeepholeCROps()
6928 MachineNode->getOperand(1)); in PeepholeCROps()
6933 if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) in PeepholeCROps()
6935 ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), in PeepholeCROps()
6938 // xor(1, y) -> nor(y, y) in PeepholeCROps()
6939 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), in PeepholeCROps()
6940 MVT::i1, MachineNode->getOperand(1), in PeepholeCROps()
6941 MachineNode->getOperand(1)); in PeepholeCROps()
6943 // xor(x, 1) -> nor(x, x) in PeepholeCROps()
6944 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), in PeepholeCROps()
6945 MVT::i1, MachineNode->getOperand(0), in PeepholeCROps()
6946 MachineNode->getOperand(0)); in PeepholeCROps()
6949 ResNode = MachineNode->getOperand(1).getNode(); in PeepholeCROps()
6952 ResNode = MachineNode->getOperand(0).getNode(); in PeepholeCROps()
6955 ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode), in PeepholeCROps()
6956 MVT::i1, MachineNode->getOperand(0). in PeepholeCROps()
6958 MachineNode->getOperand(1)); in PeepholeCROps()
6961 ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode), in PeepholeCROps()
6962 MVT::i1, MachineNode->getOperand(0), in PeepholeCROps()
6963 MachineNode->getOperand(1). in PeepholeCROps()
6966 ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode), in PeepholeCROps()
6967 MVT::i1, MachineNode->getOperand(0), in PeepholeCROps()
6968 MachineNode->getOperand(1)); in PeepholeCROps()
6974 // nor(1, y) -> 0 in PeepholeCROps()
6975 ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), in PeepholeCROps()
6978 // nor(0, y) = ~y -> nor(y, y) in PeepholeCROps()
6979 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), in PeepholeCROps()
6980 MVT::i1, MachineNode->getOperand(1), in PeepholeCROps()
6981 MachineNode->getOperand(1)); in PeepholeCROps()
6984 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), in PeepholeCROps()
6985 MVT::i1, MachineNode->getOperand(0), in PeepholeCROps()
6986 MachineNode->getOperand(0)); in PeepholeCROps()
6989 ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), in PeepholeCROps()
6990 MVT::i1, MachineNode->getOperand(0). in PeepholeCROps()
6992 MachineNode->getOperand(1)); in PeepholeCROps()
6995 ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), in PeepholeCROps()
6996 MVT::i1, MachineNode->getOperand(1). in PeepholeCROps()
6998 MachineNode->getOperand(0)); in PeepholeCROps()
7000 ResNode = CurDAG->getMachineNode(PPC::CROR, SDLoc(MachineNode), in PeepholeCROps()
7001 MVT::i1, MachineNode->getOperand(0), in PeepholeCROps()
7002 MachineNode->getOperand(1)); in PeepholeCROps()
7007 if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) in PeepholeCROps()
7009 ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), in PeepholeCROps()
7013 ResNode = MachineNode->getOperand(1).getNode(); in PeepholeCROps()
7016 ResNode = MachineNode->getOperand(0).getNode(); in PeepholeCROps()
7018 // eqv(0, y) = ~y -> nor(y, y) in PeepholeCROps()
7019 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), in PeepholeCROps()
7020 MVT::i1, MachineNode->getOperand(1), in PeepholeCROps()
7021 MachineNode->getOperand(1)); in PeepholeCROps()
7024 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), in PeepholeCROps()
7025 MVT::i1, MachineNode->getOperand(0), in PeepholeCROps()
7026 MachineNode->getOperand(0)); in PeepholeCROps()
7029 ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode), in PeepholeCROps()
7030 MVT::i1, MachineNode->getOperand(0). in PeepholeCROps()
7032 MachineNode->getOperand(1)); in PeepholeCROps()
7035 ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode), in PeepholeCROps()
7036 MVT::i1, MachineNode->getOperand(0), in PeepholeCROps()
7037 MachineNode->getOperand(1). in PeepholeCROps()
7040 ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode), in PeepholeCROps()
7041 MVT::i1, MachineNode->getOperand(0), in PeepholeCROps()
7042 MachineNode->getOperand(1)); in PeepholeCROps()
7047 if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) in PeepholeCROps()
7049 ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), in PeepholeCROps()
7053 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), in PeepholeCROps()
7054 MVT::i1, MachineNode->getOperand(1), in PeepholeCROps()
7055 MachineNode->getOperand(1)); in PeepholeCROps()
7058 ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), in PeepholeCROps()
7062 ResNode = MachineNode->getOperand(0).getNode(); in PeepholeCROps()
7065 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), in PeepholeCROps()
7066 MVT::i1, MachineNode->getOperand(0). in PeepholeCROps()
7068 MachineNode->getOperand(1)); in PeepholeCROps()
7071 ResNode = CurDAG->getMachineNode(PPC::CRAND, SDLoc(MachineNode), in PeepholeCROps()
7072 MVT::i1, MachineNode->getOperand(0), in PeepholeCROps()
7073 MachineNode->getOperand(1). in PeepholeCROps()
7076 ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), in PeepholeCROps()
7077 MVT::i1, MachineNode->getOperand(1), in PeepholeCROps()
7078 MachineNode->getOperand(0)); in PeepholeCROps()
7083 if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) in PeepholeCROps()
7085 ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), in PeepholeCROps()
7089 ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), in PeepholeCROps()
7093 ResNode = MachineNode->getOperand(0).getNode(); in PeepholeCROps()
7096 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), in PeepholeCROps()
7097 MVT::i1, MachineNode->getOperand(1), in PeepholeCROps()
7098 MachineNode->getOperand(1)); in PeepholeCROps()
7101 ResNode = CurDAG->getMachineNode(PPC::CRNAND, SDLoc(MachineNode), in PeepholeCROps()
7102 MVT::i1, MachineNode->getOperand(0). in PeepholeCROps()
7104 MachineNode->getOperand(1)); in PeepholeCROps()
7107 ResNode = CurDAG->getMachineNode(PPC::CROR, SDLoc(MachineNode), in PeepholeCROps()
7108 MVT::i1, MachineNode->getOperand(0), in PeepholeCROps()
7109 MachineNode->getOperand(1). in PeepholeCROps()
7112 ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), in PeepholeCROps()
7113 MVT::i1, MachineNode->getOperand(1), in PeepholeCROps()
7114 MachineNode->getOperand(0)); in PeepholeCROps()
7129 ResNode = MachineNode->getOperand(1).getNode(); in PeepholeCROps()
7131 ResNode = MachineNode->getOperand(2).getNode(); in PeepholeCROps()
7133 ResNode = CurDAG->getMachineNode(MachineNode->getMachineOpcode(), in PeepholeCROps()
7135 MachineNode->getValueType(0), in PeepholeCROps()
7136 MachineNode->getOperand(0). in PeepholeCROps()
7138 MachineNode->getOperand(2), in PeepholeCROps()
7139 MachineNode->getOperand(1)); in PeepholeCROps()
7144 ResNode = CurDAG->getMachineNode(Opcode == PPC::BC ? PPC::BCn : in PeepholeCROps()
7148 MachineNode->getOperand(0). in PeepholeCROps()
7150 MachineNode->getOperand(1), in PeepholeCROps()
7151 MachineNode->getOperand(2)); in PeepholeCROps()
7163 LLVM_DEBUG(MachineNode->dump(CurDAG)); in PeepholeCROps()
7165 LLVM_DEBUG(ResNode->dump(CurDAG)); in PeepholeCROps()
7173 CurDAG->RemoveDeadNodes(); in PeepholeCROps()
7177 // Gather the set of 32-bit operations that are known to have their
7178 // higher-order 32 bits zero, where ToPromote contains all such operations.
7185 // higher-order 32 bits. in PeepholePPC64ZExtGather()
7189 // higher-order bits. in PeepholePPC64ZExtGather()
7197 // SLW and SRW always clear the higher-order bits. in PeepholePPC64ZExtGather()
7215 // LHBRX and LWBRX always clear the higher-order bits. in PeepholePPC64ZExtGather()
7222 // CNT[LT]ZW always produce a 64-bit value in [0,32], and so is zero extended. in PeepholePPC64ZExtGather()
7231 // Assuming the mask does not wrap around, then the higher-order bits are in PeepholePPC64ZExtGather()
7244 // For OR, the higher-order bits are zero if that is true for both operands. in PeepholePPC64ZExtGather()
7261 // For ORI and ORIS, we need the higher-order bits of the first operand to be in PeepholePPC64ZExtGather()
7277 // The higher-order bits of AND are zero if that is true for at least one of in PeepholePPC64ZExtGather()
7299 // For ANDI and ANDIS, the higher-order bits are zero if either that is true in PeepholePPC64ZExtGather()
7323 if (!Subtarget->isPPC64()) in PeepholePPC64ZExt()
7326 // When we zero-extend from i32 to i64, we use a pattern like this: in PeepholePPC64ZExt()
7330 // There are several 32-bit shift/rotate instructions, however, that will in PeepholePPC64ZExt()
7331 // clear the higher-order bits of their output, rendering the RLDICL in PeepholePPC64ZExt()
7333 // relevant 32-bit operation to be a 64-bit operation. in PeepholePPC64ZExt()
7335 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); in PeepholePPC64ZExt()
7338 while (Position != CurDAG->allnodes_begin()) { in PeepholePPC64ZExt()
7339 SDNode *N = &*--Position; in PeepholePPC64ZExt()
7340 // Skip dead nodes and any non-machine opcodes. in PeepholePPC64ZExt()
7341 if (N->use_empty() || !N->isMachineOpcode()) in PeepholePPC64ZExt()
7344 if (N->getMachineOpcode() != PPC::RLDICL) in PeepholePPC64ZExt()
7347 if (N->getConstantOperandVal(1) != 0 || in PeepholePPC64ZExt()
7348 N->getConstantOperandVal(2) != 32) in PeepholePPC64ZExt()
7351 SDValue ISR = N->getOperand(0); in PeepholePPC64ZExt()
7367 // We now know that we're looking at a canonical i32 -> i64 zext. See if we in PeepholePPC64ZExt()
7370 SDValue Op32 = ISR->getOperand(1); in PeepholePPC64ZExt()
7374 // There are some 32-bit instructions that always clear the high-order 32 in PeepholePPC64ZExt()
7385 for (SDNode *UN : PN->uses()) { in PeepholePPC64ZExt()
7386 if (!ToPromote.count(UN) && UN != ISR.getNode()) { in PeepholePPC64ZExt()
7401 // nodes in ToPromote to 64-bit operations, where for operations in the in PeepholePPC64ZExt()
7406 switch (PN->getMachineOpcode()) { in PeepholePPC64ZExt()
7408 llvm_unreachable("Don't know the 64-bit variant of this instruction"); in PeepholePPC64ZExt()
7439 for (const SDValue &V : PN->ops()) { in PeepholePPC64ZExt()
7444 CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG, SDLoc(V), in PeepholePPC64ZExt()
7445 ISR.getNode()->getVTList(), ReplOpOps); in PeepholePPC64ZExt()
7452 // Because all to-be-promoted nodes only have users that are other in PeepholePPC64ZExt()
7457 SDVTList VTs = PN->getVTList(); in PeepholePPC64ZExt()
7465 LLVM_DEBUG(PN->dump(CurDAG)); in PeepholePPC64ZExt()
7467 CurDAG->SelectNodeTo(PN, NewOpcode, CurDAG->getVTList(NewVTs), Ops); in PeepholePPC64ZExt()
7470 LLVM_DEBUG(PN->dump(CurDAG)); in PeepholePPC64ZExt()
7479 LLVM_DEBUG(N->dump(CurDAG)); in PeepholePPC64ZExt()
7481 LLVM_DEBUG(Op32.getNode()->dump(CurDAG)); in PeepholePPC64ZExt()
7488 CurDAG->RemoveDeadNodes(); in PeepholePPC64ZExt()
7492 if (!N->isMachineOpcode()) in isVSXSwap()
7494 unsigned Opc = N->getMachineOpcode(); in isVSXSwap()
7496 // Single-operand XXPERMDI or the regular XXPERMDI/XXSLDWI where the immediate in isVSXSwap()
7499 return isa<ConstantSDNode>(N->getOperand(1)) && in isVSXSwap()
7500 N->getConstantOperandVal(1) == 2; in isVSXSwap()
7502 return N->getOperand(0) == N->getOperand(1) && in isVSXSwap()
7503 isa<ConstantSDNode>(N->getOperand(2)) && in isVSXSwap()
7504 N->getConstantOperandVal(2) == 2; in isVSXSwap()
7510 // TODO: Make this complete and replace with a table-gen bit.
7512 if (!N->isMachineOpcode()) in isLaneInsensitive()
7514 unsigned Opc = N->getMachineOpcode(); in isLaneInsensitive()
7558 // Try to simplify (xxswap (vec-op (xxswap) (xxswap))) where vec-op is
7559 // lane-insensitive.
7564 while (V->isMachineOpcode() && in reduceVSXSwap()
7565 V->getMachineOpcode() == TargetOpcode::COPY_TO_REGCLASS) { in reduceVSXSwap()
7567 if (V->use_empty() || !V->use_begin()->isOnlyUserOf(V.getNode())) in reduceVSXSwap()
7569 V = V->getOperand(0); in reduceVSXSwap()
7574 SDValue VecOp = SkipRCCopy(N->getOperand(0)); in reduceVSXSwap()
7583 // These swaps may still have chain-uses here, count on dead code elimination in reduceVSXSwap()
7585 DAG->ReplaceAllUsesOfValueWith(LHS, LHS.getOperand(0)); in reduceVSXSwap()
7586 DAG->ReplaceAllUsesOfValueWith(RHS, RHS.getOperand(0)); in reduceVSXSwap()
7587 DAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), N->getOperand(0)); in reduceVSXSwap()
7590 // Check if an SDValue has the 'aix-small-tls' global variable attribute.
7593 if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(GA->getGlobal())) in hasAIXSmallTLSAttr()
7594 if (GV->hasAttribute("aix-small-tls")) in hasAIXSmallTLSAttr()
7600 // Is an ADDI eligible for folding for non-TOC-based local-[exec|dynamic]
7604 // Check if ADDIToFold (the ADDI that we want to fold into local-exec in isEligibleToFoldADDIForFasterLocalAccesses()
7610 // Folding is only allowed for the AIX small-local-[exec|dynamic] TLS target in isEligibleToFoldADDIForFasterLocalAccesses()
7611 // attribute or when the 'aix-small-tls' global variable attribute is present. in isEligibleToFoldADDIForFasterLocalAccesses()
7613 DAG->getMachineFunction().getSubtarget<PPCSubtarget>(); in isEligibleToFoldADDIForFasterLocalAccesses()
7620 // (the local-exec TLS variable). We only perform the folding if the TLS in isEligibleToFoldADDIForFasterLocalAccesses()
7626 if (DAG->getTarget().getTLSModel(GA->getGlobal()) == TLSModel::LocalExec) { in isEligibleToFoldADDIForFasterLocalAccesses()
7632 if (!TPReg || (TPReg->getReg() != Subtarget.getThreadPointerRegister())) in isEligibleToFoldADDIForFasterLocalAccesses()
7636 // The local-[exec|dynamic] TLS variable should only have the in isEligibleToFoldADDIForFasterLocalAccesses()
7639 unsigned TargetFlags = GA->getTargetFlags(); in isEligibleToFoldADDIForFasterLocalAccesses()
7648 // For non-TOC-based local-[exec|dynamic] access where an addi is feeding into
7656 if (N->getMachineOpcode() != PPC::ADDI8) in foldADDIForFasterLocalAccesses()
7661 SDValue InitialADDI = N->getOperand(0); in foldADDIForFasterLocalAccesses()
7667 // (the local-[exec|dynamic] TLS variable), with the in foldADDIForFasterLocalAccesses()
7673 "local-[exec|dynamic] accesses!"); in foldADDIForFasterLocalAccesses()
7674 unsigned TargetFlags = GA->getTargetFlags(); in foldADDIForFasterLocalAccesses()
7679 // information during assembly printing. The offset is likely to be non-zero in foldADDIForFasterLocalAccesses()
7681 int Offset = N->getConstantOperandVal(1); in foldADDIForFasterLocalAccesses()
7682 TLSVarNode = DAG->getTargetGlobalAddress(GA->getGlobal(), SDLoc(GA), MVT::i64, in foldADDIForFasterLocalAccesses()
7685 (void)DAG->UpdateNodeOperands(N, InitialADDI.getOperand(0), TLSVarNode); in foldADDIForFasterLocalAccesses()
7686 if (InitialADDI.getNode()->use_empty()) in foldADDIForFasterLocalAccesses()
7687 DAG->RemoveDeadNode(InitialADDI.getNode()); in foldADDIForFasterLocalAccesses()
7691 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); in PeepholePPC64()
7693 while (Position != CurDAG->allnodes_begin()) { in PeepholePPC64()
7694 SDNode *N = &*--Position; in PeepholePPC64()
7695 // Skip dead nodes and any non-machine opcodes. in PeepholePPC64()
7696 if (N->use_empty() || !N->isMachineOpcode()) in PeepholePPC64()
7702 // This optimization is performed for non-TOC-based local-[exec|dynamic] in PeepholePPC64()
7707 unsigned StorageOpcode = N->getMachineOpcode(); in PeepholePPC64()
7750 // we may be able to fold an add-immediate into the memory operation. in PeepholePPC64()
7753 if (!isa<ConstantSDNode>(N->getOperand(FirstOp))) in PeepholePPC64()
7756 SDValue Base = N->getOperand(FirstOp + 1); in PeepholePPC64()
7763 // When the feeding operation is an add-immediate of some sort, in PeepholePPC64()
7791 // is used for toc-data access. in PeepholePPC64()
7792 if (Subtarget->isAIXABI()) in PeepholePPC64()
7801 // 8-byte alignment, and so we can only use offsets less than 8 (otherwise, in PeepholePPC64()
7806 const GlobalValue *GV = GA->getGlobal(); in PeepholePPC64()
7807 Align Alignment = GV->getPointerAlignment(CurDAG->getDataLayout()); in PeepholePPC64()
7808 MaxDisplacement = std::min((int)Alignment.value() - 1, MaxDisplacement); in PeepholePPC64()
7814 int Offset = N->getConstantOperandVal(FirstOp); in PeepholePPC64()
7842 const GlobalValue *GV = GA->getGlobal(); in PeepholePPC64()
7843 Align Alignment = GV->getPointerAlignment(CurDAG->getDataLayout()); in PeepholePPC64()
7852 // non-zero offset, but only if the result meets the encoding in PeepholePPC64()
7855 Offset += C->getSExtValue(); in PeepholePPC64()
7863 ImmOpnd = CurDAG->getTargetConstant(Offset, SDLoc(ImmOpnd), in PeepholePPC64()
7866 // This optimization is performed for non-TOC-based local-[exec|dynamic] in PeepholePPC64()
7869 // Add the non-zero offset information into the load or store in PeepholePPC64()
7870 // instruction to be used for non-TOC-based local-[exec|dynamic] in PeepholePPC64()
7874 "addi into local-[exec|dynamic] accesses!"); in PeepholePPC64()
7875 ImmOpnd = CurDAG->getTargetGlobalAddress(GA->getGlobal(), SDLoc(GA), in PeepholePPC64()
7877 GA->getTargetFlags()); in PeepholePPC64()
7887 LLVM_DEBUG(dbgs() << "Folding add-immediate into mem-op:\nBase: "); in PeepholePPC64()
7888 LLVM_DEBUG(Base->dump(CurDAG)); in PeepholePPC64()
7890 LLVM_DEBUG(N->dump(CurDAG)); in PeepholePPC64()
7898 const GlobalValue *GV = GA->getGlobal(); in PeepholePPC64()
7899 Align Alignment = GV->getPointerAlignment(CurDAG->getDataLayout()); in PeepholePPC64()
7906 ImmOpnd = CurDAG->getTargetGlobalAddress(GV, dl, MVT::i64, Offset, Flags); in PeepholePPC64()
7909 const Constant *C = CP->getConstVal(); in PeepholePPC64()
7910 ImmOpnd = CurDAG->getTargetConstantPool(C, MVT::i64, CP->getAlign(), in PeepholePPC64()
7916 (void)CurDAG->UpdateNodeOperands(N, N->getOperand(0), ImmOpnd, in PeepholePPC64()
7917 Base.getOperand(0), N->getOperand(3)); in PeepholePPC64()
7919 (void)CurDAG->UpdateNodeOperands(N, ImmOpnd, Base.getOperand(0), in PeepholePPC64()
7920 N->getOperand(2)); in PeepholePPC64()
7923 (void)CurDAG->UpdateNodeOperands(HBase.getNode(), HBase.getOperand(0), in PeepholePPC64()
7926 // The add-immediate may now be dead, in which case remove it. in PeepholePPC64()
7927 if (Base.getNode()->use_empty()) in PeepholePPC64()
7928 CurDAG->RemoveDeadNode(Base.getNode()); in PeepholePPC64()
7932 /// createPPCISelDag - This pass converts a legalized DAG into a
7933 /// PowerPC-specific DAG, ready for instruction scheduling.