1 //===-- PPCISelDAGToDAG.cpp - PPC --pattern matching inst selector --------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file defines a pattern matching instruction selector for PowerPC, 10 // converting from a legalized dag to a PPC dag. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "MCTargetDesc/PPCMCTargetDesc.h" 15 #include "MCTargetDesc/PPCPredicates.h" 16 #include "PPC.h" 17 #include "PPCISelLowering.h" 18 #include "PPCMachineFunctionInfo.h" 19 #include "PPCSubtarget.h" 20 #include "PPCTargetMachine.h" 21 #include "llvm/ADT/APInt.h" 22 #include "llvm/ADT/DenseMap.h" 23 #include "llvm/ADT/STLExtras.h" 24 #include "llvm/ADT/SmallPtrSet.h" 25 #include "llvm/ADT/SmallVector.h" 26 #include "llvm/ADT/Statistic.h" 27 #include "llvm/Analysis/BranchProbabilityInfo.h" 28 #include "llvm/CodeGen/FunctionLoweringInfo.h" 29 #include "llvm/CodeGen/ISDOpcodes.h" 30 #include "llvm/CodeGen/MachineBasicBlock.h" 31 #include "llvm/CodeGen/MachineFrameInfo.h" 32 #include "llvm/CodeGen/MachineFunction.h" 33 #include "llvm/CodeGen/MachineInstrBuilder.h" 34 #include "llvm/CodeGen/MachineRegisterInfo.h" 35 #include "llvm/CodeGen/SelectionDAG.h" 36 #include "llvm/CodeGen/SelectionDAGISel.h" 37 #include "llvm/CodeGen/SelectionDAGNodes.h" 38 #include "llvm/CodeGen/TargetInstrInfo.h" 39 #include "llvm/CodeGen/TargetRegisterInfo.h" 40 #include "llvm/CodeGen/ValueTypes.h" 41 #include "llvm/IR/BasicBlock.h" 42 #include "llvm/IR/DebugLoc.h" 43 #include "llvm/IR/Function.h" 44 #include "llvm/IR/GlobalValue.h" 45 #include "llvm/IR/InlineAsm.h" 46 #include "llvm/IR/InstrTypes.h" 47 #include "llvm/IR/IntrinsicsPowerPC.h" 48 #include "llvm/IR/Module.h" 49 #include "llvm/Support/Casting.h" 50 #include "llvm/Support/CodeGen.h" 51 #include "llvm/Support/CommandLine.h" 52 #include "llvm/Support/Compiler.h" 53 #include "llvm/Support/Debug.h" 54 #include "llvm/Support/ErrorHandling.h" 55 #include "llvm/Support/KnownBits.h" 56 #include "llvm/Support/MachineValueType.h" 57 #include "llvm/Support/MathExtras.h" 58 #include "llvm/Support/raw_ostream.h" 59 #include <algorithm> 60 #include <cassert> 61 #include <cstdint> 62 #include <iterator> 63 #include <limits> 64 #include <memory> 65 #include <new> 66 #include <tuple> 67 #include <utility> 68 69 using namespace llvm; 70 71 #define DEBUG_TYPE "ppc-codegen" 72 73 STATISTIC(NumSextSetcc, 74 "Number of (sext(setcc)) nodes expanded into GPR sequence."); 75 STATISTIC(NumZextSetcc, 76 "Number of (zext(setcc)) nodes expanded into GPR sequence."); 77 STATISTIC(SignExtensionsAdded, 78 "Number of sign extensions for compare inputs added."); 79 STATISTIC(ZeroExtensionsAdded, 80 "Number of zero extensions for compare inputs added."); 81 STATISTIC(NumLogicOpsOnComparison, 82 "Number of logical ops on i1 values calculated in GPR."); 83 STATISTIC(OmittedForNonExtendUses, 84 "Number of compares not eliminated as they have non-extending uses."); 85 STATISTIC(NumP9Setb, 86 "Number of compares lowered to setb."); 87 88 // FIXME: Remove this once the bug has been fixed! 89 cl::opt<bool> ANDIGlueBug("expose-ppc-andi-glue-bug", 90 cl::desc("expose the ANDI glue bug on PPC"), cl::Hidden); 91 92 static cl::opt<bool> 93 UseBitPermRewriter("ppc-use-bit-perm-rewriter", cl::init(true), 94 cl::desc("use aggressive ppc isel for bit permutations"), 95 cl::Hidden); 96 static cl::opt<bool> BPermRewriterNoMasking( 97 "ppc-bit-perm-rewriter-stress-rotates", 98 cl::desc("stress rotate selection in aggressive ppc isel for " 99 "bit permutations"), 100 cl::Hidden); 101 102 static cl::opt<bool> EnableBranchHint( 103 "ppc-use-branch-hint", cl::init(true), 104 cl::desc("Enable static hinting of branches on ppc"), 105 cl::Hidden); 106 107 static cl::opt<bool> EnableTLSOpt( 108 "ppc-tls-opt", cl::init(true), 109 cl::desc("Enable tls optimization peephole"), 110 cl::Hidden); 111 112 enum ICmpInGPRType { ICGPR_All, ICGPR_None, ICGPR_I32, ICGPR_I64, 113 ICGPR_NonExtIn, ICGPR_Zext, ICGPR_Sext, ICGPR_ZextI32, 114 ICGPR_SextI32, ICGPR_ZextI64, ICGPR_SextI64 }; 115 116 static cl::opt<ICmpInGPRType> CmpInGPR( 117 "ppc-gpr-icmps", cl::Hidden, cl::init(ICGPR_All), 118 cl::desc("Specify the types of comparisons to emit GPR-only code for."), 119 cl::values(clEnumValN(ICGPR_None, "none", "Do not modify integer comparisons."), 120 clEnumValN(ICGPR_All, "all", "All possible int comparisons in GPRs."), 121 clEnumValN(ICGPR_I32, "i32", "Only i32 comparisons in GPRs."), 122 clEnumValN(ICGPR_I64, "i64", "Only i64 comparisons in GPRs."), 123 clEnumValN(ICGPR_NonExtIn, "nonextin", 124 "Only comparisons where inputs don't need [sz]ext."), 125 clEnumValN(ICGPR_Zext, "zext", "Only comparisons with zext result."), 126 clEnumValN(ICGPR_ZextI32, "zexti32", 127 "Only i32 comparisons with zext result."), 128 clEnumValN(ICGPR_ZextI64, "zexti64", 129 "Only i64 comparisons with zext result."), 130 clEnumValN(ICGPR_Sext, "sext", "Only comparisons with sext result."), 131 clEnumValN(ICGPR_SextI32, "sexti32", 132 "Only i32 comparisons with sext result."), 133 clEnumValN(ICGPR_SextI64, "sexti64", 134 "Only i64 comparisons with sext result."))); 135 namespace { 136 137 //===--------------------------------------------------------------------===// 138 /// PPCDAGToDAGISel - PPC specific code to select PPC machine 139 /// instructions for SelectionDAG operations. 140 /// 141 class PPCDAGToDAGISel : public SelectionDAGISel { 142 const PPCTargetMachine &TM; 143 const PPCSubtarget *Subtarget = nullptr; 144 const PPCTargetLowering *PPCLowering = nullptr; 145 unsigned GlobalBaseReg = 0; 146 147 public: 148 explicit PPCDAGToDAGISel(PPCTargetMachine &tm, CodeGenOpt::Level OptLevel) 149 : SelectionDAGISel(tm, OptLevel), TM(tm) {} 150 151 bool runOnMachineFunction(MachineFunction &MF) override { 152 // Make sure we re-emit a set of the global base reg if necessary 153 GlobalBaseReg = 0; 154 Subtarget = &MF.getSubtarget<PPCSubtarget>(); 155 PPCLowering = Subtarget->getTargetLowering(); 156 if (Subtarget->hasROPProtect()) { 157 // Create a place on the stack for the ROP Protection Hash. 158 // The ROP Protection Hash will always be 8 bytes and aligned to 8 159 // bytes. 160 MachineFrameInfo &MFI = MF.getFrameInfo(); 161 PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>(); 162 const int Result = MFI.CreateStackObject(8, Align(8), false); 163 FI->setROPProtectionHashSaveIndex(Result); 164 } 165 SelectionDAGISel::runOnMachineFunction(MF); 166 167 return true; 168 } 169 170 void PreprocessISelDAG() override; 171 void PostprocessISelDAG() override; 172 173 /// getI16Imm - Return a target constant with the specified value, of type 174 /// i16. 175 inline SDValue getI16Imm(unsigned Imm, const SDLoc &dl) { 176 return CurDAG->getTargetConstant(Imm, dl, MVT::i16); 177 } 178 179 /// getI32Imm - Return a target constant with the specified value, of type 180 /// i32. 181 inline SDValue getI32Imm(unsigned Imm, const SDLoc &dl) { 182 return CurDAG->getTargetConstant(Imm, dl, MVT::i32); 183 } 184 185 /// getI64Imm - Return a target constant with the specified value, of type 186 /// i64. 187 inline SDValue getI64Imm(uint64_t Imm, const SDLoc &dl) { 188 return CurDAG->getTargetConstant(Imm, dl, MVT::i64); 189 } 190 191 /// getSmallIPtrImm - Return a target constant of pointer type. 192 inline SDValue getSmallIPtrImm(uint64_t Imm, const SDLoc &dl) { 193 return CurDAG->getTargetConstant( 194 Imm, dl, PPCLowering->getPointerTy(CurDAG->getDataLayout())); 195 } 196 197 /// isRotateAndMask - Returns true if Mask and Shift can be folded into a 198 /// rotate and mask opcode and mask operation. 199 static bool isRotateAndMask(SDNode *N, unsigned Mask, bool isShiftMask, 200 unsigned &SH, unsigned &MB, unsigned &ME); 201 202 /// getGlobalBaseReg - insert code into the entry mbb to materialize the PIC 203 /// base register. Return the virtual register that holds this value. 204 SDNode *getGlobalBaseReg(); 205 206 void selectFrameIndex(SDNode *SN, SDNode *N, uint64_t Offset = 0); 207 208 // Select - Convert the specified operand from a target-independent to a 209 // target-specific node if it hasn't already been changed. 210 void Select(SDNode *N) override; 211 212 bool tryBitfieldInsert(SDNode *N); 213 bool tryBitPermutation(SDNode *N); 214 bool tryIntCompareInGPR(SDNode *N); 215 216 // tryTLSXFormLoad - Convert an ISD::LOAD fed by a PPCISD::ADD_TLS into 217 // an X-Form load instruction with the offset being a relocation coming from 218 // the PPCISD::ADD_TLS. 219 bool tryTLSXFormLoad(LoadSDNode *N); 220 // tryTLSXFormStore - Convert an ISD::STORE fed by a PPCISD::ADD_TLS into 221 // an X-Form store instruction with the offset being a relocation coming from 222 // the PPCISD::ADD_TLS. 223 bool tryTLSXFormStore(StoreSDNode *N); 224 /// SelectCC - Select a comparison of the specified values with the 225 /// specified condition code, returning the CR# of the expression. 226 SDValue SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, 227 const SDLoc &dl, SDValue Chain = SDValue()); 228 229 /// SelectAddrImmOffs - Return true if the operand is valid for a preinc 230 /// immediate field. Note that the operand at this point is already the 231 /// result of a prior SelectAddressRegImm call. 232 bool SelectAddrImmOffs(SDValue N, SDValue &Out) const { 233 if (N.getOpcode() == ISD::TargetConstant || 234 N.getOpcode() == ISD::TargetGlobalAddress) { 235 Out = N; 236 return true; 237 } 238 239 return false; 240 } 241 242 /// SelectDSForm - Returns true if address N can be represented by the 243 /// addressing mode of DSForm instructions (a base register, plus a signed 244 /// 16-bit displacement that is a multiple of 4. 245 bool SelectDSForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) { 246 return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG, 247 Align(4)) == PPC::AM_DSForm; 248 } 249 250 /// SelectDQForm - Returns true if address N can be represented by the 251 /// addressing mode of DQForm instructions (a base register, plus a signed 252 /// 16-bit displacement that is a multiple of 16. 253 bool SelectDQForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) { 254 return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG, 255 Align(16)) == PPC::AM_DQForm; 256 } 257 258 /// SelectDForm - Returns true if address N can be represented by 259 /// the addressing mode of DForm instructions (a base register, plus a 260 /// signed 16-bit immediate. 261 bool SelectDForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) { 262 return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG, 263 None) == PPC::AM_DForm; 264 } 265 266 /// SelectPCRelForm - Returns true if address N can be represented by 267 /// PC-Relative addressing mode. 268 bool SelectPCRelForm(SDNode *Parent, SDValue N, SDValue &Disp, 269 SDValue &Base) { 270 return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG, 271 None) == PPC::AM_PCRel; 272 } 273 274 /// SelectPDForm - Returns true if address N can be represented by Prefixed 275 /// DForm addressing mode (a base register, plus a signed 34-bit immediate. 276 bool SelectPDForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) { 277 return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG, 278 None) == PPC::AM_PrefixDForm; 279 } 280 281 /// SelectXForm - Returns true if address N can be represented by the 282 /// addressing mode of XForm instructions (an indexed [r+r] operation). 283 bool SelectXForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) { 284 return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG, 285 None) == PPC::AM_XForm; 286 } 287 288 /// SelectForceXForm - Given the specified address, force it to be 289 /// represented as an indexed [r+r] operation (an XForm instruction). 290 bool SelectForceXForm(SDNode *Parent, SDValue N, SDValue &Disp, 291 SDValue &Base) { 292 return PPCLowering->SelectForceXFormMode(N, Disp, Base, *CurDAG) == 293 PPC::AM_XForm; 294 } 295 296 /// SelectAddrIdx - Given the specified address, check to see if it can be 297 /// represented as an indexed [r+r] operation. 298 /// This is for xform instructions whose associated displacement form is D. 299 /// The last parameter \p 0 means associated D form has no requirment for 16 300 /// bit signed displacement. 301 /// Returns false if it can be represented by [r+imm], which are preferred. 302 bool SelectAddrIdx(SDValue N, SDValue &Base, SDValue &Index) { 303 return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG, None); 304 } 305 306 /// SelectAddrIdx4 - Given the specified address, check to see if it can be 307 /// represented as an indexed [r+r] operation. 308 /// This is for xform instructions whose associated displacement form is DS. 309 /// The last parameter \p 4 means associated DS form 16 bit signed 310 /// displacement must be a multiple of 4. 311 /// Returns false if it can be represented by [r+imm], which are preferred. 312 bool SelectAddrIdxX4(SDValue N, SDValue &Base, SDValue &Index) { 313 return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG, 314 Align(4)); 315 } 316 317 /// SelectAddrIdx16 - Given the specified address, check to see if it can be 318 /// represented as an indexed [r+r] operation. 319 /// This is for xform instructions whose associated displacement form is DQ. 320 /// The last parameter \p 16 means associated DQ form 16 bit signed 321 /// displacement must be a multiple of 16. 322 /// Returns false if it can be represented by [r+imm], which are preferred. 323 bool SelectAddrIdxX16(SDValue N, SDValue &Base, SDValue &Index) { 324 return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG, 325 Align(16)); 326 } 327 328 /// SelectAddrIdxOnly - Given the specified address, force it to be 329 /// represented as an indexed [r+r] operation. 330 bool SelectAddrIdxOnly(SDValue N, SDValue &Base, SDValue &Index) { 331 return PPCLowering->SelectAddressRegRegOnly(N, Base, Index, *CurDAG); 332 } 333 334 /// SelectAddrImm - Returns true if the address N can be represented by 335 /// a base register plus a signed 16-bit displacement [r+imm]. 336 /// The last parameter \p 0 means D form has no requirment for 16 bit signed 337 /// displacement. 338 bool SelectAddrImm(SDValue N, SDValue &Disp, 339 SDValue &Base) { 340 return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, None); 341 } 342 343 /// SelectAddrImmX4 - Returns true if the address N can be represented by 344 /// a base register plus a signed 16-bit displacement that is a multiple of 345 /// 4 (last parameter). Suitable for use by STD and friends. 346 bool SelectAddrImmX4(SDValue N, SDValue &Disp, SDValue &Base) { 347 return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, Align(4)); 348 } 349 350 /// SelectAddrImmX16 - Returns true if the address N can be represented by 351 /// a base register plus a signed 16-bit displacement that is a multiple of 352 /// 16(last parameter). Suitable for use by STXV and friends. 353 bool SelectAddrImmX16(SDValue N, SDValue &Disp, SDValue &Base) { 354 return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, 355 Align(16)); 356 } 357 358 /// SelectAddrImmX34 - Returns true if the address N can be represented by 359 /// a base register plus a signed 34-bit displacement. Suitable for use by 360 /// PSTXVP and friends. 361 bool SelectAddrImmX34(SDValue N, SDValue &Disp, SDValue &Base) { 362 return PPCLowering->SelectAddressRegImm34(N, Disp, Base, *CurDAG); 363 } 364 365 // Select an address into a single register. 366 bool SelectAddr(SDValue N, SDValue &Base) { 367 Base = N; 368 return true; 369 } 370 371 bool SelectAddrPCRel(SDValue N, SDValue &Base) { 372 return PPCLowering->SelectAddressPCRel(N, Base); 373 } 374 375 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for 376 /// inline asm expressions. It is always correct to compute the value into 377 /// a register. The case of adding a (possibly relocatable) constant to a 378 /// register can be improved, but it is wrong to substitute Reg+Reg for 379 /// Reg in an asm, because the load or store opcode would have to change. 380 bool SelectInlineAsmMemoryOperand(const SDValue &Op, 381 unsigned ConstraintID, 382 std::vector<SDValue> &OutOps) override { 383 switch(ConstraintID) { 384 default: 385 errs() << "ConstraintID: " << ConstraintID << "\n"; 386 llvm_unreachable("Unexpected asm memory constraint"); 387 case InlineAsm::Constraint_es: 388 case InlineAsm::Constraint_m: 389 case InlineAsm::Constraint_o: 390 case InlineAsm::Constraint_Q: 391 case InlineAsm::Constraint_Z: 392 case InlineAsm::Constraint_Zy: 393 // We need to make sure that this one operand does not end up in r0 394 // (because we might end up lowering this as 0(%op)). 395 const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo(); 396 const TargetRegisterClass *TRC = TRI->getPointerRegClass(*MF, /*Kind=*/1); 397 SDLoc dl(Op); 398 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i32); 399 SDValue NewOp = 400 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, 401 dl, Op.getValueType(), 402 Op, RC), 0); 403 404 OutOps.push_back(NewOp); 405 return false; 406 } 407 return true; 408 } 409 410 StringRef getPassName() const override { 411 return "PowerPC DAG->DAG Pattern Instruction Selection"; 412 } 413 414 // Include the pieces autogenerated from the target description. 415 #include "PPCGenDAGISel.inc" 416 417 private: 418 bool trySETCC(SDNode *N); 419 bool tryFoldSWTestBRCC(SDNode *N); 420 bool tryAsSingleRLDICL(SDNode *N); 421 bool tryAsSingleRLDICR(SDNode *N); 422 bool tryAsSingleRLWINM(SDNode *N); 423 bool tryAsSingleRLWINM8(SDNode *N); 424 bool tryAsSingleRLWIMI(SDNode *N); 425 bool tryAsPairOfRLDICL(SDNode *N); 426 bool tryAsSingleRLDIMI(SDNode *N); 427 428 void PeepholePPC64(); 429 void PeepholePPC64ZExt(); 430 void PeepholeCROps(); 431 432 SDValue combineToCMPB(SDNode *N); 433 void foldBoolExts(SDValue &Res, SDNode *&N); 434 435 bool AllUsersSelectZero(SDNode *N); 436 void SwapAllSelectUsers(SDNode *N); 437 438 bool isOffsetMultipleOf(SDNode *N, unsigned Val) const; 439 void transferMemOperands(SDNode *N, SDNode *Result); 440 }; 441 442 } // end anonymous namespace 443 444 /// getGlobalBaseReg - Output the instructions required to put the 445 /// base address to use for accessing globals into a register. 446 /// 447 SDNode *PPCDAGToDAGISel::getGlobalBaseReg() { 448 if (!GlobalBaseReg) { 449 const TargetInstrInfo &TII = *Subtarget->getInstrInfo(); 450 // Insert the set of GlobalBaseReg into the first MBB of the function 451 MachineBasicBlock &FirstMBB = MF->front(); 452 MachineBasicBlock::iterator MBBI = FirstMBB.begin(); 453 const Module *M = MF->getFunction().getParent(); 454 DebugLoc dl; 455 456 if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) == MVT::i32) { 457 if (Subtarget->isTargetELF()) { 458 GlobalBaseReg = PPC::R30; 459 if (!Subtarget->isSecurePlt() && 460 M->getPICLevel() == PICLevel::SmallPIC) { 461 BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MoveGOTtoLR)); 462 BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg); 463 MF->getInfo<PPCFunctionInfo>()->setUsesPICBase(true); 464 } else { 465 BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR)); 466 BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg); 467 Register TempReg = RegInfo->createVirtualRegister(&PPC::GPRCRegClass); 468 BuildMI(FirstMBB, MBBI, dl, 469 TII.get(PPC::UpdateGBR), GlobalBaseReg) 470 .addReg(TempReg, RegState::Define).addReg(GlobalBaseReg); 471 MF->getInfo<PPCFunctionInfo>()->setUsesPICBase(true); 472 } 473 } else { 474 GlobalBaseReg = 475 RegInfo->createVirtualRegister(&PPC::GPRC_and_GPRC_NOR0RegClass); 476 BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR)); 477 BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg); 478 } 479 } else { 480 // We must ensure that this sequence is dominated by the prologue. 481 // FIXME: This is a bit of a big hammer since we don't get the benefits 482 // of shrink-wrapping whenever we emit this instruction. Considering 483 // this is used in any function where we emit a jump table, this may be 484 // a significant limitation. We should consider inserting this in the 485 // block where it is used and then commoning this sequence up if it 486 // appears in multiple places. 487 // Note: on ISA 3.0 cores, we can use lnia (addpcis) instead of 488 // MovePCtoLR8. 489 MF->getInfo<PPCFunctionInfo>()->setShrinkWrapDisabled(true); 490 GlobalBaseReg = RegInfo->createVirtualRegister(&PPC::G8RC_and_G8RC_NOX0RegClass); 491 BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR8)); 492 BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR8), GlobalBaseReg); 493 } 494 } 495 return CurDAG->getRegister(GlobalBaseReg, 496 PPCLowering->getPointerTy(CurDAG->getDataLayout())) 497 .getNode(); 498 } 499 500 // Check if a SDValue has the toc-data attribute. 501 static bool hasTocDataAttr(SDValue Val, unsigned PointerSize) { 502 GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val); 503 if (!GA) 504 return false; 505 506 const GlobalVariable *GV = dyn_cast_or_null<GlobalVariable>(GA->getGlobal()); 507 if (!GV) 508 return false; 509 510 if (!GV->hasAttribute("toc-data")) 511 return false; 512 513 // TODO: These asserts should be updated as more support for the toc data 514 // transformation is added (struct support, etc.). 515 516 assert( 517 PointerSize >= GV->getAlign().valueOrOne().value() && 518 "GlobalVariables with an alignment requirement stricter than TOC entry " 519 "size not supported by the toc data transformation."); 520 521 Type *GVType = GV->getValueType(); 522 523 assert(GVType->isSized() && "A GlobalVariable's size must be known to be " 524 "supported by the toc data transformation."); 525 526 if (GVType->isVectorTy()) 527 report_fatal_error("A GlobalVariable of Vector type is not currently " 528 "supported by the toc data transformation."); 529 530 if (GVType->isArrayTy()) 531 report_fatal_error("A GlobalVariable of Array type is not currently " 532 "supported by the toc data transformation."); 533 534 if (GVType->isStructTy()) 535 report_fatal_error("A GlobalVariable of Struct type is not currently " 536 "supported by the toc data transformation."); 537 538 assert(GVType->getPrimitiveSizeInBits() <= PointerSize * 8 && 539 "A GlobalVariable with size larger than a TOC entry is not currently " 540 "supported by the toc data transformation."); 541 542 if (GV->hasLocalLinkage() || GV->hasPrivateLinkage()) 543 report_fatal_error("A GlobalVariable with private or local linkage is not " 544 "currently supported by the toc data transformation."); 545 546 assert(!GV->hasCommonLinkage() && 547 "Tentative definitions cannot have the mapping class XMC_TD."); 548 549 return true; 550 } 551 552 /// isInt32Immediate - This method tests to see if the node is a 32-bit constant 553 /// operand. If so Imm will receive the 32-bit value. 554 static bool isInt32Immediate(SDNode *N, unsigned &Imm) { 555 if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) { 556 Imm = cast<ConstantSDNode>(N)->getZExtValue(); 557 return true; 558 } 559 return false; 560 } 561 562 /// isInt64Immediate - This method tests to see if the node is a 64-bit constant 563 /// operand. If so Imm will receive the 64-bit value. 564 static bool isInt64Immediate(SDNode *N, uint64_t &Imm) { 565 if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i64) { 566 Imm = cast<ConstantSDNode>(N)->getZExtValue(); 567 return true; 568 } 569 return false; 570 } 571 572 // isInt32Immediate - This method tests to see if a constant operand. 573 // If so Imm will receive the 32 bit value. 574 static bool isInt32Immediate(SDValue N, unsigned &Imm) { 575 return isInt32Immediate(N.getNode(), Imm); 576 } 577 578 /// isInt64Immediate - This method tests to see if the value is a 64-bit 579 /// constant operand. If so Imm will receive the 64-bit value. 580 static bool isInt64Immediate(SDValue N, uint64_t &Imm) { 581 return isInt64Immediate(N.getNode(), Imm); 582 } 583 584 static unsigned getBranchHint(unsigned PCC, 585 const FunctionLoweringInfo &FuncInfo, 586 const SDValue &DestMBB) { 587 assert(isa<BasicBlockSDNode>(DestMBB)); 588 589 if (!FuncInfo.BPI) return PPC::BR_NO_HINT; 590 591 const BasicBlock *BB = FuncInfo.MBB->getBasicBlock(); 592 const Instruction *BBTerm = BB->getTerminator(); 593 594 if (BBTerm->getNumSuccessors() != 2) return PPC::BR_NO_HINT; 595 596 const BasicBlock *TBB = BBTerm->getSuccessor(0); 597 const BasicBlock *FBB = BBTerm->getSuccessor(1); 598 599 auto TProb = FuncInfo.BPI->getEdgeProbability(BB, TBB); 600 auto FProb = FuncInfo.BPI->getEdgeProbability(BB, FBB); 601 602 // We only want to handle cases which are easy to predict at static time, e.g. 603 // C++ throw statement, that is very likely not taken, or calling never 604 // returned function, e.g. stdlib exit(). So we set Threshold to filter 605 // unwanted cases. 606 // 607 // Below is LLVM branch weight table, we only want to handle case 1, 2 608 // 609 // Case Taken:Nontaken Example 610 // 1. Unreachable 1048575:1 C++ throw, stdlib exit(), 611 // 2. Invoke-terminating 1:1048575 612 // 3. Coldblock 4:64 __builtin_expect 613 // 4. Loop Branch 124:4 For loop 614 // 5. PH/ZH/FPH 20:12 615 const uint32_t Threshold = 10000; 616 617 if (std::max(TProb, FProb) / Threshold < std::min(TProb, FProb)) 618 return PPC::BR_NO_HINT; 619 620 LLVM_DEBUG(dbgs() << "Use branch hint for '" << FuncInfo.Fn->getName() 621 << "::" << BB->getName() << "'\n" 622 << " -> " << TBB->getName() << ": " << TProb << "\n" 623 << " -> " << FBB->getName() << ": " << FProb << "\n"); 624 625 const BasicBlockSDNode *BBDN = cast<BasicBlockSDNode>(DestMBB); 626 627 // If Dest BasicBlock is False-BasicBlock (FBB), swap branch probabilities, 628 // because we want 'TProb' stands for 'branch probability' to Dest BasicBlock 629 if (BBDN->getBasicBlock()->getBasicBlock() != TBB) 630 std::swap(TProb, FProb); 631 632 return (TProb > FProb) ? PPC::BR_TAKEN_HINT : PPC::BR_NONTAKEN_HINT; 633 } 634 635 // isOpcWithIntImmediate - This method tests to see if the node is a specific 636 // opcode and that it has a immediate integer right operand. 637 // If so Imm will receive the 32 bit value. 638 static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) { 639 return N->getOpcode() == Opc 640 && isInt32Immediate(N->getOperand(1).getNode(), Imm); 641 } 642 643 void PPCDAGToDAGISel::selectFrameIndex(SDNode *SN, SDNode *N, uint64_t Offset) { 644 SDLoc dl(SN); 645 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 646 SDValue TFI = CurDAG->getTargetFrameIndex(FI, N->getValueType(0)); 647 unsigned Opc = N->getValueType(0) == MVT::i32 ? PPC::ADDI : PPC::ADDI8; 648 if (SN->hasOneUse()) 649 CurDAG->SelectNodeTo(SN, Opc, N->getValueType(0), TFI, 650 getSmallIPtrImm(Offset, dl)); 651 else 652 ReplaceNode(SN, CurDAG->getMachineNode(Opc, dl, N->getValueType(0), TFI, 653 getSmallIPtrImm(Offset, dl))); 654 } 655 656 bool PPCDAGToDAGISel::isRotateAndMask(SDNode *N, unsigned Mask, 657 bool isShiftMask, unsigned &SH, 658 unsigned &MB, unsigned &ME) { 659 // Don't even go down this path for i64, since different logic will be 660 // necessary for rldicl/rldicr/rldimi. 661 if (N->getValueType(0) != MVT::i32) 662 return false; 663 664 unsigned Shift = 32; 665 unsigned Indeterminant = ~0; // bit mask marking indeterminant results 666 unsigned Opcode = N->getOpcode(); 667 if (N->getNumOperands() != 2 || 668 !isInt32Immediate(N->getOperand(1).getNode(), Shift) || (Shift > 31)) 669 return false; 670 671 if (Opcode == ISD::SHL) { 672 // apply shift left to mask if it comes first 673 if (isShiftMask) Mask = Mask << Shift; 674 // determine which bits are made indeterminant by shift 675 Indeterminant = ~(0xFFFFFFFFu << Shift); 676 } else if (Opcode == ISD::SRL) { 677 // apply shift right to mask if it comes first 678 if (isShiftMask) Mask = Mask >> Shift; 679 // determine which bits are made indeterminant by shift 680 Indeterminant = ~(0xFFFFFFFFu >> Shift); 681 // adjust for the left rotate 682 Shift = 32 - Shift; 683 } else if (Opcode == ISD::ROTL) { 684 Indeterminant = 0; 685 } else { 686 return false; 687 } 688 689 // if the mask doesn't intersect any Indeterminant bits 690 if (Mask && !(Mask & Indeterminant)) { 691 SH = Shift & 31; 692 // make sure the mask is still a mask (wrap arounds may not be) 693 return isRunOfOnes(Mask, MB, ME); 694 } 695 return false; 696 } 697 698 bool PPCDAGToDAGISel::tryTLSXFormStore(StoreSDNode *ST) { 699 SDValue Base = ST->getBasePtr(); 700 if (Base.getOpcode() != PPCISD::ADD_TLS) 701 return false; 702 SDValue Offset = ST->getOffset(); 703 if (!Offset.isUndef()) 704 return false; 705 if (Base.getOperand(1).getOpcode() == PPCISD::TLS_LOCAL_EXEC_MAT_ADDR) 706 return false; 707 708 SDLoc dl(ST); 709 EVT MemVT = ST->getMemoryVT(); 710 EVT RegVT = ST->getValue().getValueType(); 711 712 unsigned Opcode; 713 switch (MemVT.getSimpleVT().SimpleTy) { 714 default: 715 return false; 716 case MVT::i8: { 717 Opcode = (RegVT == MVT::i32) ? PPC::STBXTLS_32 : PPC::STBXTLS; 718 break; 719 } 720 case MVT::i16: { 721 Opcode = (RegVT == MVT::i32) ? PPC::STHXTLS_32 : PPC::STHXTLS; 722 break; 723 } 724 case MVT::i32: { 725 Opcode = (RegVT == MVT::i32) ? PPC::STWXTLS_32 : PPC::STWXTLS; 726 break; 727 } 728 case MVT::i64: { 729 Opcode = PPC::STDXTLS; 730 break; 731 } 732 } 733 SDValue Chain = ST->getChain(); 734 SDVTList VTs = ST->getVTList(); 735 SDValue Ops[] = {ST->getValue(), Base.getOperand(0), Base.getOperand(1), 736 Chain}; 737 SDNode *MN = CurDAG->getMachineNode(Opcode, dl, VTs, Ops); 738 transferMemOperands(ST, MN); 739 ReplaceNode(ST, MN); 740 return true; 741 } 742 743 bool PPCDAGToDAGISel::tryTLSXFormLoad(LoadSDNode *LD) { 744 SDValue Base = LD->getBasePtr(); 745 if (Base.getOpcode() != PPCISD::ADD_TLS) 746 return false; 747 SDValue Offset = LD->getOffset(); 748 if (!Offset.isUndef()) 749 return false; 750 if (Base.getOperand(1).getOpcode() == PPCISD::TLS_LOCAL_EXEC_MAT_ADDR) 751 return false; 752 753 SDLoc dl(LD); 754 EVT MemVT = LD->getMemoryVT(); 755 EVT RegVT = LD->getValueType(0); 756 unsigned Opcode; 757 switch (MemVT.getSimpleVT().SimpleTy) { 758 default: 759 return false; 760 case MVT::i8: { 761 Opcode = (RegVT == MVT::i32) ? PPC::LBZXTLS_32 : PPC::LBZXTLS; 762 break; 763 } 764 case MVT::i16: { 765 Opcode = (RegVT == MVT::i32) ? PPC::LHZXTLS_32 : PPC::LHZXTLS; 766 break; 767 } 768 case MVT::i32: { 769 Opcode = (RegVT == MVT::i32) ? PPC::LWZXTLS_32 : PPC::LWZXTLS; 770 break; 771 } 772 case MVT::i64: { 773 Opcode = PPC::LDXTLS; 774 break; 775 } 776 } 777 SDValue Chain = LD->getChain(); 778 SDVTList VTs = LD->getVTList(); 779 SDValue Ops[] = {Base.getOperand(0), Base.getOperand(1), Chain}; 780 SDNode *MN = CurDAG->getMachineNode(Opcode, dl, VTs, Ops); 781 transferMemOperands(LD, MN); 782 ReplaceNode(LD, MN); 783 return true; 784 } 785 786 /// Turn an or of two masked values into the rotate left word immediate then 787 /// mask insert (rlwimi) instruction. 788 bool PPCDAGToDAGISel::tryBitfieldInsert(SDNode *N) { 789 SDValue Op0 = N->getOperand(0); 790 SDValue Op1 = N->getOperand(1); 791 SDLoc dl(N); 792 793 KnownBits LKnown = CurDAG->computeKnownBits(Op0); 794 KnownBits RKnown = CurDAG->computeKnownBits(Op1); 795 796 unsigned TargetMask = LKnown.Zero.getZExtValue(); 797 unsigned InsertMask = RKnown.Zero.getZExtValue(); 798 799 if ((TargetMask | InsertMask) == 0xFFFFFFFF) { 800 unsigned Op0Opc = Op0.getOpcode(); 801 unsigned Op1Opc = Op1.getOpcode(); 802 unsigned Value, SH = 0; 803 TargetMask = ~TargetMask; 804 InsertMask = ~InsertMask; 805 806 // If the LHS has a foldable shift and the RHS does not, then swap it to the 807 // RHS so that we can fold the shift into the insert. 808 if (Op0Opc == ISD::AND && Op1Opc == ISD::AND) { 809 if (Op0.getOperand(0).getOpcode() == ISD::SHL || 810 Op0.getOperand(0).getOpcode() == ISD::SRL) { 811 if (Op1.getOperand(0).getOpcode() != ISD::SHL && 812 Op1.getOperand(0).getOpcode() != ISD::SRL) { 813 std::swap(Op0, Op1); 814 std::swap(Op0Opc, Op1Opc); 815 std::swap(TargetMask, InsertMask); 816 } 817 } 818 } else if (Op0Opc == ISD::SHL || Op0Opc == ISD::SRL) { 819 if (Op1Opc == ISD::AND && Op1.getOperand(0).getOpcode() != ISD::SHL && 820 Op1.getOperand(0).getOpcode() != ISD::SRL) { 821 std::swap(Op0, Op1); 822 std::swap(Op0Opc, Op1Opc); 823 std::swap(TargetMask, InsertMask); 824 } 825 } 826 827 unsigned MB, ME; 828 if (isRunOfOnes(InsertMask, MB, ME)) { 829 if ((Op1Opc == ISD::SHL || Op1Opc == ISD::SRL) && 830 isInt32Immediate(Op1.getOperand(1), Value)) { 831 Op1 = Op1.getOperand(0); 832 SH = (Op1Opc == ISD::SHL) ? Value : 32 - Value; 833 } 834 if (Op1Opc == ISD::AND) { 835 // The AND mask might not be a constant, and we need to make sure that 836 // if we're going to fold the masking with the insert, all bits not 837 // know to be zero in the mask are known to be one. 838 KnownBits MKnown = CurDAG->computeKnownBits(Op1.getOperand(1)); 839 bool CanFoldMask = InsertMask == MKnown.One.getZExtValue(); 840 841 unsigned SHOpc = Op1.getOperand(0).getOpcode(); 842 if ((SHOpc == ISD::SHL || SHOpc == ISD::SRL) && CanFoldMask && 843 isInt32Immediate(Op1.getOperand(0).getOperand(1), Value)) { 844 // Note that Value must be in range here (less than 32) because 845 // otherwise there would not be any bits set in InsertMask. 846 Op1 = Op1.getOperand(0).getOperand(0); 847 SH = (SHOpc == ISD::SHL) ? Value : 32 - Value; 848 } 849 } 850 851 SH &= 31; 852 SDValue Ops[] = { Op0, Op1, getI32Imm(SH, dl), getI32Imm(MB, dl), 853 getI32Imm(ME, dl) }; 854 ReplaceNode(N, CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops)); 855 return true; 856 } 857 } 858 return false; 859 } 860 861 static unsigned allUsesTruncate(SelectionDAG *CurDAG, SDNode *N) { 862 unsigned MaxTruncation = 0; 863 // Cannot use range-based for loop here as we need the actual use (i.e. we 864 // need the operand number corresponding to the use). A range-based for 865 // will unbox the use and provide an SDNode*. 866 for (SDNode::use_iterator Use = N->use_begin(), UseEnd = N->use_end(); 867 Use != UseEnd; ++Use) { 868 unsigned Opc = 869 Use->isMachineOpcode() ? Use->getMachineOpcode() : Use->getOpcode(); 870 switch (Opc) { 871 default: return 0; 872 case ISD::TRUNCATE: 873 if (Use->isMachineOpcode()) 874 return 0; 875 MaxTruncation = 876 std::max(MaxTruncation, (unsigned)Use->getValueType(0).getSizeInBits()); 877 continue; 878 case ISD::STORE: { 879 if (Use->isMachineOpcode()) 880 return 0; 881 StoreSDNode *STN = cast<StoreSDNode>(*Use); 882 unsigned MemVTSize = STN->getMemoryVT().getSizeInBits(); 883 if (MemVTSize == 64 || Use.getOperandNo() != 0) 884 return 0; 885 MaxTruncation = std::max(MaxTruncation, MemVTSize); 886 continue; 887 } 888 case PPC::STW8: 889 case PPC::STWX8: 890 case PPC::STWU8: 891 case PPC::STWUX8: 892 if (Use.getOperandNo() != 0) 893 return 0; 894 MaxTruncation = std::max(MaxTruncation, 32u); 895 continue; 896 case PPC::STH8: 897 case PPC::STHX8: 898 case PPC::STHU8: 899 case PPC::STHUX8: 900 if (Use.getOperandNo() != 0) 901 return 0; 902 MaxTruncation = std::max(MaxTruncation, 16u); 903 continue; 904 case PPC::STB8: 905 case PPC::STBX8: 906 case PPC::STBU8: 907 case PPC::STBUX8: 908 if (Use.getOperandNo() != 0) 909 return 0; 910 MaxTruncation = std::max(MaxTruncation, 8u); 911 continue; 912 } 913 } 914 return MaxTruncation; 915 } 916 917 // For any 32 < Num < 64, check if the Imm contains at least Num consecutive 918 // zeros and return the number of bits by the left of these consecutive zeros. 919 static int findContiguousZerosAtLeast(uint64_t Imm, unsigned Num) { 920 unsigned HiTZ = countTrailingZeros<uint32_t>(Hi_32(Imm)); 921 unsigned LoLZ = countLeadingZeros<uint32_t>(Lo_32(Imm)); 922 if ((HiTZ + LoLZ) >= Num) 923 return (32 + HiTZ); 924 return 0; 925 } 926 927 // Direct materialization of 64-bit constants by enumerated patterns. 928 static SDNode *selectI64ImmDirect(SelectionDAG *CurDAG, const SDLoc &dl, 929 uint64_t Imm, unsigned &InstCnt) { 930 unsigned TZ = countTrailingZeros<uint64_t>(Imm); 931 unsigned LZ = countLeadingZeros<uint64_t>(Imm); 932 unsigned TO = countTrailingOnes<uint64_t>(Imm); 933 unsigned LO = countLeadingOnes<uint64_t>(Imm); 934 unsigned Hi32 = Hi_32(Imm); 935 unsigned Lo32 = Lo_32(Imm); 936 SDNode *Result = nullptr; 937 unsigned Shift = 0; 938 939 auto getI32Imm = [CurDAG, dl](unsigned Imm) { 940 return CurDAG->getTargetConstant(Imm, dl, MVT::i32); 941 }; 942 943 // Following patterns use 1 instructions to materialize the Imm. 944 InstCnt = 1; 945 // 1-1) Patterns : {zeros}{15-bit valve} 946 // {ones}{15-bit valve} 947 if (isInt<16>(Imm)) { 948 SDValue SDImm = CurDAG->getTargetConstant(Imm, dl, MVT::i64); 949 return CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, SDImm); 950 } 951 // 1-2) Patterns : {zeros}{15-bit valve}{16 zeros} 952 // {ones}{15-bit valve}{16 zeros} 953 if (TZ > 15 && (LZ > 32 || LO > 32)) 954 return CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, 955 getI32Imm((Imm >> 16) & 0xffff)); 956 957 // Following patterns use 2 instructions to materialize the Imm. 958 InstCnt = 2; 959 assert(LZ < 64 && "Unexpected leading zeros here."); 960 // Count of ones follwing the leading zeros. 961 unsigned FO = countLeadingOnes<uint64_t>(Imm << LZ); 962 // 2-1) Patterns : {zeros}{31-bit value} 963 // {ones}{31-bit value} 964 if (isInt<32>(Imm)) { 965 uint64_t ImmHi16 = (Imm >> 16) & 0xffff; 966 unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8; 967 Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16)); 968 return CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), 969 getI32Imm(Imm & 0xffff)); 970 } 971 // 2-2) Patterns : {zeros}{ones}{15-bit value}{zeros} 972 // {zeros}{15-bit value}{zeros} 973 // {zeros}{ones}{15-bit value} 974 // {ones}{15-bit value}{zeros} 975 // We can take advantage of LI's sign-extension semantics to generate leading 976 // ones, and then use RLDIC to mask off the ones in both sides after rotation. 977 if ((LZ + FO + TZ) > 48) { 978 Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, 979 getI32Imm((Imm >> TZ) & 0xffff)); 980 return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0), 981 getI32Imm(TZ), getI32Imm(LZ)); 982 } 983 // 2-3) Pattern : {zeros}{15-bit value}{ones} 984 // Shift right the Imm by (48 - LZ) bits to construct a negtive 16 bits value, 985 // therefore we can take advantage of LI's sign-extension semantics, and then 986 // mask them off after rotation. 987 // 988 // +--LZ--||-15-bit-||--TO--+ +-------------|--16-bit--+ 989 // |00000001bbbbbbbbb1111111| -> |00000000000001bbbbbbbbb1| 990 // +------------------------+ +------------------------+ 991 // 63 0 63 0 992 // Imm (Imm >> (48 - LZ) & 0xffff) 993 // +----sext-----|--16-bit--+ +clear-|-----------------+ 994 // |11111111111111bbbbbbbbb1| -> |00000001bbbbbbbbb1111111| 995 // +------------------------+ +------------------------+ 996 // 63 0 63 0 997 // LI8: sext many leading zeros RLDICL: rotate left (48 - LZ), clear left LZ 998 if ((LZ + TO) > 48) { 999 // Since the immediates with (LZ > 32) have been handled by previous 1000 // patterns, here we have (LZ <= 32) to make sure we will not shift right 1001 // the Imm by a negative value. 1002 assert(LZ <= 32 && "Unexpected shift value."); 1003 Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, 1004 getI32Imm((Imm >> (48 - LZ) & 0xffff))); 1005 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), 1006 getI32Imm(48 - LZ), getI32Imm(LZ)); 1007 } 1008 // 2-4) Patterns : {zeros}{ones}{15-bit value}{ones} 1009 // {ones}{15-bit value}{ones} 1010 // We can take advantage of LI's sign-extension semantics to generate leading 1011 // ones, and then use RLDICL to mask off the ones in left sides (if required) 1012 // after rotation. 1013 // 1014 // +-LZ-FO||-15-bit-||--TO--+ +-------------|--16-bit--+ 1015 // |00011110bbbbbbbbb1111111| -> |000000000011110bbbbbbbbb| 1016 // +------------------------+ +------------------------+ 1017 // 63 0 63 0 1018 // Imm (Imm >> TO) & 0xffff 1019 // +----sext-----|--16-bit--+ +LZ|---------------------+ 1020 // |111111111111110bbbbbbbbb| -> |00011110bbbbbbbbb1111111| 1021 // +------------------------+ +------------------------+ 1022 // 63 0 63 0 1023 // LI8: sext many leading zeros RLDICL: rotate left TO, clear left LZ 1024 if ((LZ + FO + TO) > 48) { 1025 Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, 1026 getI32Imm((Imm >> TO) & 0xffff)); 1027 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), 1028 getI32Imm(TO), getI32Imm(LZ)); 1029 } 1030 // 2-5) Pattern : {32 zeros}{****}{0}{15-bit value} 1031 // If Hi32 is zero and the Lo16(in Lo32) can be presented as a positive 16 bit 1032 // value, we can use LI for Lo16 without generating leading ones then add the 1033 // Hi16(in Lo32). 1034 if (LZ == 32 && ((Lo32 & 0x8000) == 0)) { 1035 Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, 1036 getI32Imm(Lo32 & 0xffff)); 1037 return CurDAG->getMachineNode(PPC::ORIS8, dl, MVT::i64, SDValue(Result, 0), 1038 getI32Imm(Lo32 >> 16)); 1039 } 1040 // 2-6) Patterns : {******}{49 zeros}{******} 1041 // {******}{49 ones}{******} 1042 // If the Imm contains 49 consecutive zeros/ones, it means that a total of 15 1043 // bits remain on both sides. Rotate right the Imm to construct an int<16> 1044 // value, use LI for int<16> value and then use RLDICL without mask to rotate 1045 // it back. 1046 // 1047 // 1) findContiguousZerosAtLeast(Imm, 49) 1048 // +------|--zeros-|------+ +---ones--||---15 bit--+ 1049 // |bbbbbb0000000000aaaaaa| -> |0000000000aaaaaabbbbbb| 1050 // +----------------------+ +----------------------+ 1051 // 63 0 63 0 1052 // 1053 // 2) findContiguousZerosAtLeast(~Imm, 49) 1054 // +------|--ones--|------+ +---ones--||---15 bit--+ 1055 // |bbbbbb1111111111aaaaaa| -> |1111111111aaaaaabbbbbb| 1056 // +----------------------+ +----------------------+ 1057 // 63 0 63 0 1058 if ((Shift = findContiguousZerosAtLeast(Imm, 49)) || 1059 (Shift = findContiguousZerosAtLeast(~Imm, 49))) { 1060 uint64_t RotImm = APInt(64, Imm).rotr(Shift).getZExtValue(); 1061 Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, 1062 getI32Imm(RotImm & 0xffff)); 1063 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), 1064 getI32Imm(Shift), getI32Imm(0)); 1065 } 1066 1067 // Following patterns use 3 instructions to materialize the Imm. 1068 InstCnt = 3; 1069 // 3-1) Patterns : {zeros}{ones}{31-bit value}{zeros} 1070 // {zeros}{31-bit value}{zeros} 1071 // {zeros}{ones}{31-bit value} 1072 // {ones}{31-bit value}{zeros} 1073 // We can take advantage of LIS's sign-extension semantics to generate leading 1074 // ones, add the remaining bits with ORI, and then use RLDIC to mask off the 1075 // ones in both sides after rotation. 1076 if ((LZ + FO + TZ) > 32) { 1077 uint64_t ImmHi16 = (Imm >> (TZ + 16)) & 0xffff; 1078 unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8; 1079 Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16)); 1080 Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), 1081 getI32Imm((Imm >> TZ) & 0xffff)); 1082 return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0), 1083 getI32Imm(TZ), getI32Imm(LZ)); 1084 } 1085 // 3-2) Pattern : {zeros}{31-bit value}{ones} 1086 // Shift right the Imm by (32 - LZ) bits to construct a negtive 32 bits value, 1087 // therefore we can take advantage of LIS's sign-extension semantics, add 1088 // the remaining bits with ORI, and then mask them off after rotation. 1089 // This is similar to Pattern 2-3, please refer to the diagram there. 1090 if ((LZ + TO) > 32) { 1091 // Since the immediates with (LZ > 32) have been handled by previous 1092 // patterns, here we have (LZ <= 32) to make sure we will not shift right 1093 // the Imm by a negative value. 1094 assert(LZ <= 32 && "Unexpected shift value."); 1095 Result = CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, 1096 getI32Imm((Imm >> (48 - LZ)) & 0xffff)); 1097 Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), 1098 getI32Imm((Imm >> (32 - LZ)) & 0xffff)); 1099 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), 1100 getI32Imm(32 - LZ), getI32Imm(LZ)); 1101 } 1102 // 3-3) Patterns : {zeros}{ones}{31-bit value}{ones} 1103 // {ones}{31-bit value}{ones} 1104 // We can take advantage of LIS's sign-extension semantics to generate leading 1105 // ones, add the remaining bits with ORI, and then use RLDICL to mask off the 1106 // ones in left sides (if required) after rotation. 1107 // This is similar to Pattern 2-4, please refer to the diagram there. 1108 if ((LZ + FO + TO) > 32) { 1109 Result = CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, 1110 getI32Imm((Imm >> (TO + 16)) & 0xffff)); 1111 Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), 1112 getI32Imm((Imm >> TO) & 0xffff)); 1113 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), 1114 getI32Imm(TO), getI32Imm(LZ)); 1115 } 1116 // 3-4) Patterns : High word == Low word 1117 if (Hi32 == Lo32) { 1118 // Handle the first 32 bits. 1119 uint64_t ImmHi16 = (Lo32 >> 16) & 0xffff; 1120 unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8; 1121 Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16)); 1122 Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), 1123 getI32Imm(Lo32 & 0xffff)); 1124 // Use rldimi to insert the Low word into High word. 1125 SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(32), 1126 getI32Imm(0)}; 1127 return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops); 1128 } 1129 // 3-5) Patterns : {******}{33 zeros}{******} 1130 // {******}{33 ones}{******} 1131 // If the Imm contains 33 consecutive zeros/ones, it means that a total of 31 1132 // bits remain on both sides. Rotate right the Imm to construct an int<32> 1133 // value, use LIS + ORI for int<32> value and then use RLDICL without mask to 1134 // rotate it back. 1135 // This is similar to Pattern 2-6, please refer to the diagram there. 1136 if ((Shift = findContiguousZerosAtLeast(Imm, 33)) || 1137 (Shift = findContiguousZerosAtLeast(~Imm, 33))) { 1138 uint64_t RotImm = APInt(64, Imm).rotr(Shift).getZExtValue(); 1139 uint64_t ImmHi16 = (RotImm >> 16) & 0xffff; 1140 unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8; 1141 Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16)); 1142 Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), 1143 getI32Imm(RotImm & 0xffff)); 1144 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), 1145 getI32Imm(Shift), getI32Imm(0)); 1146 } 1147 1148 InstCnt = 0; 1149 return nullptr; 1150 } 1151 1152 // Try to select instructions to generate a 64 bit immediate using prefix as 1153 // well as non prefix instructions. The function will return the SDNode 1154 // to materialize that constant or it will return nullptr if it does not 1155 // find one. The variable InstCnt is set to the number of instructions that 1156 // were selected. 1157 static SDNode *selectI64ImmDirectPrefix(SelectionDAG *CurDAG, const SDLoc &dl, 1158 uint64_t Imm, unsigned &InstCnt) { 1159 unsigned TZ = countTrailingZeros<uint64_t>(Imm); 1160 unsigned LZ = countLeadingZeros<uint64_t>(Imm); 1161 unsigned TO = countTrailingOnes<uint64_t>(Imm); 1162 unsigned FO = countLeadingOnes<uint64_t>(LZ == 64 ? 0 : (Imm << LZ)); 1163 unsigned Hi32 = Hi_32(Imm); 1164 unsigned Lo32 = Lo_32(Imm); 1165 1166 auto getI32Imm = [CurDAG, dl](unsigned Imm) { 1167 return CurDAG->getTargetConstant(Imm, dl, MVT::i32); 1168 }; 1169 1170 auto getI64Imm = [CurDAG, dl](uint64_t Imm) { 1171 return CurDAG->getTargetConstant(Imm, dl, MVT::i64); 1172 }; 1173 1174 // Following patterns use 1 instruction to materialize Imm. 1175 InstCnt = 1; 1176 1177 // The pli instruction can materialize up to 34 bits directly. 1178 // If a constant fits within 34-bits, emit the pli instruction here directly. 1179 if (isInt<34>(Imm)) 1180 return CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, 1181 CurDAG->getTargetConstant(Imm, dl, MVT::i64)); 1182 1183 // Require at least two instructions. 1184 InstCnt = 2; 1185 SDNode *Result = nullptr; 1186 // Patterns : {zeros}{ones}{33-bit value}{zeros} 1187 // {zeros}{33-bit value}{zeros} 1188 // {zeros}{ones}{33-bit value} 1189 // {ones}{33-bit value}{zeros} 1190 // We can take advantage of PLI's sign-extension semantics to generate leading 1191 // ones, and then use RLDIC to mask off the ones on both sides after rotation. 1192 if ((LZ + FO + TZ) > 30) { 1193 APInt SignedInt34 = APInt(34, (Imm >> TZ) & 0x3ffffffff); 1194 APInt Extended = SignedInt34.sext(64); 1195 Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, 1196 getI64Imm(*Extended.getRawData())); 1197 return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0), 1198 getI32Imm(TZ), getI32Imm(LZ)); 1199 } 1200 // Pattern : {zeros}{33-bit value}{ones} 1201 // Shift right the Imm by (30 - LZ) bits to construct a negative 34 bit value, 1202 // therefore we can take advantage of PLI's sign-extension semantics, and then 1203 // mask them off after rotation. 1204 // 1205 // +--LZ--||-33-bit-||--TO--+ +-------------|--34-bit--+ 1206 // |00000001bbbbbbbbb1111111| -> |00000000000001bbbbbbbbb1| 1207 // +------------------------+ +------------------------+ 1208 // 63 0 63 0 1209 // 1210 // +----sext-----|--34-bit--+ +clear-|-----------------+ 1211 // |11111111111111bbbbbbbbb1| -> |00000001bbbbbbbbb1111111| 1212 // +------------------------+ +------------------------+ 1213 // 63 0 63 0 1214 if ((LZ + TO) > 30) { 1215 APInt SignedInt34 = APInt(34, (Imm >> (30 - LZ)) & 0x3ffffffff); 1216 APInt Extended = SignedInt34.sext(64); 1217 Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, 1218 getI64Imm(*Extended.getRawData())); 1219 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), 1220 getI32Imm(30 - LZ), getI32Imm(LZ)); 1221 } 1222 // Patterns : {zeros}{ones}{33-bit value}{ones} 1223 // {ones}{33-bit value}{ones} 1224 // Similar to LI we can take advantage of PLI's sign-extension semantics to 1225 // generate leading ones, and then use RLDICL to mask off the ones in left 1226 // sides (if required) after rotation. 1227 if ((LZ + FO + TO) > 30) { 1228 APInt SignedInt34 = APInt(34, (Imm >> TO) & 0x3ffffffff); 1229 APInt Extended = SignedInt34.sext(64); 1230 Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, 1231 getI64Imm(*Extended.getRawData())); 1232 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0), 1233 getI32Imm(TO), getI32Imm(LZ)); 1234 } 1235 // Patterns : {******}{31 zeros}{******} 1236 // : {******}{31 ones}{******} 1237 // If Imm contains 31 consecutive zeros/ones then the remaining bit count 1238 // is 33. Rotate right the Imm to construct a int<33> value, we can use PLI 1239 // for the int<33> value and then use RLDICL without a mask to rotate it back. 1240 // 1241 // +------|--ones--|------+ +---ones--||---33 bit--+ 1242 // |bbbbbb1111111111aaaaaa| -> |1111111111aaaaaabbbbbb| 1243 // +----------------------+ +----------------------+ 1244 // 63 0 63 0 1245 for (unsigned Shift = 0; Shift < 63; ++Shift) { 1246 uint64_t RotImm = APInt(64, Imm).rotr(Shift).getZExtValue(); 1247 if (isInt<34>(RotImm)) { 1248 Result = 1249 CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(RotImm)); 1250 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, 1251 SDValue(Result, 0), getI32Imm(Shift), 1252 getI32Imm(0)); 1253 } 1254 } 1255 1256 // Patterns : High word == Low word 1257 // This is basically a splat of a 32 bit immediate. 1258 if (Hi32 == Lo32) { 1259 Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(Hi32)); 1260 SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(32), 1261 getI32Imm(0)}; 1262 return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops); 1263 } 1264 1265 InstCnt = 3; 1266 // Catch-all 1267 // This pattern can form any 64 bit immediate in 3 instructions. 1268 SDNode *ResultHi = 1269 CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(Hi32)); 1270 SDNode *ResultLo = 1271 CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(Lo32)); 1272 SDValue Ops[] = {SDValue(ResultLo, 0), SDValue(ResultHi, 0), getI32Imm(32), 1273 getI32Imm(0)}; 1274 return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops); 1275 } 1276 1277 static SDNode *selectI64Imm(SelectionDAG *CurDAG, const SDLoc &dl, uint64_t Imm, 1278 unsigned *InstCnt = nullptr) { 1279 unsigned InstCntDirect = 0; 1280 // No more than 3 instructions is used if we can select the i64 immediate 1281 // directly. 1282 SDNode *Result = selectI64ImmDirect(CurDAG, dl, Imm, InstCntDirect); 1283 1284 const PPCSubtarget &Subtarget = 1285 CurDAG->getMachineFunction().getSubtarget<PPCSubtarget>(); 1286 1287 // If we have prefixed instructions and there is a chance we can 1288 // materialize the constant with fewer prefixed instructions than 1289 // non-prefixed, try that. 1290 if (Subtarget.hasPrefixInstrs() && InstCntDirect != 1) { 1291 unsigned InstCntDirectP = 0; 1292 SDNode *ResultP = selectI64ImmDirectPrefix(CurDAG, dl, Imm, InstCntDirectP); 1293 // Use the prefix case in either of two cases: 1294 // 1) We have no result from the non-prefix case to use. 1295 // 2) The non-prefix case uses more instructions than the prefix case. 1296 // If the prefix and non-prefix cases use the same number of instructions 1297 // we will prefer the non-prefix case. 1298 if (ResultP && (!Result || InstCntDirectP < InstCntDirect)) { 1299 if (InstCnt) 1300 *InstCnt = InstCntDirectP; 1301 return ResultP; 1302 } 1303 } 1304 1305 if (Result) { 1306 if (InstCnt) 1307 *InstCnt = InstCntDirect; 1308 return Result; 1309 } 1310 auto getI32Imm = [CurDAG, dl](unsigned Imm) { 1311 return CurDAG->getTargetConstant(Imm, dl, MVT::i32); 1312 }; 1313 // Handle the upper 32 bit value. 1314 Result = 1315 selectI64ImmDirect(CurDAG, dl, Imm & 0xffffffff00000000, InstCntDirect); 1316 // Add in the last bits as required. 1317 if (uint32_t Hi16 = (Lo_32(Imm) >> 16) & 0xffff) { 1318 Result = CurDAG->getMachineNode(PPC::ORIS8, dl, MVT::i64, 1319 SDValue(Result, 0), getI32Imm(Hi16)); 1320 ++InstCntDirect; 1321 } 1322 if (uint32_t Lo16 = Lo_32(Imm) & 0xffff) { 1323 Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0), 1324 getI32Imm(Lo16)); 1325 ++InstCntDirect; 1326 } 1327 if (InstCnt) 1328 *InstCnt = InstCntDirect; 1329 return Result; 1330 } 1331 1332 // Select a 64-bit constant. 1333 static SDNode *selectI64Imm(SelectionDAG *CurDAG, SDNode *N) { 1334 SDLoc dl(N); 1335 1336 // Get 64 bit value. 1337 int64_t Imm = cast<ConstantSDNode>(N)->getZExtValue(); 1338 if (unsigned MinSize = allUsesTruncate(CurDAG, N)) { 1339 uint64_t SextImm = SignExtend64(Imm, MinSize); 1340 SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i64); 1341 if (isInt<16>(SextImm)) 1342 return CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, SDImm); 1343 } 1344 return selectI64Imm(CurDAG, dl, Imm); 1345 } 1346 1347 namespace { 1348 1349 class BitPermutationSelector { 1350 struct ValueBit { 1351 SDValue V; 1352 1353 // The bit number in the value, using a convention where bit 0 is the 1354 // lowest-order bit. 1355 unsigned Idx; 1356 1357 // ConstZero means a bit we need to mask off. 1358 // Variable is a bit comes from an input variable. 1359 // VariableKnownToBeZero is also a bit comes from an input variable, 1360 // but it is known to be already zero. So we do not need to mask them. 1361 enum Kind { 1362 ConstZero, 1363 Variable, 1364 VariableKnownToBeZero 1365 } K; 1366 1367 ValueBit(SDValue V, unsigned I, Kind K = Variable) 1368 : V(V), Idx(I), K(K) {} 1369 ValueBit(Kind K = Variable) : Idx(UINT32_MAX), K(K) {} 1370 1371 bool isZero() const { 1372 return K == ConstZero || K == VariableKnownToBeZero; 1373 } 1374 1375 bool hasValue() const { 1376 return K == Variable || K == VariableKnownToBeZero; 1377 } 1378 1379 SDValue getValue() const { 1380 assert(hasValue() && "Cannot get the value of a constant bit"); 1381 return V; 1382 } 1383 1384 unsigned getValueBitIndex() const { 1385 assert(hasValue() && "Cannot get the value bit index of a constant bit"); 1386 return Idx; 1387 } 1388 }; 1389 1390 // A bit group has the same underlying value and the same rotate factor. 1391 struct BitGroup { 1392 SDValue V; 1393 unsigned RLAmt; 1394 unsigned StartIdx, EndIdx; 1395 1396 // This rotation amount assumes that the lower 32 bits of the quantity are 1397 // replicated in the high 32 bits by the rotation operator (which is done 1398 // by rlwinm and friends in 64-bit mode). 1399 bool Repl32; 1400 // Did converting to Repl32 == true change the rotation factor? If it did, 1401 // it decreased it by 32. 1402 bool Repl32CR; 1403 // Was this group coalesced after setting Repl32 to true? 1404 bool Repl32Coalesced; 1405 1406 BitGroup(SDValue V, unsigned R, unsigned S, unsigned E) 1407 : V(V), RLAmt(R), StartIdx(S), EndIdx(E), Repl32(false), Repl32CR(false), 1408 Repl32Coalesced(false) { 1409 LLVM_DEBUG(dbgs() << "\tbit group for " << V.getNode() << " RLAmt = " << R 1410 << " [" << S << ", " << E << "]\n"); 1411 } 1412 }; 1413 1414 // Information on each (Value, RLAmt) pair (like the number of groups 1415 // associated with each) used to choose the lowering method. 1416 struct ValueRotInfo { 1417 SDValue V; 1418 unsigned RLAmt = std::numeric_limits<unsigned>::max(); 1419 unsigned NumGroups = 0; 1420 unsigned FirstGroupStartIdx = std::numeric_limits<unsigned>::max(); 1421 bool Repl32 = false; 1422 1423 ValueRotInfo() = default; 1424 1425 // For sorting (in reverse order) by NumGroups, and then by 1426 // FirstGroupStartIdx. 1427 bool operator < (const ValueRotInfo &Other) const { 1428 // We need to sort so that the non-Repl32 come first because, when we're 1429 // doing masking, the Repl32 bit groups might be subsumed into the 64-bit 1430 // masking operation. 1431 if (Repl32 < Other.Repl32) 1432 return true; 1433 else if (Repl32 > Other.Repl32) 1434 return false; 1435 else if (NumGroups > Other.NumGroups) 1436 return true; 1437 else if (NumGroups < Other.NumGroups) 1438 return false; 1439 else if (RLAmt == 0 && Other.RLAmt != 0) 1440 return true; 1441 else if (RLAmt != 0 && Other.RLAmt == 0) 1442 return false; 1443 else if (FirstGroupStartIdx < Other.FirstGroupStartIdx) 1444 return true; 1445 return false; 1446 } 1447 }; 1448 1449 using ValueBitsMemoizedValue = std::pair<bool, SmallVector<ValueBit, 64>>; 1450 using ValueBitsMemoizer = 1451 DenseMap<SDValue, std::unique_ptr<ValueBitsMemoizedValue>>; 1452 ValueBitsMemoizer Memoizer; 1453 1454 // Return a pair of bool and a SmallVector pointer to a memoization entry. 1455 // The bool is true if something interesting was deduced, otherwise if we're 1456 // providing only a generic representation of V (or something else likewise 1457 // uninteresting for instruction selection) through the SmallVector. 1458 std::pair<bool, SmallVector<ValueBit, 64> *> getValueBits(SDValue V, 1459 unsigned NumBits) { 1460 auto &ValueEntry = Memoizer[V]; 1461 if (ValueEntry) 1462 return std::make_pair(ValueEntry->first, &ValueEntry->second); 1463 ValueEntry.reset(new ValueBitsMemoizedValue()); 1464 bool &Interesting = ValueEntry->first; 1465 SmallVector<ValueBit, 64> &Bits = ValueEntry->second; 1466 Bits.resize(NumBits); 1467 1468 switch (V.getOpcode()) { 1469 default: break; 1470 case ISD::ROTL: 1471 if (isa<ConstantSDNode>(V.getOperand(1))) { 1472 unsigned RotAmt = V.getConstantOperandVal(1); 1473 1474 const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second; 1475 1476 for (unsigned i = 0; i < NumBits; ++i) 1477 Bits[i] = LHSBits[i < RotAmt ? i + (NumBits - RotAmt) : i - RotAmt]; 1478 1479 return std::make_pair(Interesting = true, &Bits); 1480 } 1481 break; 1482 case ISD::SHL: 1483 case PPCISD::SHL: 1484 if (isa<ConstantSDNode>(V.getOperand(1))) { 1485 unsigned ShiftAmt = V.getConstantOperandVal(1); 1486 1487 const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second; 1488 1489 for (unsigned i = ShiftAmt; i < NumBits; ++i) 1490 Bits[i] = LHSBits[i - ShiftAmt]; 1491 1492 for (unsigned i = 0; i < ShiftAmt; ++i) 1493 Bits[i] = ValueBit(ValueBit::ConstZero); 1494 1495 return std::make_pair(Interesting = true, &Bits); 1496 } 1497 break; 1498 case ISD::SRL: 1499 case PPCISD::SRL: 1500 if (isa<ConstantSDNode>(V.getOperand(1))) { 1501 unsigned ShiftAmt = V.getConstantOperandVal(1); 1502 1503 const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second; 1504 1505 for (unsigned i = 0; i < NumBits - ShiftAmt; ++i) 1506 Bits[i] = LHSBits[i + ShiftAmt]; 1507 1508 for (unsigned i = NumBits - ShiftAmt; i < NumBits; ++i) 1509 Bits[i] = ValueBit(ValueBit::ConstZero); 1510 1511 return std::make_pair(Interesting = true, &Bits); 1512 } 1513 break; 1514 case ISD::AND: 1515 if (isa<ConstantSDNode>(V.getOperand(1))) { 1516 uint64_t Mask = V.getConstantOperandVal(1); 1517 1518 const SmallVector<ValueBit, 64> *LHSBits; 1519 // Mark this as interesting, only if the LHS was also interesting. This 1520 // prevents the overall procedure from matching a single immediate 'and' 1521 // (which is non-optimal because such an and might be folded with other 1522 // things if we don't select it here). 1523 std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0), NumBits); 1524 1525 for (unsigned i = 0; i < NumBits; ++i) 1526 if (((Mask >> i) & 1) == 1) 1527 Bits[i] = (*LHSBits)[i]; 1528 else { 1529 // AND instruction masks this bit. If the input is already zero, 1530 // we have nothing to do here. Otherwise, make the bit ConstZero. 1531 if ((*LHSBits)[i].isZero()) 1532 Bits[i] = (*LHSBits)[i]; 1533 else 1534 Bits[i] = ValueBit(ValueBit::ConstZero); 1535 } 1536 1537 return std::make_pair(Interesting, &Bits); 1538 } 1539 break; 1540 case ISD::OR: { 1541 const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second; 1542 const auto &RHSBits = *getValueBits(V.getOperand(1), NumBits).second; 1543 1544 bool AllDisjoint = true; 1545 SDValue LastVal = SDValue(); 1546 unsigned LastIdx = 0; 1547 for (unsigned i = 0; i < NumBits; ++i) { 1548 if (LHSBits[i].isZero() && RHSBits[i].isZero()) { 1549 // If both inputs are known to be zero and one is ConstZero and 1550 // another is VariableKnownToBeZero, we can select whichever 1551 // we like. To minimize the number of bit groups, we select 1552 // VariableKnownToBeZero if this bit is the next bit of the same 1553 // input variable from the previous bit. Otherwise, we select 1554 // ConstZero. 1555 if (LHSBits[i].hasValue() && LHSBits[i].getValue() == LastVal && 1556 LHSBits[i].getValueBitIndex() == LastIdx + 1) 1557 Bits[i] = LHSBits[i]; 1558 else if (RHSBits[i].hasValue() && RHSBits[i].getValue() == LastVal && 1559 RHSBits[i].getValueBitIndex() == LastIdx + 1) 1560 Bits[i] = RHSBits[i]; 1561 else 1562 Bits[i] = ValueBit(ValueBit::ConstZero); 1563 } 1564 else if (LHSBits[i].isZero()) 1565 Bits[i] = RHSBits[i]; 1566 else if (RHSBits[i].isZero()) 1567 Bits[i] = LHSBits[i]; 1568 else { 1569 AllDisjoint = false; 1570 break; 1571 } 1572 // We remember the value and bit index of this bit. 1573 if (Bits[i].hasValue()) { 1574 LastVal = Bits[i].getValue(); 1575 LastIdx = Bits[i].getValueBitIndex(); 1576 } 1577 else { 1578 if (LastVal) LastVal = SDValue(); 1579 LastIdx = 0; 1580 } 1581 } 1582 1583 if (!AllDisjoint) 1584 break; 1585 1586 return std::make_pair(Interesting = true, &Bits); 1587 } 1588 case ISD::ZERO_EXTEND: { 1589 // We support only the case with zero extension from i32 to i64 so far. 1590 if (V.getValueType() != MVT::i64 || 1591 V.getOperand(0).getValueType() != MVT::i32) 1592 break; 1593 1594 const SmallVector<ValueBit, 64> *LHSBits; 1595 const unsigned NumOperandBits = 32; 1596 std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0), 1597 NumOperandBits); 1598 1599 for (unsigned i = 0; i < NumOperandBits; ++i) 1600 Bits[i] = (*LHSBits)[i]; 1601 1602 for (unsigned i = NumOperandBits; i < NumBits; ++i) 1603 Bits[i] = ValueBit(ValueBit::ConstZero); 1604 1605 return std::make_pair(Interesting, &Bits); 1606 } 1607 case ISD::TRUNCATE: { 1608 EVT FromType = V.getOperand(0).getValueType(); 1609 EVT ToType = V.getValueType(); 1610 // We support only the case with truncate from i64 to i32. 1611 if (FromType != MVT::i64 || ToType != MVT::i32) 1612 break; 1613 const unsigned NumAllBits = FromType.getSizeInBits(); 1614 SmallVector<ValueBit, 64> *InBits; 1615 std::tie(Interesting, InBits) = getValueBits(V.getOperand(0), 1616 NumAllBits); 1617 const unsigned NumValidBits = ToType.getSizeInBits(); 1618 1619 // A 32-bit instruction cannot touch upper 32-bit part of 64-bit value. 1620 // So, we cannot include this truncate. 1621 bool UseUpper32bit = false; 1622 for (unsigned i = 0; i < NumValidBits; ++i) 1623 if ((*InBits)[i].hasValue() && (*InBits)[i].getValueBitIndex() >= 32) { 1624 UseUpper32bit = true; 1625 break; 1626 } 1627 if (UseUpper32bit) 1628 break; 1629 1630 for (unsigned i = 0; i < NumValidBits; ++i) 1631 Bits[i] = (*InBits)[i]; 1632 1633 return std::make_pair(Interesting, &Bits); 1634 } 1635 case ISD::AssertZext: { 1636 // For AssertZext, we look through the operand and 1637 // mark the bits known to be zero. 1638 const SmallVector<ValueBit, 64> *LHSBits; 1639 std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0), 1640 NumBits); 1641 1642 EVT FromType = cast<VTSDNode>(V.getOperand(1))->getVT(); 1643 const unsigned NumValidBits = FromType.getSizeInBits(); 1644 for (unsigned i = 0; i < NumValidBits; ++i) 1645 Bits[i] = (*LHSBits)[i]; 1646 1647 // These bits are known to be zero but the AssertZext may be from a value 1648 // that already has some constant zero bits (i.e. from a masking and). 1649 for (unsigned i = NumValidBits; i < NumBits; ++i) 1650 Bits[i] = (*LHSBits)[i].hasValue() 1651 ? ValueBit((*LHSBits)[i].getValue(), 1652 (*LHSBits)[i].getValueBitIndex(), 1653 ValueBit::VariableKnownToBeZero) 1654 : ValueBit(ValueBit::ConstZero); 1655 1656 return std::make_pair(Interesting, &Bits); 1657 } 1658 case ISD::LOAD: 1659 LoadSDNode *LD = cast<LoadSDNode>(V); 1660 if (ISD::isZEXTLoad(V.getNode()) && V.getResNo() == 0) { 1661 EVT VT = LD->getMemoryVT(); 1662 const unsigned NumValidBits = VT.getSizeInBits(); 1663 1664 for (unsigned i = 0; i < NumValidBits; ++i) 1665 Bits[i] = ValueBit(V, i); 1666 1667 // These bits are known to be zero. 1668 for (unsigned i = NumValidBits; i < NumBits; ++i) 1669 Bits[i] = ValueBit(V, i, ValueBit::VariableKnownToBeZero); 1670 1671 // Zero-extending load itself cannot be optimized. So, it is not 1672 // interesting by itself though it gives useful information. 1673 return std::make_pair(Interesting = false, &Bits); 1674 } 1675 break; 1676 } 1677 1678 for (unsigned i = 0; i < NumBits; ++i) 1679 Bits[i] = ValueBit(V, i); 1680 1681 return std::make_pair(Interesting = false, &Bits); 1682 } 1683 1684 // For each value (except the constant ones), compute the left-rotate amount 1685 // to get it from its original to final position. 1686 void computeRotationAmounts() { 1687 NeedMask = false; 1688 RLAmt.resize(Bits.size()); 1689 for (unsigned i = 0; i < Bits.size(); ++i) 1690 if (Bits[i].hasValue()) { 1691 unsigned VBI = Bits[i].getValueBitIndex(); 1692 if (i >= VBI) 1693 RLAmt[i] = i - VBI; 1694 else 1695 RLAmt[i] = Bits.size() - (VBI - i); 1696 } else if (Bits[i].isZero()) { 1697 NeedMask = true; 1698 RLAmt[i] = UINT32_MAX; 1699 } else { 1700 llvm_unreachable("Unknown value bit type"); 1701 } 1702 } 1703 1704 // Collect groups of consecutive bits with the same underlying value and 1705 // rotation factor. If we're doing late masking, we ignore zeros, otherwise 1706 // they break up groups. 1707 void collectBitGroups(bool LateMask) { 1708 BitGroups.clear(); 1709 1710 unsigned LastRLAmt = RLAmt[0]; 1711 SDValue LastValue = Bits[0].hasValue() ? Bits[0].getValue() : SDValue(); 1712 unsigned LastGroupStartIdx = 0; 1713 bool IsGroupOfZeros = !Bits[LastGroupStartIdx].hasValue(); 1714 for (unsigned i = 1; i < Bits.size(); ++i) { 1715 unsigned ThisRLAmt = RLAmt[i]; 1716 SDValue ThisValue = Bits[i].hasValue() ? Bits[i].getValue() : SDValue(); 1717 if (LateMask && !ThisValue) { 1718 ThisValue = LastValue; 1719 ThisRLAmt = LastRLAmt; 1720 // If we're doing late masking, then the first bit group always starts 1721 // at zero (even if the first bits were zero). 1722 if (BitGroups.empty()) 1723 LastGroupStartIdx = 0; 1724 } 1725 1726 // If this bit is known to be zero and the current group is a bit group 1727 // of zeros, we do not need to terminate the current bit group even the 1728 // Value or RLAmt does not match here. Instead, we terminate this group 1729 // when the first non-zero bit appears later. 1730 if (IsGroupOfZeros && Bits[i].isZero()) 1731 continue; 1732 1733 // If this bit has the same underlying value and the same rotate factor as 1734 // the last one, then they're part of the same group. 1735 if (ThisRLAmt == LastRLAmt && ThisValue == LastValue) 1736 // We cannot continue the current group if this bits is not known to 1737 // be zero in a bit group of zeros. 1738 if (!(IsGroupOfZeros && ThisValue && !Bits[i].isZero())) 1739 continue; 1740 1741 if (LastValue.getNode()) 1742 BitGroups.push_back(BitGroup(LastValue, LastRLAmt, LastGroupStartIdx, 1743 i-1)); 1744 LastRLAmt = ThisRLAmt; 1745 LastValue = ThisValue; 1746 LastGroupStartIdx = i; 1747 IsGroupOfZeros = !Bits[LastGroupStartIdx].hasValue(); 1748 } 1749 if (LastValue.getNode()) 1750 BitGroups.push_back(BitGroup(LastValue, LastRLAmt, LastGroupStartIdx, 1751 Bits.size()-1)); 1752 1753 if (BitGroups.empty()) 1754 return; 1755 1756 // We might be able to combine the first and last groups. 1757 if (BitGroups.size() > 1) { 1758 // If the first and last groups are the same, then remove the first group 1759 // in favor of the last group, making the ending index of the last group 1760 // equal to the ending index of the to-be-removed first group. 1761 if (BitGroups[0].StartIdx == 0 && 1762 BitGroups[BitGroups.size()-1].EndIdx == Bits.size()-1 && 1763 BitGroups[0].V == BitGroups[BitGroups.size()-1].V && 1764 BitGroups[0].RLAmt == BitGroups[BitGroups.size()-1].RLAmt) { 1765 LLVM_DEBUG(dbgs() << "\tcombining final bit group with initial one\n"); 1766 BitGroups[BitGroups.size()-1].EndIdx = BitGroups[0].EndIdx; 1767 BitGroups.erase(BitGroups.begin()); 1768 } 1769 } 1770 } 1771 1772 // Take all (SDValue, RLAmt) pairs and sort them by the number of groups 1773 // associated with each. If the number of groups are same, we prefer a group 1774 // which does not require rotate, i.e. RLAmt is 0, to avoid the first rotate 1775 // instruction. If there is a degeneracy, pick the one that occurs 1776 // first (in the final value). 1777 void collectValueRotInfo() { 1778 ValueRots.clear(); 1779 1780 for (auto &BG : BitGroups) { 1781 unsigned RLAmtKey = BG.RLAmt + (BG.Repl32 ? 64 : 0); 1782 ValueRotInfo &VRI = ValueRots[std::make_pair(BG.V, RLAmtKey)]; 1783 VRI.V = BG.V; 1784 VRI.RLAmt = BG.RLAmt; 1785 VRI.Repl32 = BG.Repl32; 1786 VRI.NumGroups += 1; 1787 VRI.FirstGroupStartIdx = std::min(VRI.FirstGroupStartIdx, BG.StartIdx); 1788 } 1789 1790 // Now that we've collected the various ValueRotInfo instances, we need to 1791 // sort them. 1792 ValueRotsVec.clear(); 1793 for (auto &I : ValueRots) { 1794 ValueRotsVec.push_back(I.second); 1795 } 1796 llvm::sort(ValueRotsVec); 1797 } 1798 1799 // In 64-bit mode, rlwinm and friends have a rotation operator that 1800 // replicates the low-order 32 bits into the high-order 32-bits. The mask 1801 // indices of these instructions can only be in the lower 32 bits, so they 1802 // can only represent some 64-bit bit groups. However, when they can be used, 1803 // the 32-bit replication can be used to represent, as a single bit group, 1804 // otherwise separate bit groups. We'll convert to replicated-32-bit bit 1805 // groups when possible. Returns true if any of the bit groups were 1806 // converted. 1807 void assignRepl32BitGroups() { 1808 // If we have bits like this: 1809 // 1810 // Indices: 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0 1811 // V bits: ... 7 6 5 4 3 2 1 0 31 30 29 28 27 26 25 24 1812 // Groups: | RLAmt = 8 | RLAmt = 40 | 1813 // 1814 // But, making use of a 32-bit operation that replicates the low-order 32 1815 // bits into the high-order 32 bits, this can be one bit group with a RLAmt 1816 // of 8. 1817 1818 auto IsAllLow32 = [this](BitGroup & BG) { 1819 if (BG.StartIdx <= BG.EndIdx) { 1820 for (unsigned i = BG.StartIdx; i <= BG.EndIdx; ++i) { 1821 if (!Bits[i].hasValue()) 1822 continue; 1823 if (Bits[i].getValueBitIndex() >= 32) 1824 return false; 1825 } 1826 } else { 1827 for (unsigned i = BG.StartIdx; i < Bits.size(); ++i) { 1828 if (!Bits[i].hasValue()) 1829 continue; 1830 if (Bits[i].getValueBitIndex() >= 32) 1831 return false; 1832 } 1833 for (unsigned i = 0; i <= BG.EndIdx; ++i) { 1834 if (!Bits[i].hasValue()) 1835 continue; 1836 if (Bits[i].getValueBitIndex() >= 32) 1837 return false; 1838 } 1839 } 1840 1841 return true; 1842 }; 1843 1844 for (auto &BG : BitGroups) { 1845 // If this bit group has RLAmt of 0 and will not be merged with 1846 // another bit group, we don't benefit from Repl32. We don't mark 1847 // such group to give more freedom for later instruction selection. 1848 if (BG.RLAmt == 0) { 1849 auto PotentiallyMerged = [this](BitGroup & BG) { 1850 for (auto &BG2 : BitGroups) 1851 if (&BG != &BG2 && BG.V == BG2.V && 1852 (BG2.RLAmt == 0 || BG2.RLAmt == 32)) 1853 return true; 1854 return false; 1855 }; 1856 if (!PotentiallyMerged(BG)) 1857 continue; 1858 } 1859 if (BG.StartIdx < 32 && BG.EndIdx < 32) { 1860 if (IsAllLow32(BG)) { 1861 if (BG.RLAmt >= 32) { 1862 BG.RLAmt -= 32; 1863 BG.Repl32CR = true; 1864 } 1865 1866 BG.Repl32 = true; 1867 1868 LLVM_DEBUG(dbgs() << "\t32-bit replicated bit group for " 1869 << BG.V.getNode() << " RLAmt = " << BG.RLAmt << " [" 1870 << BG.StartIdx << ", " << BG.EndIdx << "]\n"); 1871 } 1872 } 1873 } 1874 1875 // Now walk through the bit groups, consolidating where possible. 1876 for (auto I = BitGroups.begin(); I != BitGroups.end();) { 1877 // We might want to remove this bit group by merging it with the previous 1878 // group (which might be the ending group). 1879 auto IP = (I == BitGroups.begin()) ? 1880 std::prev(BitGroups.end()) : std::prev(I); 1881 if (I->Repl32 && IP->Repl32 && I->V == IP->V && I->RLAmt == IP->RLAmt && 1882 I->StartIdx == (IP->EndIdx + 1) % 64 && I != IP) { 1883 1884 LLVM_DEBUG(dbgs() << "\tcombining 32-bit replicated bit group for " 1885 << I->V.getNode() << " RLAmt = " << I->RLAmt << " [" 1886 << I->StartIdx << ", " << I->EndIdx 1887 << "] with group with range [" << IP->StartIdx << ", " 1888 << IP->EndIdx << "]\n"); 1889 1890 IP->EndIdx = I->EndIdx; 1891 IP->Repl32CR = IP->Repl32CR || I->Repl32CR; 1892 IP->Repl32Coalesced = true; 1893 I = BitGroups.erase(I); 1894 continue; 1895 } else { 1896 // There is a special case worth handling: If there is a single group 1897 // covering the entire upper 32 bits, and it can be merged with both 1898 // the next and previous groups (which might be the same group), then 1899 // do so. If it is the same group (so there will be only one group in 1900 // total), then we need to reverse the order of the range so that it 1901 // covers the entire 64 bits. 1902 if (I->StartIdx == 32 && I->EndIdx == 63) { 1903 assert(std::next(I) == BitGroups.end() && 1904 "bit group ends at index 63 but there is another?"); 1905 auto IN = BitGroups.begin(); 1906 1907 if (IP->Repl32 && IN->Repl32 && I->V == IP->V && I->V == IN->V && 1908 (I->RLAmt % 32) == IP->RLAmt && (I->RLAmt % 32) == IN->RLAmt && 1909 IP->EndIdx == 31 && IN->StartIdx == 0 && I != IP && 1910 IsAllLow32(*I)) { 1911 1912 LLVM_DEBUG(dbgs() << "\tcombining bit group for " << I->V.getNode() 1913 << " RLAmt = " << I->RLAmt << " [" << I->StartIdx 1914 << ", " << I->EndIdx 1915 << "] with 32-bit replicated groups with ranges [" 1916 << IP->StartIdx << ", " << IP->EndIdx << "] and [" 1917 << IN->StartIdx << ", " << IN->EndIdx << "]\n"); 1918 1919 if (IP == IN) { 1920 // There is only one other group; change it to cover the whole 1921 // range (backward, so that it can still be Repl32 but cover the 1922 // whole 64-bit range). 1923 IP->StartIdx = 31; 1924 IP->EndIdx = 30; 1925 IP->Repl32CR = IP->Repl32CR || I->RLAmt >= 32; 1926 IP->Repl32Coalesced = true; 1927 I = BitGroups.erase(I); 1928 } else { 1929 // There are two separate groups, one before this group and one 1930 // after us (at the beginning). We're going to remove this group, 1931 // but also the group at the very beginning. 1932 IP->EndIdx = IN->EndIdx; 1933 IP->Repl32CR = IP->Repl32CR || IN->Repl32CR || I->RLAmt >= 32; 1934 IP->Repl32Coalesced = true; 1935 I = BitGroups.erase(I); 1936 BitGroups.erase(BitGroups.begin()); 1937 } 1938 1939 // This must be the last group in the vector (and we might have 1940 // just invalidated the iterator above), so break here. 1941 break; 1942 } 1943 } 1944 } 1945 1946 ++I; 1947 } 1948 } 1949 1950 SDValue getI32Imm(unsigned Imm, const SDLoc &dl) { 1951 return CurDAG->getTargetConstant(Imm, dl, MVT::i32); 1952 } 1953 1954 uint64_t getZerosMask() { 1955 uint64_t Mask = 0; 1956 for (unsigned i = 0; i < Bits.size(); ++i) { 1957 if (Bits[i].hasValue()) 1958 continue; 1959 Mask |= (UINT64_C(1) << i); 1960 } 1961 1962 return ~Mask; 1963 } 1964 1965 // This method extends an input value to 64 bit if input is 32-bit integer. 1966 // While selecting instructions in BitPermutationSelector in 64-bit mode, 1967 // an input value can be a 32-bit integer if a ZERO_EXTEND node is included. 1968 // In such case, we extend it to 64 bit to be consistent with other values. 1969 SDValue ExtendToInt64(SDValue V, const SDLoc &dl) { 1970 if (V.getValueSizeInBits() == 64) 1971 return V; 1972 1973 assert(V.getValueSizeInBits() == 32); 1974 SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32); 1975 SDValue ImDef = SDValue(CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, 1976 MVT::i64), 0); 1977 SDValue ExtVal = SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl, 1978 MVT::i64, ImDef, V, 1979 SubRegIdx), 0); 1980 return ExtVal; 1981 } 1982 1983 SDValue TruncateToInt32(SDValue V, const SDLoc &dl) { 1984 if (V.getValueSizeInBits() == 32) 1985 return V; 1986 1987 assert(V.getValueSizeInBits() == 64); 1988 SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32); 1989 SDValue SubVal = SDValue(CurDAG->getMachineNode(PPC::EXTRACT_SUBREG, dl, 1990 MVT::i32, V, SubRegIdx), 0); 1991 return SubVal; 1992 } 1993 1994 // Depending on the number of groups for a particular value, it might be 1995 // better to rotate, mask explicitly (using andi/andis), and then or the 1996 // result. Select this part of the result first. 1997 void SelectAndParts32(const SDLoc &dl, SDValue &Res, unsigned *InstCnt) { 1998 if (BPermRewriterNoMasking) 1999 return; 2000 2001 for (ValueRotInfo &VRI : ValueRotsVec) { 2002 unsigned Mask = 0; 2003 for (unsigned i = 0; i < Bits.size(); ++i) { 2004 if (!Bits[i].hasValue() || Bits[i].getValue() != VRI.V) 2005 continue; 2006 if (RLAmt[i] != VRI.RLAmt) 2007 continue; 2008 Mask |= (1u << i); 2009 } 2010 2011 // Compute the masks for andi/andis that would be necessary. 2012 unsigned ANDIMask = (Mask & UINT16_MAX), ANDISMask = Mask >> 16; 2013 assert((ANDIMask != 0 || ANDISMask != 0) && 2014 "No set bits in mask for value bit groups"); 2015 bool NeedsRotate = VRI.RLAmt != 0; 2016 2017 // We're trying to minimize the number of instructions. If we have one 2018 // group, using one of andi/andis can break even. If we have three 2019 // groups, we can use both andi and andis and break even (to use both 2020 // andi and andis we also need to or the results together). We need four 2021 // groups if we also need to rotate. To use andi/andis we need to do more 2022 // than break even because rotate-and-mask instructions tend to be easier 2023 // to schedule. 2024 2025 // FIXME: We've biased here against using andi/andis, which is right for 2026 // POWER cores, but not optimal everywhere. For example, on the A2, 2027 // andi/andis have single-cycle latency whereas the rotate-and-mask 2028 // instructions take two cycles, and it would be better to bias toward 2029 // andi/andis in break-even cases. 2030 2031 unsigned NumAndInsts = (unsigned) NeedsRotate + 2032 (unsigned) (ANDIMask != 0) + 2033 (unsigned) (ANDISMask != 0) + 2034 (unsigned) (ANDIMask != 0 && ANDISMask != 0) + 2035 (unsigned) (bool) Res; 2036 2037 LLVM_DEBUG(dbgs() << "\t\trotation groups for " << VRI.V.getNode() 2038 << " RL: " << VRI.RLAmt << ":" 2039 << "\n\t\t\tisel using masking: " << NumAndInsts 2040 << " using rotates: " << VRI.NumGroups << "\n"); 2041 2042 if (NumAndInsts >= VRI.NumGroups) 2043 continue; 2044 2045 LLVM_DEBUG(dbgs() << "\t\t\t\tusing masking\n"); 2046 2047 if (InstCnt) *InstCnt += NumAndInsts; 2048 2049 SDValue VRot; 2050 if (VRI.RLAmt) { 2051 SDValue Ops[] = 2052 { TruncateToInt32(VRI.V, dl), getI32Imm(VRI.RLAmt, dl), 2053 getI32Imm(0, dl), getI32Imm(31, dl) }; 2054 VRot = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, 2055 Ops), 0); 2056 } else { 2057 VRot = TruncateToInt32(VRI.V, dl); 2058 } 2059 2060 SDValue ANDIVal, ANDISVal; 2061 if (ANDIMask != 0) 2062 ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI_rec, dl, MVT::i32, 2063 VRot, getI32Imm(ANDIMask, dl)), 2064 0); 2065 if (ANDISMask != 0) 2066 ANDISVal = 2067 SDValue(CurDAG->getMachineNode(PPC::ANDIS_rec, dl, MVT::i32, VRot, 2068 getI32Imm(ANDISMask, dl)), 2069 0); 2070 2071 SDValue TotalVal; 2072 if (!ANDIVal) 2073 TotalVal = ANDISVal; 2074 else if (!ANDISVal) 2075 TotalVal = ANDIVal; 2076 else 2077 TotalVal = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32, 2078 ANDIVal, ANDISVal), 0); 2079 2080 if (!Res) 2081 Res = TotalVal; 2082 else 2083 Res = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32, 2084 Res, TotalVal), 0); 2085 2086 // Now, remove all groups with this underlying value and rotation 2087 // factor. 2088 eraseMatchingBitGroups([VRI](const BitGroup &BG) { 2089 return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt; 2090 }); 2091 } 2092 } 2093 2094 // Instruction selection for the 32-bit case. 2095 SDNode *Select32(SDNode *N, bool LateMask, unsigned *InstCnt) { 2096 SDLoc dl(N); 2097 SDValue Res; 2098 2099 if (InstCnt) *InstCnt = 0; 2100 2101 // Take care of cases that should use andi/andis first. 2102 SelectAndParts32(dl, Res, InstCnt); 2103 2104 // If we've not yet selected a 'starting' instruction, and we have no zeros 2105 // to fill in, select the (Value, RLAmt) with the highest priority (largest 2106 // number of groups), and start with this rotated value. 2107 if ((!NeedMask || LateMask) && !Res) { 2108 ValueRotInfo &VRI = ValueRotsVec[0]; 2109 if (VRI.RLAmt) { 2110 if (InstCnt) *InstCnt += 1; 2111 SDValue Ops[] = 2112 { TruncateToInt32(VRI.V, dl), getI32Imm(VRI.RLAmt, dl), 2113 getI32Imm(0, dl), getI32Imm(31, dl) }; 2114 Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 2115 0); 2116 } else { 2117 Res = TruncateToInt32(VRI.V, dl); 2118 } 2119 2120 // Now, remove all groups with this underlying value and rotation factor. 2121 eraseMatchingBitGroups([VRI](const BitGroup &BG) { 2122 return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt; 2123 }); 2124 } 2125 2126 if (InstCnt) *InstCnt += BitGroups.size(); 2127 2128 // Insert the other groups (one at a time). 2129 for (auto &BG : BitGroups) { 2130 if (!Res) { 2131 SDValue Ops[] = 2132 { TruncateToInt32(BG.V, dl), getI32Imm(BG.RLAmt, dl), 2133 getI32Imm(Bits.size() - BG.EndIdx - 1, dl), 2134 getI32Imm(Bits.size() - BG.StartIdx - 1, dl) }; 2135 Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); 2136 } else { 2137 SDValue Ops[] = 2138 { Res, TruncateToInt32(BG.V, dl), getI32Imm(BG.RLAmt, dl), 2139 getI32Imm(Bits.size() - BG.EndIdx - 1, dl), 2140 getI32Imm(Bits.size() - BG.StartIdx - 1, dl) }; 2141 Res = SDValue(CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops), 0); 2142 } 2143 } 2144 2145 if (LateMask) { 2146 unsigned Mask = (unsigned) getZerosMask(); 2147 2148 unsigned ANDIMask = (Mask & UINT16_MAX), ANDISMask = Mask >> 16; 2149 assert((ANDIMask != 0 || ANDISMask != 0) && 2150 "No set bits in zeros mask?"); 2151 2152 if (InstCnt) *InstCnt += (unsigned) (ANDIMask != 0) + 2153 (unsigned) (ANDISMask != 0) + 2154 (unsigned) (ANDIMask != 0 && ANDISMask != 0); 2155 2156 SDValue ANDIVal, ANDISVal; 2157 if (ANDIMask != 0) 2158 ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI_rec, dl, MVT::i32, 2159 Res, getI32Imm(ANDIMask, dl)), 2160 0); 2161 if (ANDISMask != 0) 2162 ANDISVal = 2163 SDValue(CurDAG->getMachineNode(PPC::ANDIS_rec, dl, MVT::i32, Res, 2164 getI32Imm(ANDISMask, dl)), 2165 0); 2166 2167 if (!ANDIVal) 2168 Res = ANDISVal; 2169 else if (!ANDISVal) 2170 Res = ANDIVal; 2171 else 2172 Res = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32, 2173 ANDIVal, ANDISVal), 0); 2174 } 2175 2176 return Res.getNode(); 2177 } 2178 2179 unsigned SelectRotMask64Count(unsigned RLAmt, bool Repl32, 2180 unsigned MaskStart, unsigned MaskEnd, 2181 bool IsIns) { 2182 // In the notation used by the instructions, 'start' and 'end' are reversed 2183 // because bits are counted from high to low order. 2184 unsigned InstMaskStart = 64 - MaskEnd - 1, 2185 InstMaskEnd = 64 - MaskStart - 1; 2186 2187 if (Repl32) 2188 return 1; 2189 2190 if ((!IsIns && (InstMaskEnd == 63 || InstMaskStart == 0)) || 2191 InstMaskEnd == 63 - RLAmt) 2192 return 1; 2193 2194 return 2; 2195 } 2196 2197 // For 64-bit values, not all combinations of rotates and masks are 2198 // available. Produce one if it is available. 2199 SDValue SelectRotMask64(SDValue V, const SDLoc &dl, unsigned RLAmt, 2200 bool Repl32, unsigned MaskStart, unsigned MaskEnd, 2201 unsigned *InstCnt = nullptr) { 2202 // In the notation used by the instructions, 'start' and 'end' are reversed 2203 // because bits are counted from high to low order. 2204 unsigned InstMaskStart = 64 - MaskEnd - 1, 2205 InstMaskEnd = 64 - MaskStart - 1; 2206 2207 if (InstCnt) *InstCnt += 1; 2208 2209 if (Repl32) { 2210 // This rotation amount assumes that the lower 32 bits of the quantity 2211 // are replicated in the high 32 bits by the rotation operator (which is 2212 // done by rlwinm and friends). 2213 assert(InstMaskStart >= 32 && "Mask cannot start out of range"); 2214 assert(InstMaskEnd >= 32 && "Mask cannot end out of range"); 2215 SDValue Ops[] = 2216 { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl), 2217 getI32Imm(InstMaskStart - 32, dl), getI32Imm(InstMaskEnd - 32, dl) }; 2218 return SDValue(CurDAG->getMachineNode(PPC::RLWINM8, dl, MVT::i64, 2219 Ops), 0); 2220 } 2221 2222 if (InstMaskEnd == 63) { 2223 SDValue Ops[] = 2224 { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl), 2225 getI32Imm(InstMaskStart, dl) }; 2226 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Ops), 0); 2227 } 2228 2229 if (InstMaskStart == 0) { 2230 SDValue Ops[] = 2231 { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl), 2232 getI32Imm(InstMaskEnd, dl) }; 2233 return SDValue(CurDAG->getMachineNode(PPC::RLDICR, dl, MVT::i64, Ops), 0); 2234 } 2235 2236 if (InstMaskEnd == 63 - RLAmt) { 2237 SDValue Ops[] = 2238 { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl), 2239 getI32Imm(InstMaskStart, dl) }; 2240 return SDValue(CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, Ops), 0); 2241 } 2242 2243 // We cannot do this with a single instruction, so we'll use two. The 2244 // problem is that we're not free to choose both a rotation amount and mask 2245 // start and end independently. We can choose an arbitrary mask start and 2246 // end, but then the rotation amount is fixed. Rotation, however, can be 2247 // inverted, and so by applying an "inverse" rotation first, we can get the 2248 // desired result. 2249 if (InstCnt) *InstCnt += 1; 2250 2251 // The rotation mask for the second instruction must be MaskStart. 2252 unsigned RLAmt2 = MaskStart; 2253 // The first instruction must rotate V so that the overall rotation amount 2254 // is RLAmt. 2255 unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64; 2256 if (RLAmt1) 2257 V = SelectRotMask64(V, dl, RLAmt1, false, 0, 63); 2258 return SelectRotMask64(V, dl, RLAmt2, false, MaskStart, MaskEnd); 2259 } 2260 2261 // For 64-bit values, not all combinations of rotates and masks are 2262 // available. Produce a rotate-mask-and-insert if one is available. 2263 SDValue SelectRotMaskIns64(SDValue Base, SDValue V, const SDLoc &dl, 2264 unsigned RLAmt, bool Repl32, unsigned MaskStart, 2265 unsigned MaskEnd, unsigned *InstCnt = nullptr) { 2266 // In the notation used by the instructions, 'start' and 'end' are reversed 2267 // because bits are counted from high to low order. 2268 unsigned InstMaskStart = 64 - MaskEnd - 1, 2269 InstMaskEnd = 64 - MaskStart - 1; 2270 2271 if (InstCnt) *InstCnt += 1; 2272 2273 if (Repl32) { 2274 // This rotation amount assumes that the lower 32 bits of the quantity 2275 // are replicated in the high 32 bits by the rotation operator (which is 2276 // done by rlwinm and friends). 2277 assert(InstMaskStart >= 32 && "Mask cannot start out of range"); 2278 assert(InstMaskEnd >= 32 && "Mask cannot end out of range"); 2279 SDValue Ops[] = 2280 { ExtendToInt64(Base, dl), ExtendToInt64(V, dl), getI32Imm(RLAmt, dl), 2281 getI32Imm(InstMaskStart - 32, dl), getI32Imm(InstMaskEnd - 32, dl) }; 2282 return SDValue(CurDAG->getMachineNode(PPC::RLWIMI8, dl, MVT::i64, 2283 Ops), 0); 2284 } 2285 2286 if (InstMaskEnd == 63 - RLAmt) { 2287 SDValue Ops[] = 2288 { ExtendToInt64(Base, dl), ExtendToInt64(V, dl), getI32Imm(RLAmt, dl), 2289 getI32Imm(InstMaskStart, dl) }; 2290 return SDValue(CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops), 0); 2291 } 2292 2293 // We cannot do this with a single instruction, so we'll use two. The 2294 // problem is that we're not free to choose both a rotation amount and mask 2295 // start and end independently. We can choose an arbitrary mask start and 2296 // end, but then the rotation amount is fixed. Rotation, however, can be 2297 // inverted, and so by applying an "inverse" rotation first, we can get the 2298 // desired result. 2299 if (InstCnt) *InstCnt += 1; 2300 2301 // The rotation mask for the second instruction must be MaskStart. 2302 unsigned RLAmt2 = MaskStart; 2303 // The first instruction must rotate V so that the overall rotation amount 2304 // is RLAmt. 2305 unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64; 2306 if (RLAmt1) 2307 V = SelectRotMask64(V, dl, RLAmt1, false, 0, 63); 2308 return SelectRotMaskIns64(Base, V, dl, RLAmt2, false, MaskStart, MaskEnd); 2309 } 2310 2311 void SelectAndParts64(const SDLoc &dl, SDValue &Res, unsigned *InstCnt) { 2312 if (BPermRewriterNoMasking) 2313 return; 2314 2315 // The idea here is the same as in the 32-bit version, but with additional 2316 // complications from the fact that Repl32 might be true. Because we 2317 // aggressively convert bit groups to Repl32 form (which, for small 2318 // rotation factors, involves no other change), and then coalesce, it might 2319 // be the case that a single 64-bit masking operation could handle both 2320 // some Repl32 groups and some non-Repl32 groups. If converting to Repl32 2321 // form allowed coalescing, then we must use a 32-bit rotaton in order to 2322 // completely capture the new combined bit group. 2323 2324 for (ValueRotInfo &VRI : ValueRotsVec) { 2325 uint64_t Mask = 0; 2326 2327 // We need to add to the mask all bits from the associated bit groups. 2328 // If Repl32 is false, we need to add bits from bit groups that have 2329 // Repl32 true, but are trivially convertable to Repl32 false. Such a 2330 // group is trivially convertable if it overlaps only with the lower 32 2331 // bits, and the group has not been coalesced. 2332 auto MatchingBG = [VRI](const BitGroup &BG) { 2333 if (VRI.V != BG.V) 2334 return false; 2335 2336 unsigned EffRLAmt = BG.RLAmt; 2337 if (!VRI.Repl32 && BG.Repl32) { 2338 if (BG.StartIdx < 32 && BG.EndIdx < 32 && BG.StartIdx <= BG.EndIdx && 2339 !BG.Repl32Coalesced) { 2340 if (BG.Repl32CR) 2341 EffRLAmt += 32; 2342 } else { 2343 return false; 2344 } 2345 } else if (VRI.Repl32 != BG.Repl32) { 2346 return false; 2347 } 2348 2349 return VRI.RLAmt == EffRLAmt; 2350 }; 2351 2352 for (auto &BG : BitGroups) { 2353 if (!MatchingBG(BG)) 2354 continue; 2355 2356 if (BG.StartIdx <= BG.EndIdx) { 2357 for (unsigned i = BG.StartIdx; i <= BG.EndIdx; ++i) 2358 Mask |= (UINT64_C(1) << i); 2359 } else { 2360 for (unsigned i = BG.StartIdx; i < Bits.size(); ++i) 2361 Mask |= (UINT64_C(1) << i); 2362 for (unsigned i = 0; i <= BG.EndIdx; ++i) 2363 Mask |= (UINT64_C(1) << i); 2364 } 2365 } 2366 2367 // We can use the 32-bit andi/andis technique if the mask does not 2368 // require any higher-order bits. This can save an instruction compared 2369 // to always using the general 64-bit technique. 2370 bool Use32BitInsts = isUInt<32>(Mask); 2371 // Compute the masks for andi/andis that would be necessary. 2372 unsigned ANDIMask = (Mask & UINT16_MAX), 2373 ANDISMask = (Mask >> 16) & UINT16_MAX; 2374 2375 bool NeedsRotate = VRI.RLAmt || (VRI.Repl32 && !isUInt<32>(Mask)); 2376 2377 unsigned NumAndInsts = (unsigned) NeedsRotate + 2378 (unsigned) (bool) Res; 2379 unsigned NumOfSelectInsts = 0; 2380 selectI64Imm(CurDAG, dl, Mask, &NumOfSelectInsts); 2381 assert(NumOfSelectInsts > 0 && "Failed to select an i64 constant."); 2382 if (Use32BitInsts) 2383 NumAndInsts += (unsigned) (ANDIMask != 0) + (unsigned) (ANDISMask != 0) + 2384 (unsigned) (ANDIMask != 0 && ANDISMask != 0); 2385 else 2386 NumAndInsts += NumOfSelectInsts + /* and */ 1; 2387 2388 unsigned NumRLInsts = 0; 2389 bool FirstBG = true; 2390 bool MoreBG = false; 2391 for (auto &BG : BitGroups) { 2392 if (!MatchingBG(BG)) { 2393 MoreBG = true; 2394 continue; 2395 } 2396 NumRLInsts += 2397 SelectRotMask64Count(BG.RLAmt, BG.Repl32, BG.StartIdx, BG.EndIdx, 2398 !FirstBG); 2399 FirstBG = false; 2400 } 2401 2402 LLVM_DEBUG(dbgs() << "\t\trotation groups for " << VRI.V.getNode() 2403 << " RL: " << VRI.RLAmt << (VRI.Repl32 ? " (32):" : ":") 2404 << "\n\t\t\tisel using masking: " << NumAndInsts 2405 << " using rotates: " << NumRLInsts << "\n"); 2406 2407 // When we'd use andi/andis, we bias toward using the rotates (andi only 2408 // has a record form, and is cracked on POWER cores). However, when using 2409 // general 64-bit constant formation, bias toward the constant form, 2410 // because that exposes more opportunities for CSE. 2411 if (NumAndInsts > NumRLInsts) 2412 continue; 2413 // When merging multiple bit groups, instruction or is used. 2414 // But when rotate is used, rldimi can inert the rotated value into any 2415 // register, so instruction or can be avoided. 2416 if ((Use32BitInsts || MoreBG) && NumAndInsts == NumRLInsts) 2417 continue; 2418 2419 LLVM_DEBUG(dbgs() << "\t\t\t\tusing masking\n"); 2420 2421 if (InstCnt) *InstCnt += NumAndInsts; 2422 2423 SDValue VRot; 2424 // We actually need to generate a rotation if we have a non-zero rotation 2425 // factor or, in the Repl32 case, if we care about any of the 2426 // higher-order replicated bits. In the latter case, we generate a mask 2427 // backward so that it actually includes the entire 64 bits. 2428 if (VRI.RLAmt || (VRI.Repl32 && !isUInt<32>(Mask))) 2429 VRot = SelectRotMask64(VRI.V, dl, VRI.RLAmt, VRI.Repl32, 2430 VRI.Repl32 ? 31 : 0, VRI.Repl32 ? 30 : 63); 2431 else 2432 VRot = VRI.V; 2433 2434 SDValue TotalVal; 2435 if (Use32BitInsts) { 2436 assert((ANDIMask != 0 || ANDISMask != 0) && 2437 "No set bits in mask when using 32-bit ands for 64-bit value"); 2438 2439 SDValue ANDIVal, ANDISVal; 2440 if (ANDIMask != 0) 2441 ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI8_rec, dl, MVT::i64, 2442 ExtendToInt64(VRot, dl), 2443 getI32Imm(ANDIMask, dl)), 2444 0); 2445 if (ANDISMask != 0) 2446 ANDISVal = 2447 SDValue(CurDAG->getMachineNode(PPC::ANDIS8_rec, dl, MVT::i64, 2448 ExtendToInt64(VRot, dl), 2449 getI32Imm(ANDISMask, dl)), 2450 0); 2451 2452 if (!ANDIVal) 2453 TotalVal = ANDISVal; 2454 else if (!ANDISVal) 2455 TotalVal = ANDIVal; 2456 else 2457 TotalVal = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, 2458 ExtendToInt64(ANDIVal, dl), ANDISVal), 0); 2459 } else { 2460 TotalVal = SDValue(selectI64Imm(CurDAG, dl, Mask), 0); 2461 TotalVal = 2462 SDValue(CurDAG->getMachineNode(PPC::AND8, dl, MVT::i64, 2463 ExtendToInt64(VRot, dl), TotalVal), 2464 0); 2465 } 2466 2467 if (!Res) 2468 Res = TotalVal; 2469 else 2470 Res = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, 2471 ExtendToInt64(Res, dl), TotalVal), 2472 0); 2473 2474 // Now, remove all groups with this underlying value and rotation 2475 // factor. 2476 eraseMatchingBitGroups(MatchingBG); 2477 } 2478 } 2479 2480 // Instruction selection for the 64-bit case. 2481 SDNode *Select64(SDNode *N, bool LateMask, unsigned *InstCnt) { 2482 SDLoc dl(N); 2483 SDValue Res; 2484 2485 if (InstCnt) *InstCnt = 0; 2486 2487 // Take care of cases that should use andi/andis first. 2488 SelectAndParts64(dl, Res, InstCnt); 2489 2490 // If we've not yet selected a 'starting' instruction, and we have no zeros 2491 // to fill in, select the (Value, RLAmt) with the highest priority (largest 2492 // number of groups), and start with this rotated value. 2493 if ((!NeedMask || LateMask) && !Res) { 2494 // If we have both Repl32 groups and non-Repl32 groups, the non-Repl32 2495 // groups will come first, and so the VRI representing the largest number 2496 // of groups might not be first (it might be the first Repl32 groups). 2497 unsigned MaxGroupsIdx = 0; 2498 if (!ValueRotsVec[0].Repl32) { 2499 for (unsigned i = 0, ie = ValueRotsVec.size(); i < ie; ++i) 2500 if (ValueRotsVec[i].Repl32) { 2501 if (ValueRotsVec[i].NumGroups > ValueRotsVec[0].NumGroups) 2502 MaxGroupsIdx = i; 2503 break; 2504 } 2505 } 2506 2507 ValueRotInfo &VRI = ValueRotsVec[MaxGroupsIdx]; 2508 bool NeedsRotate = false; 2509 if (VRI.RLAmt) { 2510 NeedsRotate = true; 2511 } else if (VRI.Repl32) { 2512 for (auto &BG : BitGroups) { 2513 if (BG.V != VRI.V || BG.RLAmt != VRI.RLAmt || 2514 BG.Repl32 != VRI.Repl32) 2515 continue; 2516 2517 // We don't need a rotate if the bit group is confined to the lower 2518 // 32 bits. 2519 if (BG.StartIdx < 32 && BG.EndIdx < 32 && BG.StartIdx < BG.EndIdx) 2520 continue; 2521 2522 NeedsRotate = true; 2523 break; 2524 } 2525 } 2526 2527 if (NeedsRotate) 2528 Res = SelectRotMask64(VRI.V, dl, VRI.RLAmt, VRI.Repl32, 2529 VRI.Repl32 ? 31 : 0, VRI.Repl32 ? 30 : 63, 2530 InstCnt); 2531 else 2532 Res = VRI.V; 2533 2534 // Now, remove all groups with this underlying value and rotation factor. 2535 if (Res) 2536 eraseMatchingBitGroups([VRI](const BitGroup &BG) { 2537 return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt && 2538 BG.Repl32 == VRI.Repl32; 2539 }); 2540 } 2541 2542 // Because 64-bit rotates are more flexible than inserts, we might have a 2543 // preference regarding which one we do first (to save one instruction). 2544 if (!Res) 2545 for (auto I = BitGroups.begin(), IE = BitGroups.end(); I != IE; ++I) { 2546 if (SelectRotMask64Count(I->RLAmt, I->Repl32, I->StartIdx, I->EndIdx, 2547 false) < 2548 SelectRotMask64Count(I->RLAmt, I->Repl32, I->StartIdx, I->EndIdx, 2549 true)) { 2550 if (I != BitGroups.begin()) { 2551 BitGroup BG = *I; 2552 BitGroups.erase(I); 2553 BitGroups.insert(BitGroups.begin(), BG); 2554 } 2555 2556 break; 2557 } 2558 } 2559 2560 // Insert the other groups (one at a time). 2561 for (auto &BG : BitGroups) { 2562 if (!Res) 2563 Res = SelectRotMask64(BG.V, dl, BG.RLAmt, BG.Repl32, BG.StartIdx, 2564 BG.EndIdx, InstCnt); 2565 else 2566 Res = SelectRotMaskIns64(Res, BG.V, dl, BG.RLAmt, BG.Repl32, 2567 BG.StartIdx, BG.EndIdx, InstCnt); 2568 } 2569 2570 if (LateMask) { 2571 uint64_t Mask = getZerosMask(); 2572 2573 // We can use the 32-bit andi/andis technique if the mask does not 2574 // require any higher-order bits. This can save an instruction compared 2575 // to always using the general 64-bit technique. 2576 bool Use32BitInsts = isUInt<32>(Mask); 2577 // Compute the masks for andi/andis that would be necessary. 2578 unsigned ANDIMask = (Mask & UINT16_MAX), 2579 ANDISMask = (Mask >> 16) & UINT16_MAX; 2580 2581 if (Use32BitInsts) { 2582 assert((ANDIMask != 0 || ANDISMask != 0) && 2583 "No set bits in mask when using 32-bit ands for 64-bit value"); 2584 2585 if (InstCnt) *InstCnt += (unsigned) (ANDIMask != 0) + 2586 (unsigned) (ANDISMask != 0) + 2587 (unsigned) (ANDIMask != 0 && ANDISMask != 0); 2588 2589 SDValue ANDIVal, ANDISVal; 2590 if (ANDIMask != 0) 2591 ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI8_rec, dl, MVT::i64, 2592 ExtendToInt64(Res, dl), 2593 getI32Imm(ANDIMask, dl)), 2594 0); 2595 if (ANDISMask != 0) 2596 ANDISVal = 2597 SDValue(CurDAG->getMachineNode(PPC::ANDIS8_rec, dl, MVT::i64, 2598 ExtendToInt64(Res, dl), 2599 getI32Imm(ANDISMask, dl)), 2600 0); 2601 2602 if (!ANDIVal) 2603 Res = ANDISVal; 2604 else if (!ANDISVal) 2605 Res = ANDIVal; 2606 else 2607 Res = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, 2608 ExtendToInt64(ANDIVal, dl), ANDISVal), 0); 2609 } else { 2610 unsigned NumOfSelectInsts = 0; 2611 SDValue MaskVal = 2612 SDValue(selectI64Imm(CurDAG, dl, Mask, &NumOfSelectInsts), 0); 2613 Res = SDValue(CurDAG->getMachineNode(PPC::AND8, dl, MVT::i64, 2614 ExtendToInt64(Res, dl), MaskVal), 2615 0); 2616 if (InstCnt) 2617 *InstCnt += NumOfSelectInsts + /* and */ 1; 2618 } 2619 } 2620 2621 return Res.getNode(); 2622 } 2623 2624 SDNode *Select(SDNode *N, bool LateMask, unsigned *InstCnt = nullptr) { 2625 // Fill in BitGroups. 2626 collectBitGroups(LateMask); 2627 if (BitGroups.empty()) 2628 return nullptr; 2629 2630 // For 64-bit values, figure out when we can use 32-bit instructions. 2631 if (Bits.size() == 64) 2632 assignRepl32BitGroups(); 2633 2634 // Fill in ValueRotsVec. 2635 collectValueRotInfo(); 2636 2637 if (Bits.size() == 32) { 2638 return Select32(N, LateMask, InstCnt); 2639 } else { 2640 assert(Bits.size() == 64 && "Not 64 bits here?"); 2641 return Select64(N, LateMask, InstCnt); 2642 } 2643 2644 return nullptr; 2645 } 2646 2647 void eraseMatchingBitGroups(function_ref<bool(const BitGroup &)> F) { 2648 erase_if(BitGroups, F); 2649 } 2650 2651 SmallVector<ValueBit, 64> Bits; 2652 2653 bool NeedMask = false; 2654 SmallVector<unsigned, 64> RLAmt; 2655 2656 SmallVector<BitGroup, 16> BitGroups; 2657 2658 DenseMap<std::pair<SDValue, unsigned>, ValueRotInfo> ValueRots; 2659 SmallVector<ValueRotInfo, 16> ValueRotsVec; 2660 2661 SelectionDAG *CurDAG = nullptr; 2662 2663 public: 2664 BitPermutationSelector(SelectionDAG *DAG) 2665 : CurDAG(DAG) {} 2666 2667 // Here we try to match complex bit permutations into a set of 2668 // rotate-and-shift/shift/and/or instructions, using a set of heuristics 2669 // known to produce optimal code for common cases (like i32 byte swapping). 2670 SDNode *Select(SDNode *N) { 2671 Memoizer.clear(); 2672 auto Result = 2673 getValueBits(SDValue(N, 0), N->getValueType(0).getSizeInBits()); 2674 if (!Result.first) 2675 return nullptr; 2676 Bits = std::move(*Result.second); 2677 2678 LLVM_DEBUG(dbgs() << "Considering bit-permutation-based instruction" 2679 " selection for: "); 2680 LLVM_DEBUG(N->dump(CurDAG)); 2681 2682 // Fill it RLAmt and set NeedMask. 2683 computeRotationAmounts(); 2684 2685 if (!NeedMask) 2686 return Select(N, false); 2687 2688 // We currently have two techniques for handling results with zeros: early 2689 // masking (the default) and late masking. Late masking is sometimes more 2690 // efficient, but because the structure of the bit groups is different, it 2691 // is hard to tell without generating both and comparing the results. With 2692 // late masking, we ignore zeros in the resulting value when inserting each 2693 // set of bit groups, and then mask in the zeros at the end. With early 2694 // masking, we only insert the non-zero parts of the result at every step. 2695 2696 unsigned InstCnt = 0, InstCntLateMask = 0; 2697 LLVM_DEBUG(dbgs() << "\tEarly masking:\n"); 2698 SDNode *RN = Select(N, false, &InstCnt); 2699 LLVM_DEBUG(dbgs() << "\t\tisel would use " << InstCnt << " instructions\n"); 2700 2701 LLVM_DEBUG(dbgs() << "\tLate masking:\n"); 2702 SDNode *RNLM = Select(N, true, &InstCntLateMask); 2703 LLVM_DEBUG(dbgs() << "\t\tisel would use " << InstCntLateMask 2704 << " instructions\n"); 2705 2706 if (InstCnt <= InstCntLateMask) { 2707 LLVM_DEBUG(dbgs() << "\tUsing early-masking for isel\n"); 2708 return RN; 2709 } 2710 2711 LLVM_DEBUG(dbgs() << "\tUsing late-masking for isel\n"); 2712 return RNLM; 2713 } 2714 }; 2715 2716 class IntegerCompareEliminator { 2717 SelectionDAG *CurDAG; 2718 PPCDAGToDAGISel *S; 2719 // Conversion type for interpreting results of a 32-bit instruction as 2720 // a 64-bit value or vice versa. 2721 enum ExtOrTruncConversion { Ext, Trunc }; 2722 2723 // Modifiers to guide how an ISD::SETCC node's result is to be computed 2724 // in a GPR. 2725 // ZExtOrig - use the original condition code, zero-extend value 2726 // ZExtInvert - invert the condition code, zero-extend value 2727 // SExtOrig - use the original condition code, sign-extend value 2728 // SExtInvert - invert the condition code, sign-extend value 2729 enum SetccInGPROpts { ZExtOrig, ZExtInvert, SExtOrig, SExtInvert }; 2730 2731 // Comparisons against zero to emit GPR code sequences for. Each of these 2732 // sequences may need to be emitted for two or more equivalent patterns. 2733 // For example (a >= 0) == (a > -1). The direction of the comparison (</>) 2734 // matters as well as the extension type: sext (-1/0), zext (1/0). 2735 // GEZExt - (zext (LHS >= 0)) 2736 // GESExt - (sext (LHS >= 0)) 2737 // LEZExt - (zext (LHS <= 0)) 2738 // LESExt - (sext (LHS <= 0)) 2739 enum ZeroCompare { GEZExt, GESExt, LEZExt, LESExt }; 2740 2741 SDNode *tryEXTEND(SDNode *N); 2742 SDNode *tryLogicOpOfCompares(SDNode *N); 2743 SDValue computeLogicOpInGPR(SDValue LogicOp); 2744 SDValue signExtendInputIfNeeded(SDValue Input); 2745 SDValue zeroExtendInputIfNeeded(SDValue Input); 2746 SDValue addExtOrTrunc(SDValue NatWidthRes, ExtOrTruncConversion Conv); 2747 SDValue getCompoundZeroComparisonInGPR(SDValue LHS, SDLoc dl, 2748 ZeroCompare CmpTy); 2749 SDValue get32BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC, 2750 int64_t RHSValue, SDLoc dl); 2751 SDValue get32BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC, 2752 int64_t RHSValue, SDLoc dl); 2753 SDValue get64BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC, 2754 int64_t RHSValue, SDLoc dl); 2755 SDValue get64BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC, 2756 int64_t RHSValue, SDLoc dl); 2757 SDValue getSETCCInGPR(SDValue Compare, SetccInGPROpts ConvOpts); 2758 2759 public: 2760 IntegerCompareEliminator(SelectionDAG *DAG, 2761 PPCDAGToDAGISel *Sel) : CurDAG(DAG), S(Sel) { 2762 assert(CurDAG->getTargetLoweringInfo() 2763 .getPointerTy(CurDAG->getDataLayout()).getSizeInBits() == 64 && 2764 "Only expecting to use this on 64 bit targets."); 2765 } 2766 SDNode *Select(SDNode *N) { 2767 if (CmpInGPR == ICGPR_None) 2768 return nullptr; 2769 switch (N->getOpcode()) { 2770 default: break; 2771 case ISD::ZERO_EXTEND: 2772 if (CmpInGPR == ICGPR_Sext || CmpInGPR == ICGPR_SextI32 || 2773 CmpInGPR == ICGPR_SextI64) 2774 return nullptr; 2775 LLVM_FALLTHROUGH; 2776 case ISD::SIGN_EXTEND: 2777 if (CmpInGPR == ICGPR_Zext || CmpInGPR == ICGPR_ZextI32 || 2778 CmpInGPR == ICGPR_ZextI64) 2779 return nullptr; 2780 return tryEXTEND(N); 2781 case ISD::AND: 2782 case ISD::OR: 2783 case ISD::XOR: 2784 return tryLogicOpOfCompares(N); 2785 } 2786 return nullptr; 2787 } 2788 }; 2789 2790 static bool isLogicOp(unsigned Opc) { 2791 return Opc == ISD::AND || Opc == ISD::OR || Opc == ISD::XOR; 2792 } 2793 // The obvious case for wanting to keep the value in a GPR. Namely, the 2794 // result of the comparison is actually needed in a GPR. 2795 SDNode *IntegerCompareEliminator::tryEXTEND(SDNode *N) { 2796 assert((N->getOpcode() == ISD::ZERO_EXTEND || 2797 N->getOpcode() == ISD::SIGN_EXTEND) && 2798 "Expecting a zero/sign extend node!"); 2799 SDValue WideRes; 2800 // If we are zero-extending the result of a logical operation on i1 2801 // values, we can keep the values in GPRs. 2802 if (isLogicOp(N->getOperand(0).getOpcode()) && 2803 N->getOperand(0).getValueType() == MVT::i1 && 2804 N->getOpcode() == ISD::ZERO_EXTEND) 2805 WideRes = computeLogicOpInGPR(N->getOperand(0)); 2806 else if (N->getOperand(0).getOpcode() != ISD::SETCC) 2807 return nullptr; 2808 else 2809 WideRes = 2810 getSETCCInGPR(N->getOperand(0), 2811 N->getOpcode() == ISD::SIGN_EXTEND ? 2812 SetccInGPROpts::SExtOrig : SetccInGPROpts::ZExtOrig); 2813 2814 if (!WideRes) 2815 return nullptr; 2816 2817 SDLoc dl(N); 2818 bool Input32Bit = WideRes.getValueType() == MVT::i32; 2819 bool Output32Bit = N->getValueType(0) == MVT::i32; 2820 2821 NumSextSetcc += N->getOpcode() == ISD::SIGN_EXTEND ? 1 : 0; 2822 NumZextSetcc += N->getOpcode() == ISD::SIGN_EXTEND ? 0 : 1; 2823 2824 SDValue ConvOp = WideRes; 2825 if (Input32Bit != Output32Bit) 2826 ConvOp = addExtOrTrunc(WideRes, Input32Bit ? ExtOrTruncConversion::Ext : 2827 ExtOrTruncConversion::Trunc); 2828 return ConvOp.getNode(); 2829 } 2830 2831 // Attempt to perform logical operations on the results of comparisons while 2832 // keeping the values in GPRs. Without doing so, these would end up being 2833 // lowered to CR-logical operations which suffer from significant latency and 2834 // low ILP. 2835 SDNode *IntegerCompareEliminator::tryLogicOpOfCompares(SDNode *N) { 2836 if (N->getValueType(0) != MVT::i1) 2837 return nullptr; 2838 assert(isLogicOp(N->getOpcode()) && 2839 "Expected a logic operation on setcc results."); 2840 SDValue LoweredLogical = computeLogicOpInGPR(SDValue(N, 0)); 2841 if (!LoweredLogical) 2842 return nullptr; 2843 2844 SDLoc dl(N); 2845 bool IsBitwiseNegate = LoweredLogical.getMachineOpcode() == PPC::XORI8; 2846 unsigned SubRegToExtract = IsBitwiseNegate ? PPC::sub_eq : PPC::sub_gt; 2847 SDValue CR0Reg = CurDAG->getRegister(PPC::CR0, MVT::i32); 2848 SDValue LHS = LoweredLogical.getOperand(0); 2849 SDValue RHS = LoweredLogical.getOperand(1); 2850 SDValue WideOp; 2851 SDValue OpToConvToRecForm; 2852 2853 // Look through any 32-bit to 64-bit implicit extend nodes to find the 2854 // opcode that is input to the XORI. 2855 if (IsBitwiseNegate && 2856 LoweredLogical.getOperand(0).getMachineOpcode() == PPC::INSERT_SUBREG) 2857 OpToConvToRecForm = LoweredLogical.getOperand(0).getOperand(1); 2858 else if (IsBitwiseNegate) 2859 // If the input to the XORI isn't an extension, that's what we're after. 2860 OpToConvToRecForm = LoweredLogical.getOperand(0); 2861 else 2862 // If this is not an XORI, it is a reg-reg logical op and we can convert 2863 // it to record-form. 2864 OpToConvToRecForm = LoweredLogical; 2865 2866 // Get the record-form version of the node we're looking to use to get the 2867 // CR result from. 2868 uint16_t NonRecOpc = OpToConvToRecForm.getMachineOpcode(); 2869 int NewOpc = PPCInstrInfo::getRecordFormOpcode(NonRecOpc); 2870 2871 // Convert the right node to record-form. This is either the logical we're 2872 // looking at or it is the input node to the negation (if we're looking at 2873 // a bitwise negation). 2874 if (NewOpc != -1 && IsBitwiseNegate) { 2875 // The input to the XORI has a record-form. Use it. 2876 assert(LoweredLogical.getConstantOperandVal(1) == 1 && 2877 "Expected a PPC::XORI8 only for bitwise negation."); 2878 // Emit the record-form instruction. 2879 std::vector<SDValue> Ops; 2880 for (int i = 0, e = OpToConvToRecForm.getNumOperands(); i < e; i++) 2881 Ops.push_back(OpToConvToRecForm.getOperand(i)); 2882 2883 WideOp = 2884 SDValue(CurDAG->getMachineNode(NewOpc, dl, 2885 OpToConvToRecForm.getValueType(), 2886 MVT::Glue, Ops), 0); 2887 } else { 2888 assert((NewOpc != -1 || !IsBitwiseNegate) && 2889 "No record form available for AND8/OR8/XOR8?"); 2890 WideOp = 2891 SDValue(CurDAG->getMachineNode(NewOpc == -1 ? PPC::ANDI8_rec : NewOpc, 2892 dl, MVT::i64, MVT::Glue, LHS, RHS), 2893 0); 2894 } 2895 2896 // Select this node to a single bit from CR0 set by the record-form node 2897 // just created. For bitwise negation, use the EQ bit which is the equivalent 2898 // of negating the result (i.e. it is a bit set when the result of the 2899 // operation is zero). 2900 SDValue SRIdxVal = 2901 CurDAG->getTargetConstant(SubRegToExtract, dl, MVT::i32); 2902 SDValue CRBit = 2903 SDValue(CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, 2904 MVT::i1, CR0Reg, SRIdxVal, 2905 WideOp.getValue(1)), 0); 2906 return CRBit.getNode(); 2907 } 2908 2909 // Lower a logical operation on i1 values into a GPR sequence if possible. 2910 // The result can be kept in a GPR if requested. 2911 // Three types of inputs can be handled: 2912 // - SETCC 2913 // - TRUNCATE 2914 // - Logical operation (AND/OR/XOR) 2915 // There is also a special case that is handled (namely a complement operation 2916 // achieved with xor %a, -1). 2917 SDValue IntegerCompareEliminator::computeLogicOpInGPR(SDValue LogicOp) { 2918 assert(isLogicOp(LogicOp.getOpcode()) && 2919 "Can only handle logic operations here."); 2920 assert(LogicOp.getValueType() == MVT::i1 && 2921 "Can only handle logic operations on i1 values here."); 2922 SDLoc dl(LogicOp); 2923 SDValue LHS, RHS; 2924 2925 // Special case: xor %a, -1 2926 bool IsBitwiseNegation = isBitwiseNot(LogicOp); 2927 2928 // Produces a GPR sequence for each operand of the binary logic operation. 2929 // For SETCC, it produces the respective comparison, for TRUNCATE it truncates 2930 // the value in a GPR and for logic operations, it will recursively produce 2931 // a GPR sequence for the operation. 2932 auto getLogicOperand = [&] (SDValue Operand) -> SDValue { 2933 unsigned OperandOpcode = Operand.getOpcode(); 2934 if (OperandOpcode == ISD::SETCC) 2935 return getSETCCInGPR(Operand, SetccInGPROpts::ZExtOrig); 2936 else if (OperandOpcode == ISD::TRUNCATE) { 2937 SDValue InputOp = Operand.getOperand(0); 2938 EVT InVT = InputOp.getValueType(); 2939 return SDValue(CurDAG->getMachineNode(InVT == MVT::i32 ? PPC::RLDICL_32 : 2940 PPC::RLDICL, dl, InVT, InputOp, 2941 S->getI64Imm(0, dl), 2942 S->getI64Imm(63, dl)), 0); 2943 } else if (isLogicOp(OperandOpcode)) 2944 return computeLogicOpInGPR(Operand); 2945 return SDValue(); 2946 }; 2947 LHS = getLogicOperand(LogicOp.getOperand(0)); 2948 RHS = getLogicOperand(LogicOp.getOperand(1)); 2949 2950 // If a GPR sequence can't be produced for the LHS we can't proceed. 2951 // Not producing a GPR sequence for the RHS is only a problem if this isn't 2952 // a bitwise negation operation. 2953 if (!LHS || (!RHS && !IsBitwiseNegation)) 2954 return SDValue(); 2955 2956 NumLogicOpsOnComparison++; 2957 2958 // We will use the inputs as 64-bit values. 2959 if (LHS.getValueType() == MVT::i32) 2960 LHS = addExtOrTrunc(LHS, ExtOrTruncConversion::Ext); 2961 if (!IsBitwiseNegation && RHS.getValueType() == MVT::i32) 2962 RHS = addExtOrTrunc(RHS, ExtOrTruncConversion::Ext); 2963 2964 unsigned NewOpc; 2965 switch (LogicOp.getOpcode()) { 2966 default: llvm_unreachable("Unknown logic operation."); 2967 case ISD::AND: NewOpc = PPC::AND8; break; 2968 case ISD::OR: NewOpc = PPC::OR8; break; 2969 case ISD::XOR: NewOpc = PPC::XOR8; break; 2970 } 2971 2972 if (IsBitwiseNegation) { 2973 RHS = S->getI64Imm(1, dl); 2974 NewOpc = PPC::XORI8; 2975 } 2976 2977 return SDValue(CurDAG->getMachineNode(NewOpc, dl, MVT::i64, LHS, RHS), 0); 2978 2979 } 2980 2981 /// If the value isn't guaranteed to be sign-extended to 64-bits, extend it. 2982 /// Otherwise just reinterpret it as a 64-bit value. 2983 /// Useful when emitting comparison code for 32-bit values without using 2984 /// the compare instruction (which only considers the lower 32-bits). 2985 SDValue IntegerCompareEliminator::signExtendInputIfNeeded(SDValue Input) { 2986 assert(Input.getValueType() == MVT::i32 && 2987 "Can only sign-extend 32-bit values here."); 2988 unsigned Opc = Input.getOpcode(); 2989 2990 // The value was sign extended and then truncated to 32-bits. No need to 2991 // sign extend it again. 2992 if (Opc == ISD::TRUNCATE && 2993 (Input.getOperand(0).getOpcode() == ISD::AssertSext || 2994 Input.getOperand(0).getOpcode() == ISD::SIGN_EXTEND)) 2995 return addExtOrTrunc(Input, ExtOrTruncConversion::Ext); 2996 2997 LoadSDNode *InputLoad = dyn_cast<LoadSDNode>(Input); 2998 // The input is a sign-extending load. All ppc sign-extending loads 2999 // sign-extend to the full 64-bits. 3000 if (InputLoad && InputLoad->getExtensionType() == ISD::SEXTLOAD) 3001 return addExtOrTrunc(Input, ExtOrTruncConversion::Ext); 3002 3003 ConstantSDNode *InputConst = dyn_cast<ConstantSDNode>(Input); 3004 // We don't sign-extend constants. 3005 if (InputConst) 3006 return addExtOrTrunc(Input, ExtOrTruncConversion::Ext); 3007 3008 SDLoc dl(Input); 3009 SignExtensionsAdded++; 3010 return SDValue(CurDAG->getMachineNode(PPC::EXTSW_32_64, dl, 3011 MVT::i64, Input), 0); 3012 } 3013 3014 /// If the value isn't guaranteed to be zero-extended to 64-bits, extend it. 3015 /// Otherwise just reinterpret it as a 64-bit value. 3016 /// Useful when emitting comparison code for 32-bit values without using 3017 /// the compare instruction (which only considers the lower 32-bits). 3018 SDValue IntegerCompareEliminator::zeroExtendInputIfNeeded(SDValue Input) { 3019 assert(Input.getValueType() == MVT::i32 && 3020 "Can only zero-extend 32-bit values here."); 3021 unsigned Opc = Input.getOpcode(); 3022 3023 // The only condition under which we can omit the actual extend instruction: 3024 // - The value is a positive constant 3025 // - The value comes from a load that isn't a sign-extending load 3026 // An ISD::TRUNCATE needs to be zero-extended unless it is fed by a zext. 3027 bool IsTruncateOfZExt = Opc == ISD::TRUNCATE && 3028 (Input.getOperand(0).getOpcode() == ISD::AssertZext || 3029 Input.getOperand(0).getOpcode() == ISD::ZERO_EXTEND); 3030 if (IsTruncateOfZExt) 3031 return addExtOrTrunc(Input, ExtOrTruncConversion::Ext); 3032 3033 ConstantSDNode *InputConst = dyn_cast<ConstantSDNode>(Input); 3034 if (InputConst && InputConst->getSExtValue() >= 0) 3035 return addExtOrTrunc(Input, ExtOrTruncConversion::Ext); 3036 3037 LoadSDNode *InputLoad = dyn_cast<LoadSDNode>(Input); 3038 // The input is a load that doesn't sign-extend (it will be zero-extended). 3039 if (InputLoad && InputLoad->getExtensionType() != ISD::SEXTLOAD) 3040 return addExtOrTrunc(Input, ExtOrTruncConversion::Ext); 3041 3042 // None of the above, need to zero-extend. 3043 SDLoc dl(Input); 3044 ZeroExtensionsAdded++; 3045 return SDValue(CurDAG->getMachineNode(PPC::RLDICL_32_64, dl, MVT::i64, Input, 3046 S->getI64Imm(0, dl), 3047 S->getI64Imm(32, dl)), 0); 3048 } 3049 3050 // Handle a 32-bit value in a 64-bit register and vice-versa. These are of 3051 // course not actual zero/sign extensions that will generate machine code, 3052 // they're just a way to reinterpret a 32 bit value in a register as a 3053 // 64 bit value and vice-versa. 3054 SDValue IntegerCompareEliminator::addExtOrTrunc(SDValue NatWidthRes, 3055 ExtOrTruncConversion Conv) { 3056 SDLoc dl(NatWidthRes); 3057 3058 // For reinterpreting 32-bit values as 64 bit values, we generate 3059 // INSERT_SUBREG IMPLICIT_DEF:i64, <input>, TargetConstant:i32<1> 3060 if (Conv == ExtOrTruncConversion::Ext) { 3061 SDValue ImDef(CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, MVT::i64), 0); 3062 SDValue SubRegIdx = 3063 CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32); 3064 return SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl, MVT::i64, 3065 ImDef, NatWidthRes, SubRegIdx), 0); 3066 } 3067 3068 assert(Conv == ExtOrTruncConversion::Trunc && 3069 "Unknown convertion between 32 and 64 bit values."); 3070 // For reinterpreting 64-bit values as 32-bit values, we just need to 3071 // EXTRACT_SUBREG (i.e. extract the low word). 3072 SDValue SubRegIdx = 3073 CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32); 3074 return SDValue(CurDAG->getMachineNode(PPC::EXTRACT_SUBREG, dl, MVT::i32, 3075 NatWidthRes, SubRegIdx), 0); 3076 } 3077 3078 // Produce a GPR sequence for compound comparisons (<=, >=) against zero. 3079 // Handle both zero-extensions and sign-extensions. 3080 SDValue 3081 IntegerCompareEliminator::getCompoundZeroComparisonInGPR(SDValue LHS, SDLoc dl, 3082 ZeroCompare CmpTy) { 3083 EVT InVT = LHS.getValueType(); 3084 bool Is32Bit = InVT == MVT::i32; 3085 SDValue ToExtend; 3086 3087 // Produce the value that needs to be either zero or sign extended. 3088 switch (CmpTy) { 3089 case ZeroCompare::GEZExt: 3090 case ZeroCompare::GESExt: 3091 ToExtend = SDValue(CurDAG->getMachineNode(Is32Bit ? PPC::NOR : PPC::NOR8, 3092 dl, InVT, LHS, LHS), 0); 3093 break; 3094 case ZeroCompare::LEZExt: 3095 case ZeroCompare::LESExt: { 3096 if (Is32Bit) { 3097 // Upper 32 bits cannot be undefined for this sequence. 3098 LHS = signExtendInputIfNeeded(LHS); 3099 SDValue Neg = 3100 SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0); 3101 ToExtend = 3102 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, 3103 Neg, S->getI64Imm(1, dl), 3104 S->getI64Imm(63, dl)), 0); 3105 } else { 3106 SDValue Addi = 3107 SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS, 3108 S->getI64Imm(~0ULL, dl)), 0); 3109 ToExtend = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64, 3110 Addi, LHS), 0); 3111 } 3112 break; 3113 } 3114 } 3115 3116 // For 64-bit sequences, the extensions are the same for the GE/LE cases. 3117 if (!Is32Bit && 3118 (CmpTy == ZeroCompare::GEZExt || CmpTy == ZeroCompare::LEZExt)) 3119 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, 3120 ToExtend, S->getI64Imm(1, dl), 3121 S->getI64Imm(63, dl)), 0); 3122 if (!Is32Bit && 3123 (CmpTy == ZeroCompare::GESExt || CmpTy == ZeroCompare::LESExt)) 3124 return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, ToExtend, 3125 S->getI64Imm(63, dl)), 0); 3126 3127 assert(Is32Bit && "Should have handled the 32-bit sequences above."); 3128 // For 32-bit sequences, the extensions differ between GE/LE cases. 3129 switch (CmpTy) { 3130 case ZeroCompare::GEZExt: { 3131 SDValue ShiftOps[] = { ToExtend, S->getI32Imm(1, dl), S->getI32Imm(31, dl), 3132 S->getI32Imm(31, dl) }; 3133 return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, 3134 ShiftOps), 0); 3135 } 3136 case ZeroCompare::GESExt: 3137 return SDValue(CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, ToExtend, 3138 S->getI32Imm(31, dl)), 0); 3139 case ZeroCompare::LEZExt: 3140 return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, ToExtend, 3141 S->getI32Imm(1, dl)), 0); 3142 case ZeroCompare::LESExt: 3143 return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, ToExtend, 3144 S->getI32Imm(-1, dl)), 0); 3145 } 3146 3147 // The above case covers all the enumerators so it can't have a default clause 3148 // to avoid compiler warnings. 3149 llvm_unreachable("Unknown zero-comparison type."); 3150 } 3151 3152 /// Produces a zero-extended result of comparing two 32-bit values according to 3153 /// the passed condition code. 3154 SDValue 3155 IntegerCompareEliminator::get32BitZExtCompare(SDValue LHS, SDValue RHS, 3156 ISD::CondCode CC, 3157 int64_t RHSValue, SDLoc dl) { 3158 if (CmpInGPR == ICGPR_I64 || CmpInGPR == ICGPR_SextI64 || 3159 CmpInGPR == ICGPR_ZextI64 || CmpInGPR == ICGPR_Sext) 3160 return SDValue(); 3161 bool IsRHSZero = RHSValue == 0; 3162 bool IsRHSOne = RHSValue == 1; 3163 bool IsRHSNegOne = RHSValue == -1LL; 3164 switch (CC) { 3165 default: return SDValue(); 3166 case ISD::SETEQ: { 3167 // (zext (setcc %a, %b, seteq)) -> (lshr (cntlzw (xor %a, %b)), 5) 3168 // (zext (setcc %a, 0, seteq)) -> (lshr (cntlzw %a), 5) 3169 SDValue Xor = IsRHSZero ? LHS : 3170 SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0); 3171 SDValue Clz = 3172 SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0); 3173 SDValue ShiftOps[] = { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl), 3174 S->getI32Imm(31, dl) }; 3175 return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, 3176 ShiftOps), 0); 3177 } 3178 case ISD::SETNE: { 3179 // (zext (setcc %a, %b, setne)) -> (xor (lshr (cntlzw (xor %a, %b)), 5), 1) 3180 // (zext (setcc %a, 0, setne)) -> (xor (lshr (cntlzw %a), 5), 1) 3181 SDValue Xor = IsRHSZero ? LHS : 3182 SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0); 3183 SDValue Clz = 3184 SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0); 3185 SDValue ShiftOps[] = { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl), 3186 S->getI32Imm(31, dl) }; 3187 SDValue Shift = 3188 SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, ShiftOps), 0); 3189 return SDValue(CurDAG->getMachineNode(PPC::XORI, dl, MVT::i32, Shift, 3190 S->getI32Imm(1, dl)), 0); 3191 } 3192 case ISD::SETGE: { 3193 // (zext (setcc %a, %b, setge)) -> (xor (lshr (sub %a, %b), 63), 1) 3194 // (zext (setcc %a, 0, setge)) -> (lshr (~ %a), 31) 3195 if(IsRHSZero) 3196 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt); 3197 3198 // Not a special case (i.e. RHS == 0). Handle (%a >= %b) as (%b <= %a) 3199 // by swapping inputs and falling through. 3200 std::swap(LHS, RHS); 3201 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS); 3202 IsRHSZero = RHSConst && RHSConst->isZero(); 3203 LLVM_FALLTHROUGH; 3204 } 3205 case ISD::SETLE: { 3206 if (CmpInGPR == ICGPR_NonExtIn) 3207 return SDValue(); 3208 // (zext (setcc %a, %b, setle)) -> (xor (lshr (sub %b, %a), 63), 1) 3209 // (zext (setcc %a, 0, setle)) -> (xor (lshr (- %a), 63), 1) 3210 if(IsRHSZero) { 3211 if (CmpInGPR == ICGPR_NonExtIn) 3212 return SDValue(); 3213 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt); 3214 } 3215 3216 // The upper 32-bits of the register can't be undefined for this sequence. 3217 LHS = signExtendInputIfNeeded(LHS); 3218 RHS = signExtendInputIfNeeded(RHS); 3219 SDValue Sub = 3220 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0); 3221 SDValue Shift = 3222 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Sub, 3223 S->getI64Imm(1, dl), S->getI64Imm(63, dl)), 3224 0); 3225 return 3226 SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, 3227 MVT::i64, Shift, S->getI32Imm(1, dl)), 0); 3228 } 3229 case ISD::SETGT: { 3230 // (zext (setcc %a, %b, setgt)) -> (lshr (sub %b, %a), 63) 3231 // (zext (setcc %a, -1, setgt)) -> (lshr (~ %a), 31) 3232 // (zext (setcc %a, 0, setgt)) -> (lshr (- %a), 63) 3233 // Handle SETLT -1 (which is equivalent to SETGE 0). 3234 if (IsRHSNegOne) 3235 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt); 3236 3237 if (IsRHSZero) { 3238 if (CmpInGPR == ICGPR_NonExtIn) 3239 return SDValue(); 3240 // The upper 32-bits of the register can't be undefined for this sequence. 3241 LHS = signExtendInputIfNeeded(LHS); 3242 RHS = signExtendInputIfNeeded(RHS); 3243 SDValue Neg = 3244 SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0); 3245 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, 3246 Neg, S->getI32Imm(1, dl), S->getI32Imm(63, dl)), 0); 3247 } 3248 // Not a special case (i.e. RHS == 0 or RHS == -1). Handle (%a > %b) as 3249 // (%b < %a) by swapping inputs and falling through. 3250 std::swap(LHS, RHS); 3251 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS); 3252 IsRHSZero = RHSConst && RHSConst->isZero(); 3253 IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1; 3254 LLVM_FALLTHROUGH; 3255 } 3256 case ISD::SETLT: { 3257 // (zext (setcc %a, %b, setlt)) -> (lshr (sub %a, %b), 63) 3258 // (zext (setcc %a, 1, setlt)) -> (xor (lshr (- %a), 63), 1) 3259 // (zext (setcc %a, 0, setlt)) -> (lshr %a, 31) 3260 // Handle SETLT 1 (which is equivalent to SETLE 0). 3261 if (IsRHSOne) { 3262 if (CmpInGPR == ICGPR_NonExtIn) 3263 return SDValue(); 3264 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt); 3265 } 3266 3267 if (IsRHSZero) { 3268 SDValue ShiftOps[] = { LHS, S->getI32Imm(1, dl), S->getI32Imm(31, dl), 3269 S->getI32Imm(31, dl) }; 3270 return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, 3271 ShiftOps), 0); 3272 } 3273 3274 if (CmpInGPR == ICGPR_NonExtIn) 3275 return SDValue(); 3276 // The upper 32-bits of the register can't be undefined for this sequence. 3277 LHS = signExtendInputIfNeeded(LHS); 3278 RHS = signExtendInputIfNeeded(RHS); 3279 SDValue SUBFNode = 3280 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0); 3281 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, 3282 SUBFNode, S->getI64Imm(1, dl), 3283 S->getI64Imm(63, dl)), 0); 3284 } 3285 case ISD::SETUGE: 3286 // (zext (setcc %a, %b, setuge)) -> (xor (lshr (sub %b, %a), 63), 1) 3287 // (zext (setcc %a, %b, setule)) -> (xor (lshr (sub %a, %b), 63), 1) 3288 std::swap(LHS, RHS); 3289 LLVM_FALLTHROUGH; 3290 case ISD::SETULE: { 3291 if (CmpInGPR == ICGPR_NonExtIn) 3292 return SDValue(); 3293 // The upper 32-bits of the register can't be undefined for this sequence. 3294 LHS = zeroExtendInputIfNeeded(LHS); 3295 RHS = zeroExtendInputIfNeeded(RHS); 3296 SDValue Subtract = 3297 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0); 3298 SDValue SrdiNode = 3299 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, 3300 Subtract, S->getI64Imm(1, dl), 3301 S->getI64Imm(63, dl)), 0); 3302 return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, SrdiNode, 3303 S->getI32Imm(1, dl)), 0); 3304 } 3305 case ISD::SETUGT: 3306 // (zext (setcc %a, %b, setugt)) -> (lshr (sub %b, %a), 63) 3307 // (zext (setcc %a, %b, setult)) -> (lshr (sub %a, %b), 63) 3308 std::swap(LHS, RHS); 3309 LLVM_FALLTHROUGH; 3310 case ISD::SETULT: { 3311 if (CmpInGPR == ICGPR_NonExtIn) 3312 return SDValue(); 3313 // The upper 32-bits of the register can't be undefined for this sequence. 3314 LHS = zeroExtendInputIfNeeded(LHS); 3315 RHS = zeroExtendInputIfNeeded(RHS); 3316 SDValue Subtract = 3317 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0); 3318 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, 3319 Subtract, S->getI64Imm(1, dl), 3320 S->getI64Imm(63, dl)), 0); 3321 } 3322 } 3323 } 3324 3325 /// Produces a sign-extended result of comparing two 32-bit values according to 3326 /// the passed condition code. 3327 SDValue 3328 IntegerCompareEliminator::get32BitSExtCompare(SDValue LHS, SDValue RHS, 3329 ISD::CondCode CC, 3330 int64_t RHSValue, SDLoc dl) { 3331 if (CmpInGPR == ICGPR_I64 || CmpInGPR == ICGPR_SextI64 || 3332 CmpInGPR == ICGPR_ZextI64 || CmpInGPR == ICGPR_Zext) 3333 return SDValue(); 3334 bool IsRHSZero = RHSValue == 0; 3335 bool IsRHSOne = RHSValue == 1; 3336 bool IsRHSNegOne = RHSValue == -1LL; 3337 3338 switch (CC) { 3339 default: return SDValue(); 3340 case ISD::SETEQ: { 3341 // (sext (setcc %a, %b, seteq)) -> 3342 // (ashr (shl (ctlz (xor %a, %b)), 58), 63) 3343 // (sext (setcc %a, 0, seteq)) -> 3344 // (ashr (shl (ctlz %a), 58), 63) 3345 SDValue CountInput = IsRHSZero ? LHS : 3346 SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0); 3347 SDValue Cntlzw = 3348 SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, CountInput), 0); 3349 SDValue SHLOps[] = { Cntlzw, S->getI32Imm(27, dl), 3350 S->getI32Imm(5, dl), S->getI32Imm(31, dl) }; 3351 SDValue Slwi = 3352 SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, SHLOps), 0); 3353 return SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Slwi), 0); 3354 } 3355 case ISD::SETNE: { 3356 // Bitwise xor the operands, count leading zeros, shift right by 5 bits and 3357 // flip the bit, finally take 2's complement. 3358 // (sext (setcc %a, %b, setne)) -> 3359 // (neg (xor (lshr (ctlz (xor %a, %b)), 5), 1)) 3360 // Same as above, but the first xor is not needed. 3361 // (sext (setcc %a, 0, setne)) -> 3362 // (neg (xor (lshr (ctlz %a), 5), 1)) 3363 SDValue Xor = IsRHSZero ? LHS : 3364 SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0); 3365 SDValue Clz = 3366 SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0); 3367 SDValue ShiftOps[] = 3368 { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl), S->getI32Imm(31, dl) }; 3369 SDValue Shift = 3370 SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, ShiftOps), 0); 3371 SDValue Xori = 3372 SDValue(CurDAG->getMachineNode(PPC::XORI, dl, MVT::i32, Shift, 3373 S->getI32Imm(1, dl)), 0); 3374 return SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Xori), 0); 3375 } 3376 case ISD::SETGE: { 3377 // (sext (setcc %a, %b, setge)) -> (add (lshr (sub %a, %b), 63), -1) 3378 // (sext (setcc %a, 0, setge)) -> (ashr (~ %a), 31) 3379 if (IsRHSZero) 3380 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt); 3381 3382 // Not a special case (i.e. RHS == 0). Handle (%a >= %b) as (%b <= %a) 3383 // by swapping inputs and falling through. 3384 std::swap(LHS, RHS); 3385 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS); 3386 IsRHSZero = RHSConst && RHSConst->isZero(); 3387 LLVM_FALLTHROUGH; 3388 } 3389 case ISD::SETLE: { 3390 if (CmpInGPR == ICGPR_NonExtIn) 3391 return SDValue(); 3392 // (sext (setcc %a, %b, setge)) -> (add (lshr (sub %b, %a), 63), -1) 3393 // (sext (setcc %a, 0, setle)) -> (add (lshr (- %a), 63), -1) 3394 if (IsRHSZero) 3395 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt); 3396 3397 // The upper 32-bits of the register can't be undefined for this sequence. 3398 LHS = signExtendInputIfNeeded(LHS); 3399 RHS = signExtendInputIfNeeded(RHS); 3400 SDValue SUBFNode = 3401 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, MVT::Glue, 3402 LHS, RHS), 0); 3403 SDValue Srdi = 3404 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, 3405 SUBFNode, S->getI64Imm(1, dl), 3406 S->getI64Imm(63, dl)), 0); 3407 return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, Srdi, 3408 S->getI32Imm(-1, dl)), 0); 3409 } 3410 case ISD::SETGT: { 3411 // (sext (setcc %a, %b, setgt)) -> (ashr (sub %b, %a), 63) 3412 // (sext (setcc %a, -1, setgt)) -> (ashr (~ %a), 31) 3413 // (sext (setcc %a, 0, setgt)) -> (ashr (- %a), 63) 3414 if (IsRHSNegOne) 3415 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt); 3416 if (IsRHSZero) { 3417 if (CmpInGPR == ICGPR_NonExtIn) 3418 return SDValue(); 3419 // The upper 32-bits of the register can't be undefined for this sequence. 3420 LHS = signExtendInputIfNeeded(LHS); 3421 RHS = signExtendInputIfNeeded(RHS); 3422 SDValue Neg = 3423 SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0); 3424 return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, Neg, 3425 S->getI64Imm(63, dl)), 0); 3426 } 3427 // Not a special case (i.e. RHS == 0 or RHS == -1). Handle (%a > %b) as 3428 // (%b < %a) by swapping inputs and falling through. 3429 std::swap(LHS, RHS); 3430 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS); 3431 IsRHSZero = RHSConst && RHSConst->isZero(); 3432 IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1; 3433 LLVM_FALLTHROUGH; 3434 } 3435 case ISD::SETLT: { 3436 // (sext (setcc %a, %b, setgt)) -> (ashr (sub %a, %b), 63) 3437 // (sext (setcc %a, 1, setgt)) -> (add (lshr (- %a), 63), -1) 3438 // (sext (setcc %a, 0, setgt)) -> (ashr %a, 31) 3439 if (IsRHSOne) { 3440 if (CmpInGPR == ICGPR_NonExtIn) 3441 return SDValue(); 3442 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt); 3443 } 3444 if (IsRHSZero) 3445 return SDValue(CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, LHS, 3446 S->getI32Imm(31, dl)), 0); 3447 3448 if (CmpInGPR == ICGPR_NonExtIn) 3449 return SDValue(); 3450 // The upper 32-bits of the register can't be undefined for this sequence. 3451 LHS = signExtendInputIfNeeded(LHS); 3452 RHS = signExtendInputIfNeeded(RHS); 3453 SDValue SUBFNode = 3454 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0); 3455 return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, 3456 SUBFNode, S->getI64Imm(63, dl)), 0); 3457 } 3458 case ISD::SETUGE: 3459 // (sext (setcc %a, %b, setuge)) -> (add (lshr (sub %a, %b), 63), -1) 3460 // (sext (setcc %a, %b, setule)) -> (add (lshr (sub %b, %a), 63), -1) 3461 std::swap(LHS, RHS); 3462 LLVM_FALLTHROUGH; 3463 case ISD::SETULE: { 3464 if (CmpInGPR == ICGPR_NonExtIn) 3465 return SDValue(); 3466 // The upper 32-bits of the register can't be undefined for this sequence. 3467 LHS = zeroExtendInputIfNeeded(LHS); 3468 RHS = zeroExtendInputIfNeeded(RHS); 3469 SDValue Subtract = 3470 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0); 3471 SDValue Shift = 3472 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Subtract, 3473 S->getI32Imm(1, dl), S->getI32Imm(63,dl)), 3474 0); 3475 return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, Shift, 3476 S->getI32Imm(-1, dl)), 0); 3477 } 3478 case ISD::SETUGT: 3479 // (sext (setcc %a, %b, setugt)) -> (ashr (sub %b, %a), 63) 3480 // (sext (setcc %a, %b, setugt)) -> (ashr (sub %a, %b), 63) 3481 std::swap(LHS, RHS); 3482 LLVM_FALLTHROUGH; 3483 case ISD::SETULT: { 3484 if (CmpInGPR == ICGPR_NonExtIn) 3485 return SDValue(); 3486 // The upper 32-bits of the register can't be undefined for this sequence. 3487 LHS = zeroExtendInputIfNeeded(LHS); 3488 RHS = zeroExtendInputIfNeeded(RHS); 3489 SDValue Subtract = 3490 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0); 3491 return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, 3492 Subtract, S->getI64Imm(63, dl)), 0); 3493 } 3494 } 3495 } 3496 3497 /// Produces a zero-extended result of comparing two 64-bit values according to 3498 /// the passed condition code. 3499 SDValue 3500 IntegerCompareEliminator::get64BitZExtCompare(SDValue LHS, SDValue RHS, 3501 ISD::CondCode CC, 3502 int64_t RHSValue, SDLoc dl) { 3503 if (CmpInGPR == ICGPR_I32 || CmpInGPR == ICGPR_SextI32 || 3504 CmpInGPR == ICGPR_ZextI32 || CmpInGPR == ICGPR_Sext) 3505 return SDValue(); 3506 bool IsRHSZero = RHSValue == 0; 3507 bool IsRHSOne = RHSValue == 1; 3508 bool IsRHSNegOne = RHSValue == -1LL; 3509 switch (CC) { 3510 default: return SDValue(); 3511 case ISD::SETEQ: { 3512 // (zext (setcc %a, %b, seteq)) -> (lshr (ctlz (xor %a, %b)), 6) 3513 // (zext (setcc %a, 0, seteq)) -> (lshr (ctlz %a), 6) 3514 SDValue Xor = IsRHSZero ? LHS : 3515 SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0); 3516 SDValue Clz = 3517 SDValue(CurDAG->getMachineNode(PPC::CNTLZD, dl, MVT::i64, Xor), 0); 3518 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Clz, 3519 S->getI64Imm(58, dl), 3520 S->getI64Imm(63, dl)), 0); 3521 } 3522 case ISD::SETNE: { 3523 // {addc.reg, addc.CA} = (addcarry (xor %a, %b), -1) 3524 // (zext (setcc %a, %b, setne)) -> (sube addc.reg, addc.reg, addc.CA) 3525 // {addcz.reg, addcz.CA} = (addcarry %a, -1) 3526 // (zext (setcc %a, 0, setne)) -> (sube addcz.reg, addcz.reg, addcz.CA) 3527 SDValue Xor = IsRHSZero ? LHS : 3528 SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0); 3529 SDValue AC = 3530 SDValue(CurDAG->getMachineNode(PPC::ADDIC8, dl, MVT::i64, MVT::Glue, 3531 Xor, S->getI32Imm(~0U, dl)), 0); 3532 return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, AC, 3533 Xor, AC.getValue(1)), 0); 3534 } 3535 case ISD::SETGE: { 3536 // {subc.reg, subc.CA} = (subcarry %a, %b) 3537 // (zext (setcc %a, %b, setge)) -> 3538 // (adde (lshr %b, 63), (ashr %a, 63), subc.CA) 3539 // (zext (setcc %a, 0, setge)) -> (lshr (~ %a), 63) 3540 if (IsRHSZero) 3541 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt); 3542 std::swap(LHS, RHS); 3543 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS); 3544 IsRHSZero = RHSConst && RHSConst->isZero(); 3545 LLVM_FALLTHROUGH; 3546 } 3547 case ISD::SETLE: { 3548 // {subc.reg, subc.CA} = (subcarry %b, %a) 3549 // (zext (setcc %a, %b, setge)) -> 3550 // (adde (lshr %a, 63), (ashr %b, 63), subc.CA) 3551 // (zext (setcc %a, 0, setge)) -> (lshr (or %a, (add %a, -1)), 63) 3552 if (IsRHSZero) 3553 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt); 3554 SDValue ShiftL = 3555 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS, 3556 S->getI64Imm(1, dl), 3557 S->getI64Imm(63, dl)), 0); 3558 SDValue ShiftR = 3559 SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, RHS, 3560 S->getI64Imm(63, dl)), 0); 3561 SDValue SubtractCarry = 3562 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, 3563 LHS, RHS), 1); 3564 return SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue, 3565 ShiftR, ShiftL, SubtractCarry), 0); 3566 } 3567 case ISD::SETGT: { 3568 // {subc.reg, subc.CA} = (subcarry %b, %a) 3569 // (zext (setcc %a, %b, setgt)) -> 3570 // (xor (adde (lshr %a, 63), (ashr %b, 63), subc.CA), 1) 3571 // (zext (setcc %a, 0, setgt)) -> (lshr (nor (add %a, -1), %a), 63) 3572 if (IsRHSNegOne) 3573 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt); 3574 if (IsRHSZero) { 3575 SDValue Addi = 3576 SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS, 3577 S->getI64Imm(~0ULL, dl)), 0); 3578 SDValue Nor = 3579 SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, Addi, LHS), 0); 3580 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Nor, 3581 S->getI64Imm(1, dl), 3582 S->getI64Imm(63, dl)), 0); 3583 } 3584 std::swap(LHS, RHS); 3585 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS); 3586 IsRHSZero = RHSConst && RHSConst->isZero(); 3587 IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1; 3588 LLVM_FALLTHROUGH; 3589 } 3590 case ISD::SETLT: { 3591 // {subc.reg, subc.CA} = (subcarry %a, %b) 3592 // (zext (setcc %a, %b, setlt)) -> 3593 // (xor (adde (lshr %b, 63), (ashr %a, 63), subc.CA), 1) 3594 // (zext (setcc %a, 0, setlt)) -> (lshr %a, 63) 3595 if (IsRHSOne) 3596 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt); 3597 if (IsRHSZero) 3598 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS, 3599 S->getI64Imm(1, dl), 3600 S->getI64Imm(63, dl)), 0); 3601 SDValue SRADINode = 3602 SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, 3603 LHS, S->getI64Imm(63, dl)), 0); 3604 SDValue SRDINode = 3605 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, 3606 RHS, S->getI64Imm(1, dl), 3607 S->getI64Imm(63, dl)), 0); 3608 SDValue SUBFC8Carry = 3609 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, 3610 RHS, LHS), 1); 3611 SDValue ADDE8Node = 3612 SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue, 3613 SRDINode, SRADINode, SUBFC8Carry), 0); 3614 return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, 3615 ADDE8Node, S->getI64Imm(1, dl)), 0); 3616 } 3617 case ISD::SETUGE: 3618 // {subc.reg, subc.CA} = (subcarry %a, %b) 3619 // (zext (setcc %a, %b, setuge)) -> (add (sube %b, %b, subc.CA), 1) 3620 std::swap(LHS, RHS); 3621 LLVM_FALLTHROUGH; 3622 case ISD::SETULE: { 3623 // {subc.reg, subc.CA} = (subcarry %b, %a) 3624 // (zext (setcc %a, %b, setule)) -> (add (sube %a, %a, subc.CA), 1) 3625 SDValue SUBFC8Carry = 3626 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, 3627 LHS, RHS), 1); 3628 SDValue SUBFE8Node = 3629 SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, MVT::Glue, 3630 LHS, LHS, SUBFC8Carry), 0); 3631 return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, 3632 SUBFE8Node, S->getI64Imm(1, dl)), 0); 3633 } 3634 case ISD::SETUGT: 3635 // {subc.reg, subc.CA} = (subcarry %b, %a) 3636 // (zext (setcc %a, %b, setugt)) -> -(sube %b, %b, subc.CA) 3637 std::swap(LHS, RHS); 3638 LLVM_FALLTHROUGH; 3639 case ISD::SETULT: { 3640 // {subc.reg, subc.CA} = (subcarry %a, %b) 3641 // (zext (setcc %a, %b, setult)) -> -(sube %a, %a, subc.CA) 3642 SDValue SubtractCarry = 3643 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, 3644 RHS, LHS), 1); 3645 SDValue ExtSub = 3646 SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, 3647 LHS, LHS, SubtractCarry), 0); 3648 return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, 3649 ExtSub), 0); 3650 } 3651 } 3652 } 3653 3654 /// Produces a sign-extended result of comparing two 64-bit values according to 3655 /// the passed condition code. 3656 SDValue 3657 IntegerCompareEliminator::get64BitSExtCompare(SDValue LHS, SDValue RHS, 3658 ISD::CondCode CC, 3659 int64_t RHSValue, SDLoc dl) { 3660 if (CmpInGPR == ICGPR_I32 || CmpInGPR == ICGPR_SextI32 || 3661 CmpInGPR == ICGPR_ZextI32 || CmpInGPR == ICGPR_Zext) 3662 return SDValue(); 3663 bool IsRHSZero = RHSValue == 0; 3664 bool IsRHSOne = RHSValue == 1; 3665 bool IsRHSNegOne = RHSValue == -1LL; 3666 switch (CC) { 3667 default: return SDValue(); 3668 case ISD::SETEQ: { 3669 // {addc.reg, addc.CA} = (addcarry (xor %a, %b), -1) 3670 // (sext (setcc %a, %b, seteq)) -> (sube addc.reg, addc.reg, addc.CA) 3671 // {addcz.reg, addcz.CA} = (addcarry %a, -1) 3672 // (sext (setcc %a, 0, seteq)) -> (sube addcz.reg, addcz.reg, addcz.CA) 3673 SDValue AddInput = IsRHSZero ? LHS : 3674 SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0); 3675 SDValue Addic = 3676 SDValue(CurDAG->getMachineNode(PPC::ADDIC8, dl, MVT::i64, MVT::Glue, 3677 AddInput, S->getI32Imm(~0U, dl)), 0); 3678 return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, Addic, 3679 Addic, Addic.getValue(1)), 0); 3680 } 3681 case ISD::SETNE: { 3682 // {subfc.reg, subfc.CA} = (subcarry 0, (xor %a, %b)) 3683 // (sext (setcc %a, %b, setne)) -> (sube subfc.reg, subfc.reg, subfc.CA) 3684 // {subfcz.reg, subfcz.CA} = (subcarry 0, %a) 3685 // (sext (setcc %a, 0, setne)) -> (sube subfcz.reg, subfcz.reg, subfcz.CA) 3686 SDValue Xor = IsRHSZero ? LHS : 3687 SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0); 3688 SDValue SC = 3689 SDValue(CurDAG->getMachineNode(PPC::SUBFIC8, dl, MVT::i64, MVT::Glue, 3690 Xor, S->getI32Imm(0, dl)), 0); 3691 return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, SC, 3692 SC, SC.getValue(1)), 0); 3693 } 3694 case ISD::SETGE: { 3695 // {subc.reg, subc.CA} = (subcarry %a, %b) 3696 // (zext (setcc %a, %b, setge)) -> 3697 // (- (adde (lshr %b, 63), (ashr %a, 63), subc.CA)) 3698 // (zext (setcc %a, 0, setge)) -> (~ (ashr %a, 63)) 3699 if (IsRHSZero) 3700 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt); 3701 std::swap(LHS, RHS); 3702 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS); 3703 IsRHSZero = RHSConst && RHSConst->isZero(); 3704 LLVM_FALLTHROUGH; 3705 } 3706 case ISD::SETLE: { 3707 // {subc.reg, subc.CA} = (subcarry %b, %a) 3708 // (zext (setcc %a, %b, setge)) -> 3709 // (- (adde (lshr %a, 63), (ashr %b, 63), subc.CA)) 3710 // (zext (setcc %a, 0, setge)) -> (ashr (or %a, (add %a, -1)), 63) 3711 if (IsRHSZero) 3712 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt); 3713 SDValue ShiftR = 3714 SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, RHS, 3715 S->getI64Imm(63, dl)), 0); 3716 SDValue ShiftL = 3717 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS, 3718 S->getI64Imm(1, dl), 3719 S->getI64Imm(63, dl)), 0); 3720 SDValue SubtractCarry = 3721 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, 3722 LHS, RHS), 1); 3723 SDValue Adde = 3724 SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue, 3725 ShiftR, ShiftL, SubtractCarry), 0); 3726 return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, Adde), 0); 3727 } 3728 case ISD::SETGT: { 3729 // {subc.reg, subc.CA} = (subcarry %b, %a) 3730 // (zext (setcc %a, %b, setgt)) -> 3731 // -(xor (adde (lshr %a, 63), (ashr %b, 63), subc.CA), 1) 3732 // (zext (setcc %a, 0, setgt)) -> (ashr (nor (add %a, -1), %a), 63) 3733 if (IsRHSNegOne) 3734 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt); 3735 if (IsRHSZero) { 3736 SDValue Add = 3737 SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS, 3738 S->getI64Imm(-1, dl)), 0); 3739 SDValue Nor = 3740 SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, Add, LHS), 0); 3741 return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, Nor, 3742 S->getI64Imm(63, dl)), 0); 3743 } 3744 std::swap(LHS, RHS); 3745 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS); 3746 IsRHSZero = RHSConst && RHSConst->isZero(); 3747 IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1; 3748 LLVM_FALLTHROUGH; 3749 } 3750 case ISD::SETLT: { 3751 // {subc.reg, subc.CA} = (subcarry %a, %b) 3752 // (zext (setcc %a, %b, setlt)) -> 3753 // -(xor (adde (lshr %b, 63), (ashr %a, 63), subc.CA), 1) 3754 // (zext (setcc %a, 0, setlt)) -> (ashr %a, 63) 3755 if (IsRHSOne) 3756 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt); 3757 if (IsRHSZero) { 3758 return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, LHS, 3759 S->getI64Imm(63, dl)), 0); 3760 } 3761 SDValue SRADINode = 3762 SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, 3763 LHS, S->getI64Imm(63, dl)), 0); 3764 SDValue SRDINode = 3765 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, 3766 RHS, S->getI64Imm(1, dl), 3767 S->getI64Imm(63, dl)), 0); 3768 SDValue SUBFC8Carry = 3769 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, 3770 RHS, LHS), 1); 3771 SDValue ADDE8Node = 3772 SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, 3773 SRDINode, SRADINode, SUBFC8Carry), 0); 3774 SDValue XORI8Node = 3775 SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, 3776 ADDE8Node, S->getI64Imm(1, dl)), 0); 3777 return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, 3778 XORI8Node), 0); 3779 } 3780 case ISD::SETUGE: 3781 // {subc.reg, subc.CA} = (subcarry %a, %b) 3782 // (sext (setcc %a, %b, setuge)) -> ~(sube %b, %b, subc.CA) 3783 std::swap(LHS, RHS); 3784 LLVM_FALLTHROUGH; 3785 case ISD::SETULE: { 3786 // {subc.reg, subc.CA} = (subcarry %b, %a) 3787 // (sext (setcc %a, %b, setule)) -> ~(sube %a, %a, subc.CA) 3788 SDValue SubtractCarry = 3789 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, 3790 LHS, RHS), 1); 3791 SDValue ExtSub = 3792 SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, MVT::Glue, LHS, 3793 LHS, SubtractCarry), 0); 3794 return SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, 3795 ExtSub, ExtSub), 0); 3796 } 3797 case ISD::SETUGT: 3798 // {subc.reg, subc.CA} = (subcarry %b, %a) 3799 // (sext (setcc %a, %b, setugt)) -> (sube %b, %b, subc.CA) 3800 std::swap(LHS, RHS); 3801 LLVM_FALLTHROUGH; 3802 case ISD::SETULT: { 3803 // {subc.reg, subc.CA} = (subcarry %a, %b) 3804 // (sext (setcc %a, %b, setult)) -> (sube %a, %a, subc.CA) 3805 SDValue SubCarry = 3806 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue, 3807 RHS, LHS), 1); 3808 return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, 3809 LHS, LHS, SubCarry), 0); 3810 } 3811 } 3812 } 3813 3814 /// Do all uses of this SDValue need the result in a GPR? 3815 /// This is meant to be used on values that have type i1 since 3816 /// it is somewhat meaningless to ask if values of other types 3817 /// should be kept in GPR's. 3818 static bool allUsesExtend(SDValue Compare, SelectionDAG *CurDAG) { 3819 assert(Compare.getOpcode() == ISD::SETCC && 3820 "An ISD::SETCC node required here."); 3821 3822 // For values that have a single use, the caller should obviously already have 3823 // checked if that use is an extending use. We check the other uses here. 3824 if (Compare.hasOneUse()) 3825 return true; 3826 // We want the value in a GPR if it is being extended, used for a select, or 3827 // used in logical operations. 3828 for (auto CompareUse : Compare.getNode()->uses()) 3829 if (CompareUse->getOpcode() != ISD::SIGN_EXTEND && 3830 CompareUse->getOpcode() != ISD::ZERO_EXTEND && 3831 CompareUse->getOpcode() != ISD::SELECT && 3832 !isLogicOp(CompareUse->getOpcode())) { 3833 OmittedForNonExtendUses++; 3834 return false; 3835 } 3836 return true; 3837 } 3838 3839 /// Returns an equivalent of a SETCC node but with the result the same width as 3840 /// the inputs. This can also be used for SELECT_CC if either the true or false 3841 /// values is a power of two while the other is zero. 3842 SDValue IntegerCompareEliminator::getSETCCInGPR(SDValue Compare, 3843 SetccInGPROpts ConvOpts) { 3844 assert((Compare.getOpcode() == ISD::SETCC || 3845 Compare.getOpcode() == ISD::SELECT_CC) && 3846 "An ISD::SETCC node required here."); 3847 3848 // Don't convert this comparison to a GPR sequence because there are uses 3849 // of the i1 result (i.e. uses that require the result in the CR). 3850 if ((Compare.getOpcode() == ISD::SETCC) && !allUsesExtend(Compare, CurDAG)) 3851 return SDValue(); 3852 3853 SDValue LHS = Compare.getOperand(0); 3854 SDValue RHS = Compare.getOperand(1); 3855 3856 // The condition code is operand 2 for SETCC and operand 4 for SELECT_CC. 3857 int CCOpNum = Compare.getOpcode() == ISD::SELECT_CC ? 4 : 2; 3858 ISD::CondCode CC = 3859 cast<CondCodeSDNode>(Compare.getOperand(CCOpNum))->get(); 3860 EVT InputVT = LHS.getValueType(); 3861 if (InputVT != MVT::i32 && InputVT != MVT::i64) 3862 return SDValue(); 3863 3864 if (ConvOpts == SetccInGPROpts::ZExtInvert || 3865 ConvOpts == SetccInGPROpts::SExtInvert) 3866 CC = ISD::getSetCCInverse(CC, InputVT); 3867 3868 bool Inputs32Bit = InputVT == MVT::i32; 3869 3870 SDLoc dl(Compare); 3871 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS); 3872 int64_t RHSValue = RHSConst ? RHSConst->getSExtValue() : INT64_MAX; 3873 bool IsSext = ConvOpts == SetccInGPROpts::SExtOrig || 3874 ConvOpts == SetccInGPROpts::SExtInvert; 3875 3876 if (IsSext && Inputs32Bit) 3877 return get32BitSExtCompare(LHS, RHS, CC, RHSValue, dl); 3878 else if (Inputs32Bit) 3879 return get32BitZExtCompare(LHS, RHS, CC, RHSValue, dl); 3880 else if (IsSext) 3881 return get64BitSExtCompare(LHS, RHS, CC, RHSValue, dl); 3882 return get64BitZExtCompare(LHS, RHS, CC, RHSValue, dl); 3883 } 3884 3885 } // end anonymous namespace 3886 3887 bool PPCDAGToDAGISel::tryIntCompareInGPR(SDNode *N) { 3888 if (N->getValueType(0) != MVT::i32 && 3889 N->getValueType(0) != MVT::i64) 3890 return false; 3891 3892 // This optimization will emit code that assumes 64-bit registers 3893 // so we don't want to run it in 32-bit mode. Also don't run it 3894 // on functions that are not to be optimized. 3895 if (TM.getOptLevel() == CodeGenOpt::None || !TM.isPPC64()) 3896 return false; 3897 3898 // For POWER10, it is more profitable to use the set boolean extension 3899 // instructions rather than the integer compare elimination codegen. 3900 // Users can override this via the command line option, `--ppc-gpr-icmps`. 3901 if (!(CmpInGPR.getNumOccurrences() > 0) && Subtarget->isISA3_1()) 3902 return false; 3903 3904 switch (N->getOpcode()) { 3905 default: break; 3906 case ISD::ZERO_EXTEND: 3907 case ISD::SIGN_EXTEND: 3908 case ISD::AND: 3909 case ISD::OR: 3910 case ISD::XOR: { 3911 IntegerCompareEliminator ICmpElim(CurDAG, this); 3912 if (SDNode *New = ICmpElim.Select(N)) { 3913 ReplaceNode(N, New); 3914 return true; 3915 } 3916 } 3917 } 3918 return false; 3919 } 3920 3921 bool PPCDAGToDAGISel::tryBitPermutation(SDNode *N) { 3922 if (N->getValueType(0) != MVT::i32 && 3923 N->getValueType(0) != MVT::i64) 3924 return false; 3925 3926 if (!UseBitPermRewriter) 3927 return false; 3928 3929 switch (N->getOpcode()) { 3930 default: break; 3931 case ISD::ROTL: 3932 case ISD::SHL: 3933 case ISD::SRL: 3934 case ISD::AND: 3935 case ISD::OR: { 3936 BitPermutationSelector BPS(CurDAG); 3937 if (SDNode *New = BPS.Select(N)) { 3938 ReplaceNode(N, New); 3939 return true; 3940 } 3941 return false; 3942 } 3943 } 3944 3945 return false; 3946 } 3947 3948 /// SelectCC - Select a comparison of the specified values with the specified 3949 /// condition code, returning the CR# of the expression. 3950 SDValue PPCDAGToDAGISel::SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, 3951 const SDLoc &dl, SDValue Chain) { 3952 // Always select the LHS. 3953 unsigned Opc; 3954 3955 if (LHS.getValueType() == MVT::i32) { 3956 unsigned Imm; 3957 if (CC == ISD::SETEQ || CC == ISD::SETNE) { 3958 if (isInt32Immediate(RHS, Imm)) { 3959 // SETEQ/SETNE comparison with 16-bit immediate, fold it. 3960 if (isUInt<16>(Imm)) 3961 return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS, 3962 getI32Imm(Imm & 0xFFFF, dl)), 3963 0); 3964 // If this is a 16-bit signed immediate, fold it. 3965 if (isInt<16>((int)Imm)) 3966 return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS, 3967 getI32Imm(Imm & 0xFFFF, dl)), 3968 0); 3969 3970 // For non-equality comparisons, the default code would materialize the 3971 // constant, then compare against it, like this: 3972 // lis r2, 4660 3973 // ori r2, r2, 22136 3974 // cmpw cr0, r3, r2 3975 // Since we are just comparing for equality, we can emit this instead: 3976 // xoris r0,r3,0x1234 3977 // cmplwi cr0,r0,0x5678 3978 // beq cr0,L6 3979 SDValue Xor(CurDAG->getMachineNode(PPC::XORIS, dl, MVT::i32, LHS, 3980 getI32Imm(Imm >> 16, dl)), 0); 3981 return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, Xor, 3982 getI32Imm(Imm & 0xFFFF, dl)), 0); 3983 } 3984 Opc = PPC::CMPLW; 3985 } else if (ISD::isUnsignedIntSetCC(CC)) { 3986 if (isInt32Immediate(RHS, Imm) && isUInt<16>(Imm)) 3987 return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS, 3988 getI32Imm(Imm & 0xFFFF, dl)), 0); 3989 Opc = PPC::CMPLW; 3990 } else { 3991 int16_t SImm; 3992 if (isIntS16Immediate(RHS, SImm)) 3993 return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS, 3994 getI32Imm((int)SImm & 0xFFFF, 3995 dl)), 3996 0); 3997 Opc = PPC::CMPW; 3998 } 3999 } else if (LHS.getValueType() == MVT::i64) { 4000 uint64_t Imm; 4001 if (CC == ISD::SETEQ || CC == ISD::SETNE) { 4002 if (isInt64Immediate(RHS.getNode(), Imm)) { 4003 // SETEQ/SETNE comparison with 16-bit immediate, fold it. 4004 if (isUInt<16>(Imm)) 4005 return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS, 4006 getI32Imm(Imm & 0xFFFF, dl)), 4007 0); 4008 // If this is a 16-bit signed immediate, fold it. 4009 if (isInt<16>(Imm)) 4010 return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS, 4011 getI32Imm(Imm & 0xFFFF, dl)), 4012 0); 4013 4014 // For non-equality comparisons, the default code would materialize the 4015 // constant, then compare against it, like this: 4016 // lis r2, 4660 4017 // ori r2, r2, 22136 4018 // cmpd cr0, r3, r2 4019 // Since we are just comparing for equality, we can emit this instead: 4020 // xoris r0,r3,0x1234 4021 // cmpldi cr0,r0,0x5678 4022 // beq cr0,L6 4023 if (isUInt<32>(Imm)) { 4024 SDValue Xor(CurDAG->getMachineNode(PPC::XORIS8, dl, MVT::i64, LHS, 4025 getI64Imm(Imm >> 16, dl)), 0); 4026 return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, Xor, 4027 getI64Imm(Imm & 0xFFFF, dl)), 4028 0); 4029 } 4030 } 4031 Opc = PPC::CMPLD; 4032 } else if (ISD::isUnsignedIntSetCC(CC)) { 4033 if (isInt64Immediate(RHS.getNode(), Imm) && isUInt<16>(Imm)) 4034 return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS, 4035 getI64Imm(Imm & 0xFFFF, dl)), 0); 4036 Opc = PPC::CMPLD; 4037 } else { 4038 int16_t SImm; 4039 if (isIntS16Immediate(RHS, SImm)) 4040 return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS, 4041 getI64Imm(SImm & 0xFFFF, dl)), 4042 0); 4043 Opc = PPC::CMPD; 4044 } 4045 } else if (LHS.getValueType() == MVT::f32) { 4046 if (Subtarget->hasSPE()) { 4047 switch (CC) { 4048 default: 4049 case ISD::SETEQ: 4050 case ISD::SETNE: 4051 Opc = PPC::EFSCMPEQ; 4052 break; 4053 case ISD::SETLT: 4054 case ISD::SETGE: 4055 case ISD::SETOLT: 4056 case ISD::SETOGE: 4057 case ISD::SETULT: 4058 case ISD::SETUGE: 4059 Opc = PPC::EFSCMPLT; 4060 break; 4061 case ISD::SETGT: 4062 case ISD::SETLE: 4063 case ISD::SETOGT: 4064 case ISD::SETOLE: 4065 case ISD::SETUGT: 4066 case ISD::SETULE: 4067 Opc = PPC::EFSCMPGT; 4068 break; 4069 } 4070 } else 4071 Opc = PPC::FCMPUS; 4072 } else if (LHS.getValueType() == MVT::f64) { 4073 if (Subtarget->hasSPE()) { 4074 switch (CC) { 4075 default: 4076 case ISD::SETEQ: 4077 case ISD::SETNE: 4078 Opc = PPC::EFDCMPEQ; 4079 break; 4080 case ISD::SETLT: 4081 case ISD::SETGE: 4082 case ISD::SETOLT: 4083 case ISD::SETOGE: 4084 case ISD::SETULT: 4085 case ISD::SETUGE: 4086 Opc = PPC::EFDCMPLT; 4087 break; 4088 case ISD::SETGT: 4089 case ISD::SETLE: 4090 case ISD::SETOGT: 4091 case ISD::SETOLE: 4092 case ISD::SETUGT: 4093 case ISD::SETULE: 4094 Opc = PPC::EFDCMPGT; 4095 break; 4096 } 4097 } else 4098 Opc = Subtarget->hasVSX() ? PPC::XSCMPUDP : PPC::FCMPUD; 4099 } else { 4100 assert(LHS.getValueType() == MVT::f128 && "Unknown vt!"); 4101 assert(Subtarget->hasP9Vector() && "XSCMPUQP requires Power9 Vector"); 4102 Opc = PPC::XSCMPUQP; 4103 } 4104 if (Chain) 4105 return SDValue( 4106 CurDAG->getMachineNode(Opc, dl, MVT::i32, MVT::Other, LHS, RHS, Chain), 4107 0); 4108 else 4109 return SDValue(CurDAG->getMachineNode(Opc, dl, MVT::i32, LHS, RHS), 0); 4110 } 4111 4112 static PPC::Predicate getPredicateForSetCC(ISD::CondCode CC, const EVT &VT, 4113 const PPCSubtarget *Subtarget) { 4114 // For SPE instructions, the result is in GT bit of the CR 4115 bool UseSPE = Subtarget->hasSPE() && VT.isFloatingPoint(); 4116 4117 switch (CC) { 4118 case ISD::SETUEQ: 4119 case ISD::SETONE: 4120 case ISD::SETOLE: 4121 case ISD::SETOGE: 4122 llvm_unreachable("Should be lowered by legalize!"); 4123 default: llvm_unreachable("Unknown condition!"); 4124 case ISD::SETOEQ: 4125 case ISD::SETEQ: 4126 return UseSPE ? PPC::PRED_GT : PPC::PRED_EQ; 4127 case ISD::SETUNE: 4128 case ISD::SETNE: 4129 return UseSPE ? PPC::PRED_LE : PPC::PRED_NE; 4130 case ISD::SETOLT: 4131 case ISD::SETLT: 4132 return UseSPE ? PPC::PRED_GT : PPC::PRED_LT; 4133 case ISD::SETULE: 4134 case ISD::SETLE: 4135 return PPC::PRED_LE; 4136 case ISD::SETOGT: 4137 case ISD::SETGT: 4138 return PPC::PRED_GT; 4139 case ISD::SETUGE: 4140 case ISD::SETGE: 4141 return UseSPE ? PPC::PRED_LE : PPC::PRED_GE; 4142 case ISD::SETO: return PPC::PRED_NU; 4143 case ISD::SETUO: return PPC::PRED_UN; 4144 // These two are invalid for floating point. Assume we have int. 4145 case ISD::SETULT: return PPC::PRED_LT; 4146 case ISD::SETUGT: return PPC::PRED_GT; 4147 } 4148 } 4149 4150 /// getCRIdxForSetCC - Return the index of the condition register field 4151 /// associated with the SetCC condition, and whether or not the field is 4152 /// treated as inverted. That is, lt = 0; ge = 0 inverted. 4153 static unsigned getCRIdxForSetCC(ISD::CondCode CC, bool &Invert) { 4154 Invert = false; 4155 switch (CC) { 4156 default: llvm_unreachable("Unknown condition!"); 4157 case ISD::SETOLT: 4158 case ISD::SETLT: return 0; // Bit #0 = SETOLT 4159 case ISD::SETOGT: 4160 case ISD::SETGT: return 1; // Bit #1 = SETOGT 4161 case ISD::SETOEQ: 4162 case ISD::SETEQ: return 2; // Bit #2 = SETOEQ 4163 case ISD::SETUO: return 3; // Bit #3 = SETUO 4164 case ISD::SETUGE: 4165 case ISD::SETGE: Invert = true; return 0; // !Bit #0 = SETUGE 4166 case ISD::SETULE: 4167 case ISD::SETLE: Invert = true; return 1; // !Bit #1 = SETULE 4168 case ISD::SETUNE: 4169 case ISD::SETNE: Invert = true; return 2; // !Bit #2 = SETUNE 4170 case ISD::SETO: Invert = true; return 3; // !Bit #3 = SETO 4171 case ISD::SETUEQ: 4172 case ISD::SETOGE: 4173 case ISD::SETOLE: 4174 case ISD::SETONE: 4175 llvm_unreachable("Invalid branch code: should be expanded by legalize"); 4176 // These are invalid for floating point. Assume integer. 4177 case ISD::SETULT: return 0; 4178 case ISD::SETUGT: return 1; 4179 } 4180 } 4181 4182 // getVCmpInst: return the vector compare instruction for the specified 4183 // vector type and condition code. Since this is for altivec specific code, 4184 // only support the altivec types (v16i8, v8i16, v4i32, v2i64, v1i128, 4185 // and v4f32). 4186 static unsigned int getVCmpInst(MVT VecVT, ISD::CondCode CC, 4187 bool HasVSX, bool &Swap, bool &Negate) { 4188 Swap = false; 4189 Negate = false; 4190 4191 if (VecVT.isFloatingPoint()) { 4192 /* Handle some cases by swapping input operands. */ 4193 switch (CC) { 4194 case ISD::SETLE: CC = ISD::SETGE; Swap = true; break; 4195 case ISD::SETLT: CC = ISD::SETGT; Swap = true; break; 4196 case ISD::SETOLE: CC = ISD::SETOGE; Swap = true; break; 4197 case ISD::SETOLT: CC = ISD::SETOGT; Swap = true; break; 4198 case ISD::SETUGE: CC = ISD::SETULE; Swap = true; break; 4199 case ISD::SETUGT: CC = ISD::SETULT; Swap = true; break; 4200 default: break; 4201 } 4202 /* Handle some cases by negating the result. */ 4203 switch (CC) { 4204 case ISD::SETNE: CC = ISD::SETEQ; Negate = true; break; 4205 case ISD::SETUNE: CC = ISD::SETOEQ; Negate = true; break; 4206 case ISD::SETULE: CC = ISD::SETOGT; Negate = true; break; 4207 case ISD::SETULT: CC = ISD::SETOGE; Negate = true; break; 4208 default: break; 4209 } 4210 /* We have instructions implementing the remaining cases. */ 4211 switch (CC) { 4212 case ISD::SETEQ: 4213 case ISD::SETOEQ: 4214 if (VecVT == MVT::v4f32) 4215 return HasVSX ? PPC::XVCMPEQSP : PPC::VCMPEQFP; 4216 else if (VecVT == MVT::v2f64) 4217 return PPC::XVCMPEQDP; 4218 break; 4219 case ISD::SETGT: 4220 case ISD::SETOGT: 4221 if (VecVT == MVT::v4f32) 4222 return HasVSX ? PPC::XVCMPGTSP : PPC::VCMPGTFP; 4223 else if (VecVT == MVT::v2f64) 4224 return PPC::XVCMPGTDP; 4225 break; 4226 case ISD::SETGE: 4227 case ISD::SETOGE: 4228 if (VecVT == MVT::v4f32) 4229 return HasVSX ? PPC::XVCMPGESP : PPC::VCMPGEFP; 4230 else if (VecVT == MVT::v2f64) 4231 return PPC::XVCMPGEDP; 4232 break; 4233 default: 4234 break; 4235 } 4236 llvm_unreachable("Invalid floating-point vector compare condition"); 4237 } else { 4238 /* Handle some cases by swapping input operands. */ 4239 switch (CC) { 4240 case ISD::SETGE: CC = ISD::SETLE; Swap = true; break; 4241 case ISD::SETLT: CC = ISD::SETGT; Swap = true; break; 4242 case ISD::SETUGE: CC = ISD::SETULE; Swap = true; break; 4243 case ISD::SETULT: CC = ISD::SETUGT; Swap = true; break; 4244 default: break; 4245 } 4246 /* Handle some cases by negating the result. */ 4247 switch (CC) { 4248 case ISD::SETNE: CC = ISD::SETEQ; Negate = true; break; 4249 case ISD::SETUNE: CC = ISD::SETUEQ; Negate = true; break; 4250 case ISD::SETLE: CC = ISD::SETGT; Negate = true; break; 4251 case ISD::SETULE: CC = ISD::SETUGT; Negate = true; break; 4252 default: break; 4253 } 4254 /* We have instructions implementing the remaining cases. */ 4255 switch (CC) { 4256 case ISD::SETEQ: 4257 case ISD::SETUEQ: 4258 if (VecVT == MVT::v16i8) 4259 return PPC::VCMPEQUB; 4260 else if (VecVT == MVT::v8i16) 4261 return PPC::VCMPEQUH; 4262 else if (VecVT == MVT::v4i32) 4263 return PPC::VCMPEQUW; 4264 else if (VecVT == MVT::v2i64) 4265 return PPC::VCMPEQUD; 4266 else if (VecVT == MVT::v1i128) 4267 return PPC::VCMPEQUQ; 4268 break; 4269 case ISD::SETGT: 4270 if (VecVT == MVT::v16i8) 4271 return PPC::VCMPGTSB; 4272 else if (VecVT == MVT::v8i16) 4273 return PPC::VCMPGTSH; 4274 else if (VecVT == MVT::v4i32) 4275 return PPC::VCMPGTSW; 4276 else if (VecVT == MVT::v2i64) 4277 return PPC::VCMPGTSD; 4278 else if (VecVT == MVT::v1i128) 4279 return PPC::VCMPGTSQ; 4280 break; 4281 case ISD::SETUGT: 4282 if (VecVT == MVT::v16i8) 4283 return PPC::VCMPGTUB; 4284 else if (VecVT == MVT::v8i16) 4285 return PPC::VCMPGTUH; 4286 else if (VecVT == MVT::v4i32) 4287 return PPC::VCMPGTUW; 4288 else if (VecVT == MVT::v2i64) 4289 return PPC::VCMPGTUD; 4290 else if (VecVT == MVT::v1i128) 4291 return PPC::VCMPGTUQ; 4292 break; 4293 default: 4294 break; 4295 } 4296 llvm_unreachable("Invalid integer vector compare condition"); 4297 } 4298 } 4299 4300 bool PPCDAGToDAGISel::trySETCC(SDNode *N) { 4301 SDLoc dl(N); 4302 unsigned Imm; 4303 bool IsStrict = N->isStrictFPOpcode(); 4304 ISD::CondCode CC = 4305 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get(); 4306 EVT PtrVT = 4307 CurDAG->getTargetLoweringInfo().getPointerTy(CurDAG->getDataLayout()); 4308 bool isPPC64 = (PtrVT == MVT::i64); 4309 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 4310 4311 SDValue LHS = N->getOperand(IsStrict ? 1 : 0); 4312 SDValue RHS = N->getOperand(IsStrict ? 2 : 1); 4313 4314 if (!IsStrict && !Subtarget->useCRBits() && isInt32Immediate(RHS, Imm)) { 4315 // We can codegen setcc op, imm very efficiently compared to a brcond. 4316 // Check for those cases here. 4317 // setcc op, 0 4318 if (Imm == 0) { 4319 SDValue Op = LHS; 4320 switch (CC) { 4321 default: break; 4322 case ISD::SETEQ: { 4323 Op = SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Op), 0); 4324 SDValue Ops[] = { Op, getI32Imm(27, dl), getI32Imm(5, dl), 4325 getI32Imm(31, dl) }; 4326 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); 4327 return true; 4328 } 4329 case ISD::SETNE: { 4330 if (isPPC64) break; 4331 SDValue AD = 4332 SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, 4333 Op, getI32Imm(~0U, dl)), 0); 4334 CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, AD, Op, AD.getValue(1)); 4335 return true; 4336 } 4337 case ISD::SETLT: { 4338 SDValue Ops[] = { Op, getI32Imm(1, dl), getI32Imm(31, dl), 4339 getI32Imm(31, dl) }; 4340 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); 4341 return true; 4342 } 4343 case ISD::SETGT: { 4344 SDValue T = 4345 SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Op), 0); 4346 T = SDValue(CurDAG->getMachineNode(PPC::ANDC, dl, MVT::i32, T, Op), 0); 4347 SDValue Ops[] = { T, getI32Imm(1, dl), getI32Imm(31, dl), 4348 getI32Imm(31, dl) }; 4349 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); 4350 return true; 4351 } 4352 } 4353 } else if (Imm == ~0U) { // setcc op, -1 4354 SDValue Op = LHS; 4355 switch (CC) { 4356 default: break; 4357 case ISD::SETEQ: 4358 if (isPPC64) break; 4359 Op = SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, 4360 Op, getI32Imm(1, dl)), 0); 4361 CurDAG->SelectNodeTo(N, PPC::ADDZE, MVT::i32, 4362 SDValue(CurDAG->getMachineNode(PPC::LI, dl, 4363 MVT::i32, 4364 getI32Imm(0, dl)), 4365 0), Op.getValue(1)); 4366 return true; 4367 case ISD::SETNE: { 4368 if (isPPC64) break; 4369 Op = SDValue(CurDAG->getMachineNode(PPC::NOR, dl, MVT::i32, Op, Op), 0); 4370 SDNode *AD = CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, 4371 Op, getI32Imm(~0U, dl)); 4372 CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, SDValue(AD, 0), Op, 4373 SDValue(AD, 1)); 4374 return true; 4375 } 4376 case ISD::SETLT: { 4377 SDValue AD = SDValue(CurDAG->getMachineNode(PPC::ADDI, dl, MVT::i32, Op, 4378 getI32Imm(1, dl)), 0); 4379 SDValue AN = SDValue(CurDAG->getMachineNode(PPC::AND, dl, MVT::i32, AD, 4380 Op), 0); 4381 SDValue Ops[] = { AN, getI32Imm(1, dl), getI32Imm(31, dl), 4382 getI32Imm(31, dl) }; 4383 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); 4384 return true; 4385 } 4386 case ISD::SETGT: { 4387 SDValue Ops[] = { Op, getI32Imm(1, dl), getI32Imm(31, dl), 4388 getI32Imm(31, dl) }; 4389 Op = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); 4390 CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Op, getI32Imm(1, dl)); 4391 return true; 4392 } 4393 } 4394 } 4395 } 4396 4397 // Altivec Vector compare instructions do not set any CR register by default and 4398 // vector compare operations return the same type as the operands. 4399 if (!IsStrict && LHS.getValueType().isVector()) { 4400 if (Subtarget->hasSPE()) 4401 return false; 4402 4403 EVT VecVT = LHS.getValueType(); 4404 bool Swap, Negate; 4405 unsigned int VCmpInst = 4406 getVCmpInst(VecVT.getSimpleVT(), CC, Subtarget->hasVSX(), Swap, Negate); 4407 if (Swap) 4408 std::swap(LHS, RHS); 4409 4410 EVT ResVT = VecVT.changeVectorElementTypeToInteger(); 4411 if (Negate) { 4412 SDValue VCmp(CurDAG->getMachineNode(VCmpInst, dl, ResVT, LHS, RHS), 0); 4413 CurDAG->SelectNodeTo(N, Subtarget->hasVSX() ? PPC::XXLNOR : PPC::VNOR, 4414 ResVT, VCmp, VCmp); 4415 return true; 4416 } 4417 4418 CurDAG->SelectNodeTo(N, VCmpInst, ResVT, LHS, RHS); 4419 return true; 4420 } 4421 4422 if (Subtarget->useCRBits()) 4423 return false; 4424 4425 bool Inv; 4426 unsigned Idx = getCRIdxForSetCC(CC, Inv); 4427 SDValue CCReg = SelectCC(LHS, RHS, CC, dl, Chain); 4428 if (IsStrict) 4429 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 1), CCReg.getValue(1)); 4430 SDValue IntCR; 4431 4432 // SPE e*cmp* instructions only set the 'gt' bit, so hard-code that 4433 // The correct compare instruction is already set by SelectCC() 4434 if (Subtarget->hasSPE() && LHS.getValueType().isFloatingPoint()) { 4435 Idx = 1; 4436 } 4437 4438 // Force the ccreg into CR7. 4439 SDValue CR7Reg = CurDAG->getRegister(PPC::CR7, MVT::i32); 4440 4441 SDValue InFlag; // Null incoming flag value. 4442 CCReg = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, CR7Reg, CCReg, 4443 InFlag).getValue(1); 4444 4445 IntCR = SDValue(CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, CR7Reg, 4446 CCReg), 0); 4447 4448 SDValue Ops[] = { IntCR, getI32Imm((32 - (3 - Idx)) & 31, dl), 4449 getI32Imm(31, dl), getI32Imm(31, dl) }; 4450 if (!Inv) { 4451 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); 4452 return true; 4453 } 4454 4455 // Get the specified bit. 4456 SDValue Tmp = 4457 SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); 4458 CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Tmp, getI32Imm(1, dl)); 4459 return true; 4460 } 4461 4462 /// Does this node represent a load/store node whose address can be represented 4463 /// with a register plus an immediate that's a multiple of \p Val: 4464 bool PPCDAGToDAGISel::isOffsetMultipleOf(SDNode *N, unsigned Val) const { 4465 LoadSDNode *LDN = dyn_cast<LoadSDNode>(N); 4466 StoreSDNode *STN = dyn_cast<StoreSDNode>(N); 4467 MemIntrinsicSDNode *MIN = dyn_cast<MemIntrinsicSDNode>(N); 4468 SDValue AddrOp; 4469 if (LDN || (MIN && MIN->getOpcode() == PPCISD::LD_SPLAT)) 4470 AddrOp = N->getOperand(1); 4471 else if (STN) 4472 AddrOp = STN->getOperand(2); 4473 4474 // If the address points a frame object or a frame object with an offset, 4475 // we need to check the object alignment. 4476 short Imm = 0; 4477 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>( 4478 AddrOp.getOpcode() == ISD::ADD ? AddrOp.getOperand(0) : 4479 AddrOp)) { 4480 // If op0 is a frame index that is under aligned, we can't do it either, 4481 // because it is translated to r31 or r1 + slot + offset. We won't know the 4482 // slot number until the stack frame is finalized. 4483 const MachineFrameInfo &MFI = CurDAG->getMachineFunction().getFrameInfo(); 4484 unsigned SlotAlign = MFI.getObjectAlign(FI->getIndex()).value(); 4485 if ((SlotAlign % Val) != 0) 4486 return false; 4487 4488 // If we have an offset, we need further check on the offset. 4489 if (AddrOp.getOpcode() != ISD::ADD) 4490 return true; 4491 } 4492 4493 if (AddrOp.getOpcode() == ISD::ADD) 4494 return isIntS16Immediate(AddrOp.getOperand(1), Imm) && !(Imm % Val); 4495 4496 // If the address comes from the outside, the offset will be zero. 4497 return AddrOp.getOpcode() == ISD::CopyFromReg; 4498 } 4499 4500 void PPCDAGToDAGISel::transferMemOperands(SDNode *N, SDNode *Result) { 4501 // Transfer memoperands. 4502 MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand(); 4503 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {MemOp}); 4504 } 4505 4506 static bool mayUseP9Setb(SDNode *N, const ISD::CondCode &CC, SelectionDAG *DAG, 4507 bool &NeedSwapOps, bool &IsUnCmp) { 4508 4509 assert(N->getOpcode() == ISD::SELECT_CC && "Expecting a SELECT_CC here."); 4510 4511 SDValue LHS = N->getOperand(0); 4512 SDValue RHS = N->getOperand(1); 4513 SDValue TrueRes = N->getOperand(2); 4514 SDValue FalseRes = N->getOperand(3); 4515 ConstantSDNode *TrueConst = dyn_cast<ConstantSDNode>(TrueRes); 4516 if (!TrueConst || (N->getSimpleValueType(0) != MVT::i64 && 4517 N->getSimpleValueType(0) != MVT::i32)) 4518 return false; 4519 4520 // We are looking for any of: 4521 // (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, cc2)), cc1) 4522 // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, cc2)), cc1) 4523 // (select_cc lhs, rhs, 0, (select_cc [lr]hs, [lr]hs, 1, -1, cc2), seteq) 4524 // (select_cc lhs, rhs, 0, (select_cc [lr]hs, [lr]hs, -1, 1, cc2), seteq) 4525 int64_t TrueResVal = TrueConst->getSExtValue(); 4526 if ((TrueResVal < -1 || TrueResVal > 1) || 4527 (TrueResVal == -1 && FalseRes.getOpcode() != ISD::ZERO_EXTEND) || 4528 (TrueResVal == 1 && FalseRes.getOpcode() != ISD::SIGN_EXTEND) || 4529 (TrueResVal == 0 && 4530 (FalseRes.getOpcode() != ISD::SELECT_CC || CC != ISD::SETEQ))) 4531 return false; 4532 4533 SDValue SetOrSelCC = FalseRes.getOpcode() == ISD::SELECT_CC 4534 ? FalseRes 4535 : FalseRes.getOperand(0); 4536 bool InnerIsSel = SetOrSelCC.getOpcode() == ISD::SELECT_CC; 4537 if (SetOrSelCC.getOpcode() != ISD::SETCC && 4538 SetOrSelCC.getOpcode() != ISD::SELECT_CC) 4539 return false; 4540 4541 // Without this setb optimization, the outer SELECT_CC will be manually 4542 // selected to SELECT_CC_I4/SELECT_CC_I8 Pseudo, then expand-isel-pseudos pass 4543 // transforms pseudo instruction to isel instruction. When there are more than 4544 // one use for result like zext/sext, with current optimization we only see 4545 // isel is replaced by setb but can't see any significant gain. Since 4546 // setb has longer latency than original isel, we should avoid this. Another 4547 // point is that setb requires comparison always kept, it can break the 4548 // opportunity to get the comparison away if we have in future. 4549 if (!SetOrSelCC.hasOneUse() || (!InnerIsSel && !FalseRes.hasOneUse())) 4550 return false; 4551 4552 SDValue InnerLHS = SetOrSelCC.getOperand(0); 4553 SDValue InnerRHS = SetOrSelCC.getOperand(1); 4554 ISD::CondCode InnerCC = 4555 cast<CondCodeSDNode>(SetOrSelCC.getOperand(InnerIsSel ? 4 : 2))->get(); 4556 // If the inner comparison is a select_cc, make sure the true/false values are 4557 // 1/-1 and canonicalize it if needed. 4558 if (InnerIsSel) { 4559 ConstantSDNode *SelCCTrueConst = 4560 dyn_cast<ConstantSDNode>(SetOrSelCC.getOperand(2)); 4561 ConstantSDNode *SelCCFalseConst = 4562 dyn_cast<ConstantSDNode>(SetOrSelCC.getOperand(3)); 4563 if (!SelCCTrueConst || !SelCCFalseConst) 4564 return false; 4565 int64_t SelCCTVal = SelCCTrueConst->getSExtValue(); 4566 int64_t SelCCFVal = SelCCFalseConst->getSExtValue(); 4567 // The values must be -1/1 (requiring a swap) or 1/-1. 4568 if (SelCCTVal == -1 && SelCCFVal == 1) { 4569 std::swap(InnerLHS, InnerRHS); 4570 } else if (SelCCTVal != 1 || SelCCFVal != -1) 4571 return false; 4572 } 4573 4574 // Canonicalize unsigned case 4575 if (InnerCC == ISD::SETULT || InnerCC == ISD::SETUGT) { 4576 IsUnCmp = true; 4577 InnerCC = (InnerCC == ISD::SETULT) ? ISD::SETLT : ISD::SETGT; 4578 } 4579 4580 bool InnerSwapped = false; 4581 if (LHS == InnerRHS && RHS == InnerLHS) 4582 InnerSwapped = true; 4583 else if (LHS != InnerLHS || RHS != InnerRHS) 4584 return false; 4585 4586 switch (CC) { 4587 // (select_cc lhs, rhs, 0, \ 4588 // (select_cc [lr]hs, [lr]hs, 1, -1, setlt/setgt), seteq) 4589 case ISD::SETEQ: 4590 if (!InnerIsSel) 4591 return false; 4592 if (InnerCC != ISD::SETLT && InnerCC != ISD::SETGT) 4593 return false; 4594 NeedSwapOps = (InnerCC == ISD::SETGT) ? InnerSwapped : !InnerSwapped; 4595 break; 4596 4597 // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, setne)), setu?lt) 4598 // (select_cc lhs, rhs, -1, (zext (setcc lhs, rhs, setgt)), setu?lt) 4599 // (select_cc lhs, rhs, -1, (zext (setcc rhs, lhs, setlt)), setu?lt) 4600 // (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, setne)), setu?lt) 4601 // (select_cc lhs, rhs, 1, (sext (setcc lhs, rhs, setgt)), setu?lt) 4602 // (select_cc lhs, rhs, 1, (sext (setcc rhs, lhs, setlt)), setu?lt) 4603 case ISD::SETULT: 4604 if (!IsUnCmp && InnerCC != ISD::SETNE) 4605 return false; 4606 IsUnCmp = true; 4607 LLVM_FALLTHROUGH; 4608 case ISD::SETLT: 4609 if (InnerCC == ISD::SETNE || (InnerCC == ISD::SETGT && !InnerSwapped) || 4610 (InnerCC == ISD::SETLT && InnerSwapped)) 4611 NeedSwapOps = (TrueResVal == 1); 4612 else 4613 return false; 4614 break; 4615 4616 // (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, setne)), setu?gt) 4617 // (select_cc lhs, rhs, 1, (sext (setcc lhs, rhs, setlt)), setu?gt) 4618 // (select_cc lhs, rhs, 1, (sext (setcc rhs, lhs, setgt)), setu?gt) 4619 // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, setne)), setu?gt) 4620 // (select_cc lhs, rhs, -1, (zext (setcc lhs, rhs, setlt)), setu?gt) 4621 // (select_cc lhs, rhs, -1, (zext (setcc rhs, lhs, setgt)), setu?gt) 4622 case ISD::SETUGT: 4623 if (!IsUnCmp && InnerCC != ISD::SETNE) 4624 return false; 4625 IsUnCmp = true; 4626 LLVM_FALLTHROUGH; 4627 case ISD::SETGT: 4628 if (InnerCC == ISD::SETNE || (InnerCC == ISD::SETLT && !InnerSwapped) || 4629 (InnerCC == ISD::SETGT && InnerSwapped)) 4630 NeedSwapOps = (TrueResVal == -1); 4631 else 4632 return false; 4633 break; 4634 4635 default: 4636 return false; 4637 } 4638 4639 LLVM_DEBUG(dbgs() << "Found a node that can be lowered to a SETB: "); 4640 LLVM_DEBUG(N->dump()); 4641 4642 return true; 4643 } 4644 4645 // Return true if it's a software square-root/divide operand. 4646 static bool isSWTestOp(SDValue N) { 4647 if (N.getOpcode() == PPCISD::FTSQRT) 4648 return true; 4649 if (N.getNumOperands() < 1 || !isa<ConstantSDNode>(N.getOperand(0)) || 4650 N.getOpcode() != ISD::INTRINSIC_WO_CHAIN) 4651 return false; 4652 switch (N.getConstantOperandVal(0)) { 4653 case Intrinsic::ppc_vsx_xvtdivdp: 4654 case Intrinsic::ppc_vsx_xvtdivsp: 4655 case Intrinsic::ppc_vsx_xvtsqrtdp: 4656 case Intrinsic::ppc_vsx_xvtsqrtsp: 4657 return true; 4658 } 4659 return false; 4660 } 4661 4662 bool PPCDAGToDAGISel::tryFoldSWTestBRCC(SDNode *N) { 4663 assert(N->getOpcode() == ISD::BR_CC && "ISD::BR_CC is expected."); 4664 // We are looking for following patterns, where `truncate to i1` actually has 4665 // the same semantic with `and 1`. 4666 // (br_cc seteq, (truncateToi1 SWTestOp), 0) -> (BCC PRED_NU, SWTestOp) 4667 // (br_cc seteq, (and SWTestOp, 2), 0) -> (BCC PRED_NE, SWTestOp) 4668 // (br_cc seteq, (and SWTestOp, 4), 0) -> (BCC PRED_LE, SWTestOp) 4669 // (br_cc seteq, (and SWTestOp, 8), 0) -> (BCC PRED_GE, SWTestOp) 4670 // (br_cc setne, (truncateToi1 SWTestOp), 0) -> (BCC PRED_UN, SWTestOp) 4671 // (br_cc setne, (and SWTestOp, 2), 0) -> (BCC PRED_EQ, SWTestOp) 4672 // (br_cc setne, (and SWTestOp, 4), 0) -> (BCC PRED_GT, SWTestOp) 4673 // (br_cc setne, (and SWTestOp, 8), 0) -> (BCC PRED_LT, SWTestOp) 4674 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(1))->get(); 4675 if (CC != ISD::SETEQ && CC != ISD::SETNE) 4676 return false; 4677 4678 SDValue CmpRHS = N->getOperand(3); 4679 if (!isa<ConstantSDNode>(CmpRHS) || 4680 cast<ConstantSDNode>(CmpRHS)->getSExtValue() != 0) 4681 return false; 4682 4683 SDValue CmpLHS = N->getOperand(2); 4684 if (CmpLHS.getNumOperands() < 1 || !isSWTestOp(CmpLHS.getOperand(0))) 4685 return false; 4686 4687 unsigned PCC = 0; 4688 bool IsCCNE = CC == ISD::SETNE; 4689 if (CmpLHS.getOpcode() == ISD::AND && 4690 isa<ConstantSDNode>(CmpLHS.getOperand(1))) 4691 switch (CmpLHS.getConstantOperandVal(1)) { 4692 case 1: 4693 PCC = IsCCNE ? PPC::PRED_UN : PPC::PRED_NU; 4694 break; 4695 case 2: 4696 PCC = IsCCNE ? PPC::PRED_EQ : PPC::PRED_NE; 4697 break; 4698 case 4: 4699 PCC = IsCCNE ? PPC::PRED_GT : PPC::PRED_LE; 4700 break; 4701 case 8: 4702 PCC = IsCCNE ? PPC::PRED_LT : PPC::PRED_GE; 4703 break; 4704 default: 4705 return false; 4706 } 4707 else if (CmpLHS.getOpcode() == ISD::TRUNCATE && 4708 CmpLHS.getValueType() == MVT::i1) 4709 PCC = IsCCNE ? PPC::PRED_UN : PPC::PRED_NU; 4710 4711 if (PCC) { 4712 SDLoc dl(N); 4713 SDValue Ops[] = {getI32Imm(PCC, dl), CmpLHS.getOperand(0), N->getOperand(4), 4714 N->getOperand(0)}; 4715 CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops); 4716 return true; 4717 } 4718 return false; 4719 } 4720 4721 bool PPCDAGToDAGISel::tryAsSingleRLWINM(SDNode *N) { 4722 assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected"); 4723 unsigned Imm; 4724 if (!isInt32Immediate(N->getOperand(1), Imm)) 4725 return false; 4726 4727 SDLoc dl(N); 4728 SDValue Val = N->getOperand(0); 4729 unsigned SH, MB, ME; 4730 // If this is an and of a value rotated between 0 and 31 bits and then and'd 4731 // with a mask, emit rlwinm 4732 if (isRotateAndMask(Val.getNode(), Imm, false, SH, MB, ME)) { 4733 Val = Val.getOperand(0); 4734 SDValue Ops[] = {Val, getI32Imm(SH, dl), getI32Imm(MB, dl), 4735 getI32Imm(ME, dl)}; 4736 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); 4737 return true; 4738 } 4739 4740 // If this is just a masked value where the input is not handled, and 4741 // is not a rotate-left (handled by a pattern in the .td file), emit rlwinm 4742 if (isRunOfOnes(Imm, MB, ME) && Val.getOpcode() != ISD::ROTL) { 4743 SDValue Ops[] = {Val, getI32Imm(0, dl), getI32Imm(MB, dl), 4744 getI32Imm(ME, dl)}; 4745 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); 4746 return true; 4747 } 4748 4749 // AND X, 0 -> 0, not "rlwinm 32". 4750 if (Imm == 0) { 4751 ReplaceUses(SDValue(N, 0), N->getOperand(1)); 4752 return true; 4753 } 4754 4755 return false; 4756 } 4757 4758 bool PPCDAGToDAGISel::tryAsSingleRLWINM8(SDNode *N) { 4759 assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected"); 4760 uint64_t Imm64; 4761 if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64)) 4762 return false; 4763 4764 unsigned MB, ME; 4765 if (isRunOfOnes64(Imm64, MB, ME) && MB >= 32 && MB <= ME) { 4766 // MB ME 4767 // +----------------------+ 4768 // |xxxxxxxxxxx00011111000| 4769 // +----------------------+ 4770 // 0 32 64 4771 // We can only do it if the MB is larger than 32 and MB <= ME 4772 // as RLWINM will replace the contents of [0 - 32) with [32 - 64) even 4773 // we didn't rotate it. 4774 SDLoc dl(N); 4775 SDValue Ops[] = {N->getOperand(0), getI64Imm(0, dl), getI64Imm(MB - 32, dl), 4776 getI64Imm(ME - 32, dl)}; 4777 CurDAG->SelectNodeTo(N, PPC::RLWINM8, MVT::i64, Ops); 4778 return true; 4779 } 4780 4781 return false; 4782 } 4783 4784 bool PPCDAGToDAGISel::tryAsPairOfRLDICL(SDNode *N) { 4785 assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected"); 4786 uint64_t Imm64; 4787 if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64)) 4788 return false; 4789 4790 // Do nothing if it is 16-bit imm as the pattern in the .td file handle 4791 // it well with "andi.". 4792 if (isUInt<16>(Imm64)) 4793 return false; 4794 4795 SDLoc Loc(N); 4796 SDValue Val = N->getOperand(0); 4797 4798 // Optimized with two rldicl's as follows: 4799 // Add missing bits on left to the mask and check that the mask is a 4800 // wrapped run of ones, i.e. 4801 // Change pattern |0001111100000011111111| 4802 // to |1111111100000011111111|. 4803 unsigned NumOfLeadingZeros = countLeadingZeros(Imm64); 4804 if (NumOfLeadingZeros != 0) 4805 Imm64 |= maskLeadingOnes<uint64_t>(NumOfLeadingZeros); 4806 4807 unsigned MB, ME; 4808 if (!isRunOfOnes64(Imm64, MB, ME)) 4809 return false; 4810 4811 // ME MB MB-ME+63 4812 // +----------------------+ +----------------------+ 4813 // |1111111100000011111111| -> |0000001111111111111111| 4814 // +----------------------+ +----------------------+ 4815 // 0 63 0 63 4816 // There are ME + 1 ones on the left and (MB - ME + 63) & 63 zeros in between. 4817 unsigned OnesOnLeft = ME + 1; 4818 unsigned ZerosInBetween = (MB - ME + 63) & 63; 4819 // Rotate left by OnesOnLeft (so leading ones are now trailing ones) and clear 4820 // on the left the bits that are already zeros in the mask. 4821 Val = SDValue(CurDAG->getMachineNode(PPC::RLDICL, Loc, MVT::i64, Val, 4822 getI64Imm(OnesOnLeft, Loc), 4823 getI64Imm(ZerosInBetween, Loc)), 4824 0); 4825 // MB-ME+63 ME MB 4826 // +----------------------+ +----------------------+ 4827 // |0000001111111111111111| -> |0001111100000011111111| 4828 // +----------------------+ +----------------------+ 4829 // 0 63 0 63 4830 // Rotate back by 64 - OnesOnLeft to undo previous rotate. Then clear on the 4831 // left the number of ones we previously added. 4832 SDValue Ops[] = {Val, getI64Imm(64 - OnesOnLeft, Loc), 4833 getI64Imm(NumOfLeadingZeros, Loc)}; 4834 CurDAG->SelectNodeTo(N, PPC::RLDICL, MVT::i64, Ops); 4835 return true; 4836 } 4837 4838 bool PPCDAGToDAGISel::tryAsSingleRLWIMI(SDNode *N) { 4839 assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected"); 4840 unsigned Imm; 4841 if (!isInt32Immediate(N->getOperand(1), Imm)) 4842 return false; 4843 4844 SDValue Val = N->getOperand(0); 4845 unsigned Imm2; 4846 // ISD::OR doesn't get all the bitfield insertion fun. 4847 // (and (or x, c1), c2) where isRunOfOnes(~(c1^c2)) might be a 4848 // bitfield insert. 4849 if (Val.getOpcode() != ISD::OR || !isInt32Immediate(Val.getOperand(1), Imm2)) 4850 return false; 4851 4852 // The idea here is to check whether this is equivalent to: 4853 // (c1 & m) | (x & ~m) 4854 // where m is a run-of-ones mask. The logic here is that, for each bit in 4855 // c1 and c2: 4856 // - if both are 1, then the output will be 1. 4857 // - if both are 0, then the output will be 0. 4858 // - if the bit in c1 is 0, and the bit in c2 is 1, then the output will 4859 // come from x. 4860 // - if the bit in c1 is 1, and the bit in c2 is 0, then the output will 4861 // be 0. 4862 // If that last condition is never the case, then we can form m from the 4863 // bits that are the same between c1 and c2. 4864 unsigned MB, ME; 4865 if (isRunOfOnes(~(Imm ^ Imm2), MB, ME) && !(~Imm & Imm2)) { 4866 SDLoc dl(N); 4867 SDValue Ops[] = {Val.getOperand(0), Val.getOperand(1), getI32Imm(0, dl), 4868 getI32Imm(MB, dl), getI32Imm(ME, dl)}; 4869 ReplaceNode(N, CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops)); 4870 return true; 4871 } 4872 4873 return false; 4874 } 4875 4876 bool PPCDAGToDAGISel::tryAsSingleRLDICL(SDNode *N) { 4877 assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected"); 4878 uint64_t Imm64; 4879 if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64) || !isMask_64(Imm64)) 4880 return false; 4881 4882 // If this is a 64-bit zero-extension mask, emit rldicl. 4883 unsigned MB = 64 - countTrailingOnes(Imm64); 4884 unsigned SH = 0; 4885 unsigned Imm; 4886 SDValue Val = N->getOperand(0); 4887 SDLoc dl(N); 4888 4889 if (Val.getOpcode() == ISD::ANY_EXTEND) { 4890 auto Op0 = Val.getOperand(0); 4891 if (Op0.getOpcode() == ISD::SRL && 4892 isInt32Immediate(Op0.getOperand(1).getNode(), Imm) && Imm <= MB) { 4893 4894 auto ResultType = Val.getNode()->getValueType(0); 4895 auto ImDef = CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, ResultType); 4896 SDValue IDVal(ImDef, 0); 4897 4898 Val = SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl, ResultType, 4899 IDVal, Op0.getOperand(0), 4900 getI32Imm(1, dl)), 4901 0); 4902 SH = 64 - Imm; 4903 } 4904 } 4905 4906 // If the operand is a logical right shift, we can fold it into this 4907 // instruction: rldicl(rldicl(x, 64-n, n), 0, mb) -> rldicl(x, 64-n, mb) 4908 // for n <= mb. The right shift is really a left rotate followed by a 4909 // mask, and this mask is a more-restrictive sub-mask of the mask implied 4910 // by the shift. 4911 if (Val.getOpcode() == ISD::SRL && 4912 isInt32Immediate(Val.getOperand(1).getNode(), Imm) && Imm <= MB) { 4913 assert(Imm < 64 && "Illegal shift amount"); 4914 Val = Val.getOperand(0); 4915 SH = 64 - Imm; 4916 } 4917 4918 SDValue Ops[] = {Val, getI32Imm(SH, dl), getI32Imm(MB, dl)}; 4919 CurDAG->SelectNodeTo(N, PPC::RLDICL, MVT::i64, Ops); 4920 return true; 4921 } 4922 4923 bool PPCDAGToDAGISel::tryAsSingleRLDICR(SDNode *N) { 4924 assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected"); 4925 uint64_t Imm64; 4926 if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64) || 4927 !isMask_64(~Imm64)) 4928 return false; 4929 4930 // If this is a negated 64-bit zero-extension mask, 4931 // i.e. the immediate is a sequence of ones from most significant side 4932 // and all zero for reminder, we should use rldicr. 4933 unsigned MB = 63 - countTrailingOnes(~Imm64); 4934 unsigned SH = 0; 4935 SDLoc dl(N); 4936 SDValue Ops[] = {N->getOperand(0), getI32Imm(SH, dl), getI32Imm(MB, dl)}; 4937 CurDAG->SelectNodeTo(N, PPC::RLDICR, MVT::i64, Ops); 4938 return true; 4939 } 4940 4941 bool PPCDAGToDAGISel::tryAsSingleRLDIMI(SDNode *N) { 4942 assert(N->getOpcode() == ISD::OR && "ISD::OR SDNode expected"); 4943 uint64_t Imm64; 4944 unsigned MB, ME; 4945 SDValue N0 = N->getOperand(0); 4946 4947 // We won't get fewer instructions if the imm is 32-bit integer. 4948 // rldimi requires the imm to have consecutive ones with both sides zero. 4949 // Also, make sure the first Op has only one use, otherwise this may increase 4950 // register pressure since rldimi is destructive. 4951 if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64) || 4952 isUInt<32>(Imm64) || !isRunOfOnes64(Imm64, MB, ME) || !N0.hasOneUse()) 4953 return false; 4954 4955 unsigned SH = 63 - ME; 4956 SDLoc Dl(N); 4957 // Use select64Imm for making LI instr instead of directly putting Imm64 4958 SDValue Ops[] = { 4959 N->getOperand(0), 4960 SDValue(selectI64Imm(CurDAG, getI64Imm(-1, Dl).getNode()), 0), 4961 getI32Imm(SH, Dl), getI32Imm(MB, Dl)}; 4962 CurDAG->SelectNodeTo(N, PPC::RLDIMI, MVT::i64, Ops); 4963 return true; 4964 } 4965 4966 // Select - Convert the specified operand from a target-independent to a 4967 // target-specific node if it hasn't already been changed. 4968 void PPCDAGToDAGISel::Select(SDNode *N) { 4969 SDLoc dl(N); 4970 if (N->isMachineOpcode()) { 4971 N->setNodeId(-1); 4972 return; // Already selected. 4973 } 4974 4975 // In case any misguided DAG-level optimizations form an ADD with a 4976 // TargetConstant operand, crash here instead of miscompiling (by selecting 4977 // an r+r add instead of some kind of r+i add). 4978 if (N->getOpcode() == ISD::ADD && 4979 N->getOperand(1).getOpcode() == ISD::TargetConstant) 4980 llvm_unreachable("Invalid ADD with TargetConstant operand"); 4981 4982 // Try matching complex bit permutations before doing anything else. 4983 if (tryBitPermutation(N)) 4984 return; 4985 4986 // Try to emit integer compares as GPR-only sequences (i.e. no use of CR). 4987 if (tryIntCompareInGPR(N)) 4988 return; 4989 4990 switch (N->getOpcode()) { 4991 default: break; 4992 4993 case ISD::Constant: 4994 if (N->getValueType(0) == MVT::i64) { 4995 ReplaceNode(N, selectI64Imm(CurDAG, N)); 4996 return; 4997 } 4998 break; 4999 5000 case ISD::INTRINSIC_VOID: { 5001 auto IntrinsicID = N->getConstantOperandVal(1); 5002 if (IntrinsicID == Intrinsic::ppc_tdw || IntrinsicID == Intrinsic::ppc_tw) { 5003 unsigned Opcode = IntrinsicID == Intrinsic::ppc_tdw ? PPC::TDI : PPC::TWI; 5004 SDValue Ops[] = {N->getOperand(4), N->getOperand(2), N->getOperand(3)}; 5005 int16_t SImmOperand2; 5006 int16_t SImmOperand3; 5007 int16_t SImmOperand4; 5008 bool isOperand2IntS16Immediate = 5009 isIntS16Immediate(N->getOperand(2), SImmOperand2); 5010 bool isOperand3IntS16Immediate = 5011 isIntS16Immediate(N->getOperand(3), SImmOperand3); 5012 // We will emit PPC::TD or PPC::TW if the 2nd and 3rd operands are reg + 5013 // reg or imm + imm. The imm + imm form will be optimized to either an 5014 // unconditional trap or a nop in a later pass. 5015 if (isOperand2IntS16Immediate == isOperand3IntS16Immediate) 5016 Opcode = IntrinsicID == Intrinsic::ppc_tdw ? PPC::TD : PPC::TW; 5017 else if (isOperand3IntS16Immediate) 5018 // The 2nd and 3rd operands are reg + imm. 5019 Ops[2] = getI32Imm(int(SImmOperand3) & 0xFFFF, dl); 5020 else { 5021 // The 2nd and 3rd operands are imm + reg. 5022 bool isOperand4IntS16Immediate = 5023 isIntS16Immediate(N->getOperand(4), SImmOperand4); 5024 (void)isOperand4IntS16Immediate; 5025 assert(isOperand4IntS16Immediate && 5026 "The 4th operand is not an Immediate"); 5027 // We need to flip the condition immediate TO. 5028 int16_t TO = int(SImmOperand4) & 0x1F; 5029 // We swap the first and second bit of TO if they are not same. 5030 if ((TO & 0x1) != ((TO & 0x2) >> 1)) 5031 TO = (TO & 0x1) ? TO + 1 : TO - 1; 5032 // We swap the fourth and fifth bit of TO if they are not same. 5033 if ((TO & 0x8) != ((TO & 0x10) >> 1)) 5034 TO = (TO & 0x8) ? TO + 8 : TO - 8; 5035 Ops[0] = getI32Imm(TO, dl); 5036 Ops[1] = N->getOperand(3); 5037 Ops[2] = getI32Imm(int(SImmOperand2) & 0xFFFF, dl); 5038 } 5039 CurDAG->SelectNodeTo(N, Opcode, MVT::Other, Ops); 5040 return; 5041 } 5042 break; 5043 } 5044 5045 case ISD::INTRINSIC_WO_CHAIN: { 5046 // We emit the PPC::FSELS instruction here because of type conflicts with 5047 // the comparison operand. The FSELS instruction is defined to use an 8-byte 5048 // comparison like the FSELD version. The fsels intrinsic takes a 4-byte 5049 // value for the comparison. When selecting through a .td file, a type 5050 // error is raised. Must check this first so we never break on the 5051 // !Subtarget->isISA3_1() check. 5052 auto IntID = N->getConstantOperandVal(0); 5053 if (IntID == Intrinsic::ppc_fsels) { 5054 SDValue Ops[] = {N->getOperand(1), N->getOperand(2), N->getOperand(3)}; 5055 CurDAG->SelectNodeTo(N, PPC::FSELS, MVT::f32, Ops); 5056 return; 5057 } 5058 5059 if (IntID == Intrinsic::ppc_bcdadd_p || IntID == Intrinsic::ppc_bcdsub_p) { 5060 auto Pred = N->getConstantOperandVal(1); 5061 unsigned Opcode = 5062 IntID == Intrinsic::ppc_bcdadd_p ? PPC::BCDADD_rec : PPC::BCDSUB_rec; 5063 unsigned SubReg = 0; 5064 unsigned ShiftVal = 0; 5065 bool Reverse = false; 5066 switch (Pred) { 5067 case 0: 5068 SubReg = PPC::sub_eq; 5069 ShiftVal = 1; 5070 break; 5071 case 1: 5072 SubReg = PPC::sub_eq; 5073 ShiftVal = 1; 5074 Reverse = true; 5075 break; 5076 case 2: 5077 SubReg = PPC::sub_lt; 5078 ShiftVal = 3; 5079 break; 5080 case 3: 5081 SubReg = PPC::sub_lt; 5082 ShiftVal = 3; 5083 Reverse = true; 5084 break; 5085 case 4: 5086 SubReg = PPC::sub_gt; 5087 ShiftVal = 2; 5088 break; 5089 case 5: 5090 SubReg = PPC::sub_gt; 5091 ShiftVal = 2; 5092 Reverse = true; 5093 break; 5094 case 6: 5095 SubReg = PPC::sub_un; 5096 break; 5097 case 7: 5098 SubReg = PPC::sub_un; 5099 Reverse = true; 5100 break; 5101 } 5102 5103 EVT VTs[] = {MVT::v16i8, MVT::Glue}; 5104 SDValue Ops[] = {N->getOperand(2), N->getOperand(3), 5105 CurDAG->getTargetConstant(0, dl, MVT::i32)}; 5106 SDValue BCDOp = SDValue(CurDAG->getMachineNode(Opcode, dl, VTs, Ops), 0); 5107 SDValue CR6Reg = CurDAG->getRegister(PPC::CR6, MVT::i32); 5108 // On Power10, we can use SETBC[R]. On prior architectures, we have to use 5109 // MFOCRF and shift/negate the value. 5110 if (Subtarget->isISA3_1()) { 5111 SDValue SubRegIdx = CurDAG->getTargetConstant(SubReg, dl, MVT::i32); 5112 SDValue CRBit = SDValue( 5113 CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, MVT::i1, 5114 CR6Reg, SubRegIdx, BCDOp.getValue(1)), 5115 0); 5116 CurDAG->SelectNodeTo(N, Reverse ? PPC::SETBCR : PPC::SETBC, MVT::i32, 5117 CRBit); 5118 } else { 5119 SDValue Move = 5120 SDValue(CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, CR6Reg, 5121 BCDOp.getValue(1)), 5122 0); 5123 SDValue Ops[] = {Move, getI32Imm((32 - (4 + ShiftVal)) & 31, dl), 5124 getI32Imm(31, dl), getI32Imm(31, dl)}; 5125 if (!Reverse) 5126 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); 5127 else { 5128 SDValue Shift = SDValue( 5129 CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0); 5130 CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Shift, getI32Imm(1, dl)); 5131 } 5132 } 5133 return; 5134 } 5135 5136 if (!Subtarget->isISA3_1()) 5137 break; 5138 unsigned Opcode = 0; 5139 switch (IntID) { 5140 default: 5141 break; 5142 case Intrinsic::ppc_altivec_vstribr_p: 5143 Opcode = PPC::VSTRIBR_rec; 5144 break; 5145 case Intrinsic::ppc_altivec_vstribl_p: 5146 Opcode = PPC::VSTRIBL_rec; 5147 break; 5148 case Intrinsic::ppc_altivec_vstrihr_p: 5149 Opcode = PPC::VSTRIHR_rec; 5150 break; 5151 case Intrinsic::ppc_altivec_vstrihl_p: 5152 Opcode = PPC::VSTRIHL_rec; 5153 break; 5154 } 5155 if (!Opcode) 5156 break; 5157 5158 // Generate the appropriate vector string isolate intrinsic to match. 5159 EVT VTs[] = {MVT::v16i8, MVT::Glue}; 5160 SDValue VecStrOp = 5161 SDValue(CurDAG->getMachineNode(Opcode, dl, VTs, N->getOperand(2)), 0); 5162 // Vector string isolate instructions update the EQ bit of CR6. 5163 // Generate a SETBC instruction to extract the bit and place it in a GPR. 5164 SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_eq, dl, MVT::i32); 5165 SDValue CR6Reg = CurDAG->getRegister(PPC::CR6, MVT::i32); 5166 SDValue CRBit = SDValue( 5167 CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, MVT::i1, 5168 CR6Reg, SubRegIdx, VecStrOp.getValue(1)), 5169 0); 5170 CurDAG->SelectNodeTo(N, PPC::SETBC, MVT::i32, CRBit); 5171 return; 5172 } 5173 5174 case ISD::SETCC: 5175 case ISD::STRICT_FSETCC: 5176 case ISD::STRICT_FSETCCS: 5177 if (trySETCC(N)) 5178 return; 5179 break; 5180 // These nodes will be transformed into GETtlsADDR32 node, which 5181 // later becomes BL_TLS __tls_get_addr(sym at tlsgd)@PLT 5182 case PPCISD::ADDI_TLSLD_L_ADDR: 5183 case PPCISD::ADDI_TLSGD_L_ADDR: { 5184 const Module *Mod = MF->getFunction().getParent(); 5185 if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) != MVT::i32 || 5186 !Subtarget->isSecurePlt() || !Subtarget->isTargetELF() || 5187 Mod->getPICLevel() == PICLevel::SmallPIC) 5188 break; 5189 // Attach global base pointer on GETtlsADDR32 node in order to 5190 // generate secure plt code for TLS symbols. 5191 getGlobalBaseReg(); 5192 } break; 5193 case PPCISD::CALL: { 5194 if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) != MVT::i32 || 5195 !TM.isPositionIndependent() || !Subtarget->isSecurePlt() || 5196 !Subtarget->isTargetELF()) 5197 break; 5198 5199 SDValue Op = N->getOperand(1); 5200 5201 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op)) { 5202 if (GA->getTargetFlags() == PPCII::MO_PLT) 5203 getGlobalBaseReg(); 5204 } 5205 else if (ExternalSymbolSDNode *ES = dyn_cast<ExternalSymbolSDNode>(Op)) { 5206 if (ES->getTargetFlags() == PPCII::MO_PLT) 5207 getGlobalBaseReg(); 5208 } 5209 } 5210 break; 5211 5212 case PPCISD::GlobalBaseReg: 5213 ReplaceNode(N, getGlobalBaseReg()); 5214 return; 5215 5216 case ISD::FrameIndex: 5217 selectFrameIndex(N, N); 5218 return; 5219 5220 case PPCISD::MFOCRF: { 5221 SDValue InFlag = N->getOperand(1); 5222 ReplaceNode(N, CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, 5223 N->getOperand(0), InFlag)); 5224 return; 5225 } 5226 5227 case PPCISD::READ_TIME_BASE: 5228 ReplaceNode(N, CurDAG->getMachineNode(PPC::ReadTB, dl, MVT::i32, MVT::i32, 5229 MVT::Other, N->getOperand(0))); 5230 return; 5231 5232 case PPCISD::SRA_ADDZE: { 5233 SDValue N0 = N->getOperand(0); 5234 SDValue ShiftAmt = 5235 CurDAG->getTargetConstant(*cast<ConstantSDNode>(N->getOperand(1))-> 5236 getConstantIntValue(), dl, 5237 N->getValueType(0)); 5238 if (N->getValueType(0) == MVT::i64) { 5239 SDNode *Op = 5240 CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, MVT::Glue, 5241 N0, ShiftAmt); 5242 CurDAG->SelectNodeTo(N, PPC::ADDZE8, MVT::i64, SDValue(Op, 0), 5243 SDValue(Op, 1)); 5244 return; 5245 } else { 5246 assert(N->getValueType(0) == MVT::i32 && 5247 "Expecting i64 or i32 in PPCISD::SRA_ADDZE"); 5248 SDNode *Op = 5249 CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, MVT::Glue, 5250 N0, ShiftAmt); 5251 CurDAG->SelectNodeTo(N, PPC::ADDZE, MVT::i32, SDValue(Op, 0), 5252 SDValue(Op, 1)); 5253 return; 5254 } 5255 } 5256 5257 case ISD::STORE: { 5258 // Change TLS initial-exec D-form stores to X-form stores. 5259 StoreSDNode *ST = cast<StoreSDNode>(N); 5260 if (EnableTLSOpt && Subtarget->isELFv2ABI() && 5261 ST->getAddressingMode() != ISD::PRE_INC) 5262 if (tryTLSXFormStore(ST)) 5263 return; 5264 break; 5265 } 5266 case ISD::LOAD: { 5267 // Handle preincrement loads. 5268 LoadSDNode *LD = cast<LoadSDNode>(N); 5269 EVT LoadedVT = LD->getMemoryVT(); 5270 5271 // Normal loads are handled by code generated from the .td file. 5272 if (LD->getAddressingMode() != ISD::PRE_INC) { 5273 // Change TLS initial-exec D-form loads to X-form loads. 5274 if (EnableTLSOpt && Subtarget->isELFv2ABI()) 5275 if (tryTLSXFormLoad(LD)) 5276 return; 5277 break; 5278 } 5279 5280 SDValue Offset = LD->getOffset(); 5281 if (Offset.getOpcode() == ISD::TargetConstant || 5282 Offset.getOpcode() == ISD::TargetGlobalAddress) { 5283 5284 unsigned Opcode; 5285 bool isSExt = LD->getExtensionType() == ISD::SEXTLOAD; 5286 if (LD->getValueType(0) != MVT::i64) { 5287 // Handle PPC32 integer and normal FP loads. 5288 assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load"); 5289 switch (LoadedVT.getSimpleVT().SimpleTy) { 5290 default: llvm_unreachable("Invalid PPC load type!"); 5291 case MVT::f64: Opcode = PPC::LFDU; break; 5292 case MVT::f32: Opcode = PPC::LFSU; break; 5293 case MVT::i32: Opcode = PPC::LWZU; break; 5294 case MVT::i16: Opcode = isSExt ? PPC::LHAU : PPC::LHZU; break; 5295 case MVT::i1: 5296 case MVT::i8: Opcode = PPC::LBZU; break; 5297 } 5298 } else { 5299 assert(LD->getValueType(0) == MVT::i64 && "Unknown load result type!"); 5300 assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load"); 5301 switch (LoadedVT.getSimpleVT().SimpleTy) { 5302 default: llvm_unreachable("Invalid PPC load type!"); 5303 case MVT::i64: Opcode = PPC::LDU; break; 5304 case MVT::i32: Opcode = PPC::LWZU8; break; 5305 case MVT::i16: Opcode = isSExt ? PPC::LHAU8 : PPC::LHZU8; break; 5306 case MVT::i1: 5307 case MVT::i8: Opcode = PPC::LBZU8; break; 5308 } 5309 } 5310 5311 SDValue Chain = LD->getChain(); 5312 SDValue Base = LD->getBasePtr(); 5313 SDValue Ops[] = { Offset, Base, Chain }; 5314 SDNode *MN = CurDAG->getMachineNode( 5315 Opcode, dl, LD->getValueType(0), 5316 PPCLowering->getPointerTy(CurDAG->getDataLayout()), MVT::Other, Ops); 5317 transferMemOperands(N, MN); 5318 ReplaceNode(N, MN); 5319 return; 5320 } else { 5321 unsigned Opcode; 5322 bool isSExt = LD->getExtensionType() == ISD::SEXTLOAD; 5323 if (LD->getValueType(0) != MVT::i64) { 5324 // Handle PPC32 integer and normal FP loads. 5325 assert((!isSExt || LoadedVT == MVT::i16) && "Invalid sext update load"); 5326 switch (LoadedVT.getSimpleVT().SimpleTy) { 5327 default: llvm_unreachable("Invalid PPC load type!"); 5328 case MVT::f64: Opcode = PPC::LFDUX; break; 5329 case MVT::f32: Opcode = PPC::LFSUX; break; 5330 case MVT::i32: Opcode = PPC::LWZUX; break; 5331 case MVT::i16: Opcode = isSExt ? PPC::LHAUX : PPC::LHZUX; break; 5332 case MVT::i1: 5333 case MVT::i8: Opcode = PPC::LBZUX; break; 5334 } 5335 } else { 5336 assert(LD->getValueType(0) == MVT::i64 && "Unknown load result type!"); 5337 assert((!isSExt || LoadedVT == MVT::i16 || LoadedVT == MVT::i32) && 5338 "Invalid sext update load"); 5339 switch (LoadedVT.getSimpleVT().SimpleTy) { 5340 default: llvm_unreachable("Invalid PPC load type!"); 5341 case MVT::i64: Opcode = PPC::LDUX; break; 5342 case MVT::i32: Opcode = isSExt ? PPC::LWAUX : PPC::LWZUX8; break; 5343 case MVT::i16: Opcode = isSExt ? PPC::LHAUX8 : PPC::LHZUX8; break; 5344 case MVT::i1: 5345 case MVT::i8: Opcode = PPC::LBZUX8; break; 5346 } 5347 } 5348 5349 SDValue Chain = LD->getChain(); 5350 SDValue Base = LD->getBasePtr(); 5351 SDValue Ops[] = { Base, Offset, Chain }; 5352 SDNode *MN = CurDAG->getMachineNode( 5353 Opcode, dl, LD->getValueType(0), 5354 PPCLowering->getPointerTy(CurDAG->getDataLayout()), MVT::Other, Ops); 5355 transferMemOperands(N, MN); 5356 ReplaceNode(N, MN); 5357 return; 5358 } 5359 } 5360 5361 case ISD::AND: 5362 // If this is an 'and' with a mask, try to emit rlwinm/rldicl/rldicr 5363 if (tryAsSingleRLWINM(N) || tryAsSingleRLWIMI(N) || tryAsSingleRLDICL(N) || 5364 tryAsSingleRLDICR(N) || tryAsSingleRLWINM8(N) || tryAsPairOfRLDICL(N)) 5365 return; 5366 5367 // Other cases are autogenerated. 5368 break; 5369 case ISD::OR: { 5370 if (N->getValueType(0) == MVT::i32) 5371 if (tryBitfieldInsert(N)) 5372 return; 5373 5374 int16_t Imm; 5375 if (N->getOperand(0)->getOpcode() == ISD::FrameIndex && 5376 isIntS16Immediate(N->getOperand(1), Imm)) { 5377 KnownBits LHSKnown = CurDAG->computeKnownBits(N->getOperand(0)); 5378 5379 // If this is equivalent to an add, then we can fold it with the 5380 // FrameIndex calculation. 5381 if ((LHSKnown.Zero.getZExtValue()|~(uint64_t)Imm) == ~0ULL) { 5382 selectFrameIndex(N, N->getOperand(0).getNode(), (int64_t)Imm); 5383 return; 5384 } 5385 } 5386 5387 // If this is 'or' against an imm with consecutive ones and both sides zero, 5388 // try to emit rldimi 5389 if (tryAsSingleRLDIMI(N)) 5390 return; 5391 5392 // OR with a 32-bit immediate can be handled by ori + oris 5393 // without creating an immediate in a GPR. 5394 uint64_t Imm64 = 0; 5395 bool IsPPC64 = Subtarget->isPPC64(); 5396 if (IsPPC64 && isInt64Immediate(N->getOperand(1), Imm64) && 5397 (Imm64 & ~0xFFFFFFFFuLL) == 0) { 5398 // If ImmHi (ImmHi) is zero, only one ori (oris) is generated later. 5399 uint64_t ImmHi = Imm64 >> 16; 5400 uint64_t ImmLo = Imm64 & 0xFFFF; 5401 if (ImmHi != 0 && ImmLo != 0) { 5402 SDNode *Lo = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, 5403 N->getOperand(0), 5404 getI16Imm(ImmLo, dl)); 5405 SDValue Ops1[] = { SDValue(Lo, 0), getI16Imm(ImmHi, dl)}; 5406 CurDAG->SelectNodeTo(N, PPC::ORIS8, MVT::i64, Ops1); 5407 return; 5408 } 5409 } 5410 5411 // Other cases are autogenerated. 5412 break; 5413 } 5414 case ISD::XOR: { 5415 // XOR with a 32-bit immediate can be handled by xori + xoris 5416 // without creating an immediate in a GPR. 5417 uint64_t Imm64 = 0; 5418 bool IsPPC64 = Subtarget->isPPC64(); 5419 if (IsPPC64 && isInt64Immediate(N->getOperand(1), Imm64) && 5420 (Imm64 & ~0xFFFFFFFFuLL) == 0) { 5421 // If ImmHi (ImmHi) is zero, only one xori (xoris) is generated later. 5422 uint64_t ImmHi = Imm64 >> 16; 5423 uint64_t ImmLo = Imm64 & 0xFFFF; 5424 if (ImmHi != 0 && ImmLo != 0) { 5425 SDNode *Lo = CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, 5426 N->getOperand(0), 5427 getI16Imm(ImmLo, dl)); 5428 SDValue Ops1[] = { SDValue(Lo, 0), getI16Imm(ImmHi, dl)}; 5429 CurDAG->SelectNodeTo(N, PPC::XORIS8, MVT::i64, Ops1); 5430 return; 5431 } 5432 } 5433 5434 break; 5435 } 5436 case ISD::ADD: { 5437 int16_t Imm; 5438 if (N->getOperand(0)->getOpcode() == ISD::FrameIndex && 5439 isIntS16Immediate(N->getOperand(1), Imm)) { 5440 selectFrameIndex(N, N->getOperand(0).getNode(), (int64_t)Imm); 5441 return; 5442 } 5443 5444 break; 5445 } 5446 case ISD::SHL: { 5447 unsigned Imm, SH, MB, ME; 5448 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, Imm) && 5449 isRotateAndMask(N, Imm, true, SH, MB, ME)) { 5450 SDValue Ops[] = { N->getOperand(0).getOperand(0), 5451 getI32Imm(SH, dl), getI32Imm(MB, dl), 5452 getI32Imm(ME, dl) }; 5453 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); 5454 return; 5455 } 5456 5457 // Other cases are autogenerated. 5458 break; 5459 } 5460 case ISD::SRL: { 5461 unsigned Imm, SH, MB, ME; 5462 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, Imm) && 5463 isRotateAndMask(N, Imm, true, SH, MB, ME)) { 5464 SDValue Ops[] = { N->getOperand(0).getOperand(0), 5465 getI32Imm(SH, dl), getI32Imm(MB, dl), 5466 getI32Imm(ME, dl) }; 5467 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); 5468 return; 5469 } 5470 5471 // Other cases are autogenerated. 5472 break; 5473 } 5474 case ISD::MUL: { 5475 SDValue Op1 = N->getOperand(1); 5476 if (Op1.getOpcode() != ISD::Constant || 5477 (Op1.getValueType() != MVT::i64 && Op1.getValueType() != MVT::i32)) 5478 break; 5479 5480 // If the multiplier fits int16, we can handle it with mulli. 5481 int64_t Imm = cast<ConstantSDNode>(Op1)->getZExtValue(); 5482 unsigned Shift = countTrailingZeros<uint64_t>(Imm); 5483 if (isInt<16>(Imm) || !Shift) 5484 break; 5485 5486 // If the shifted value fits int16, we can do this transformation: 5487 // (mul X, c1 << c2) -> (rldicr (mulli X, c1) c2). We do this in ISEL due to 5488 // DAGCombiner prefers (shl (mul X, c1), c2) -> (mul X, c1 << c2). 5489 uint64_t ImmSh = Imm >> Shift; 5490 if (!isInt<16>(ImmSh)) 5491 break; 5492 5493 uint64_t SextImm = SignExtend64(ImmSh & 0xFFFF, 16); 5494 if (Op1.getValueType() == MVT::i64) { 5495 SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i64); 5496 SDNode *MulNode = CurDAG->getMachineNode(PPC::MULLI8, dl, MVT::i64, 5497 N->getOperand(0), SDImm); 5498 5499 SDValue Ops[] = {SDValue(MulNode, 0), getI32Imm(Shift, dl), 5500 getI32Imm(63 - Shift, dl)}; 5501 CurDAG->SelectNodeTo(N, PPC::RLDICR, MVT::i64, Ops); 5502 return; 5503 } else { 5504 SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i32); 5505 SDNode *MulNode = CurDAG->getMachineNode(PPC::MULLI, dl, MVT::i32, 5506 N->getOperand(0), SDImm); 5507 5508 SDValue Ops[] = {SDValue(MulNode, 0), getI32Imm(Shift, dl), 5509 getI32Imm(0, dl), getI32Imm(31 - Shift, dl)}; 5510 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops); 5511 return; 5512 } 5513 break; 5514 } 5515 // FIXME: Remove this once the ANDI glue bug is fixed: 5516 case PPCISD::ANDI_rec_1_EQ_BIT: 5517 case PPCISD::ANDI_rec_1_GT_BIT: { 5518 if (!ANDIGlueBug) 5519 break; 5520 5521 EVT InVT = N->getOperand(0).getValueType(); 5522 assert((InVT == MVT::i64 || InVT == MVT::i32) && 5523 "Invalid input type for ANDI_rec_1_EQ_BIT"); 5524 5525 unsigned Opcode = (InVT == MVT::i64) ? PPC::ANDI8_rec : PPC::ANDI_rec; 5526 SDValue AndI(CurDAG->getMachineNode(Opcode, dl, InVT, MVT::Glue, 5527 N->getOperand(0), 5528 CurDAG->getTargetConstant(1, dl, InVT)), 5529 0); 5530 SDValue CR0Reg = CurDAG->getRegister(PPC::CR0, MVT::i32); 5531 SDValue SRIdxVal = CurDAG->getTargetConstant( 5532 N->getOpcode() == PPCISD::ANDI_rec_1_EQ_BIT ? PPC::sub_eq : PPC::sub_gt, 5533 dl, MVT::i32); 5534 5535 CurDAG->SelectNodeTo(N, TargetOpcode::EXTRACT_SUBREG, MVT::i1, CR0Reg, 5536 SRIdxVal, SDValue(AndI.getNode(), 1) /* glue */); 5537 return; 5538 } 5539 case ISD::SELECT_CC: { 5540 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(4))->get(); 5541 EVT PtrVT = 5542 CurDAG->getTargetLoweringInfo().getPointerTy(CurDAG->getDataLayout()); 5543 bool isPPC64 = (PtrVT == MVT::i64); 5544 5545 // If this is a select of i1 operands, we'll pattern match it. 5546 if (Subtarget->useCRBits() && N->getOperand(0).getValueType() == MVT::i1) 5547 break; 5548 5549 if (Subtarget->isISA3_0() && Subtarget->isPPC64()) { 5550 bool NeedSwapOps = false; 5551 bool IsUnCmp = false; 5552 if (mayUseP9Setb(N, CC, CurDAG, NeedSwapOps, IsUnCmp)) { 5553 SDValue LHS = N->getOperand(0); 5554 SDValue RHS = N->getOperand(1); 5555 if (NeedSwapOps) 5556 std::swap(LHS, RHS); 5557 5558 // Make use of SelectCC to generate the comparison to set CR bits, for 5559 // equality comparisons having one literal operand, SelectCC probably 5560 // doesn't need to materialize the whole literal and just use xoris to 5561 // check it first, it leads the following comparison result can't 5562 // exactly represent GT/LT relationship. So to avoid this we specify 5563 // SETGT/SETUGT here instead of SETEQ. 5564 SDValue GenCC = 5565 SelectCC(LHS, RHS, IsUnCmp ? ISD::SETUGT : ISD::SETGT, dl); 5566 CurDAG->SelectNodeTo( 5567 N, N->getSimpleValueType(0) == MVT::i64 ? PPC::SETB8 : PPC::SETB, 5568 N->getValueType(0), GenCC); 5569 NumP9Setb++; 5570 return; 5571 } 5572 } 5573 5574 // Handle the setcc cases here. select_cc lhs, 0, 1, 0, cc 5575 if (!isPPC64) 5576 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1))) 5577 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N->getOperand(2))) 5578 if (ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N->getOperand(3))) 5579 if (N1C->isZero() && N3C->isZero() && N2C->getZExtValue() == 1ULL && 5580 CC == ISD::SETNE && 5581 // FIXME: Implement this optzn for PPC64. 5582 N->getValueType(0) == MVT::i32) { 5583 SDNode *Tmp = 5584 CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue, 5585 N->getOperand(0), getI32Imm(~0U, dl)); 5586 CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, SDValue(Tmp, 0), 5587 N->getOperand(0), SDValue(Tmp, 1)); 5588 return; 5589 } 5590 5591 SDValue CCReg = SelectCC(N->getOperand(0), N->getOperand(1), CC, dl); 5592 5593 if (N->getValueType(0) == MVT::i1) { 5594 // An i1 select is: (c & t) | (!c & f). 5595 bool Inv; 5596 unsigned Idx = getCRIdxForSetCC(CC, Inv); 5597 5598 unsigned SRI; 5599 switch (Idx) { 5600 default: llvm_unreachable("Invalid CC index"); 5601 case 0: SRI = PPC::sub_lt; break; 5602 case 1: SRI = PPC::sub_gt; break; 5603 case 2: SRI = PPC::sub_eq; break; 5604 case 3: SRI = PPC::sub_un; break; 5605 } 5606 5607 SDValue CCBit = CurDAG->getTargetExtractSubreg(SRI, dl, MVT::i1, CCReg); 5608 5609 SDValue NotCCBit(CurDAG->getMachineNode(PPC::CRNOR, dl, MVT::i1, 5610 CCBit, CCBit), 0); 5611 SDValue C = Inv ? NotCCBit : CCBit, 5612 NotC = Inv ? CCBit : NotCCBit; 5613 5614 SDValue CAndT(CurDAG->getMachineNode(PPC::CRAND, dl, MVT::i1, 5615 C, N->getOperand(2)), 0); 5616 SDValue NotCAndF(CurDAG->getMachineNode(PPC::CRAND, dl, MVT::i1, 5617 NotC, N->getOperand(3)), 0); 5618 5619 CurDAG->SelectNodeTo(N, PPC::CROR, MVT::i1, CAndT, NotCAndF); 5620 return; 5621 } 5622 5623 unsigned BROpc = 5624 getPredicateForSetCC(CC, N->getOperand(0).getValueType(), Subtarget); 5625 5626 unsigned SelectCCOp; 5627 if (N->getValueType(0) == MVT::i32) 5628 SelectCCOp = PPC::SELECT_CC_I4; 5629 else if (N->getValueType(0) == MVT::i64) 5630 SelectCCOp = PPC::SELECT_CC_I8; 5631 else if (N->getValueType(0) == MVT::f32) { 5632 if (Subtarget->hasP8Vector()) 5633 SelectCCOp = PPC::SELECT_CC_VSSRC; 5634 else if (Subtarget->hasSPE()) 5635 SelectCCOp = PPC::SELECT_CC_SPE4; 5636 else 5637 SelectCCOp = PPC::SELECT_CC_F4; 5638 } else if (N->getValueType(0) == MVT::f64) { 5639 if (Subtarget->hasVSX()) 5640 SelectCCOp = PPC::SELECT_CC_VSFRC; 5641 else if (Subtarget->hasSPE()) 5642 SelectCCOp = PPC::SELECT_CC_SPE; 5643 else 5644 SelectCCOp = PPC::SELECT_CC_F8; 5645 } else if (N->getValueType(0) == MVT::f128) 5646 SelectCCOp = PPC::SELECT_CC_F16; 5647 else if (Subtarget->hasSPE()) 5648 SelectCCOp = PPC::SELECT_CC_SPE; 5649 else if (N->getValueType(0) == MVT::v2f64 || 5650 N->getValueType(0) == MVT::v2i64) 5651 SelectCCOp = PPC::SELECT_CC_VSRC; 5652 else 5653 SelectCCOp = PPC::SELECT_CC_VRRC; 5654 5655 SDValue Ops[] = { CCReg, N->getOperand(2), N->getOperand(3), 5656 getI32Imm(BROpc, dl) }; 5657 CurDAG->SelectNodeTo(N, SelectCCOp, N->getValueType(0), Ops); 5658 return; 5659 } 5660 case ISD::VECTOR_SHUFFLE: 5661 if (Subtarget->hasVSX() && (N->getValueType(0) == MVT::v2f64 || 5662 N->getValueType(0) == MVT::v2i64)) { 5663 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N); 5664 5665 SDValue Op1 = N->getOperand(SVN->getMaskElt(0) < 2 ? 0 : 1), 5666 Op2 = N->getOperand(SVN->getMaskElt(1) < 2 ? 0 : 1); 5667 unsigned DM[2]; 5668 5669 for (int i = 0; i < 2; ++i) 5670 if (SVN->getMaskElt(i) <= 0 || SVN->getMaskElt(i) == 2) 5671 DM[i] = 0; 5672 else 5673 DM[i] = 1; 5674 5675 if (Op1 == Op2 && DM[0] == 0 && DM[1] == 0 && 5676 Op1.getOpcode() == ISD::SCALAR_TO_VECTOR && 5677 isa<LoadSDNode>(Op1.getOperand(0))) { 5678 LoadSDNode *LD = cast<LoadSDNode>(Op1.getOperand(0)); 5679 SDValue Base, Offset; 5680 5681 if (LD->isUnindexed() && LD->hasOneUse() && Op1.hasOneUse() && 5682 (LD->getMemoryVT() == MVT::f64 || 5683 LD->getMemoryVT() == MVT::i64) && 5684 SelectAddrIdxOnly(LD->getBasePtr(), Base, Offset)) { 5685 SDValue Chain = LD->getChain(); 5686 SDValue Ops[] = { Base, Offset, Chain }; 5687 MachineMemOperand *MemOp = LD->getMemOperand(); 5688 SDNode *NewN = CurDAG->SelectNodeTo(N, PPC::LXVDSX, 5689 N->getValueType(0), Ops); 5690 CurDAG->setNodeMemRefs(cast<MachineSDNode>(NewN), {MemOp}); 5691 return; 5692 } 5693 } 5694 5695 // For little endian, we must swap the input operands and adjust 5696 // the mask elements (reverse and invert them). 5697 if (Subtarget->isLittleEndian()) { 5698 std::swap(Op1, Op2); 5699 unsigned tmp = DM[0]; 5700 DM[0] = 1 - DM[1]; 5701 DM[1] = 1 - tmp; 5702 } 5703 5704 SDValue DMV = CurDAG->getTargetConstant(DM[1] | (DM[0] << 1), dl, 5705 MVT::i32); 5706 SDValue Ops[] = { Op1, Op2, DMV }; 5707 CurDAG->SelectNodeTo(N, PPC::XXPERMDI, N->getValueType(0), Ops); 5708 return; 5709 } 5710 5711 break; 5712 case PPCISD::BDNZ: 5713 case PPCISD::BDZ: { 5714 bool IsPPC64 = Subtarget->isPPC64(); 5715 SDValue Ops[] = { N->getOperand(1), N->getOperand(0) }; 5716 CurDAG->SelectNodeTo(N, N->getOpcode() == PPCISD::BDNZ 5717 ? (IsPPC64 ? PPC::BDNZ8 : PPC::BDNZ) 5718 : (IsPPC64 ? PPC::BDZ8 : PPC::BDZ), 5719 MVT::Other, Ops); 5720 return; 5721 } 5722 case PPCISD::COND_BRANCH: { 5723 // Op #0 is the Chain. 5724 // Op #1 is the PPC::PRED_* number. 5725 // Op #2 is the CR# 5726 // Op #3 is the Dest MBB 5727 // Op #4 is the Flag. 5728 // Prevent PPC::PRED_* from being selected into LI. 5729 unsigned PCC = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 5730 if (EnableBranchHint) 5731 PCC |= getBranchHint(PCC, *FuncInfo, N->getOperand(3)); 5732 5733 SDValue Pred = getI32Imm(PCC, dl); 5734 SDValue Ops[] = { Pred, N->getOperand(2), N->getOperand(3), 5735 N->getOperand(0), N->getOperand(4) }; 5736 CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops); 5737 return; 5738 } 5739 case ISD::BR_CC: { 5740 if (tryFoldSWTestBRCC(N)) 5741 return; 5742 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(1))->get(); 5743 unsigned PCC = 5744 getPredicateForSetCC(CC, N->getOperand(2).getValueType(), Subtarget); 5745 5746 if (N->getOperand(2).getValueType() == MVT::i1) { 5747 unsigned Opc; 5748 bool Swap; 5749 switch (PCC) { 5750 default: llvm_unreachable("Unexpected Boolean-operand predicate"); 5751 case PPC::PRED_LT: Opc = PPC::CRANDC; Swap = true; break; 5752 case PPC::PRED_LE: Opc = PPC::CRORC; Swap = true; break; 5753 case PPC::PRED_EQ: Opc = PPC::CREQV; Swap = false; break; 5754 case PPC::PRED_GE: Opc = PPC::CRORC; Swap = false; break; 5755 case PPC::PRED_GT: Opc = PPC::CRANDC; Swap = false; break; 5756 case PPC::PRED_NE: Opc = PPC::CRXOR; Swap = false; break; 5757 } 5758 5759 // A signed comparison of i1 values produces the opposite result to an 5760 // unsigned one if the condition code includes less-than or greater-than. 5761 // This is because 1 is the most negative signed i1 number and the most 5762 // positive unsigned i1 number. The CR-logical operations used for such 5763 // comparisons are non-commutative so for signed comparisons vs. unsigned 5764 // ones, the input operands just need to be swapped. 5765 if (ISD::isSignedIntSetCC(CC)) 5766 Swap = !Swap; 5767 5768 SDValue BitComp(CurDAG->getMachineNode(Opc, dl, MVT::i1, 5769 N->getOperand(Swap ? 3 : 2), 5770 N->getOperand(Swap ? 2 : 3)), 0); 5771 CurDAG->SelectNodeTo(N, PPC::BC, MVT::Other, BitComp, N->getOperand(4), 5772 N->getOperand(0)); 5773 return; 5774 } 5775 5776 if (EnableBranchHint) 5777 PCC |= getBranchHint(PCC, *FuncInfo, N->getOperand(4)); 5778 5779 SDValue CondCode = SelectCC(N->getOperand(2), N->getOperand(3), CC, dl); 5780 SDValue Ops[] = { getI32Imm(PCC, dl), CondCode, 5781 N->getOperand(4), N->getOperand(0) }; 5782 CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops); 5783 return; 5784 } 5785 case ISD::BRIND: { 5786 // FIXME: Should custom lower this. 5787 SDValue Chain = N->getOperand(0); 5788 SDValue Target = N->getOperand(1); 5789 unsigned Opc = Target.getValueType() == MVT::i32 ? PPC::MTCTR : PPC::MTCTR8; 5790 unsigned Reg = Target.getValueType() == MVT::i32 ? PPC::BCTR : PPC::BCTR8; 5791 Chain = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, Target, 5792 Chain), 0); 5793 CurDAG->SelectNodeTo(N, Reg, MVT::Other, Chain); 5794 return; 5795 } 5796 case PPCISD::TOC_ENTRY: { 5797 const bool isPPC64 = Subtarget->isPPC64(); 5798 const bool isELFABI = Subtarget->isSVR4ABI(); 5799 const bool isAIXABI = Subtarget->isAIXABI(); 5800 5801 // PowerPC only support small, medium and large code model. 5802 const CodeModel::Model CModel = TM.getCodeModel(); 5803 assert(!(CModel == CodeModel::Tiny || CModel == CodeModel::Kernel) && 5804 "PowerPC doesn't support tiny or kernel code models."); 5805 5806 if (isAIXABI && CModel == CodeModel::Medium) 5807 report_fatal_error("Medium code model is not supported on AIX."); 5808 5809 // For 64-bit ELF small code model, we allow SelectCodeCommon to handle 5810 // this, selecting one of LDtoc, LDtocJTI, LDtocCPT, and LDtocBA. For AIX 5811 // small code model, we need to check for a toc-data attribute. 5812 if (isPPC64 && !isAIXABI && CModel == CodeModel::Small) 5813 break; 5814 5815 auto replaceWith = [this, &dl](unsigned OpCode, SDNode *TocEntry, 5816 EVT OperandTy) { 5817 SDValue GA = TocEntry->getOperand(0); 5818 SDValue TocBase = TocEntry->getOperand(1); 5819 SDNode *MN = CurDAG->getMachineNode(OpCode, dl, OperandTy, GA, TocBase); 5820 transferMemOperands(TocEntry, MN); 5821 ReplaceNode(TocEntry, MN); 5822 }; 5823 5824 // Handle 32-bit small code model. 5825 if (!isPPC64 && CModel == CodeModel::Small) { 5826 // Transforms the ISD::TOC_ENTRY node to passed in Opcode, either 5827 // PPC::ADDItoc, or PPC::LWZtoc 5828 if (isELFABI) { 5829 assert(TM.isPositionIndependent() && 5830 "32-bit ELF can only have TOC entries in position independent" 5831 " code."); 5832 // 32-bit ELF always uses a small code model toc access. 5833 replaceWith(PPC::LWZtoc, N, MVT::i32); 5834 return; 5835 } 5836 5837 assert(isAIXABI && "ELF ABI already handled"); 5838 5839 if (hasTocDataAttr(N->getOperand(0), 5840 CurDAG->getDataLayout().getPointerSize())) { 5841 replaceWith(PPC::ADDItoc, N, MVT::i32); 5842 return; 5843 } 5844 5845 replaceWith(PPC::LWZtoc, N, MVT::i32); 5846 return; 5847 } 5848 5849 if (isPPC64 && CModel == CodeModel::Small) { 5850 assert(isAIXABI && "ELF ABI handled in common SelectCode"); 5851 5852 if (hasTocDataAttr(N->getOperand(0), 5853 CurDAG->getDataLayout().getPointerSize())) { 5854 replaceWith(PPC::ADDItoc8, N, MVT::i64); 5855 return; 5856 } 5857 // Break if it doesn't have toc data attribute. Proceed with common 5858 // SelectCode. 5859 break; 5860 } 5861 5862 assert(CModel != CodeModel::Small && "All small code models handled."); 5863 5864 assert((isPPC64 || (isAIXABI && !isPPC64)) && "We are dealing with 64-bit" 5865 " ELF/AIX or 32-bit AIX in the following."); 5866 5867 // Transforms the ISD::TOC_ENTRY node for 32-bit AIX large code model mode 5868 // or 64-bit medium (ELF-only) or large (ELF and AIX) code model code. We 5869 // generate two instructions as described below. The first source operand 5870 // is a symbol reference. If it must be toc-referenced according to 5871 // Subtarget, we generate: 5872 // [32-bit AIX] 5873 // LWZtocL(@sym, ADDIStocHA(%r2, @sym)) 5874 // [64-bit ELF/AIX] 5875 // LDtocL(@sym, ADDIStocHA8(%x2, @sym)) 5876 // Otherwise we generate: 5877 // ADDItocL(ADDIStocHA8(%x2, @sym), @sym) 5878 SDValue GA = N->getOperand(0); 5879 SDValue TOCbase = N->getOperand(1); 5880 5881 EVT VT = isPPC64 ? MVT::i64 : MVT::i32; 5882 SDNode *Tmp = CurDAG->getMachineNode( 5883 isPPC64 ? PPC::ADDIStocHA8 : PPC::ADDIStocHA, dl, VT, TOCbase, GA); 5884 5885 if (PPCLowering->isAccessedAsGotIndirect(GA)) { 5886 // If it is accessed as got-indirect, we need an extra LWZ/LD to load 5887 // the address. 5888 SDNode *MN = CurDAG->getMachineNode( 5889 isPPC64 ? PPC::LDtocL : PPC::LWZtocL, dl, VT, GA, SDValue(Tmp, 0)); 5890 5891 transferMemOperands(N, MN); 5892 ReplaceNode(N, MN); 5893 return; 5894 } 5895 5896 // Build the address relative to the TOC-pointer. 5897 ReplaceNode(N, CurDAG->getMachineNode(PPC::ADDItocL, dl, MVT::i64, 5898 SDValue(Tmp, 0), GA)); 5899 return; 5900 } 5901 case PPCISD::PPC32_PICGOT: 5902 // Generate a PIC-safe GOT reference. 5903 assert(Subtarget->is32BitELFABI() && 5904 "PPCISD::PPC32_PICGOT is only supported for 32-bit SVR4"); 5905 CurDAG->SelectNodeTo(N, PPC::PPC32PICGOT, 5906 PPCLowering->getPointerTy(CurDAG->getDataLayout()), 5907 MVT::i32); 5908 return; 5909 5910 case PPCISD::VADD_SPLAT: { 5911 // This expands into one of three sequences, depending on whether 5912 // the first operand is odd or even, positive or negative. 5913 assert(isa<ConstantSDNode>(N->getOperand(0)) && 5914 isa<ConstantSDNode>(N->getOperand(1)) && 5915 "Invalid operand on VADD_SPLAT!"); 5916 5917 int Elt = N->getConstantOperandVal(0); 5918 int EltSize = N->getConstantOperandVal(1); 5919 unsigned Opc1, Opc2, Opc3; 5920 EVT VT; 5921 5922 if (EltSize == 1) { 5923 Opc1 = PPC::VSPLTISB; 5924 Opc2 = PPC::VADDUBM; 5925 Opc3 = PPC::VSUBUBM; 5926 VT = MVT::v16i8; 5927 } else if (EltSize == 2) { 5928 Opc1 = PPC::VSPLTISH; 5929 Opc2 = PPC::VADDUHM; 5930 Opc3 = PPC::VSUBUHM; 5931 VT = MVT::v8i16; 5932 } else { 5933 assert(EltSize == 4 && "Invalid element size on VADD_SPLAT!"); 5934 Opc1 = PPC::VSPLTISW; 5935 Opc2 = PPC::VADDUWM; 5936 Opc3 = PPC::VSUBUWM; 5937 VT = MVT::v4i32; 5938 } 5939 5940 if ((Elt & 1) == 0) { 5941 // Elt is even, in the range [-32,-18] + [16,30]. 5942 // 5943 // Convert: VADD_SPLAT elt, size 5944 // Into: tmp = VSPLTIS[BHW] elt 5945 // VADDU[BHW]M tmp, tmp 5946 // Where: [BHW] = B for size = 1, H for size = 2, W for size = 4 5947 SDValue EltVal = getI32Imm(Elt >> 1, dl); 5948 SDNode *Tmp = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); 5949 SDValue TmpVal = SDValue(Tmp, 0); 5950 ReplaceNode(N, CurDAG->getMachineNode(Opc2, dl, VT, TmpVal, TmpVal)); 5951 return; 5952 } else if (Elt > 0) { 5953 // Elt is odd and positive, in the range [17,31]. 5954 // 5955 // Convert: VADD_SPLAT elt, size 5956 // Into: tmp1 = VSPLTIS[BHW] elt-16 5957 // tmp2 = VSPLTIS[BHW] -16 5958 // VSUBU[BHW]M tmp1, tmp2 5959 SDValue EltVal = getI32Imm(Elt - 16, dl); 5960 SDNode *Tmp1 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); 5961 EltVal = getI32Imm(-16, dl); 5962 SDNode *Tmp2 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); 5963 ReplaceNode(N, CurDAG->getMachineNode(Opc3, dl, VT, SDValue(Tmp1, 0), 5964 SDValue(Tmp2, 0))); 5965 return; 5966 } else { 5967 // Elt is odd and negative, in the range [-31,-17]. 5968 // 5969 // Convert: VADD_SPLAT elt, size 5970 // Into: tmp1 = VSPLTIS[BHW] elt+16 5971 // tmp2 = VSPLTIS[BHW] -16 5972 // VADDU[BHW]M tmp1, tmp2 5973 SDValue EltVal = getI32Imm(Elt + 16, dl); 5974 SDNode *Tmp1 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); 5975 EltVal = getI32Imm(-16, dl); 5976 SDNode *Tmp2 = CurDAG->getMachineNode(Opc1, dl, VT, EltVal); 5977 ReplaceNode(N, CurDAG->getMachineNode(Opc2, dl, VT, SDValue(Tmp1, 0), 5978 SDValue(Tmp2, 0))); 5979 return; 5980 } 5981 } 5982 case PPCISD::LD_SPLAT: { 5983 // Here we want to handle splat load for type v16i8 and v8i16 when there is 5984 // no direct move, we don't need to use stack for this case. If target has 5985 // direct move, we should be able to get the best selection in the .td file. 5986 if (!Subtarget->hasAltivec() || Subtarget->hasDirectMove()) 5987 break; 5988 5989 EVT Type = N->getValueType(0); 5990 if (Type != MVT::v16i8 && Type != MVT::v8i16) 5991 break; 5992 5993 // If the alignment for the load is 16 or bigger, we don't need the 5994 // permutated mask to get the required value. The value must be the 0 5995 // element in big endian target or 7/15 in little endian target in the 5996 // result vsx register of lvx instruction. 5997 // Select the instruction in the .td file. 5998 if (cast<MemIntrinsicSDNode>(N)->getAlign() >= Align(16) && 5999 isOffsetMultipleOf(N, 16)) 6000 break; 6001 6002 SDValue ZeroReg = 6003 CurDAG->getRegister(Subtarget->isPPC64() ? PPC::ZERO8 : PPC::ZERO, 6004 Subtarget->isPPC64() ? MVT::i64 : MVT::i32); 6005 unsigned LIOpcode = Subtarget->isPPC64() ? PPC::LI8 : PPC::LI; 6006 // v16i8 LD_SPLAT addr 6007 // ======> 6008 // Mask = LVSR/LVSL 0, addr 6009 // LoadLow = LVX 0, addr 6010 // Perm = VPERM LoadLow, LoadLow, Mask 6011 // Splat = VSPLTB 15/0, Perm 6012 // 6013 // v8i16 LD_SPLAT addr 6014 // ======> 6015 // Mask = LVSR/LVSL 0, addr 6016 // LoadLow = LVX 0, addr 6017 // LoadHigh = LVX (LI, 1), addr 6018 // Perm = VPERM LoadLow, LoadHigh, Mask 6019 // Splat = VSPLTH 7/0, Perm 6020 unsigned SplatOp = (Type == MVT::v16i8) ? PPC::VSPLTB : PPC::VSPLTH; 6021 unsigned SplatElemIndex = 6022 Subtarget->isLittleEndian() ? ((Type == MVT::v16i8) ? 15 : 7) : 0; 6023 6024 SDNode *Mask = CurDAG->getMachineNode( 6025 Subtarget->isLittleEndian() ? PPC::LVSR : PPC::LVSL, dl, Type, ZeroReg, 6026 N->getOperand(1)); 6027 6028 SDNode *LoadLow = 6029 CurDAG->getMachineNode(PPC::LVX, dl, MVT::v16i8, MVT::Other, 6030 {ZeroReg, N->getOperand(1), N->getOperand(0)}); 6031 6032 SDNode *LoadHigh = LoadLow; 6033 if (Type == MVT::v8i16) { 6034 LoadHigh = CurDAG->getMachineNode( 6035 PPC::LVX, dl, MVT::v16i8, MVT::Other, 6036 {SDValue(CurDAG->getMachineNode( 6037 LIOpcode, dl, MVT::i32, 6038 CurDAG->getTargetConstant(1, dl, MVT::i8)), 6039 0), 6040 N->getOperand(1), SDValue(LoadLow, 1)}); 6041 } 6042 6043 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 1), SDValue(LoadHigh, 1)); 6044 transferMemOperands(N, LoadHigh); 6045 6046 SDNode *Perm = 6047 CurDAG->getMachineNode(PPC::VPERM, dl, Type, SDValue(LoadLow, 0), 6048 SDValue(LoadHigh, 0), SDValue(Mask, 0)); 6049 CurDAG->SelectNodeTo(N, SplatOp, Type, 6050 CurDAG->getTargetConstant(SplatElemIndex, dl, MVT::i8), 6051 SDValue(Perm, 0)); 6052 return; 6053 } 6054 } 6055 6056 SelectCode(N); 6057 } 6058 6059 // If the target supports the cmpb instruction, do the idiom recognition here. 6060 // We don't do this as a DAG combine because we don't want to do it as nodes 6061 // are being combined (because we might miss part of the eventual idiom). We 6062 // don't want to do it during instruction selection because we want to reuse 6063 // the logic for lowering the masking operations already part of the 6064 // instruction selector. 6065 SDValue PPCDAGToDAGISel::combineToCMPB(SDNode *N) { 6066 SDLoc dl(N); 6067 6068 assert(N->getOpcode() == ISD::OR && 6069 "Only OR nodes are supported for CMPB"); 6070 6071 SDValue Res; 6072 if (!Subtarget->hasCMPB()) 6073 return Res; 6074 6075 if (N->getValueType(0) != MVT::i32 && 6076 N->getValueType(0) != MVT::i64) 6077 return Res; 6078 6079 EVT VT = N->getValueType(0); 6080 6081 SDValue RHS, LHS; 6082 bool BytesFound[8] = {false, false, false, false, false, false, false, false}; 6083 uint64_t Mask = 0, Alt = 0; 6084 6085 auto IsByteSelectCC = [this](SDValue O, unsigned &b, 6086 uint64_t &Mask, uint64_t &Alt, 6087 SDValue &LHS, SDValue &RHS) { 6088 if (O.getOpcode() != ISD::SELECT_CC) 6089 return false; 6090 ISD::CondCode CC = cast<CondCodeSDNode>(O.getOperand(4))->get(); 6091 6092 if (!isa<ConstantSDNode>(O.getOperand(2)) || 6093 !isa<ConstantSDNode>(O.getOperand(3))) 6094 return false; 6095 6096 uint64_t PM = O.getConstantOperandVal(2); 6097 uint64_t PAlt = O.getConstantOperandVal(3); 6098 for (b = 0; b < 8; ++b) { 6099 uint64_t Mask = UINT64_C(0xFF) << (8*b); 6100 if (PM && (PM & Mask) == PM && (PAlt & Mask) == PAlt) 6101 break; 6102 } 6103 6104 if (b == 8) 6105 return false; 6106 Mask |= PM; 6107 Alt |= PAlt; 6108 6109 if (!isa<ConstantSDNode>(O.getOperand(1)) || 6110 O.getConstantOperandVal(1) != 0) { 6111 SDValue Op0 = O.getOperand(0), Op1 = O.getOperand(1); 6112 if (Op0.getOpcode() == ISD::TRUNCATE) 6113 Op0 = Op0.getOperand(0); 6114 if (Op1.getOpcode() == ISD::TRUNCATE) 6115 Op1 = Op1.getOperand(0); 6116 6117 if (Op0.getOpcode() == ISD::SRL && Op1.getOpcode() == ISD::SRL && 6118 Op0.getOperand(1) == Op1.getOperand(1) && CC == ISD::SETEQ && 6119 isa<ConstantSDNode>(Op0.getOperand(1))) { 6120 6121 unsigned Bits = Op0.getValueSizeInBits(); 6122 if (b != Bits/8-1) 6123 return false; 6124 if (Op0.getConstantOperandVal(1) != Bits-8) 6125 return false; 6126 6127 LHS = Op0.getOperand(0); 6128 RHS = Op1.getOperand(0); 6129 return true; 6130 } 6131 6132 // When we have small integers (i16 to be specific), the form present 6133 // post-legalization uses SETULT in the SELECT_CC for the 6134 // higher-order byte, depending on the fact that the 6135 // even-higher-order bytes are known to all be zero, for example: 6136 // select_cc (xor $lhs, $rhs), 256, 65280, 0, setult 6137 // (so when the second byte is the same, because all higher-order 6138 // bits from bytes 3 and 4 are known to be zero, the result of the 6139 // xor can be at most 255) 6140 if (Op0.getOpcode() == ISD::XOR && CC == ISD::SETULT && 6141 isa<ConstantSDNode>(O.getOperand(1))) { 6142 6143 uint64_t ULim = O.getConstantOperandVal(1); 6144 if (ULim != (UINT64_C(1) << b*8)) 6145 return false; 6146 6147 // Now we need to make sure that the upper bytes are known to be 6148 // zero. 6149 unsigned Bits = Op0.getValueSizeInBits(); 6150 if (!CurDAG->MaskedValueIsZero( 6151 Op0, APInt::getHighBitsSet(Bits, Bits - (b + 1) * 8))) 6152 return false; 6153 6154 LHS = Op0.getOperand(0); 6155 RHS = Op0.getOperand(1); 6156 return true; 6157 } 6158 6159 return false; 6160 } 6161 6162 if (CC != ISD::SETEQ) 6163 return false; 6164 6165 SDValue Op = O.getOperand(0); 6166 if (Op.getOpcode() == ISD::AND) { 6167 if (!isa<ConstantSDNode>(Op.getOperand(1))) 6168 return false; 6169 if (Op.getConstantOperandVal(1) != (UINT64_C(0xFF) << (8*b))) 6170 return false; 6171 6172 SDValue XOR = Op.getOperand(0); 6173 if (XOR.getOpcode() == ISD::TRUNCATE) 6174 XOR = XOR.getOperand(0); 6175 if (XOR.getOpcode() != ISD::XOR) 6176 return false; 6177 6178 LHS = XOR.getOperand(0); 6179 RHS = XOR.getOperand(1); 6180 return true; 6181 } else if (Op.getOpcode() == ISD::SRL) { 6182 if (!isa<ConstantSDNode>(Op.getOperand(1))) 6183 return false; 6184 unsigned Bits = Op.getValueSizeInBits(); 6185 if (b != Bits/8-1) 6186 return false; 6187 if (Op.getConstantOperandVal(1) != Bits-8) 6188 return false; 6189 6190 SDValue XOR = Op.getOperand(0); 6191 if (XOR.getOpcode() == ISD::TRUNCATE) 6192 XOR = XOR.getOperand(0); 6193 if (XOR.getOpcode() != ISD::XOR) 6194 return false; 6195 6196 LHS = XOR.getOperand(0); 6197 RHS = XOR.getOperand(1); 6198 return true; 6199 } 6200 6201 return false; 6202 }; 6203 6204 SmallVector<SDValue, 8> Queue(1, SDValue(N, 0)); 6205 while (!Queue.empty()) { 6206 SDValue V = Queue.pop_back_val(); 6207 6208 for (const SDValue &O : V.getNode()->ops()) { 6209 unsigned b = 0; 6210 uint64_t M = 0, A = 0; 6211 SDValue OLHS, ORHS; 6212 if (O.getOpcode() == ISD::OR) { 6213 Queue.push_back(O); 6214 } else if (IsByteSelectCC(O, b, M, A, OLHS, ORHS)) { 6215 if (!LHS) { 6216 LHS = OLHS; 6217 RHS = ORHS; 6218 BytesFound[b] = true; 6219 Mask |= M; 6220 Alt |= A; 6221 } else if ((LHS == ORHS && RHS == OLHS) || 6222 (RHS == ORHS && LHS == OLHS)) { 6223 BytesFound[b] = true; 6224 Mask |= M; 6225 Alt |= A; 6226 } else { 6227 return Res; 6228 } 6229 } else { 6230 return Res; 6231 } 6232 } 6233 } 6234 6235 unsigned LastB = 0, BCnt = 0; 6236 for (unsigned i = 0; i < 8; ++i) 6237 if (BytesFound[LastB]) { 6238 ++BCnt; 6239 LastB = i; 6240 } 6241 6242 if (!LastB || BCnt < 2) 6243 return Res; 6244 6245 // Because we'll be zero-extending the output anyway if don't have a specific 6246 // value for each input byte (via the Mask), we can 'anyext' the inputs. 6247 if (LHS.getValueType() != VT) { 6248 LHS = CurDAG->getAnyExtOrTrunc(LHS, dl, VT); 6249 RHS = CurDAG->getAnyExtOrTrunc(RHS, dl, VT); 6250 } 6251 6252 Res = CurDAG->getNode(PPCISD::CMPB, dl, VT, LHS, RHS); 6253 6254 bool NonTrivialMask = ((int64_t) Mask) != INT64_C(-1); 6255 if (NonTrivialMask && !Alt) { 6256 // Res = Mask & CMPB 6257 Res = CurDAG->getNode(ISD::AND, dl, VT, Res, 6258 CurDAG->getConstant(Mask, dl, VT)); 6259 } else if (Alt) { 6260 // Res = (CMPB & Mask) | (~CMPB & Alt) 6261 // Which, as suggested here: 6262 // https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge 6263 // can be written as: 6264 // Res = Alt ^ ((Alt ^ Mask) & CMPB) 6265 // useful because the (Alt ^ Mask) can be pre-computed. 6266 Res = CurDAG->getNode(ISD::AND, dl, VT, Res, 6267 CurDAG->getConstant(Mask ^ Alt, dl, VT)); 6268 Res = CurDAG->getNode(ISD::XOR, dl, VT, Res, 6269 CurDAG->getConstant(Alt, dl, VT)); 6270 } 6271 6272 return Res; 6273 } 6274 6275 // When CR bit registers are enabled, an extension of an i1 variable to a i32 6276 // or i64 value is lowered in terms of a SELECT_I[48] operation, and thus 6277 // involves constant materialization of a 0 or a 1 or both. If the result of 6278 // the extension is then operated upon by some operator that can be constant 6279 // folded with a constant 0 or 1, and that constant can be materialized using 6280 // only one instruction (like a zero or one), then we should fold in those 6281 // operations with the select. 6282 void PPCDAGToDAGISel::foldBoolExts(SDValue &Res, SDNode *&N) { 6283 if (!Subtarget->useCRBits()) 6284 return; 6285 6286 if (N->getOpcode() != ISD::ZERO_EXTEND && 6287 N->getOpcode() != ISD::SIGN_EXTEND && 6288 N->getOpcode() != ISD::ANY_EXTEND) 6289 return; 6290 6291 if (N->getOperand(0).getValueType() != MVT::i1) 6292 return; 6293 6294 if (!N->hasOneUse()) 6295 return; 6296 6297 SDLoc dl(N); 6298 EVT VT = N->getValueType(0); 6299 SDValue Cond = N->getOperand(0); 6300 SDValue ConstTrue = 6301 CurDAG->getConstant(N->getOpcode() == ISD::SIGN_EXTEND ? -1 : 1, dl, VT); 6302 SDValue ConstFalse = CurDAG->getConstant(0, dl, VT); 6303 6304 do { 6305 SDNode *User = *N->use_begin(); 6306 if (User->getNumOperands() != 2) 6307 break; 6308 6309 auto TryFold = [this, N, User, dl](SDValue Val) { 6310 SDValue UserO0 = User->getOperand(0), UserO1 = User->getOperand(1); 6311 SDValue O0 = UserO0.getNode() == N ? Val : UserO0; 6312 SDValue O1 = UserO1.getNode() == N ? Val : UserO1; 6313 6314 return CurDAG->FoldConstantArithmetic(User->getOpcode(), dl, 6315 User->getValueType(0), {O0, O1}); 6316 }; 6317 6318 // FIXME: When the semantics of the interaction between select and undef 6319 // are clearly defined, it may turn out to be unnecessary to break here. 6320 SDValue TrueRes = TryFold(ConstTrue); 6321 if (!TrueRes || TrueRes.isUndef()) 6322 break; 6323 SDValue FalseRes = TryFold(ConstFalse); 6324 if (!FalseRes || FalseRes.isUndef()) 6325 break; 6326 6327 // For us to materialize these using one instruction, we must be able to 6328 // represent them as signed 16-bit integers. 6329 uint64_t True = cast<ConstantSDNode>(TrueRes)->getZExtValue(), 6330 False = cast<ConstantSDNode>(FalseRes)->getZExtValue(); 6331 if (!isInt<16>(True) || !isInt<16>(False)) 6332 break; 6333 6334 // We can replace User with a new SELECT node, and try again to see if we 6335 // can fold the select with its user. 6336 Res = CurDAG->getSelect(dl, User->getValueType(0), Cond, TrueRes, FalseRes); 6337 N = User; 6338 ConstTrue = TrueRes; 6339 ConstFalse = FalseRes; 6340 } while (N->hasOneUse()); 6341 } 6342 6343 void PPCDAGToDAGISel::PreprocessISelDAG() { 6344 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); 6345 6346 bool MadeChange = false; 6347 while (Position != CurDAG->allnodes_begin()) { 6348 SDNode *N = &*--Position; 6349 if (N->use_empty()) 6350 continue; 6351 6352 SDValue Res; 6353 switch (N->getOpcode()) { 6354 default: break; 6355 case ISD::OR: 6356 Res = combineToCMPB(N); 6357 break; 6358 } 6359 6360 if (!Res) 6361 foldBoolExts(Res, N); 6362 6363 if (Res) { 6364 LLVM_DEBUG(dbgs() << "PPC DAG preprocessing replacing:\nOld: "); 6365 LLVM_DEBUG(N->dump(CurDAG)); 6366 LLVM_DEBUG(dbgs() << "\nNew: "); 6367 LLVM_DEBUG(Res.getNode()->dump(CurDAG)); 6368 LLVM_DEBUG(dbgs() << "\n"); 6369 6370 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res); 6371 MadeChange = true; 6372 } 6373 } 6374 6375 if (MadeChange) 6376 CurDAG->RemoveDeadNodes(); 6377 } 6378 6379 /// PostprocessISelDAG - Perform some late peephole optimizations 6380 /// on the DAG representation. 6381 void PPCDAGToDAGISel::PostprocessISelDAG() { 6382 // Skip peepholes at -O0. 6383 if (TM.getOptLevel() == CodeGenOpt::None) 6384 return; 6385 6386 PeepholePPC64(); 6387 PeepholeCROps(); 6388 PeepholePPC64ZExt(); 6389 } 6390 6391 // Check if all users of this node will become isel where the second operand 6392 // is the constant zero. If this is so, and if we can negate the condition, 6393 // then we can flip the true and false operands. This will allow the zero to 6394 // be folded with the isel so that we don't need to materialize a register 6395 // containing zero. 6396 bool PPCDAGToDAGISel::AllUsersSelectZero(SDNode *N) { 6397 for (const SDNode *User : N->uses()) { 6398 if (!User->isMachineOpcode()) 6399 return false; 6400 if (User->getMachineOpcode() != PPC::SELECT_I4 && 6401 User->getMachineOpcode() != PPC::SELECT_I8) 6402 return false; 6403 6404 SDNode *Op1 = User->getOperand(1).getNode(); 6405 SDNode *Op2 = User->getOperand(2).getNode(); 6406 // If we have a degenerate select with two equal operands, swapping will 6407 // not do anything, and we may run into an infinite loop. 6408 if (Op1 == Op2) 6409 return false; 6410 6411 if (!Op2->isMachineOpcode()) 6412 return false; 6413 6414 if (Op2->getMachineOpcode() != PPC::LI && 6415 Op2->getMachineOpcode() != PPC::LI8) 6416 return false; 6417 6418 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op2->getOperand(0)); 6419 if (!C) 6420 return false; 6421 6422 if (!C->isZero()) 6423 return false; 6424 } 6425 6426 return true; 6427 } 6428 6429 void PPCDAGToDAGISel::SwapAllSelectUsers(SDNode *N) { 6430 SmallVector<SDNode *, 4> ToReplace; 6431 for (SDNode *User : N->uses()) { 6432 assert((User->getMachineOpcode() == PPC::SELECT_I4 || 6433 User->getMachineOpcode() == PPC::SELECT_I8) && 6434 "Must have all select users"); 6435 ToReplace.push_back(User); 6436 } 6437 6438 for (SDNode *User : ToReplace) { 6439 SDNode *ResNode = 6440 CurDAG->getMachineNode(User->getMachineOpcode(), SDLoc(User), 6441 User->getValueType(0), User->getOperand(0), 6442 User->getOperand(2), 6443 User->getOperand(1)); 6444 6445 LLVM_DEBUG(dbgs() << "CR Peephole replacing:\nOld: "); 6446 LLVM_DEBUG(User->dump(CurDAG)); 6447 LLVM_DEBUG(dbgs() << "\nNew: "); 6448 LLVM_DEBUG(ResNode->dump(CurDAG)); 6449 LLVM_DEBUG(dbgs() << "\n"); 6450 6451 ReplaceUses(User, ResNode); 6452 } 6453 } 6454 6455 void PPCDAGToDAGISel::PeepholeCROps() { 6456 bool IsModified; 6457 do { 6458 IsModified = false; 6459 for (SDNode &Node : CurDAG->allnodes()) { 6460 MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(&Node); 6461 if (!MachineNode || MachineNode->use_empty()) 6462 continue; 6463 SDNode *ResNode = MachineNode; 6464 6465 bool Op1Set = false, Op1Unset = false, 6466 Op1Not = false, 6467 Op2Set = false, Op2Unset = false, 6468 Op2Not = false; 6469 6470 unsigned Opcode = MachineNode->getMachineOpcode(); 6471 switch (Opcode) { 6472 default: break; 6473 case PPC::CRAND: 6474 case PPC::CRNAND: 6475 case PPC::CROR: 6476 case PPC::CRXOR: 6477 case PPC::CRNOR: 6478 case PPC::CREQV: 6479 case PPC::CRANDC: 6480 case PPC::CRORC: { 6481 SDValue Op = MachineNode->getOperand(1); 6482 if (Op.isMachineOpcode()) { 6483 if (Op.getMachineOpcode() == PPC::CRSET) 6484 Op2Set = true; 6485 else if (Op.getMachineOpcode() == PPC::CRUNSET) 6486 Op2Unset = true; 6487 else if (Op.getMachineOpcode() == PPC::CRNOR && 6488 Op.getOperand(0) == Op.getOperand(1)) 6489 Op2Not = true; 6490 } 6491 LLVM_FALLTHROUGH; 6492 } 6493 case PPC::BC: 6494 case PPC::BCn: 6495 case PPC::SELECT_I4: 6496 case PPC::SELECT_I8: 6497 case PPC::SELECT_F4: 6498 case PPC::SELECT_F8: 6499 case PPC::SELECT_SPE: 6500 case PPC::SELECT_SPE4: 6501 case PPC::SELECT_VRRC: 6502 case PPC::SELECT_VSFRC: 6503 case PPC::SELECT_VSSRC: 6504 case PPC::SELECT_VSRC: { 6505 SDValue Op = MachineNode->getOperand(0); 6506 if (Op.isMachineOpcode()) { 6507 if (Op.getMachineOpcode() == PPC::CRSET) 6508 Op1Set = true; 6509 else if (Op.getMachineOpcode() == PPC::CRUNSET) 6510 Op1Unset = true; 6511 else if (Op.getMachineOpcode() == PPC::CRNOR && 6512 Op.getOperand(0) == Op.getOperand(1)) 6513 Op1Not = true; 6514 } 6515 } 6516 break; 6517 } 6518 6519 bool SelectSwap = false; 6520 switch (Opcode) { 6521 default: break; 6522 case PPC::CRAND: 6523 if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) 6524 // x & x = x 6525 ResNode = MachineNode->getOperand(0).getNode(); 6526 else if (Op1Set) 6527 // 1 & y = y 6528 ResNode = MachineNode->getOperand(1).getNode(); 6529 else if (Op2Set) 6530 // x & 1 = x 6531 ResNode = MachineNode->getOperand(0).getNode(); 6532 else if (Op1Unset || Op2Unset) 6533 // x & 0 = 0 & y = 0 6534 ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), 6535 MVT::i1); 6536 else if (Op1Not) 6537 // ~x & y = andc(y, x) 6538 ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), 6539 MVT::i1, MachineNode->getOperand(1), 6540 MachineNode->getOperand(0). 6541 getOperand(0)); 6542 else if (Op2Not) 6543 // x & ~y = andc(x, y) 6544 ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), 6545 MVT::i1, MachineNode->getOperand(0), 6546 MachineNode->getOperand(1). 6547 getOperand(0)); 6548 else if (AllUsersSelectZero(MachineNode)) { 6549 ResNode = CurDAG->getMachineNode(PPC::CRNAND, SDLoc(MachineNode), 6550 MVT::i1, MachineNode->getOperand(0), 6551 MachineNode->getOperand(1)); 6552 SelectSwap = true; 6553 } 6554 break; 6555 case PPC::CRNAND: 6556 if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) 6557 // nand(x, x) -> nor(x, x) 6558 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 6559 MVT::i1, MachineNode->getOperand(0), 6560 MachineNode->getOperand(0)); 6561 else if (Op1Set) 6562 // nand(1, y) -> nor(y, y) 6563 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 6564 MVT::i1, MachineNode->getOperand(1), 6565 MachineNode->getOperand(1)); 6566 else if (Op2Set) 6567 // nand(x, 1) -> nor(x, x) 6568 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 6569 MVT::i1, MachineNode->getOperand(0), 6570 MachineNode->getOperand(0)); 6571 else if (Op1Unset || Op2Unset) 6572 // nand(x, 0) = nand(0, y) = 1 6573 ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), 6574 MVT::i1); 6575 else if (Op1Not) 6576 // nand(~x, y) = ~(~x & y) = x | ~y = orc(x, y) 6577 ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), 6578 MVT::i1, MachineNode->getOperand(0). 6579 getOperand(0), 6580 MachineNode->getOperand(1)); 6581 else if (Op2Not) 6582 // nand(x, ~y) = ~x | y = orc(y, x) 6583 ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), 6584 MVT::i1, MachineNode->getOperand(1). 6585 getOperand(0), 6586 MachineNode->getOperand(0)); 6587 else if (AllUsersSelectZero(MachineNode)) { 6588 ResNode = CurDAG->getMachineNode(PPC::CRAND, SDLoc(MachineNode), 6589 MVT::i1, MachineNode->getOperand(0), 6590 MachineNode->getOperand(1)); 6591 SelectSwap = true; 6592 } 6593 break; 6594 case PPC::CROR: 6595 if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) 6596 // x | x = x 6597 ResNode = MachineNode->getOperand(0).getNode(); 6598 else if (Op1Set || Op2Set) 6599 // x | 1 = 1 | y = 1 6600 ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), 6601 MVT::i1); 6602 else if (Op1Unset) 6603 // 0 | y = y 6604 ResNode = MachineNode->getOperand(1).getNode(); 6605 else if (Op2Unset) 6606 // x | 0 = x 6607 ResNode = MachineNode->getOperand(0).getNode(); 6608 else if (Op1Not) 6609 // ~x | y = orc(y, x) 6610 ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), 6611 MVT::i1, MachineNode->getOperand(1), 6612 MachineNode->getOperand(0). 6613 getOperand(0)); 6614 else if (Op2Not) 6615 // x | ~y = orc(x, y) 6616 ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), 6617 MVT::i1, MachineNode->getOperand(0), 6618 MachineNode->getOperand(1). 6619 getOperand(0)); 6620 else if (AllUsersSelectZero(MachineNode)) { 6621 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 6622 MVT::i1, MachineNode->getOperand(0), 6623 MachineNode->getOperand(1)); 6624 SelectSwap = true; 6625 } 6626 break; 6627 case PPC::CRXOR: 6628 if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) 6629 // xor(x, x) = 0 6630 ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), 6631 MVT::i1); 6632 else if (Op1Set) 6633 // xor(1, y) -> nor(y, y) 6634 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 6635 MVT::i1, MachineNode->getOperand(1), 6636 MachineNode->getOperand(1)); 6637 else if (Op2Set) 6638 // xor(x, 1) -> nor(x, x) 6639 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 6640 MVT::i1, MachineNode->getOperand(0), 6641 MachineNode->getOperand(0)); 6642 else if (Op1Unset) 6643 // xor(0, y) = y 6644 ResNode = MachineNode->getOperand(1).getNode(); 6645 else if (Op2Unset) 6646 // xor(x, 0) = x 6647 ResNode = MachineNode->getOperand(0).getNode(); 6648 else if (Op1Not) 6649 // xor(~x, y) = eqv(x, y) 6650 ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode), 6651 MVT::i1, MachineNode->getOperand(0). 6652 getOperand(0), 6653 MachineNode->getOperand(1)); 6654 else if (Op2Not) 6655 // xor(x, ~y) = eqv(x, y) 6656 ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode), 6657 MVT::i1, MachineNode->getOperand(0), 6658 MachineNode->getOperand(1). 6659 getOperand(0)); 6660 else if (AllUsersSelectZero(MachineNode)) { 6661 ResNode = CurDAG->getMachineNode(PPC::CREQV, SDLoc(MachineNode), 6662 MVT::i1, MachineNode->getOperand(0), 6663 MachineNode->getOperand(1)); 6664 SelectSwap = true; 6665 } 6666 break; 6667 case PPC::CRNOR: 6668 if (Op1Set || Op2Set) 6669 // nor(1, y) -> 0 6670 ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), 6671 MVT::i1); 6672 else if (Op1Unset) 6673 // nor(0, y) = ~y -> nor(y, y) 6674 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 6675 MVT::i1, MachineNode->getOperand(1), 6676 MachineNode->getOperand(1)); 6677 else if (Op2Unset) 6678 // nor(x, 0) = ~x 6679 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 6680 MVT::i1, MachineNode->getOperand(0), 6681 MachineNode->getOperand(0)); 6682 else if (Op1Not) 6683 // nor(~x, y) = andc(x, y) 6684 ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), 6685 MVT::i1, MachineNode->getOperand(0). 6686 getOperand(0), 6687 MachineNode->getOperand(1)); 6688 else if (Op2Not) 6689 // nor(x, ~y) = andc(y, x) 6690 ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), 6691 MVT::i1, MachineNode->getOperand(1). 6692 getOperand(0), 6693 MachineNode->getOperand(0)); 6694 else if (AllUsersSelectZero(MachineNode)) { 6695 ResNode = CurDAG->getMachineNode(PPC::CROR, SDLoc(MachineNode), 6696 MVT::i1, MachineNode->getOperand(0), 6697 MachineNode->getOperand(1)); 6698 SelectSwap = true; 6699 } 6700 break; 6701 case PPC::CREQV: 6702 if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) 6703 // eqv(x, x) = 1 6704 ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), 6705 MVT::i1); 6706 else if (Op1Set) 6707 // eqv(1, y) = y 6708 ResNode = MachineNode->getOperand(1).getNode(); 6709 else if (Op2Set) 6710 // eqv(x, 1) = x 6711 ResNode = MachineNode->getOperand(0).getNode(); 6712 else if (Op1Unset) 6713 // eqv(0, y) = ~y -> nor(y, y) 6714 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 6715 MVT::i1, MachineNode->getOperand(1), 6716 MachineNode->getOperand(1)); 6717 else if (Op2Unset) 6718 // eqv(x, 0) = ~x 6719 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 6720 MVT::i1, MachineNode->getOperand(0), 6721 MachineNode->getOperand(0)); 6722 else if (Op1Not) 6723 // eqv(~x, y) = xor(x, y) 6724 ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode), 6725 MVT::i1, MachineNode->getOperand(0). 6726 getOperand(0), 6727 MachineNode->getOperand(1)); 6728 else if (Op2Not) 6729 // eqv(x, ~y) = xor(x, y) 6730 ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode), 6731 MVT::i1, MachineNode->getOperand(0), 6732 MachineNode->getOperand(1). 6733 getOperand(0)); 6734 else if (AllUsersSelectZero(MachineNode)) { 6735 ResNode = CurDAG->getMachineNode(PPC::CRXOR, SDLoc(MachineNode), 6736 MVT::i1, MachineNode->getOperand(0), 6737 MachineNode->getOperand(1)); 6738 SelectSwap = true; 6739 } 6740 break; 6741 case PPC::CRANDC: 6742 if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) 6743 // andc(x, x) = 0 6744 ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), 6745 MVT::i1); 6746 else if (Op1Set) 6747 // andc(1, y) = ~y 6748 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 6749 MVT::i1, MachineNode->getOperand(1), 6750 MachineNode->getOperand(1)); 6751 else if (Op1Unset || Op2Set) 6752 // andc(0, y) = andc(x, 1) = 0 6753 ResNode = CurDAG->getMachineNode(PPC::CRUNSET, SDLoc(MachineNode), 6754 MVT::i1); 6755 else if (Op2Unset) 6756 // andc(x, 0) = x 6757 ResNode = MachineNode->getOperand(0).getNode(); 6758 else if (Op1Not) 6759 // andc(~x, y) = ~(x | y) = nor(x, y) 6760 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 6761 MVT::i1, MachineNode->getOperand(0). 6762 getOperand(0), 6763 MachineNode->getOperand(1)); 6764 else if (Op2Not) 6765 // andc(x, ~y) = x & y 6766 ResNode = CurDAG->getMachineNode(PPC::CRAND, SDLoc(MachineNode), 6767 MVT::i1, MachineNode->getOperand(0), 6768 MachineNode->getOperand(1). 6769 getOperand(0)); 6770 else if (AllUsersSelectZero(MachineNode)) { 6771 ResNode = CurDAG->getMachineNode(PPC::CRORC, SDLoc(MachineNode), 6772 MVT::i1, MachineNode->getOperand(1), 6773 MachineNode->getOperand(0)); 6774 SelectSwap = true; 6775 } 6776 break; 6777 case PPC::CRORC: 6778 if (MachineNode->getOperand(0) == MachineNode->getOperand(1)) 6779 // orc(x, x) = 1 6780 ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), 6781 MVT::i1); 6782 else if (Op1Set || Op2Unset) 6783 // orc(1, y) = orc(x, 0) = 1 6784 ResNode = CurDAG->getMachineNode(PPC::CRSET, SDLoc(MachineNode), 6785 MVT::i1); 6786 else if (Op2Set) 6787 // orc(x, 1) = x 6788 ResNode = MachineNode->getOperand(0).getNode(); 6789 else if (Op1Unset) 6790 // orc(0, y) = ~y 6791 ResNode = CurDAG->getMachineNode(PPC::CRNOR, SDLoc(MachineNode), 6792 MVT::i1, MachineNode->getOperand(1), 6793 MachineNode->getOperand(1)); 6794 else if (Op1Not) 6795 // orc(~x, y) = ~(x & y) = nand(x, y) 6796 ResNode = CurDAG->getMachineNode(PPC::CRNAND, SDLoc(MachineNode), 6797 MVT::i1, MachineNode->getOperand(0). 6798 getOperand(0), 6799 MachineNode->getOperand(1)); 6800 else if (Op2Not) 6801 // orc(x, ~y) = x | y 6802 ResNode = CurDAG->getMachineNode(PPC::CROR, SDLoc(MachineNode), 6803 MVT::i1, MachineNode->getOperand(0), 6804 MachineNode->getOperand(1). 6805 getOperand(0)); 6806 else if (AllUsersSelectZero(MachineNode)) { 6807 ResNode = CurDAG->getMachineNode(PPC::CRANDC, SDLoc(MachineNode), 6808 MVT::i1, MachineNode->getOperand(1), 6809 MachineNode->getOperand(0)); 6810 SelectSwap = true; 6811 } 6812 break; 6813 case PPC::SELECT_I4: 6814 case PPC::SELECT_I8: 6815 case PPC::SELECT_F4: 6816 case PPC::SELECT_F8: 6817 case PPC::SELECT_SPE: 6818 case PPC::SELECT_SPE4: 6819 case PPC::SELECT_VRRC: 6820 case PPC::SELECT_VSFRC: 6821 case PPC::SELECT_VSSRC: 6822 case PPC::SELECT_VSRC: 6823 if (Op1Set) 6824 ResNode = MachineNode->getOperand(1).getNode(); 6825 else if (Op1Unset) 6826 ResNode = MachineNode->getOperand(2).getNode(); 6827 else if (Op1Not) 6828 ResNode = CurDAG->getMachineNode(MachineNode->getMachineOpcode(), 6829 SDLoc(MachineNode), 6830 MachineNode->getValueType(0), 6831 MachineNode->getOperand(0). 6832 getOperand(0), 6833 MachineNode->getOperand(2), 6834 MachineNode->getOperand(1)); 6835 break; 6836 case PPC::BC: 6837 case PPC::BCn: 6838 if (Op1Not) 6839 ResNode = CurDAG->getMachineNode(Opcode == PPC::BC ? PPC::BCn : 6840 PPC::BC, 6841 SDLoc(MachineNode), 6842 MVT::Other, 6843 MachineNode->getOperand(0). 6844 getOperand(0), 6845 MachineNode->getOperand(1), 6846 MachineNode->getOperand(2)); 6847 // FIXME: Handle Op1Set, Op1Unset here too. 6848 break; 6849 } 6850 6851 // If we're inverting this node because it is used only by selects that 6852 // we'd like to swap, then swap the selects before the node replacement. 6853 if (SelectSwap) 6854 SwapAllSelectUsers(MachineNode); 6855 6856 if (ResNode != MachineNode) { 6857 LLVM_DEBUG(dbgs() << "CR Peephole replacing:\nOld: "); 6858 LLVM_DEBUG(MachineNode->dump(CurDAG)); 6859 LLVM_DEBUG(dbgs() << "\nNew: "); 6860 LLVM_DEBUG(ResNode->dump(CurDAG)); 6861 LLVM_DEBUG(dbgs() << "\n"); 6862 6863 ReplaceUses(MachineNode, ResNode); 6864 IsModified = true; 6865 } 6866 } 6867 if (IsModified) 6868 CurDAG->RemoveDeadNodes(); 6869 } while (IsModified); 6870 } 6871 6872 // Gather the set of 32-bit operations that are known to have their 6873 // higher-order 32 bits zero, where ToPromote contains all such operations. 6874 static bool PeepholePPC64ZExtGather(SDValue Op32, 6875 SmallPtrSetImpl<SDNode *> &ToPromote) { 6876 if (!Op32.isMachineOpcode()) 6877 return false; 6878 6879 // First, check for the "frontier" instructions (those that will clear the 6880 // higher-order 32 bits. 6881 6882 // For RLWINM and RLWNM, we need to make sure that the mask does not wrap 6883 // around. If it does not, then these instructions will clear the 6884 // higher-order bits. 6885 if ((Op32.getMachineOpcode() == PPC::RLWINM || 6886 Op32.getMachineOpcode() == PPC::RLWNM) && 6887 Op32.getConstantOperandVal(2) <= Op32.getConstantOperandVal(3)) { 6888 ToPromote.insert(Op32.getNode()); 6889 return true; 6890 } 6891 6892 // SLW and SRW always clear the higher-order bits. 6893 if (Op32.getMachineOpcode() == PPC::SLW || 6894 Op32.getMachineOpcode() == PPC::SRW) { 6895 ToPromote.insert(Op32.getNode()); 6896 return true; 6897 } 6898 6899 // For LI and LIS, we need the immediate to be positive (so that it is not 6900 // sign extended). 6901 if (Op32.getMachineOpcode() == PPC::LI || 6902 Op32.getMachineOpcode() == PPC::LIS) { 6903 if (!isUInt<15>(Op32.getConstantOperandVal(0))) 6904 return false; 6905 6906 ToPromote.insert(Op32.getNode()); 6907 return true; 6908 } 6909 6910 // LHBRX and LWBRX always clear the higher-order bits. 6911 if (Op32.getMachineOpcode() == PPC::LHBRX || 6912 Op32.getMachineOpcode() == PPC::LWBRX) { 6913 ToPromote.insert(Op32.getNode()); 6914 return true; 6915 } 6916 6917 // CNT[LT]ZW always produce a 64-bit value in [0,32], and so is zero extended. 6918 if (Op32.getMachineOpcode() == PPC::CNTLZW || 6919 Op32.getMachineOpcode() == PPC::CNTTZW) { 6920 ToPromote.insert(Op32.getNode()); 6921 return true; 6922 } 6923 6924 // Next, check for those instructions we can look through. 6925 6926 // Assuming the mask does not wrap around, then the higher-order bits are 6927 // taken directly from the first operand. 6928 if (Op32.getMachineOpcode() == PPC::RLWIMI && 6929 Op32.getConstantOperandVal(3) <= Op32.getConstantOperandVal(4)) { 6930 SmallPtrSet<SDNode *, 16> ToPromote1; 6931 if (!PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1)) 6932 return false; 6933 6934 ToPromote.insert(Op32.getNode()); 6935 ToPromote.insert(ToPromote1.begin(), ToPromote1.end()); 6936 return true; 6937 } 6938 6939 // For OR, the higher-order bits are zero if that is true for both operands. 6940 // For SELECT_I4, the same is true (but the relevant operand numbers are 6941 // shifted by 1). 6942 if (Op32.getMachineOpcode() == PPC::OR || 6943 Op32.getMachineOpcode() == PPC::SELECT_I4) { 6944 unsigned B = Op32.getMachineOpcode() == PPC::SELECT_I4 ? 1 : 0; 6945 SmallPtrSet<SDNode *, 16> ToPromote1; 6946 if (!PeepholePPC64ZExtGather(Op32.getOperand(B+0), ToPromote1)) 6947 return false; 6948 if (!PeepholePPC64ZExtGather(Op32.getOperand(B+1), ToPromote1)) 6949 return false; 6950 6951 ToPromote.insert(Op32.getNode()); 6952 ToPromote.insert(ToPromote1.begin(), ToPromote1.end()); 6953 return true; 6954 } 6955 6956 // For ORI and ORIS, we need the higher-order bits of the first operand to be 6957 // zero, and also for the constant to be positive (so that it is not sign 6958 // extended). 6959 if (Op32.getMachineOpcode() == PPC::ORI || 6960 Op32.getMachineOpcode() == PPC::ORIS) { 6961 SmallPtrSet<SDNode *, 16> ToPromote1; 6962 if (!PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1)) 6963 return false; 6964 if (!isUInt<15>(Op32.getConstantOperandVal(1))) 6965 return false; 6966 6967 ToPromote.insert(Op32.getNode()); 6968 ToPromote.insert(ToPromote1.begin(), ToPromote1.end()); 6969 return true; 6970 } 6971 6972 // The higher-order bits of AND are zero if that is true for at least one of 6973 // the operands. 6974 if (Op32.getMachineOpcode() == PPC::AND) { 6975 SmallPtrSet<SDNode *, 16> ToPromote1, ToPromote2; 6976 bool Op0OK = 6977 PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1); 6978 bool Op1OK = 6979 PeepholePPC64ZExtGather(Op32.getOperand(1), ToPromote2); 6980 if (!Op0OK && !Op1OK) 6981 return false; 6982 6983 ToPromote.insert(Op32.getNode()); 6984 6985 if (Op0OK) 6986 ToPromote.insert(ToPromote1.begin(), ToPromote1.end()); 6987 6988 if (Op1OK) 6989 ToPromote.insert(ToPromote2.begin(), ToPromote2.end()); 6990 6991 return true; 6992 } 6993 6994 // For ANDI and ANDIS, the higher-order bits are zero if either that is true 6995 // of the first operand, or if the second operand is positive (so that it is 6996 // not sign extended). 6997 if (Op32.getMachineOpcode() == PPC::ANDI_rec || 6998 Op32.getMachineOpcode() == PPC::ANDIS_rec) { 6999 SmallPtrSet<SDNode *, 16> ToPromote1; 7000 bool Op0OK = 7001 PeepholePPC64ZExtGather(Op32.getOperand(0), ToPromote1); 7002 bool Op1OK = isUInt<15>(Op32.getConstantOperandVal(1)); 7003 if (!Op0OK && !Op1OK) 7004 return false; 7005 7006 ToPromote.insert(Op32.getNode()); 7007 7008 if (Op0OK) 7009 ToPromote.insert(ToPromote1.begin(), ToPromote1.end()); 7010 7011 return true; 7012 } 7013 7014 return false; 7015 } 7016 7017 void PPCDAGToDAGISel::PeepholePPC64ZExt() { 7018 if (!Subtarget->isPPC64()) 7019 return; 7020 7021 // When we zero-extend from i32 to i64, we use a pattern like this: 7022 // def : Pat<(i64 (zext i32:$in)), 7023 // (RLDICL (INSERT_SUBREG (i64 (IMPLICIT_DEF)), $in, sub_32), 7024 // 0, 32)>; 7025 // There are several 32-bit shift/rotate instructions, however, that will 7026 // clear the higher-order bits of their output, rendering the RLDICL 7027 // unnecessary. When that happens, we remove it here, and redefine the 7028 // relevant 32-bit operation to be a 64-bit operation. 7029 7030 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); 7031 7032 bool MadeChange = false; 7033 while (Position != CurDAG->allnodes_begin()) { 7034 SDNode *N = &*--Position; 7035 // Skip dead nodes and any non-machine opcodes. 7036 if (N->use_empty() || !N->isMachineOpcode()) 7037 continue; 7038 7039 if (N->getMachineOpcode() != PPC::RLDICL) 7040 continue; 7041 7042 if (N->getConstantOperandVal(1) != 0 || 7043 N->getConstantOperandVal(2) != 32) 7044 continue; 7045 7046 SDValue ISR = N->getOperand(0); 7047 if (!ISR.isMachineOpcode() || 7048 ISR.getMachineOpcode() != TargetOpcode::INSERT_SUBREG) 7049 continue; 7050 7051 if (!ISR.hasOneUse()) 7052 continue; 7053 7054 if (ISR.getConstantOperandVal(2) != PPC::sub_32) 7055 continue; 7056 7057 SDValue IDef = ISR.getOperand(0); 7058 if (!IDef.isMachineOpcode() || 7059 IDef.getMachineOpcode() != TargetOpcode::IMPLICIT_DEF) 7060 continue; 7061 7062 // We now know that we're looking at a canonical i32 -> i64 zext. See if we 7063 // can get rid of it. 7064 7065 SDValue Op32 = ISR->getOperand(1); 7066 if (!Op32.isMachineOpcode()) 7067 continue; 7068 7069 // There are some 32-bit instructions that always clear the high-order 32 7070 // bits, there are also some instructions (like AND) that we can look 7071 // through. 7072 SmallPtrSet<SDNode *, 16> ToPromote; 7073 if (!PeepholePPC64ZExtGather(Op32, ToPromote)) 7074 continue; 7075 7076 // If the ToPromote set contains nodes that have uses outside of the set 7077 // (except for the original INSERT_SUBREG), then abort the transformation. 7078 bool OutsideUse = false; 7079 for (SDNode *PN : ToPromote) { 7080 for (SDNode *UN : PN->uses()) { 7081 if (!ToPromote.count(UN) && UN != ISR.getNode()) { 7082 OutsideUse = true; 7083 break; 7084 } 7085 } 7086 7087 if (OutsideUse) 7088 break; 7089 } 7090 if (OutsideUse) 7091 continue; 7092 7093 MadeChange = true; 7094 7095 // We now know that this zero extension can be removed by promoting to 7096 // nodes in ToPromote to 64-bit operations, where for operations in the 7097 // frontier of the set, we need to insert INSERT_SUBREGs for their 7098 // operands. 7099 for (SDNode *PN : ToPromote) { 7100 unsigned NewOpcode; 7101 switch (PN->getMachineOpcode()) { 7102 default: 7103 llvm_unreachable("Don't know the 64-bit variant of this instruction"); 7104 case PPC::RLWINM: NewOpcode = PPC::RLWINM8; break; 7105 case PPC::RLWNM: NewOpcode = PPC::RLWNM8; break; 7106 case PPC::SLW: NewOpcode = PPC::SLW8; break; 7107 case PPC::SRW: NewOpcode = PPC::SRW8; break; 7108 case PPC::LI: NewOpcode = PPC::LI8; break; 7109 case PPC::LIS: NewOpcode = PPC::LIS8; break; 7110 case PPC::LHBRX: NewOpcode = PPC::LHBRX8; break; 7111 case PPC::LWBRX: NewOpcode = PPC::LWBRX8; break; 7112 case PPC::CNTLZW: NewOpcode = PPC::CNTLZW8; break; 7113 case PPC::CNTTZW: NewOpcode = PPC::CNTTZW8; break; 7114 case PPC::RLWIMI: NewOpcode = PPC::RLWIMI8; break; 7115 case PPC::OR: NewOpcode = PPC::OR8; break; 7116 case PPC::SELECT_I4: NewOpcode = PPC::SELECT_I8; break; 7117 case PPC::ORI: NewOpcode = PPC::ORI8; break; 7118 case PPC::ORIS: NewOpcode = PPC::ORIS8; break; 7119 case PPC::AND: NewOpcode = PPC::AND8; break; 7120 case PPC::ANDI_rec: 7121 NewOpcode = PPC::ANDI8_rec; 7122 break; 7123 case PPC::ANDIS_rec: 7124 NewOpcode = PPC::ANDIS8_rec; 7125 break; 7126 } 7127 7128 // Note: During the replacement process, the nodes will be in an 7129 // inconsistent state (some instructions will have operands with values 7130 // of the wrong type). Once done, however, everything should be right 7131 // again. 7132 7133 SmallVector<SDValue, 4> Ops; 7134 for (const SDValue &V : PN->ops()) { 7135 if (!ToPromote.count(V.getNode()) && V.getValueType() == MVT::i32 && 7136 !isa<ConstantSDNode>(V)) { 7137 SDValue ReplOpOps[] = { ISR.getOperand(0), V, ISR.getOperand(2) }; 7138 SDNode *ReplOp = 7139 CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG, SDLoc(V), 7140 ISR.getNode()->getVTList(), ReplOpOps); 7141 Ops.push_back(SDValue(ReplOp, 0)); 7142 } else { 7143 Ops.push_back(V); 7144 } 7145 } 7146 7147 // Because all to-be-promoted nodes only have users that are other 7148 // promoted nodes (or the original INSERT_SUBREG), we can safely replace 7149 // the i32 result value type with i64. 7150 7151 SmallVector<EVT, 2> NewVTs; 7152 SDVTList VTs = PN->getVTList(); 7153 for (unsigned i = 0, ie = VTs.NumVTs; i != ie; ++i) 7154 if (VTs.VTs[i] == MVT::i32) 7155 NewVTs.push_back(MVT::i64); 7156 else 7157 NewVTs.push_back(VTs.VTs[i]); 7158 7159 LLVM_DEBUG(dbgs() << "PPC64 ZExt Peephole morphing:\nOld: "); 7160 LLVM_DEBUG(PN->dump(CurDAG)); 7161 7162 CurDAG->SelectNodeTo(PN, NewOpcode, CurDAG->getVTList(NewVTs), Ops); 7163 7164 LLVM_DEBUG(dbgs() << "\nNew: "); 7165 LLVM_DEBUG(PN->dump(CurDAG)); 7166 LLVM_DEBUG(dbgs() << "\n"); 7167 } 7168 7169 // Now we replace the original zero extend and its associated INSERT_SUBREG 7170 // with the value feeding the INSERT_SUBREG (which has now been promoted to 7171 // return an i64). 7172 7173 LLVM_DEBUG(dbgs() << "PPC64 ZExt Peephole replacing:\nOld: "); 7174 LLVM_DEBUG(N->dump(CurDAG)); 7175 LLVM_DEBUG(dbgs() << "\nNew: "); 7176 LLVM_DEBUG(Op32.getNode()->dump(CurDAG)); 7177 LLVM_DEBUG(dbgs() << "\n"); 7178 7179 ReplaceUses(N, Op32.getNode()); 7180 } 7181 7182 if (MadeChange) 7183 CurDAG->RemoveDeadNodes(); 7184 } 7185 7186 static bool isVSXSwap(SDValue N) { 7187 if (!N->isMachineOpcode()) 7188 return false; 7189 unsigned Opc = N->getMachineOpcode(); 7190 7191 // Single-operand XXPERMDI or the regular XXPERMDI/XXSLDWI where the immediate 7192 // operand is 2. 7193 if (Opc == PPC::XXPERMDIs) { 7194 return isa<ConstantSDNode>(N->getOperand(1)) && 7195 N->getConstantOperandVal(1) == 2; 7196 } else if (Opc == PPC::XXPERMDI || Opc == PPC::XXSLDWI) { 7197 return N->getOperand(0) == N->getOperand(1) && 7198 isa<ConstantSDNode>(N->getOperand(2)) && 7199 N->getConstantOperandVal(2) == 2; 7200 } 7201 7202 return false; 7203 } 7204 7205 // TODO: Make this complete and replace with a table-gen bit. 7206 static bool isLaneInsensitive(SDValue N) { 7207 if (!N->isMachineOpcode()) 7208 return false; 7209 unsigned Opc = N->getMachineOpcode(); 7210 7211 switch (Opc) { 7212 default: 7213 return false; 7214 case PPC::VAVGSB: 7215 case PPC::VAVGUB: 7216 case PPC::VAVGSH: 7217 case PPC::VAVGUH: 7218 case PPC::VAVGSW: 7219 case PPC::VAVGUW: 7220 case PPC::VMAXFP: 7221 case PPC::VMAXSB: 7222 case PPC::VMAXUB: 7223 case PPC::VMAXSH: 7224 case PPC::VMAXUH: 7225 case PPC::VMAXSW: 7226 case PPC::VMAXUW: 7227 case PPC::VMINFP: 7228 case PPC::VMINSB: 7229 case PPC::VMINUB: 7230 case PPC::VMINSH: 7231 case PPC::VMINUH: 7232 case PPC::VMINSW: 7233 case PPC::VMINUW: 7234 case PPC::VADDFP: 7235 case PPC::VADDUBM: 7236 case PPC::VADDUHM: 7237 case PPC::VADDUWM: 7238 case PPC::VSUBFP: 7239 case PPC::VSUBUBM: 7240 case PPC::VSUBUHM: 7241 case PPC::VSUBUWM: 7242 case PPC::VAND: 7243 case PPC::VANDC: 7244 case PPC::VOR: 7245 case PPC::VORC: 7246 case PPC::VXOR: 7247 case PPC::VNOR: 7248 case PPC::VMULUWM: 7249 return true; 7250 } 7251 } 7252 7253 // Try to simplify (xxswap (vec-op (xxswap) (xxswap))) where vec-op is 7254 // lane-insensitive. 7255 static void reduceVSXSwap(SDNode *N, SelectionDAG *DAG) { 7256 // Our desired xxswap might be source of COPY_TO_REGCLASS. 7257 // TODO: Can we put this a common method for DAG? 7258 auto SkipRCCopy = [](SDValue V) { 7259 while (V->isMachineOpcode() && 7260 V->getMachineOpcode() == TargetOpcode::COPY_TO_REGCLASS) { 7261 // All values in the chain should have single use. 7262 if (V->use_empty() || !V->use_begin()->isOnlyUserOf(V.getNode())) 7263 return SDValue(); 7264 V = V->getOperand(0); 7265 } 7266 return V.hasOneUse() ? V : SDValue(); 7267 }; 7268 7269 SDValue VecOp = SkipRCCopy(N->getOperand(0)); 7270 if (!VecOp || !isLaneInsensitive(VecOp)) 7271 return; 7272 7273 SDValue LHS = SkipRCCopy(VecOp.getOperand(0)), 7274 RHS = SkipRCCopy(VecOp.getOperand(1)); 7275 if (!LHS || !RHS || !isVSXSwap(LHS) || !isVSXSwap(RHS)) 7276 return; 7277 7278 // These swaps may still have chain-uses here, count on dead code elimination 7279 // in following passes to remove them. 7280 DAG->ReplaceAllUsesOfValueWith(LHS, LHS.getOperand(0)); 7281 DAG->ReplaceAllUsesOfValueWith(RHS, RHS.getOperand(0)); 7282 DAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), N->getOperand(0)); 7283 } 7284 7285 void PPCDAGToDAGISel::PeepholePPC64() { 7286 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); 7287 7288 while (Position != CurDAG->allnodes_begin()) { 7289 SDNode *N = &*--Position; 7290 // Skip dead nodes and any non-machine opcodes. 7291 if (N->use_empty() || !N->isMachineOpcode()) 7292 continue; 7293 7294 if (isVSXSwap(SDValue(N, 0))) 7295 reduceVSXSwap(N, CurDAG); 7296 7297 unsigned FirstOp; 7298 unsigned StorageOpcode = N->getMachineOpcode(); 7299 bool RequiresMod4Offset = false; 7300 7301 switch (StorageOpcode) { 7302 default: continue; 7303 7304 case PPC::LWA: 7305 case PPC::LD: 7306 case PPC::DFLOADf64: 7307 case PPC::DFLOADf32: 7308 RequiresMod4Offset = true; 7309 LLVM_FALLTHROUGH; 7310 case PPC::LBZ: 7311 case PPC::LBZ8: 7312 case PPC::LFD: 7313 case PPC::LFS: 7314 case PPC::LHA: 7315 case PPC::LHA8: 7316 case PPC::LHZ: 7317 case PPC::LHZ8: 7318 case PPC::LWZ: 7319 case PPC::LWZ8: 7320 FirstOp = 0; 7321 break; 7322 7323 case PPC::STD: 7324 case PPC::DFSTOREf64: 7325 case PPC::DFSTOREf32: 7326 RequiresMod4Offset = true; 7327 LLVM_FALLTHROUGH; 7328 case PPC::STB: 7329 case PPC::STB8: 7330 case PPC::STFD: 7331 case PPC::STFS: 7332 case PPC::STH: 7333 case PPC::STH8: 7334 case PPC::STW: 7335 case PPC::STW8: 7336 FirstOp = 1; 7337 break; 7338 } 7339 7340 // If this is a load or store with a zero offset, or within the alignment, 7341 // we may be able to fold an add-immediate into the memory operation. 7342 // The check against alignment is below, as it can't occur until we check 7343 // the arguments to N 7344 if (!isa<ConstantSDNode>(N->getOperand(FirstOp))) 7345 continue; 7346 7347 SDValue Base = N->getOperand(FirstOp + 1); 7348 if (!Base.isMachineOpcode()) 7349 continue; 7350 7351 unsigned Flags = 0; 7352 bool ReplaceFlags = true; 7353 7354 // When the feeding operation is an add-immediate of some sort, 7355 // determine whether we need to add relocation information to the 7356 // target flags on the immediate operand when we fold it into the 7357 // load instruction. 7358 // 7359 // For something like ADDItocL, the relocation information is 7360 // inferred from the opcode; when we process it in the AsmPrinter, 7361 // we add the necessary relocation there. A load, though, can receive 7362 // relocation from various flavors of ADDIxxx, so we need to carry 7363 // the relocation information in the target flags. 7364 switch (Base.getMachineOpcode()) { 7365 default: continue; 7366 7367 case PPC::ADDI8: 7368 case PPC::ADDI: 7369 // In some cases (such as TLS) the relocation information 7370 // is already in place on the operand, so copying the operand 7371 // is sufficient. 7372 ReplaceFlags = false; 7373 // For these cases, the immediate may not be divisible by 4, in 7374 // which case the fold is illegal for DS-form instructions. (The 7375 // other cases provide aligned addresses and are always safe.) 7376 if (RequiresMod4Offset && 7377 (!isa<ConstantSDNode>(Base.getOperand(1)) || 7378 Base.getConstantOperandVal(1) % 4 != 0)) 7379 continue; 7380 break; 7381 case PPC::ADDIdtprelL: 7382 Flags = PPCII::MO_DTPREL_LO; 7383 break; 7384 case PPC::ADDItlsldL: 7385 Flags = PPCII::MO_TLSLD_LO; 7386 break; 7387 case PPC::ADDItocL: 7388 Flags = PPCII::MO_TOC_LO; 7389 break; 7390 } 7391 7392 SDValue ImmOpnd = Base.getOperand(1); 7393 7394 // On PPC64, the TOC base pointer is guaranteed by the ABI only to have 7395 // 8-byte alignment, and so we can only use offsets less than 8 (otherwise, 7396 // we might have needed different @ha relocation values for the offset 7397 // pointers). 7398 int MaxDisplacement = 7; 7399 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(ImmOpnd)) { 7400 const GlobalValue *GV = GA->getGlobal(); 7401 Align Alignment = GV->getPointerAlignment(CurDAG->getDataLayout()); 7402 MaxDisplacement = std::min((int)Alignment.value() - 1, MaxDisplacement); 7403 } 7404 7405 bool UpdateHBase = false; 7406 SDValue HBase = Base.getOperand(0); 7407 7408 int Offset = N->getConstantOperandVal(FirstOp); 7409 if (ReplaceFlags) { 7410 if (Offset < 0 || Offset > MaxDisplacement) { 7411 // If we have a addi(toc@l)/addis(toc@ha) pair, and the addis has only 7412 // one use, then we can do this for any offset, we just need to also 7413 // update the offset (i.e. the symbol addend) on the addis also. 7414 if (Base.getMachineOpcode() != PPC::ADDItocL) 7415 continue; 7416 7417 if (!HBase.isMachineOpcode() || 7418 HBase.getMachineOpcode() != PPC::ADDIStocHA8) 7419 continue; 7420 7421 if (!Base.hasOneUse() || !HBase.hasOneUse()) 7422 continue; 7423 7424 SDValue HImmOpnd = HBase.getOperand(1); 7425 if (HImmOpnd != ImmOpnd) 7426 continue; 7427 7428 UpdateHBase = true; 7429 } 7430 } else { 7431 // If we're directly folding the addend from an addi instruction, then: 7432 // 1. In general, the offset on the memory access must be zero. 7433 // 2. If the addend is a constant, then it can be combined with a 7434 // non-zero offset, but only if the result meets the encoding 7435 // requirements. 7436 if (auto *C = dyn_cast<ConstantSDNode>(ImmOpnd)) { 7437 Offset += C->getSExtValue(); 7438 7439 if (RequiresMod4Offset && (Offset % 4) != 0) 7440 continue; 7441 7442 if (!isInt<16>(Offset)) 7443 continue; 7444 7445 ImmOpnd = CurDAG->getTargetConstant(Offset, SDLoc(ImmOpnd), 7446 ImmOpnd.getValueType()); 7447 } else if (Offset != 0) { 7448 continue; 7449 } 7450 } 7451 7452 // We found an opportunity. Reverse the operands from the add 7453 // immediate and substitute them into the load or store. If 7454 // needed, update the target flags for the immediate operand to 7455 // reflect the necessary relocation information. 7456 LLVM_DEBUG(dbgs() << "Folding add-immediate into mem-op:\nBase: "); 7457 LLVM_DEBUG(Base->dump(CurDAG)); 7458 LLVM_DEBUG(dbgs() << "\nN: "); 7459 LLVM_DEBUG(N->dump(CurDAG)); 7460 LLVM_DEBUG(dbgs() << "\n"); 7461 7462 // If the relocation information isn't already present on the 7463 // immediate operand, add it now. 7464 if (ReplaceFlags) { 7465 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(ImmOpnd)) { 7466 SDLoc dl(GA); 7467 const GlobalValue *GV = GA->getGlobal(); 7468 Align Alignment = GV->getPointerAlignment(CurDAG->getDataLayout()); 7469 // We can't perform this optimization for data whose alignment 7470 // is insufficient for the instruction encoding. 7471 if (Alignment < 4 && (RequiresMod4Offset || (Offset % 4) != 0)) { 7472 LLVM_DEBUG(dbgs() << "Rejected this candidate for alignment.\n\n"); 7473 continue; 7474 } 7475 ImmOpnd = CurDAG->getTargetGlobalAddress(GV, dl, MVT::i64, Offset, Flags); 7476 } else if (ConstantPoolSDNode *CP = 7477 dyn_cast<ConstantPoolSDNode>(ImmOpnd)) { 7478 const Constant *C = CP->getConstVal(); 7479 ImmOpnd = CurDAG->getTargetConstantPool(C, MVT::i64, CP->getAlign(), 7480 Offset, Flags); 7481 } 7482 } 7483 7484 if (FirstOp == 1) // Store 7485 (void)CurDAG->UpdateNodeOperands(N, N->getOperand(0), ImmOpnd, 7486 Base.getOperand(0), N->getOperand(3)); 7487 else // Load 7488 (void)CurDAG->UpdateNodeOperands(N, ImmOpnd, Base.getOperand(0), 7489 N->getOperand(2)); 7490 7491 if (UpdateHBase) 7492 (void)CurDAG->UpdateNodeOperands(HBase.getNode(), HBase.getOperand(0), 7493 ImmOpnd); 7494 7495 // The add-immediate may now be dead, in which case remove it. 7496 if (Base.getNode()->use_empty()) 7497 CurDAG->RemoveDeadNode(Base.getNode()); 7498 } 7499 } 7500 7501 /// createPPCISelDag - This pass converts a legalized DAG into a 7502 /// PowerPC-specific DAG, ready for instruction scheduling. 7503 /// 7504 FunctionPass *llvm::createPPCISelDag(PPCTargetMachine &TM, 7505 CodeGenOpt::Level OptLevel) { 7506 return new PPCDAGToDAGISel(TM, OptLevel); 7507 } 7508