1 //===-- PPCInstrInfo.cpp - PowerPC Instruction Information ----------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains the PowerPC implementation of the TargetInstrInfo class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "PPCInstrInfo.h" 14 #include "MCTargetDesc/PPCPredicates.h" 15 #include "PPC.h" 16 #include "PPCHazardRecognizers.h" 17 #include "PPCInstrBuilder.h" 18 #include "PPCMachineFunctionInfo.h" 19 #include "PPCTargetMachine.h" 20 #include "llvm/ADT/STLExtras.h" 21 #include "llvm/ADT/Statistic.h" 22 #include "llvm/Analysis/AliasAnalysis.h" 23 #include "llvm/CodeGen/LiveIntervals.h" 24 #include "llvm/CodeGen/MachineConstantPool.h" 25 #include "llvm/CodeGen/MachineFrameInfo.h" 26 #include "llvm/CodeGen/MachineFunctionPass.h" 27 #include "llvm/CodeGen/MachineInstrBuilder.h" 28 #include "llvm/CodeGen/MachineMemOperand.h" 29 #include "llvm/CodeGen/MachineRegisterInfo.h" 30 #include "llvm/CodeGen/PseudoSourceValue.h" 31 #include "llvm/CodeGen/RegisterClassInfo.h" 32 #include "llvm/CodeGen/RegisterPressure.h" 33 #include "llvm/CodeGen/ScheduleDAG.h" 34 #include "llvm/CodeGen/SlotIndexes.h" 35 #include "llvm/CodeGen/StackMaps.h" 36 #include "llvm/MC/MCAsmInfo.h" 37 #include "llvm/MC/MCInst.h" 38 #include "llvm/MC/TargetRegistry.h" 39 #include "llvm/Support/CommandLine.h" 40 #include "llvm/Support/Debug.h" 41 #include "llvm/Support/ErrorHandling.h" 42 #include "llvm/Support/raw_ostream.h" 43 44 using namespace llvm; 45 46 #define DEBUG_TYPE "ppc-instr-info" 47 48 #define GET_INSTRMAP_INFO 49 #define GET_INSTRINFO_CTOR_DTOR 50 #include "PPCGenInstrInfo.inc" 51 52 STATISTIC(NumStoreSPILLVSRRCAsVec, 53 "Number of spillvsrrc spilled to stack as vec"); 54 STATISTIC(NumStoreSPILLVSRRCAsGpr, 55 "Number of spillvsrrc spilled to stack as gpr"); 56 STATISTIC(NumGPRtoVSRSpill, "Number of gpr spills to spillvsrrc"); 57 STATISTIC(CmpIselsConverted, 58 "Number of ISELs that depend on comparison of constants converted"); 59 STATISTIC(MissedConvertibleImmediateInstrs, 60 "Number of compare-immediate instructions fed by constants"); 61 STATISTIC(NumRcRotatesConvertedToRcAnd, 62 "Number of record-form rotates converted to record-form andi"); 63 64 static cl:: 65 opt<bool> DisableCTRLoopAnal("disable-ppc-ctrloop-analysis", cl::Hidden, 66 cl::desc("Disable analysis for CTR loops")); 67 68 static cl::opt<bool> DisableCmpOpt("disable-ppc-cmp-opt", 69 cl::desc("Disable compare instruction optimization"), cl::Hidden); 70 71 static cl::opt<bool> VSXSelfCopyCrash("crash-on-ppc-vsx-self-copy", 72 cl::desc("Causes the backend to crash instead of generating a nop VSX copy"), 73 cl::Hidden); 74 75 static cl::opt<bool> 76 UseOldLatencyCalc("ppc-old-latency-calc", cl::Hidden, 77 cl::desc("Use the old (incorrect) instruction latency calculation")); 78 79 static cl::opt<float> 80 FMARPFactor("ppc-fma-rp-factor", cl::Hidden, cl::init(1.5), 81 cl::desc("register pressure factor for the transformations.")); 82 83 static cl::opt<bool> EnableFMARegPressureReduction( 84 "ppc-fma-rp-reduction", cl::Hidden, cl::init(true), 85 cl::desc("enable register pressure reduce in machine combiner pass.")); 86 87 // Pin the vtable to this file. 88 void PPCInstrInfo::anchor() {} 89 90 PPCInstrInfo::PPCInstrInfo(PPCSubtarget &STI) 91 : PPCGenInstrInfo(PPC::ADJCALLSTACKDOWN, PPC::ADJCALLSTACKUP, 92 /* CatchRetOpcode */ -1, 93 STI.isPPC64() ? PPC::BLR8 : PPC::BLR), 94 Subtarget(STI), RI(STI.getTargetMachine()) {} 95 96 /// CreateTargetHazardRecognizer - Return the hazard recognizer to use for 97 /// this target when scheduling the DAG. 98 ScheduleHazardRecognizer * 99 PPCInstrInfo::CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI, 100 const ScheduleDAG *DAG) const { 101 unsigned Directive = 102 static_cast<const PPCSubtarget *>(STI)->getCPUDirective(); 103 if (Directive == PPC::DIR_440 || Directive == PPC::DIR_A2 || 104 Directive == PPC::DIR_E500mc || Directive == PPC::DIR_E5500) { 105 const InstrItineraryData *II = 106 static_cast<const PPCSubtarget *>(STI)->getInstrItineraryData(); 107 return new ScoreboardHazardRecognizer(II, DAG); 108 } 109 110 return TargetInstrInfo::CreateTargetHazardRecognizer(STI, DAG); 111 } 112 113 /// CreateTargetPostRAHazardRecognizer - Return the postRA hazard recognizer 114 /// to use for this target when scheduling the DAG. 115 ScheduleHazardRecognizer * 116 PPCInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II, 117 const ScheduleDAG *DAG) const { 118 unsigned Directive = 119 DAG->MF.getSubtarget<PPCSubtarget>().getCPUDirective(); 120 121 // FIXME: Leaving this as-is until we have POWER9 scheduling info 122 if (Directive == PPC::DIR_PWR7 || Directive == PPC::DIR_PWR8) 123 return new PPCDispatchGroupSBHazardRecognizer(II, DAG); 124 125 // Most subtargets use a PPC970 recognizer. 126 if (Directive != PPC::DIR_440 && Directive != PPC::DIR_A2 && 127 Directive != PPC::DIR_E500mc && Directive != PPC::DIR_E5500) { 128 assert(DAG->TII && "No InstrInfo?"); 129 130 return new PPCHazardRecognizer970(*DAG); 131 } 132 133 return new ScoreboardHazardRecognizer(II, DAG); 134 } 135 136 unsigned PPCInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, 137 const MachineInstr &MI, 138 unsigned *PredCost) const { 139 if (!ItinData || UseOldLatencyCalc) 140 return PPCGenInstrInfo::getInstrLatency(ItinData, MI, PredCost); 141 142 // The default implementation of getInstrLatency calls getStageLatency, but 143 // getStageLatency does not do the right thing for us. While we have 144 // itinerary, most cores are fully pipelined, and so the itineraries only 145 // express the first part of the pipeline, not every stage. Instead, we need 146 // to use the listed output operand cycle number (using operand 0 here, which 147 // is an output). 148 149 unsigned Latency = 1; 150 unsigned DefClass = MI.getDesc().getSchedClass(); 151 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 152 const MachineOperand &MO = MI.getOperand(i); 153 if (!MO.isReg() || !MO.isDef() || MO.isImplicit()) 154 continue; 155 156 int Cycle = ItinData->getOperandCycle(DefClass, i); 157 if (Cycle < 0) 158 continue; 159 160 Latency = std::max(Latency, (unsigned) Cycle); 161 } 162 163 return Latency; 164 } 165 166 int PPCInstrInfo::getOperandLatency(const InstrItineraryData *ItinData, 167 const MachineInstr &DefMI, unsigned DefIdx, 168 const MachineInstr &UseMI, 169 unsigned UseIdx) const { 170 int Latency = PPCGenInstrInfo::getOperandLatency(ItinData, DefMI, DefIdx, 171 UseMI, UseIdx); 172 173 if (!DefMI.getParent()) 174 return Latency; 175 176 const MachineOperand &DefMO = DefMI.getOperand(DefIdx); 177 Register Reg = DefMO.getReg(); 178 179 bool IsRegCR; 180 if (Register::isVirtualRegister(Reg)) { 181 const MachineRegisterInfo *MRI = 182 &DefMI.getParent()->getParent()->getRegInfo(); 183 IsRegCR = MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRRCRegClass) || 184 MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRBITRCRegClass); 185 } else { 186 IsRegCR = PPC::CRRCRegClass.contains(Reg) || 187 PPC::CRBITRCRegClass.contains(Reg); 188 } 189 190 if (UseMI.isBranch() && IsRegCR) { 191 if (Latency < 0) 192 Latency = getInstrLatency(ItinData, DefMI); 193 194 // On some cores, there is an additional delay between writing to a condition 195 // register, and using it from a branch. 196 unsigned Directive = Subtarget.getCPUDirective(); 197 switch (Directive) { 198 default: break; 199 case PPC::DIR_7400: 200 case PPC::DIR_750: 201 case PPC::DIR_970: 202 case PPC::DIR_E5500: 203 case PPC::DIR_PWR4: 204 case PPC::DIR_PWR5: 205 case PPC::DIR_PWR5X: 206 case PPC::DIR_PWR6: 207 case PPC::DIR_PWR6X: 208 case PPC::DIR_PWR7: 209 case PPC::DIR_PWR8: 210 // FIXME: Is this needed for POWER9? 211 Latency += 2; 212 break; 213 } 214 } 215 216 return Latency; 217 } 218 219 /// This is an architecture-specific helper function of reassociateOps. 220 /// Set special operand attributes for new instructions after reassociation. 221 void PPCInstrInfo::setSpecialOperandAttr(MachineInstr &OldMI1, 222 MachineInstr &OldMI2, 223 MachineInstr &NewMI1, 224 MachineInstr &NewMI2) const { 225 // Propagate FP flags from the original instructions. 226 // But clear poison-generating flags because those may not be valid now. 227 uint16_t IntersectedFlags = OldMI1.getFlags() & OldMI2.getFlags(); 228 NewMI1.setFlags(IntersectedFlags); 229 NewMI1.clearFlag(MachineInstr::MIFlag::NoSWrap); 230 NewMI1.clearFlag(MachineInstr::MIFlag::NoUWrap); 231 NewMI1.clearFlag(MachineInstr::MIFlag::IsExact); 232 233 NewMI2.setFlags(IntersectedFlags); 234 NewMI2.clearFlag(MachineInstr::MIFlag::NoSWrap); 235 NewMI2.clearFlag(MachineInstr::MIFlag::NoUWrap); 236 NewMI2.clearFlag(MachineInstr::MIFlag::IsExact); 237 } 238 239 void PPCInstrInfo::setSpecialOperandAttr(MachineInstr &MI, 240 uint16_t Flags) const { 241 MI.setFlags(Flags); 242 MI.clearFlag(MachineInstr::MIFlag::NoSWrap); 243 MI.clearFlag(MachineInstr::MIFlag::NoUWrap); 244 MI.clearFlag(MachineInstr::MIFlag::IsExact); 245 } 246 247 // This function does not list all associative and commutative operations, but 248 // only those worth feeding through the machine combiner in an attempt to 249 // reduce the critical path. Mostly, this means floating-point operations, 250 // because they have high latencies(>=5) (compared to other operations, such as 251 // and/or, which are also associative and commutative, but have low latencies). 252 bool PPCInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst) const { 253 switch (Inst.getOpcode()) { 254 // Floating point: 255 // FP Add: 256 case PPC::FADD: 257 case PPC::FADDS: 258 // FP Multiply: 259 case PPC::FMUL: 260 case PPC::FMULS: 261 // Altivec Add: 262 case PPC::VADDFP: 263 // VSX Add: 264 case PPC::XSADDDP: 265 case PPC::XVADDDP: 266 case PPC::XVADDSP: 267 case PPC::XSADDSP: 268 // VSX Multiply: 269 case PPC::XSMULDP: 270 case PPC::XVMULDP: 271 case PPC::XVMULSP: 272 case PPC::XSMULSP: 273 return Inst.getFlag(MachineInstr::MIFlag::FmReassoc) && 274 Inst.getFlag(MachineInstr::MIFlag::FmNsz); 275 // Fixed point: 276 // Multiply: 277 case PPC::MULHD: 278 case PPC::MULLD: 279 case PPC::MULHW: 280 case PPC::MULLW: 281 return true; 282 default: 283 return false; 284 } 285 } 286 287 #define InfoArrayIdxFMAInst 0 288 #define InfoArrayIdxFAddInst 1 289 #define InfoArrayIdxFMULInst 2 290 #define InfoArrayIdxAddOpIdx 3 291 #define InfoArrayIdxMULOpIdx 4 292 #define InfoArrayIdxFSubInst 5 293 // Array keeps info for FMA instructions: 294 // Index 0(InfoArrayIdxFMAInst): FMA instruction; 295 // Index 1(InfoArrayIdxFAddInst): ADD instruction associated with FMA; 296 // Index 2(InfoArrayIdxFMULInst): MUL instruction associated with FMA; 297 // Index 3(InfoArrayIdxAddOpIdx): ADD operand index in FMA operands; 298 // Index 4(InfoArrayIdxMULOpIdx): first MUL operand index in FMA operands; 299 // second MUL operand index is plus 1; 300 // Index 5(InfoArrayIdxFSubInst): SUB instruction associated with FMA. 301 static const uint16_t FMAOpIdxInfo[][6] = { 302 // FIXME: Add more FMA instructions like XSNMADDADP and so on. 303 {PPC::XSMADDADP, PPC::XSADDDP, PPC::XSMULDP, 1, 2, PPC::XSSUBDP}, 304 {PPC::XSMADDASP, PPC::XSADDSP, PPC::XSMULSP, 1, 2, PPC::XSSUBSP}, 305 {PPC::XVMADDADP, PPC::XVADDDP, PPC::XVMULDP, 1, 2, PPC::XVSUBDP}, 306 {PPC::XVMADDASP, PPC::XVADDSP, PPC::XVMULSP, 1, 2, PPC::XVSUBSP}, 307 {PPC::FMADD, PPC::FADD, PPC::FMUL, 3, 1, PPC::FSUB}, 308 {PPC::FMADDS, PPC::FADDS, PPC::FMULS, 3, 1, PPC::FSUBS}}; 309 310 // Check if an opcode is a FMA instruction. If it is, return the index in array 311 // FMAOpIdxInfo. Otherwise, return -1. 312 int16_t PPCInstrInfo::getFMAOpIdxInfo(unsigned Opcode) const { 313 for (unsigned I = 0; I < array_lengthof(FMAOpIdxInfo); I++) 314 if (FMAOpIdxInfo[I][InfoArrayIdxFMAInst] == Opcode) 315 return I; 316 return -1; 317 } 318 319 // On PowerPC target, we have two kinds of patterns related to FMA: 320 // 1: Improve ILP. 321 // Try to reassociate FMA chains like below: 322 // 323 // Pattern 1: 324 // A = FADD X, Y (Leaf) 325 // B = FMA A, M21, M22 (Prev) 326 // C = FMA B, M31, M32 (Root) 327 // --> 328 // A = FMA X, M21, M22 329 // B = FMA Y, M31, M32 330 // C = FADD A, B 331 // 332 // Pattern 2: 333 // A = FMA X, M11, M12 (Leaf) 334 // B = FMA A, M21, M22 (Prev) 335 // C = FMA B, M31, M32 (Root) 336 // --> 337 // A = FMUL M11, M12 338 // B = FMA X, M21, M22 339 // D = FMA A, M31, M32 340 // C = FADD B, D 341 // 342 // breaking the dependency between A and B, allowing FMA to be executed in 343 // parallel (or back-to-back in a pipeline) instead of depending on each other. 344 // 345 // 2: Reduce register pressure. 346 // Try to reassociate FMA with FSUB and a constant like below: 347 // C is a floating point const. 348 // 349 // Pattern 1: 350 // A = FSUB X, Y (Leaf) 351 // D = FMA B, C, A (Root) 352 // --> 353 // A = FMA B, Y, -C 354 // D = FMA A, X, C 355 // 356 // Pattern 2: 357 // A = FSUB X, Y (Leaf) 358 // D = FMA B, A, C (Root) 359 // --> 360 // A = FMA B, Y, -C 361 // D = FMA A, X, C 362 // 363 // Before the transformation, A must be assigned with different hardware 364 // register with D. After the transformation, A and D must be assigned with 365 // same hardware register due to TIE attribute of FMA instructions. 366 // 367 bool PPCInstrInfo::getFMAPatterns( 368 MachineInstr &Root, SmallVectorImpl<MachineCombinerPattern> &Patterns, 369 bool DoRegPressureReduce) const { 370 MachineBasicBlock *MBB = Root.getParent(); 371 const MachineRegisterInfo *MRI = &MBB->getParent()->getRegInfo(); 372 const TargetRegisterInfo *TRI = &getRegisterInfo(); 373 374 auto IsAllOpsVirtualReg = [](const MachineInstr &Instr) { 375 for (const auto &MO : Instr.explicit_operands()) 376 if (!(MO.isReg() && Register::isVirtualRegister(MO.getReg()))) 377 return false; 378 return true; 379 }; 380 381 auto IsReassociableAddOrSub = [&](const MachineInstr &Instr, 382 unsigned OpType) { 383 if (Instr.getOpcode() != 384 FMAOpIdxInfo[getFMAOpIdxInfo(Root.getOpcode())][OpType]) 385 return false; 386 387 // Instruction can be reassociated. 388 // fast math flags may prohibit reassociation. 389 if (!(Instr.getFlag(MachineInstr::MIFlag::FmReassoc) && 390 Instr.getFlag(MachineInstr::MIFlag::FmNsz))) 391 return false; 392 393 // Instruction operands are virtual registers for reassociation. 394 if (!IsAllOpsVirtualReg(Instr)) 395 return false; 396 397 // For register pressure reassociation, the FSub must have only one use as 398 // we want to delete the sub to save its def. 399 if (OpType == InfoArrayIdxFSubInst && 400 !MRI->hasOneNonDBGUse(Instr.getOperand(0).getReg())) 401 return false; 402 403 return true; 404 }; 405 406 auto IsReassociableFMA = [&](const MachineInstr &Instr, int16_t &AddOpIdx, 407 int16_t &MulOpIdx, bool IsLeaf) { 408 int16_t Idx = getFMAOpIdxInfo(Instr.getOpcode()); 409 if (Idx < 0) 410 return false; 411 412 // Instruction can be reassociated. 413 // fast math flags may prohibit reassociation. 414 if (!(Instr.getFlag(MachineInstr::MIFlag::FmReassoc) && 415 Instr.getFlag(MachineInstr::MIFlag::FmNsz))) 416 return false; 417 418 // Instruction operands are virtual registers for reassociation. 419 if (!IsAllOpsVirtualReg(Instr)) 420 return false; 421 422 MulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx]; 423 if (IsLeaf) 424 return true; 425 426 AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx]; 427 428 const MachineOperand &OpAdd = Instr.getOperand(AddOpIdx); 429 MachineInstr *MIAdd = MRI->getUniqueVRegDef(OpAdd.getReg()); 430 // If 'add' operand's def is not in current block, don't do ILP related opt. 431 if (!MIAdd || MIAdd->getParent() != MBB) 432 return false; 433 434 // If this is not Leaf FMA Instr, its 'add' operand should only have one use 435 // as this fma will be changed later. 436 return IsLeaf ? true : MRI->hasOneNonDBGUse(OpAdd.getReg()); 437 }; 438 439 int16_t AddOpIdx = -1; 440 int16_t MulOpIdx = -1; 441 442 bool IsUsedOnceL = false; 443 bool IsUsedOnceR = false; 444 MachineInstr *MULInstrL = nullptr; 445 MachineInstr *MULInstrR = nullptr; 446 447 auto IsRPReductionCandidate = [&]() { 448 // Currently, we only support float and double. 449 // FIXME: add support for other types. 450 unsigned Opcode = Root.getOpcode(); 451 if (Opcode != PPC::XSMADDASP && Opcode != PPC::XSMADDADP) 452 return false; 453 454 // Root must be a valid FMA like instruction. 455 // Treat it as leaf as we don't care its add operand. 456 if (IsReassociableFMA(Root, AddOpIdx, MulOpIdx, true)) { 457 assert((MulOpIdx >= 0) && "mul operand index not right!"); 458 Register MULRegL = TRI->lookThruSingleUseCopyChain( 459 Root.getOperand(MulOpIdx).getReg(), MRI); 460 Register MULRegR = TRI->lookThruSingleUseCopyChain( 461 Root.getOperand(MulOpIdx + 1).getReg(), MRI); 462 if (!MULRegL && !MULRegR) 463 return false; 464 465 if (MULRegL && !MULRegR) { 466 MULRegR = 467 TRI->lookThruCopyLike(Root.getOperand(MulOpIdx + 1).getReg(), MRI); 468 IsUsedOnceL = true; 469 } else if (!MULRegL && MULRegR) { 470 MULRegL = 471 TRI->lookThruCopyLike(Root.getOperand(MulOpIdx).getReg(), MRI); 472 IsUsedOnceR = true; 473 } else { 474 IsUsedOnceL = true; 475 IsUsedOnceR = true; 476 } 477 478 if (!Register::isVirtualRegister(MULRegL) || 479 !Register::isVirtualRegister(MULRegR)) 480 return false; 481 482 MULInstrL = MRI->getVRegDef(MULRegL); 483 MULInstrR = MRI->getVRegDef(MULRegR); 484 return true; 485 } 486 return false; 487 }; 488 489 // Register pressure fma reassociation patterns. 490 if (DoRegPressureReduce && IsRPReductionCandidate()) { 491 assert((MULInstrL && MULInstrR) && "wrong register preduction candidate!"); 492 // Register pressure pattern 1 493 if (isLoadFromConstantPool(MULInstrL) && IsUsedOnceR && 494 IsReassociableAddOrSub(*MULInstrR, InfoArrayIdxFSubInst)) { 495 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_BCA\n"); 496 Patterns.push_back(MachineCombinerPattern::REASSOC_XY_BCA); 497 return true; 498 } 499 500 // Register pressure pattern 2 501 if ((isLoadFromConstantPool(MULInstrR) && IsUsedOnceL && 502 IsReassociableAddOrSub(*MULInstrL, InfoArrayIdxFSubInst))) { 503 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_BAC\n"); 504 Patterns.push_back(MachineCombinerPattern::REASSOC_XY_BAC); 505 return true; 506 } 507 } 508 509 // ILP fma reassociation patterns. 510 // Root must be a valid FMA like instruction. 511 AddOpIdx = -1; 512 if (!IsReassociableFMA(Root, AddOpIdx, MulOpIdx, false)) 513 return false; 514 515 assert((AddOpIdx >= 0) && "add operand index not right!"); 516 517 Register RegB = Root.getOperand(AddOpIdx).getReg(); 518 MachineInstr *Prev = MRI->getUniqueVRegDef(RegB); 519 520 // Prev must be a valid FMA like instruction. 521 AddOpIdx = -1; 522 if (!IsReassociableFMA(*Prev, AddOpIdx, MulOpIdx, false)) 523 return false; 524 525 assert((AddOpIdx >= 0) && "add operand index not right!"); 526 527 Register RegA = Prev->getOperand(AddOpIdx).getReg(); 528 MachineInstr *Leaf = MRI->getUniqueVRegDef(RegA); 529 AddOpIdx = -1; 530 if (IsReassociableFMA(*Leaf, AddOpIdx, MulOpIdx, true)) { 531 Patterns.push_back(MachineCombinerPattern::REASSOC_XMM_AMM_BMM); 532 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XMM_AMM_BMM\n"); 533 return true; 534 } 535 if (IsReassociableAddOrSub(*Leaf, InfoArrayIdxFAddInst)) { 536 Patterns.push_back(MachineCombinerPattern::REASSOC_XY_AMM_BMM); 537 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_AMM_BMM\n"); 538 return true; 539 } 540 return false; 541 } 542 543 void PPCInstrInfo::finalizeInsInstrs( 544 MachineInstr &Root, MachineCombinerPattern &P, 545 SmallVectorImpl<MachineInstr *> &InsInstrs) const { 546 assert(!InsInstrs.empty() && "Instructions set to be inserted is empty!"); 547 548 MachineFunction *MF = Root.getMF(); 549 MachineRegisterInfo *MRI = &MF->getRegInfo(); 550 const TargetRegisterInfo *TRI = &getRegisterInfo(); 551 MachineConstantPool *MCP = MF->getConstantPool(); 552 553 int16_t Idx = getFMAOpIdxInfo(Root.getOpcode()); 554 if (Idx < 0) 555 return; 556 557 uint16_t FirstMulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx]; 558 559 // For now we only need to fix up placeholder for register pressure reduce 560 // patterns. 561 Register ConstReg = 0; 562 switch (P) { 563 case MachineCombinerPattern::REASSOC_XY_BCA: 564 ConstReg = 565 TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx).getReg(), MRI); 566 break; 567 case MachineCombinerPattern::REASSOC_XY_BAC: 568 ConstReg = 569 TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx + 1).getReg(), MRI); 570 break; 571 default: 572 // Not register pressure reduce patterns. 573 return; 574 } 575 576 MachineInstr *ConstDefInstr = MRI->getVRegDef(ConstReg); 577 // Get const value from const pool. 578 const Constant *C = getConstantFromConstantPool(ConstDefInstr); 579 assert(isa<llvm::ConstantFP>(C) && "not a valid constant!"); 580 581 // Get negative fp const. 582 APFloat F1((dyn_cast<ConstantFP>(C))->getValueAPF()); 583 F1.changeSign(); 584 Constant *NegC = ConstantFP::get(dyn_cast<ConstantFP>(C)->getContext(), F1); 585 Align Alignment = MF->getDataLayout().getPrefTypeAlign(C->getType()); 586 587 // Put negative fp const into constant pool. 588 unsigned ConstPoolIdx = MCP->getConstantPoolIndex(NegC, Alignment); 589 590 MachineOperand *Placeholder = nullptr; 591 // Record the placeholder PPC::ZERO8 we add in reassociateFMA. 592 for (auto *Inst : InsInstrs) { 593 for (MachineOperand &Operand : Inst->explicit_operands()) { 594 assert(Operand.isReg() && "Invalid instruction in InsInstrs!"); 595 if (Operand.getReg() == PPC::ZERO8) { 596 Placeholder = &Operand; 597 break; 598 } 599 } 600 } 601 602 assert(Placeholder && "Placeholder does not exist!"); 603 604 // Generate instructions to load the const fp from constant pool. 605 // We only support PPC64 and medium code model. 606 Register LoadNewConst = 607 generateLoadForNewConst(ConstPoolIdx, &Root, C->getType(), InsInstrs); 608 609 // Fill the placeholder with the new load from constant pool. 610 Placeholder->setReg(LoadNewConst); 611 } 612 613 bool PPCInstrInfo::shouldReduceRegisterPressure( 614 MachineBasicBlock *MBB, RegisterClassInfo *RegClassInfo) const { 615 616 if (!EnableFMARegPressureReduction) 617 return false; 618 619 // Currently, we only enable register pressure reducing in machine combiner 620 // for: 1: PPC64; 2: Code Model is Medium; 3: Power9 which also has vector 621 // support. 622 // 623 // So we need following instructions to access a TOC entry: 624 // 625 // %6:g8rc_and_g8rc_nox0 = ADDIStocHA8 $x2, %const.0 626 // %7:vssrc = DFLOADf32 target-flags(ppc-toc-lo) %const.0, 627 // killed %6:g8rc_and_g8rc_nox0, implicit $x2 :: (load 4 from constant-pool) 628 // 629 // FIXME: add more supported targets, like Small and Large code model, PPC32, 630 // AIX. 631 if (!(Subtarget.isPPC64() && Subtarget.hasP9Vector() && 632 Subtarget.getTargetMachine().getCodeModel() == CodeModel::Medium)) 633 return false; 634 635 const TargetRegisterInfo *TRI = &getRegisterInfo(); 636 MachineFunction *MF = MBB->getParent(); 637 MachineRegisterInfo *MRI = &MF->getRegInfo(); 638 639 auto GetMBBPressure = [&](MachineBasicBlock *MBB) -> std::vector<unsigned> { 640 RegionPressure Pressure; 641 RegPressureTracker RPTracker(Pressure); 642 643 // Initialize the register pressure tracker. 644 RPTracker.init(MBB->getParent(), RegClassInfo, nullptr, MBB, MBB->end(), 645 /*TrackLaneMasks*/ false, /*TrackUntiedDefs=*/true); 646 647 for (MachineBasicBlock::iterator MII = MBB->instr_end(), 648 MIE = MBB->instr_begin(); 649 MII != MIE; --MII) { 650 MachineInstr &MI = *std::prev(MII); 651 if (MI.isDebugValue() || MI.isDebugLabel()) 652 continue; 653 RegisterOperands RegOpers; 654 RegOpers.collect(MI, *TRI, *MRI, false, false); 655 RPTracker.recedeSkipDebugValues(); 656 assert(&*RPTracker.getPos() == &MI && "RPTracker sync error!"); 657 RPTracker.recede(RegOpers); 658 } 659 660 // Close the RPTracker to finalize live ins. 661 RPTracker.closeRegion(); 662 663 return RPTracker.getPressure().MaxSetPressure; 664 }; 665 666 // For now we only care about float and double type fma. 667 unsigned VSSRCLimit = TRI->getRegPressureSetLimit( 668 *MBB->getParent(), PPC::RegisterPressureSets::VSSRC); 669 670 // Only reduce register pressure when pressure is high. 671 return GetMBBPressure(MBB)[PPC::RegisterPressureSets::VSSRC] > 672 (float)VSSRCLimit * FMARPFactor; 673 } 674 675 bool PPCInstrInfo::isLoadFromConstantPool(MachineInstr *I) const { 676 // I has only one memory operand which is load from constant pool. 677 if (!I->hasOneMemOperand()) 678 return false; 679 680 MachineMemOperand *Op = I->memoperands()[0]; 681 return Op->isLoad() && Op->getPseudoValue() && 682 Op->getPseudoValue()->kind() == PseudoSourceValue::ConstantPool; 683 } 684 685 Register PPCInstrInfo::generateLoadForNewConst( 686 unsigned Idx, MachineInstr *MI, Type *Ty, 687 SmallVectorImpl<MachineInstr *> &InsInstrs) const { 688 // Now we only support PPC64, Medium code model and P9 with vector. 689 // We have immutable pattern to access const pool. See function 690 // shouldReduceRegisterPressure. 691 assert((Subtarget.isPPC64() && Subtarget.hasP9Vector() && 692 Subtarget.getTargetMachine().getCodeModel() == CodeModel::Medium) && 693 "Target not supported!\n"); 694 695 MachineFunction *MF = MI->getMF(); 696 MachineRegisterInfo *MRI = &MF->getRegInfo(); 697 698 // Generate ADDIStocHA8 699 Register VReg1 = MRI->createVirtualRegister(&PPC::G8RC_and_G8RC_NOX0RegClass); 700 MachineInstrBuilder TOCOffset = 701 BuildMI(*MF, MI->getDebugLoc(), get(PPC::ADDIStocHA8), VReg1) 702 .addReg(PPC::X2) 703 .addConstantPoolIndex(Idx); 704 705 assert((Ty->isFloatTy() || Ty->isDoubleTy()) && 706 "Only float and double are supported!"); 707 708 unsigned LoadOpcode; 709 // Should be float type or double type. 710 if (Ty->isFloatTy()) 711 LoadOpcode = PPC::DFLOADf32; 712 else 713 LoadOpcode = PPC::DFLOADf64; 714 715 const TargetRegisterClass *RC = MRI->getRegClass(MI->getOperand(0).getReg()); 716 Register VReg2 = MRI->createVirtualRegister(RC); 717 MachineMemOperand *MMO = MF->getMachineMemOperand( 718 MachinePointerInfo::getConstantPool(*MF), MachineMemOperand::MOLoad, 719 Ty->getScalarSizeInBits() / 8, MF->getDataLayout().getPrefTypeAlign(Ty)); 720 721 // Generate Load from constant pool. 722 MachineInstrBuilder Load = 723 BuildMI(*MF, MI->getDebugLoc(), get(LoadOpcode), VReg2) 724 .addConstantPoolIndex(Idx) 725 .addReg(VReg1, getKillRegState(true)) 726 .addMemOperand(MMO); 727 728 Load->getOperand(1).setTargetFlags(PPCII::MO_TOC_LO); 729 730 // Insert the toc load instructions into InsInstrs. 731 InsInstrs.insert(InsInstrs.begin(), Load); 732 InsInstrs.insert(InsInstrs.begin(), TOCOffset); 733 return VReg2; 734 } 735 736 // This function returns the const value in constant pool if the \p I is a load 737 // from constant pool. 738 const Constant * 739 PPCInstrInfo::getConstantFromConstantPool(MachineInstr *I) const { 740 MachineFunction *MF = I->getMF(); 741 MachineRegisterInfo *MRI = &MF->getRegInfo(); 742 MachineConstantPool *MCP = MF->getConstantPool(); 743 assert(I->mayLoad() && "Should be a load instruction.\n"); 744 for (auto MO : I->uses()) { 745 if (!MO.isReg()) 746 continue; 747 Register Reg = MO.getReg(); 748 if (Reg == 0 || !Register::isVirtualRegister(Reg)) 749 continue; 750 // Find the toc address. 751 MachineInstr *DefMI = MRI->getVRegDef(Reg); 752 for (auto MO2 : DefMI->uses()) 753 if (MO2.isCPI()) 754 return (MCP->getConstants())[MO2.getIndex()].Val.ConstVal; 755 } 756 return nullptr; 757 } 758 759 bool PPCInstrInfo::getMachineCombinerPatterns( 760 MachineInstr &Root, SmallVectorImpl<MachineCombinerPattern> &Patterns, 761 bool DoRegPressureReduce) const { 762 // Using the machine combiner in this way is potentially expensive, so 763 // restrict to when aggressive optimizations are desired. 764 if (Subtarget.getTargetMachine().getOptLevel() != CodeGenOpt::Aggressive) 765 return false; 766 767 if (getFMAPatterns(Root, Patterns, DoRegPressureReduce)) 768 return true; 769 770 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns, 771 DoRegPressureReduce); 772 } 773 774 void PPCInstrInfo::genAlternativeCodeSequence( 775 MachineInstr &Root, MachineCombinerPattern Pattern, 776 SmallVectorImpl<MachineInstr *> &InsInstrs, 777 SmallVectorImpl<MachineInstr *> &DelInstrs, 778 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const { 779 switch (Pattern) { 780 case MachineCombinerPattern::REASSOC_XY_AMM_BMM: 781 case MachineCombinerPattern::REASSOC_XMM_AMM_BMM: 782 case MachineCombinerPattern::REASSOC_XY_BCA: 783 case MachineCombinerPattern::REASSOC_XY_BAC: 784 reassociateFMA(Root, Pattern, InsInstrs, DelInstrs, InstrIdxForVirtReg); 785 break; 786 default: 787 // Reassociate default patterns. 788 TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs, 789 DelInstrs, InstrIdxForVirtReg); 790 break; 791 } 792 } 793 794 void PPCInstrInfo::reassociateFMA( 795 MachineInstr &Root, MachineCombinerPattern Pattern, 796 SmallVectorImpl<MachineInstr *> &InsInstrs, 797 SmallVectorImpl<MachineInstr *> &DelInstrs, 798 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const { 799 MachineFunction *MF = Root.getMF(); 800 MachineRegisterInfo &MRI = MF->getRegInfo(); 801 const TargetRegisterInfo *TRI = &getRegisterInfo(); 802 MachineOperand &OpC = Root.getOperand(0); 803 Register RegC = OpC.getReg(); 804 const TargetRegisterClass *RC = MRI.getRegClass(RegC); 805 MRI.constrainRegClass(RegC, RC); 806 807 unsigned FmaOp = Root.getOpcode(); 808 int16_t Idx = getFMAOpIdxInfo(FmaOp); 809 assert(Idx >= 0 && "Root must be a FMA instruction"); 810 811 bool IsILPReassociate = 812 (Pattern == MachineCombinerPattern::REASSOC_XY_AMM_BMM) || 813 (Pattern == MachineCombinerPattern::REASSOC_XMM_AMM_BMM); 814 815 uint16_t AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx]; 816 uint16_t FirstMulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx]; 817 818 MachineInstr *Prev = nullptr; 819 MachineInstr *Leaf = nullptr; 820 switch (Pattern) { 821 default: 822 llvm_unreachable("not recognized pattern!"); 823 case MachineCombinerPattern::REASSOC_XY_AMM_BMM: 824 case MachineCombinerPattern::REASSOC_XMM_AMM_BMM: 825 Prev = MRI.getUniqueVRegDef(Root.getOperand(AddOpIdx).getReg()); 826 Leaf = MRI.getUniqueVRegDef(Prev->getOperand(AddOpIdx).getReg()); 827 break; 828 case MachineCombinerPattern::REASSOC_XY_BAC: { 829 Register MULReg = 830 TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx).getReg(), &MRI); 831 Leaf = MRI.getVRegDef(MULReg); 832 break; 833 } 834 case MachineCombinerPattern::REASSOC_XY_BCA: { 835 Register MULReg = TRI->lookThruCopyLike( 836 Root.getOperand(FirstMulOpIdx + 1).getReg(), &MRI); 837 Leaf = MRI.getVRegDef(MULReg); 838 break; 839 } 840 } 841 842 uint16_t IntersectedFlags = 0; 843 if (IsILPReassociate) 844 IntersectedFlags = Root.getFlags() & Prev->getFlags() & Leaf->getFlags(); 845 else 846 IntersectedFlags = Root.getFlags() & Leaf->getFlags(); 847 848 auto GetOperandInfo = [&](const MachineOperand &Operand, Register &Reg, 849 bool &KillFlag) { 850 Reg = Operand.getReg(); 851 MRI.constrainRegClass(Reg, RC); 852 KillFlag = Operand.isKill(); 853 }; 854 855 auto GetFMAInstrInfo = [&](const MachineInstr &Instr, Register &MulOp1, 856 Register &MulOp2, Register &AddOp, 857 bool &MulOp1KillFlag, bool &MulOp2KillFlag, 858 bool &AddOpKillFlag) { 859 GetOperandInfo(Instr.getOperand(FirstMulOpIdx), MulOp1, MulOp1KillFlag); 860 GetOperandInfo(Instr.getOperand(FirstMulOpIdx + 1), MulOp2, MulOp2KillFlag); 861 GetOperandInfo(Instr.getOperand(AddOpIdx), AddOp, AddOpKillFlag); 862 }; 863 864 Register RegM11, RegM12, RegX, RegY, RegM21, RegM22, RegM31, RegM32, RegA11, 865 RegA21, RegB; 866 bool KillX = false, KillY = false, KillM11 = false, KillM12 = false, 867 KillM21 = false, KillM22 = false, KillM31 = false, KillM32 = false, 868 KillA11 = false, KillA21 = false, KillB = false; 869 870 GetFMAInstrInfo(Root, RegM31, RegM32, RegB, KillM31, KillM32, KillB); 871 872 if (IsILPReassociate) 873 GetFMAInstrInfo(*Prev, RegM21, RegM22, RegA21, KillM21, KillM22, KillA21); 874 875 if (Pattern == MachineCombinerPattern::REASSOC_XMM_AMM_BMM) { 876 GetFMAInstrInfo(*Leaf, RegM11, RegM12, RegA11, KillM11, KillM12, KillA11); 877 GetOperandInfo(Leaf->getOperand(AddOpIdx), RegX, KillX); 878 } else if (Pattern == MachineCombinerPattern::REASSOC_XY_AMM_BMM) { 879 GetOperandInfo(Leaf->getOperand(1), RegX, KillX); 880 GetOperandInfo(Leaf->getOperand(2), RegY, KillY); 881 } else { 882 // Get FSUB instruction info. 883 GetOperandInfo(Leaf->getOperand(1), RegX, KillX); 884 GetOperandInfo(Leaf->getOperand(2), RegY, KillY); 885 } 886 887 // Create new virtual registers for the new results instead of 888 // recycling legacy ones because the MachineCombiner's computation of the 889 // critical path requires a new register definition rather than an existing 890 // one. 891 // For register pressure reassociation, we only need create one virtual 892 // register for the new fma. 893 Register NewVRA = MRI.createVirtualRegister(RC); 894 InstrIdxForVirtReg.insert(std::make_pair(NewVRA, 0)); 895 896 Register NewVRB = 0; 897 if (IsILPReassociate) { 898 NewVRB = MRI.createVirtualRegister(RC); 899 InstrIdxForVirtReg.insert(std::make_pair(NewVRB, 1)); 900 } 901 902 Register NewVRD = 0; 903 if (Pattern == MachineCombinerPattern::REASSOC_XMM_AMM_BMM) { 904 NewVRD = MRI.createVirtualRegister(RC); 905 InstrIdxForVirtReg.insert(std::make_pair(NewVRD, 2)); 906 } 907 908 auto AdjustOperandOrder = [&](MachineInstr *MI, Register RegAdd, bool KillAdd, 909 Register RegMul1, bool KillRegMul1, 910 Register RegMul2, bool KillRegMul2) { 911 MI->getOperand(AddOpIdx).setReg(RegAdd); 912 MI->getOperand(AddOpIdx).setIsKill(KillAdd); 913 MI->getOperand(FirstMulOpIdx).setReg(RegMul1); 914 MI->getOperand(FirstMulOpIdx).setIsKill(KillRegMul1); 915 MI->getOperand(FirstMulOpIdx + 1).setReg(RegMul2); 916 MI->getOperand(FirstMulOpIdx + 1).setIsKill(KillRegMul2); 917 }; 918 919 MachineInstrBuilder NewARegPressure, NewCRegPressure; 920 switch (Pattern) { 921 default: 922 llvm_unreachable("not recognized pattern!"); 923 case MachineCombinerPattern::REASSOC_XY_AMM_BMM: { 924 // Create new instructions for insertion. 925 MachineInstrBuilder MINewB = 926 BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB) 927 .addReg(RegX, getKillRegState(KillX)) 928 .addReg(RegM21, getKillRegState(KillM21)) 929 .addReg(RegM22, getKillRegState(KillM22)); 930 MachineInstrBuilder MINewA = 931 BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRA) 932 .addReg(RegY, getKillRegState(KillY)) 933 .addReg(RegM31, getKillRegState(KillM31)) 934 .addReg(RegM32, getKillRegState(KillM32)); 935 // If AddOpIdx is not 1, adjust the order. 936 if (AddOpIdx != 1) { 937 AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22); 938 AdjustOperandOrder(MINewA, RegY, KillY, RegM31, KillM31, RegM32, KillM32); 939 } 940 941 MachineInstrBuilder MINewC = 942 BuildMI(*MF, Root.getDebugLoc(), 943 get(FMAOpIdxInfo[Idx][InfoArrayIdxFAddInst]), RegC) 944 .addReg(NewVRB, getKillRegState(true)) 945 .addReg(NewVRA, getKillRegState(true)); 946 947 // Update flags for newly created instructions. 948 setSpecialOperandAttr(*MINewA, IntersectedFlags); 949 setSpecialOperandAttr(*MINewB, IntersectedFlags); 950 setSpecialOperandAttr(*MINewC, IntersectedFlags); 951 952 // Record new instructions for insertion. 953 InsInstrs.push_back(MINewA); 954 InsInstrs.push_back(MINewB); 955 InsInstrs.push_back(MINewC); 956 break; 957 } 958 case MachineCombinerPattern::REASSOC_XMM_AMM_BMM: { 959 assert(NewVRD && "new FMA register not created!"); 960 // Create new instructions for insertion. 961 MachineInstrBuilder MINewA = 962 BuildMI(*MF, Leaf->getDebugLoc(), 963 get(FMAOpIdxInfo[Idx][InfoArrayIdxFMULInst]), NewVRA) 964 .addReg(RegM11, getKillRegState(KillM11)) 965 .addReg(RegM12, getKillRegState(KillM12)); 966 MachineInstrBuilder MINewB = 967 BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB) 968 .addReg(RegX, getKillRegState(KillX)) 969 .addReg(RegM21, getKillRegState(KillM21)) 970 .addReg(RegM22, getKillRegState(KillM22)); 971 MachineInstrBuilder MINewD = 972 BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRD) 973 .addReg(NewVRA, getKillRegState(true)) 974 .addReg(RegM31, getKillRegState(KillM31)) 975 .addReg(RegM32, getKillRegState(KillM32)); 976 // If AddOpIdx is not 1, adjust the order. 977 if (AddOpIdx != 1) { 978 AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22); 979 AdjustOperandOrder(MINewD, NewVRA, true, RegM31, KillM31, RegM32, 980 KillM32); 981 } 982 983 MachineInstrBuilder MINewC = 984 BuildMI(*MF, Root.getDebugLoc(), 985 get(FMAOpIdxInfo[Idx][InfoArrayIdxFAddInst]), RegC) 986 .addReg(NewVRB, getKillRegState(true)) 987 .addReg(NewVRD, getKillRegState(true)); 988 989 // Update flags for newly created instructions. 990 setSpecialOperandAttr(*MINewA, IntersectedFlags); 991 setSpecialOperandAttr(*MINewB, IntersectedFlags); 992 setSpecialOperandAttr(*MINewD, IntersectedFlags); 993 setSpecialOperandAttr(*MINewC, IntersectedFlags); 994 995 // Record new instructions for insertion. 996 InsInstrs.push_back(MINewA); 997 InsInstrs.push_back(MINewB); 998 InsInstrs.push_back(MINewD); 999 InsInstrs.push_back(MINewC); 1000 break; 1001 } 1002 case MachineCombinerPattern::REASSOC_XY_BAC: 1003 case MachineCombinerPattern::REASSOC_XY_BCA: { 1004 Register VarReg; 1005 bool KillVarReg = false; 1006 if (Pattern == MachineCombinerPattern::REASSOC_XY_BCA) { 1007 VarReg = RegM31; 1008 KillVarReg = KillM31; 1009 } else { 1010 VarReg = RegM32; 1011 KillVarReg = KillM32; 1012 } 1013 // We don't want to get negative const from memory pool too early, as the 1014 // created entry will not be deleted even if it has no users. Since all 1015 // operand of Leaf and Root are virtual register, we use zero register 1016 // here as a placeholder. When the InsInstrs is selected in 1017 // MachineCombiner, we call finalizeInsInstrs to replace the zero register 1018 // with a virtual register which is a load from constant pool. 1019 NewARegPressure = BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRA) 1020 .addReg(RegB, getKillRegState(RegB)) 1021 .addReg(RegY, getKillRegState(KillY)) 1022 .addReg(PPC::ZERO8); 1023 NewCRegPressure = BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), RegC) 1024 .addReg(NewVRA, getKillRegState(true)) 1025 .addReg(RegX, getKillRegState(KillX)) 1026 .addReg(VarReg, getKillRegState(KillVarReg)); 1027 // For now, we only support xsmaddadp/xsmaddasp, their add operand are 1028 // both at index 1, no need to adjust. 1029 // FIXME: when add more fma instructions support, like fma/fmas, adjust 1030 // the operand index here. 1031 break; 1032 } 1033 } 1034 1035 if (!IsILPReassociate) { 1036 setSpecialOperandAttr(*NewARegPressure, IntersectedFlags); 1037 setSpecialOperandAttr(*NewCRegPressure, IntersectedFlags); 1038 1039 InsInstrs.push_back(NewARegPressure); 1040 InsInstrs.push_back(NewCRegPressure); 1041 } 1042 1043 assert(!InsInstrs.empty() && 1044 "Insertion instructions set should not be empty!"); 1045 1046 // Record old instructions for deletion. 1047 DelInstrs.push_back(Leaf); 1048 if (IsILPReassociate) 1049 DelInstrs.push_back(Prev); 1050 DelInstrs.push_back(&Root); 1051 } 1052 1053 // Detect 32 -> 64-bit extensions where we may reuse the low sub-register. 1054 bool PPCInstrInfo::isCoalescableExtInstr(const MachineInstr &MI, 1055 Register &SrcReg, Register &DstReg, 1056 unsigned &SubIdx) const { 1057 switch (MI.getOpcode()) { 1058 default: return false; 1059 case PPC::EXTSW: 1060 case PPC::EXTSW_32: 1061 case PPC::EXTSW_32_64: 1062 SrcReg = MI.getOperand(1).getReg(); 1063 DstReg = MI.getOperand(0).getReg(); 1064 SubIdx = PPC::sub_32; 1065 return true; 1066 } 1067 } 1068 1069 unsigned PPCInstrInfo::isLoadFromStackSlot(const MachineInstr &MI, 1070 int &FrameIndex) const { 1071 unsigned Opcode = MI.getOpcode(); 1072 const unsigned *OpcodesForSpill = getLoadOpcodesForSpillArray(); 1073 const unsigned *End = OpcodesForSpill + SOK_LastOpcodeSpill; 1074 1075 if (End != std::find(OpcodesForSpill, End, Opcode)) { 1076 // Check for the operands added by addFrameReference (the immediate is the 1077 // offset which defaults to 0). 1078 if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() && 1079 MI.getOperand(2).isFI()) { 1080 FrameIndex = MI.getOperand(2).getIndex(); 1081 return MI.getOperand(0).getReg(); 1082 } 1083 } 1084 return 0; 1085 } 1086 1087 // For opcodes with the ReMaterializable flag set, this function is called to 1088 // verify the instruction is really rematable. 1089 bool PPCInstrInfo::isReallyTriviallyReMaterializable( 1090 const MachineInstr &MI) const { 1091 switch (MI.getOpcode()) { 1092 default: 1093 // This function should only be called for opcodes with the ReMaterializable 1094 // flag set. 1095 llvm_unreachable("Unknown rematerializable operation!"); 1096 break; 1097 case PPC::LI: 1098 case PPC::LI8: 1099 case PPC::PLI: 1100 case PPC::PLI8: 1101 case PPC::LIS: 1102 case PPC::LIS8: 1103 case PPC::ADDIStocHA: 1104 case PPC::ADDIStocHA8: 1105 case PPC::ADDItocL: 1106 case PPC::LOAD_STACK_GUARD: 1107 case PPC::XXLXORz: 1108 case PPC::XXLXORspz: 1109 case PPC::XXLXORdpz: 1110 case PPC::XXLEQVOnes: 1111 case PPC::XXSPLTI32DX: 1112 case PPC::XXSPLTIW: 1113 case PPC::XXSPLTIDP: 1114 case PPC::V_SET0B: 1115 case PPC::V_SET0H: 1116 case PPC::V_SET0: 1117 case PPC::V_SETALLONESB: 1118 case PPC::V_SETALLONESH: 1119 case PPC::V_SETALLONES: 1120 case PPC::CRSET: 1121 case PPC::CRUNSET: 1122 case PPC::XXSETACCZ: 1123 return true; 1124 } 1125 return false; 1126 } 1127 1128 unsigned PPCInstrInfo::isStoreToStackSlot(const MachineInstr &MI, 1129 int &FrameIndex) const { 1130 unsigned Opcode = MI.getOpcode(); 1131 const unsigned *OpcodesForSpill = getStoreOpcodesForSpillArray(); 1132 const unsigned *End = OpcodesForSpill + SOK_LastOpcodeSpill; 1133 1134 if (End != std::find(OpcodesForSpill, End, Opcode)) { 1135 if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() && 1136 MI.getOperand(2).isFI()) { 1137 FrameIndex = MI.getOperand(2).getIndex(); 1138 return MI.getOperand(0).getReg(); 1139 } 1140 } 1141 return 0; 1142 } 1143 1144 MachineInstr *PPCInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI, 1145 unsigned OpIdx1, 1146 unsigned OpIdx2) const { 1147 MachineFunction &MF = *MI.getParent()->getParent(); 1148 1149 // Normal instructions can be commuted the obvious way. 1150 if (MI.getOpcode() != PPC::RLWIMI && MI.getOpcode() != PPC::RLWIMI_rec) 1151 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2); 1152 // Note that RLWIMI can be commuted as a 32-bit instruction, but not as a 1153 // 64-bit instruction (so we don't handle PPC::RLWIMI8 here), because 1154 // changing the relative order of the mask operands might change what happens 1155 // to the high-bits of the mask (and, thus, the result). 1156 1157 // Cannot commute if it has a non-zero rotate count. 1158 if (MI.getOperand(3).getImm() != 0) 1159 return nullptr; 1160 1161 // If we have a zero rotate count, we have: 1162 // M = mask(MB,ME) 1163 // Op0 = (Op1 & ~M) | (Op2 & M) 1164 // Change this to: 1165 // M = mask((ME+1)&31, (MB-1)&31) 1166 // Op0 = (Op2 & ~M) | (Op1 & M) 1167 1168 // Swap op1/op2 1169 assert(((OpIdx1 == 1 && OpIdx2 == 2) || (OpIdx1 == 2 && OpIdx2 == 1)) && 1170 "Only the operands 1 and 2 can be swapped in RLSIMI/RLWIMI_rec."); 1171 Register Reg0 = MI.getOperand(0).getReg(); 1172 Register Reg1 = MI.getOperand(1).getReg(); 1173 Register Reg2 = MI.getOperand(2).getReg(); 1174 unsigned SubReg1 = MI.getOperand(1).getSubReg(); 1175 unsigned SubReg2 = MI.getOperand(2).getSubReg(); 1176 bool Reg1IsKill = MI.getOperand(1).isKill(); 1177 bool Reg2IsKill = MI.getOperand(2).isKill(); 1178 bool ChangeReg0 = false; 1179 // If machine instrs are no longer in two-address forms, update 1180 // destination register as well. 1181 if (Reg0 == Reg1) { 1182 // Must be two address instruction! 1183 assert(MI.getDesc().getOperandConstraint(0, MCOI::TIED_TO) && 1184 "Expecting a two-address instruction!"); 1185 assert(MI.getOperand(0).getSubReg() == SubReg1 && "Tied subreg mismatch"); 1186 Reg2IsKill = false; 1187 ChangeReg0 = true; 1188 } 1189 1190 // Masks. 1191 unsigned MB = MI.getOperand(4).getImm(); 1192 unsigned ME = MI.getOperand(5).getImm(); 1193 1194 // We can't commute a trivial mask (there is no way to represent an all-zero 1195 // mask). 1196 if (MB == 0 && ME == 31) 1197 return nullptr; 1198 1199 if (NewMI) { 1200 // Create a new instruction. 1201 Register Reg0 = ChangeReg0 ? Reg2 : MI.getOperand(0).getReg(); 1202 bool Reg0IsDead = MI.getOperand(0).isDead(); 1203 return BuildMI(MF, MI.getDebugLoc(), MI.getDesc()) 1204 .addReg(Reg0, RegState::Define | getDeadRegState(Reg0IsDead)) 1205 .addReg(Reg2, getKillRegState(Reg2IsKill)) 1206 .addReg(Reg1, getKillRegState(Reg1IsKill)) 1207 .addImm((ME + 1) & 31) 1208 .addImm((MB - 1) & 31); 1209 } 1210 1211 if (ChangeReg0) { 1212 MI.getOperand(0).setReg(Reg2); 1213 MI.getOperand(0).setSubReg(SubReg2); 1214 } 1215 MI.getOperand(2).setReg(Reg1); 1216 MI.getOperand(1).setReg(Reg2); 1217 MI.getOperand(2).setSubReg(SubReg1); 1218 MI.getOperand(1).setSubReg(SubReg2); 1219 MI.getOperand(2).setIsKill(Reg1IsKill); 1220 MI.getOperand(1).setIsKill(Reg2IsKill); 1221 1222 // Swap the mask around. 1223 MI.getOperand(4).setImm((ME + 1) & 31); 1224 MI.getOperand(5).setImm((MB - 1) & 31); 1225 return &MI; 1226 } 1227 1228 bool PPCInstrInfo::findCommutedOpIndices(const MachineInstr &MI, 1229 unsigned &SrcOpIdx1, 1230 unsigned &SrcOpIdx2) const { 1231 // For VSX A-Type FMA instructions, it is the first two operands that can be 1232 // commuted, however, because the non-encoded tied input operand is listed 1233 // first, the operands to swap are actually the second and third. 1234 1235 int AltOpc = PPC::getAltVSXFMAOpcode(MI.getOpcode()); 1236 if (AltOpc == -1) 1237 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2); 1238 1239 // The commutable operand indices are 2 and 3. Return them in SrcOpIdx1 1240 // and SrcOpIdx2. 1241 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3); 1242 } 1243 1244 void PPCInstrInfo::insertNoop(MachineBasicBlock &MBB, 1245 MachineBasicBlock::iterator MI) const { 1246 // This function is used for scheduling, and the nop wanted here is the type 1247 // that terminates dispatch groups on the POWER cores. 1248 unsigned Directive = Subtarget.getCPUDirective(); 1249 unsigned Opcode; 1250 switch (Directive) { 1251 default: Opcode = PPC::NOP; break; 1252 case PPC::DIR_PWR6: Opcode = PPC::NOP_GT_PWR6; break; 1253 case PPC::DIR_PWR7: Opcode = PPC::NOP_GT_PWR7; break; 1254 case PPC::DIR_PWR8: Opcode = PPC::NOP_GT_PWR7; break; /* FIXME: Update when P8 InstrScheduling model is ready */ 1255 // FIXME: Update when POWER9 scheduling model is ready. 1256 case PPC::DIR_PWR9: Opcode = PPC::NOP_GT_PWR7; break; 1257 } 1258 1259 DebugLoc DL; 1260 BuildMI(MBB, MI, DL, get(Opcode)); 1261 } 1262 1263 /// Return the noop instruction to use for a noop. 1264 MCInst PPCInstrInfo::getNop() const { 1265 MCInst Nop; 1266 Nop.setOpcode(PPC::NOP); 1267 return Nop; 1268 } 1269 1270 // Branch analysis. 1271 // Note: If the condition register is set to CTR or CTR8 then this is a 1272 // BDNZ (imm == 1) or BDZ (imm == 0) branch. 1273 bool PPCInstrInfo::analyzeBranch(MachineBasicBlock &MBB, 1274 MachineBasicBlock *&TBB, 1275 MachineBasicBlock *&FBB, 1276 SmallVectorImpl<MachineOperand> &Cond, 1277 bool AllowModify) const { 1278 bool isPPC64 = Subtarget.isPPC64(); 1279 1280 // If the block has no terminators, it just falls into the block after it. 1281 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr(); 1282 if (I == MBB.end()) 1283 return false; 1284 1285 if (!isUnpredicatedTerminator(*I)) 1286 return false; 1287 1288 if (AllowModify) { 1289 // If the BB ends with an unconditional branch to the fallthrough BB, 1290 // we eliminate the branch instruction. 1291 if (I->getOpcode() == PPC::B && 1292 MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) { 1293 I->eraseFromParent(); 1294 1295 // We update iterator after deleting the last branch. 1296 I = MBB.getLastNonDebugInstr(); 1297 if (I == MBB.end() || !isUnpredicatedTerminator(*I)) 1298 return false; 1299 } 1300 } 1301 1302 // Get the last instruction in the block. 1303 MachineInstr &LastInst = *I; 1304 1305 // If there is only one terminator instruction, process it. 1306 if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) { 1307 if (LastInst.getOpcode() == PPC::B) { 1308 if (!LastInst.getOperand(0).isMBB()) 1309 return true; 1310 TBB = LastInst.getOperand(0).getMBB(); 1311 return false; 1312 } else if (LastInst.getOpcode() == PPC::BCC) { 1313 if (!LastInst.getOperand(2).isMBB()) 1314 return true; 1315 // Block ends with fall-through condbranch. 1316 TBB = LastInst.getOperand(2).getMBB(); 1317 Cond.push_back(LastInst.getOperand(0)); 1318 Cond.push_back(LastInst.getOperand(1)); 1319 return false; 1320 } else if (LastInst.getOpcode() == PPC::BC) { 1321 if (!LastInst.getOperand(1).isMBB()) 1322 return true; 1323 // Block ends with fall-through condbranch. 1324 TBB = LastInst.getOperand(1).getMBB(); 1325 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET)); 1326 Cond.push_back(LastInst.getOperand(0)); 1327 return false; 1328 } else if (LastInst.getOpcode() == PPC::BCn) { 1329 if (!LastInst.getOperand(1).isMBB()) 1330 return true; 1331 // Block ends with fall-through condbranch. 1332 TBB = LastInst.getOperand(1).getMBB(); 1333 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_UNSET)); 1334 Cond.push_back(LastInst.getOperand(0)); 1335 return false; 1336 } else if (LastInst.getOpcode() == PPC::BDNZ8 || 1337 LastInst.getOpcode() == PPC::BDNZ) { 1338 if (!LastInst.getOperand(0).isMBB()) 1339 return true; 1340 if (DisableCTRLoopAnal) 1341 return true; 1342 TBB = LastInst.getOperand(0).getMBB(); 1343 Cond.push_back(MachineOperand::CreateImm(1)); 1344 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR, 1345 true)); 1346 return false; 1347 } else if (LastInst.getOpcode() == PPC::BDZ8 || 1348 LastInst.getOpcode() == PPC::BDZ) { 1349 if (!LastInst.getOperand(0).isMBB()) 1350 return true; 1351 if (DisableCTRLoopAnal) 1352 return true; 1353 TBB = LastInst.getOperand(0).getMBB(); 1354 Cond.push_back(MachineOperand::CreateImm(0)); 1355 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR, 1356 true)); 1357 return false; 1358 } 1359 1360 // Otherwise, don't know what this is. 1361 return true; 1362 } 1363 1364 // Get the instruction before it if it's a terminator. 1365 MachineInstr &SecondLastInst = *I; 1366 1367 // If there are three terminators, we don't know what sort of block this is. 1368 if (I != MBB.begin() && isUnpredicatedTerminator(*--I)) 1369 return true; 1370 1371 // If the block ends with PPC::B and PPC:BCC, handle it. 1372 if (SecondLastInst.getOpcode() == PPC::BCC && 1373 LastInst.getOpcode() == PPC::B) { 1374 if (!SecondLastInst.getOperand(2).isMBB() || 1375 !LastInst.getOperand(0).isMBB()) 1376 return true; 1377 TBB = SecondLastInst.getOperand(2).getMBB(); 1378 Cond.push_back(SecondLastInst.getOperand(0)); 1379 Cond.push_back(SecondLastInst.getOperand(1)); 1380 FBB = LastInst.getOperand(0).getMBB(); 1381 return false; 1382 } else if (SecondLastInst.getOpcode() == PPC::BC && 1383 LastInst.getOpcode() == PPC::B) { 1384 if (!SecondLastInst.getOperand(1).isMBB() || 1385 !LastInst.getOperand(0).isMBB()) 1386 return true; 1387 TBB = SecondLastInst.getOperand(1).getMBB(); 1388 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET)); 1389 Cond.push_back(SecondLastInst.getOperand(0)); 1390 FBB = LastInst.getOperand(0).getMBB(); 1391 return false; 1392 } else if (SecondLastInst.getOpcode() == PPC::BCn && 1393 LastInst.getOpcode() == PPC::B) { 1394 if (!SecondLastInst.getOperand(1).isMBB() || 1395 !LastInst.getOperand(0).isMBB()) 1396 return true; 1397 TBB = SecondLastInst.getOperand(1).getMBB(); 1398 Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_UNSET)); 1399 Cond.push_back(SecondLastInst.getOperand(0)); 1400 FBB = LastInst.getOperand(0).getMBB(); 1401 return false; 1402 } else if ((SecondLastInst.getOpcode() == PPC::BDNZ8 || 1403 SecondLastInst.getOpcode() == PPC::BDNZ) && 1404 LastInst.getOpcode() == PPC::B) { 1405 if (!SecondLastInst.getOperand(0).isMBB() || 1406 !LastInst.getOperand(0).isMBB()) 1407 return true; 1408 if (DisableCTRLoopAnal) 1409 return true; 1410 TBB = SecondLastInst.getOperand(0).getMBB(); 1411 Cond.push_back(MachineOperand::CreateImm(1)); 1412 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR, 1413 true)); 1414 FBB = LastInst.getOperand(0).getMBB(); 1415 return false; 1416 } else if ((SecondLastInst.getOpcode() == PPC::BDZ8 || 1417 SecondLastInst.getOpcode() == PPC::BDZ) && 1418 LastInst.getOpcode() == PPC::B) { 1419 if (!SecondLastInst.getOperand(0).isMBB() || 1420 !LastInst.getOperand(0).isMBB()) 1421 return true; 1422 if (DisableCTRLoopAnal) 1423 return true; 1424 TBB = SecondLastInst.getOperand(0).getMBB(); 1425 Cond.push_back(MachineOperand::CreateImm(0)); 1426 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR, 1427 true)); 1428 FBB = LastInst.getOperand(0).getMBB(); 1429 return false; 1430 } 1431 1432 // If the block ends with two PPC:Bs, handle it. The second one is not 1433 // executed, so remove it. 1434 if (SecondLastInst.getOpcode() == PPC::B && LastInst.getOpcode() == PPC::B) { 1435 if (!SecondLastInst.getOperand(0).isMBB()) 1436 return true; 1437 TBB = SecondLastInst.getOperand(0).getMBB(); 1438 I = LastInst; 1439 if (AllowModify) 1440 I->eraseFromParent(); 1441 return false; 1442 } 1443 1444 // Otherwise, can't handle this. 1445 return true; 1446 } 1447 1448 unsigned PPCInstrInfo::removeBranch(MachineBasicBlock &MBB, 1449 int *BytesRemoved) const { 1450 assert(!BytesRemoved && "code size not handled"); 1451 1452 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr(); 1453 if (I == MBB.end()) 1454 return 0; 1455 1456 if (I->getOpcode() != PPC::B && I->getOpcode() != PPC::BCC && 1457 I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn && 1458 I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ && 1459 I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ) 1460 return 0; 1461 1462 // Remove the branch. 1463 I->eraseFromParent(); 1464 1465 I = MBB.end(); 1466 1467 if (I == MBB.begin()) return 1; 1468 --I; 1469 if (I->getOpcode() != PPC::BCC && 1470 I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn && 1471 I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ && 1472 I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ) 1473 return 1; 1474 1475 // Remove the branch. 1476 I->eraseFromParent(); 1477 return 2; 1478 } 1479 1480 unsigned PPCInstrInfo::insertBranch(MachineBasicBlock &MBB, 1481 MachineBasicBlock *TBB, 1482 MachineBasicBlock *FBB, 1483 ArrayRef<MachineOperand> Cond, 1484 const DebugLoc &DL, 1485 int *BytesAdded) const { 1486 // Shouldn't be a fall through. 1487 assert(TBB && "insertBranch must not be told to insert a fallthrough"); 1488 assert((Cond.size() == 2 || Cond.size() == 0) && 1489 "PPC branch conditions have two components!"); 1490 assert(!BytesAdded && "code size not handled"); 1491 1492 bool isPPC64 = Subtarget.isPPC64(); 1493 1494 // One-way branch. 1495 if (!FBB) { 1496 if (Cond.empty()) // Unconditional branch 1497 BuildMI(&MBB, DL, get(PPC::B)).addMBB(TBB); 1498 else if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8) 1499 BuildMI(&MBB, DL, get(Cond[0].getImm() ? 1500 (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) : 1501 (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB); 1502 else if (Cond[0].getImm() == PPC::PRED_BIT_SET) 1503 BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB); 1504 else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET) 1505 BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB); 1506 else // Conditional branch 1507 BuildMI(&MBB, DL, get(PPC::BCC)) 1508 .addImm(Cond[0].getImm()) 1509 .add(Cond[1]) 1510 .addMBB(TBB); 1511 return 1; 1512 } 1513 1514 // Two-way Conditional Branch. 1515 if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8) 1516 BuildMI(&MBB, DL, get(Cond[0].getImm() ? 1517 (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) : 1518 (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB); 1519 else if (Cond[0].getImm() == PPC::PRED_BIT_SET) 1520 BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB); 1521 else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET) 1522 BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB); 1523 else 1524 BuildMI(&MBB, DL, get(PPC::BCC)) 1525 .addImm(Cond[0].getImm()) 1526 .add(Cond[1]) 1527 .addMBB(TBB); 1528 BuildMI(&MBB, DL, get(PPC::B)).addMBB(FBB); 1529 return 2; 1530 } 1531 1532 // Select analysis. 1533 bool PPCInstrInfo::canInsertSelect(const MachineBasicBlock &MBB, 1534 ArrayRef<MachineOperand> Cond, 1535 Register DstReg, Register TrueReg, 1536 Register FalseReg, int &CondCycles, 1537 int &TrueCycles, int &FalseCycles) const { 1538 if (Cond.size() != 2) 1539 return false; 1540 1541 // If this is really a bdnz-like condition, then it cannot be turned into a 1542 // select. 1543 if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8) 1544 return false; 1545 1546 // If the conditional branch uses a physical register, then it cannot be 1547 // turned into a select. 1548 if (Register::isPhysicalRegister(Cond[1].getReg())) 1549 return false; 1550 1551 // Check register classes. 1552 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 1553 const TargetRegisterClass *RC = 1554 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg)); 1555 if (!RC) 1556 return false; 1557 1558 // isel is for regular integer GPRs only. 1559 if (!PPC::GPRCRegClass.hasSubClassEq(RC) && 1560 !PPC::GPRC_NOR0RegClass.hasSubClassEq(RC) && 1561 !PPC::G8RCRegClass.hasSubClassEq(RC) && 1562 !PPC::G8RC_NOX0RegClass.hasSubClassEq(RC)) 1563 return false; 1564 1565 // FIXME: These numbers are for the A2, how well they work for other cores is 1566 // an open question. On the A2, the isel instruction has a 2-cycle latency 1567 // but single-cycle throughput. These numbers are used in combination with 1568 // the MispredictPenalty setting from the active SchedMachineModel. 1569 CondCycles = 1; 1570 TrueCycles = 1; 1571 FalseCycles = 1; 1572 1573 return true; 1574 } 1575 1576 void PPCInstrInfo::insertSelect(MachineBasicBlock &MBB, 1577 MachineBasicBlock::iterator MI, 1578 const DebugLoc &dl, Register DestReg, 1579 ArrayRef<MachineOperand> Cond, Register TrueReg, 1580 Register FalseReg) const { 1581 assert(Cond.size() == 2 && 1582 "PPC branch conditions have two components!"); 1583 1584 // Get the register classes. 1585 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 1586 const TargetRegisterClass *RC = 1587 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg)); 1588 assert(RC && "TrueReg and FalseReg must have overlapping register classes"); 1589 1590 bool Is64Bit = PPC::G8RCRegClass.hasSubClassEq(RC) || 1591 PPC::G8RC_NOX0RegClass.hasSubClassEq(RC); 1592 assert((Is64Bit || 1593 PPC::GPRCRegClass.hasSubClassEq(RC) || 1594 PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) && 1595 "isel is for regular integer GPRs only"); 1596 1597 unsigned OpCode = Is64Bit ? PPC::ISEL8 : PPC::ISEL; 1598 auto SelectPred = static_cast<PPC::Predicate>(Cond[0].getImm()); 1599 1600 unsigned SubIdx = 0; 1601 bool SwapOps = false; 1602 switch (SelectPred) { 1603 case PPC::PRED_EQ: 1604 case PPC::PRED_EQ_MINUS: 1605 case PPC::PRED_EQ_PLUS: 1606 SubIdx = PPC::sub_eq; SwapOps = false; break; 1607 case PPC::PRED_NE: 1608 case PPC::PRED_NE_MINUS: 1609 case PPC::PRED_NE_PLUS: 1610 SubIdx = PPC::sub_eq; SwapOps = true; break; 1611 case PPC::PRED_LT: 1612 case PPC::PRED_LT_MINUS: 1613 case PPC::PRED_LT_PLUS: 1614 SubIdx = PPC::sub_lt; SwapOps = false; break; 1615 case PPC::PRED_GE: 1616 case PPC::PRED_GE_MINUS: 1617 case PPC::PRED_GE_PLUS: 1618 SubIdx = PPC::sub_lt; SwapOps = true; break; 1619 case PPC::PRED_GT: 1620 case PPC::PRED_GT_MINUS: 1621 case PPC::PRED_GT_PLUS: 1622 SubIdx = PPC::sub_gt; SwapOps = false; break; 1623 case PPC::PRED_LE: 1624 case PPC::PRED_LE_MINUS: 1625 case PPC::PRED_LE_PLUS: 1626 SubIdx = PPC::sub_gt; SwapOps = true; break; 1627 case PPC::PRED_UN: 1628 case PPC::PRED_UN_MINUS: 1629 case PPC::PRED_UN_PLUS: 1630 SubIdx = PPC::sub_un; SwapOps = false; break; 1631 case PPC::PRED_NU: 1632 case PPC::PRED_NU_MINUS: 1633 case PPC::PRED_NU_PLUS: 1634 SubIdx = PPC::sub_un; SwapOps = true; break; 1635 case PPC::PRED_BIT_SET: SubIdx = 0; SwapOps = false; break; 1636 case PPC::PRED_BIT_UNSET: SubIdx = 0; SwapOps = true; break; 1637 } 1638 1639 Register FirstReg = SwapOps ? FalseReg : TrueReg, 1640 SecondReg = SwapOps ? TrueReg : FalseReg; 1641 1642 // The first input register of isel cannot be r0. If it is a member 1643 // of a register class that can be r0, then copy it first (the 1644 // register allocator should eliminate the copy). 1645 if (MRI.getRegClass(FirstReg)->contains(PPC::R0) || 1646 MRI.getRegClass(FirstReg)->contains(PPC::X0)) { 1647 const TargetRegisterClass *FirstRC = 1648 MRI.getRegClass(FirstReg)->contains(PPC::X0) ? 1649 &PPC::G8RC_NOX0RegClass : &PPC::GPRC_NOR0RegClass; 1650 Register OldFirstReg = FirstReg; 1651 FirstReg = MRI.createVirtualRegister(FirstRC); 1652 BuildMI(MBB, MI, dl, get(TargetOpcode::COPY), FirstReg) 1653 .addReg(OldFirstReg); 1654 } 1655 1656 BuildMI(MBB, MI, dl, get(OpCode), DestReg) 1657 .addReg(FirstReg).addReg(SecondReg) 1658 .addReg(Cond[1].getReg(), 0, SubIdx); 1659 } 1660 1661 static unsigned getCRBitValue(unsigned CRBit) { 1662 unsigned Ret = 4; 1663 if (CRBit == PPC::CR0LT || CRBit == PPC::CR1LT || 1664 CRBit == PPC::CR2LT || CRBit == PPC::CR3LT || 1665 CRBit == PPC::CR4LT || CRBit == PPC::CR5LT || 1666 CRBit == PPC::CR6LT || CRBit == PPC::CR7LT) 1667 Ret = 3; 1668 if (CRBit == PPC::CR0GT || CRBit == PPC::CR1GT || 1669 CRBit == PPC::CR2GT || CRBit == PPC::CR3GT || 1670 CRBit == PPC::CR4GT || CRBit == PPC::CR5GT || 1671 CRBit == PPC::CR6GT || CRBit == PPC::CR7GT) 1672 Ret = 2; 1673 if (CRBit == PPC::CR0EQ || CRBit == PPC::CR1EQ || 1674 CRBit == PPC::CR2EQ || CRBit == PPC::CR3EQ || 1675 CRBit == PPC::CR4EQ || CRBit == PPC::CR5EQ || 1676 CRBit == PPC::CR6EQ || CRBit == PPC::CR7EQ) 1677 Ret = 1; 1678 if (CRBit == PPC::CR0UN || CRBit == PPC::CR1UN || 1679 CRBit == PPC::CR2UN || CRBit == PPC::CR3UN || 1680 CRBit == PPC::CR4UN || CRBit == PPC::CR5UN || 1681 CRBit == PPC::CR6UN || CRBit == PPC::CR7UN) 1682 Ret = 0; 1683 1684 assert(Ret != 4 && "Invalid CR bit register"); 1685 return Ret; 1686 } 1687 1688 void PPCInstrInfo::copyPhysReg(MachineBasicBlock &MBB, 1689 MachineBasicBlock::iterator I, 1690 const DebugLoc &DL, MCRegister DestReg, 1691 MCRegister SrcReg, bool KillSrc) const { 1692 // We can end up with self copies and similar things as a result of VSX copy 1693 // legalization. Promote them here. 1694 const TargetRegisterInfo *TRI = &getRegisterInfo(); 1695 if (PPC::F8RCRegClass.contains(DestReg) && 1696 PPC::VSRCRegClass.contains(SrcReg)) { 1697 MCRegister SuperReg = 1698 TRI->getMatchingSuperReg(DestReg, PPC::sub_64, &PPC::VSRCRegClass); 1699 1700 if (VSXSelfCopyCrash && SrcReg == SuperReg) 1701 llvm_unreachable("nop VSX copy"); 1702 1703 DestReg = SuperReg; 1704 } else if (PPC::F8RCRegClass.contains(SrcReg) && 1705 PPC::VSRCRegClass.contains(DestReg)) { 1706 MCRegister SuperReg = 1707 TRI->getMatchingSuperReg(SrcReg, PPC::sub_64, &PPC::VSRCRegClass); 1708 1709 if (VSXSelfCopyCrash && DestReg == SuperReg) 1710 llvm_unreachable("nop VSX copy"); 1711 1712 SrcReg = SuperReg; 1713 } 1714 1715 // Different class register copy 1716 if (PPC::CRBITRCRegClass.contains(SrcReg) && 1717 PPC::GPRCRegClass.contains(DestReg)) { 1718 MCRegister CRReg = getCRFromCRBit(SrcReg); 1719 BuildMI(MBB, I, DL, get(PPC::MFOCRF), DestReg).addReg(CRReg); 1720 getKillRegState(KillSrc); 1721 // Rotate the CR bit in the CR fields to be the least significant bit and 1722 // then mask with 0x1 (MB = ME = 31). 1723 BuildMI(MBB, I, DL, get(PPC::RLWINM), DestReg) 1724 .addReg(DestReg, RegState::Kill) 1725 .addImm(TRI->getEncodingValue(CRReg) * 4 + (4 - getCRBitValue(SrcReg))) 1726 .addImm(31) 1727 .addImm(31); 1728 return; 1729 } else if (PPC::CRRCRegClass.contains(SrcReg) && 1730 (PPC::G8RCRegClass.contains(DestReg) || 1731 PPC::GPRCRegClass.contains(DestReg))) { 1732 bool Is64Bit = PPC::G8RCRegClass.contains(DestReg); 1733 unsigned MvCode = Is64Bit ? PPC::MFOCRF8 : PPC::MFOCRF; 1734 unsigned ShCode = Is64Bit ? PPC::RLWINM8 : PPC::RLWINM; 1735 unsigned CRNum = TRI->getEncodingValue(SrcReg); 1736 BuildMI(MBB, I, DL, get(MvCode), DestReg).addReg(SrcReg); 1737 getKillRegState(KillSrc); 1738 if (CRNum == 7) 1739 return; 1740 // Shift the CR bits to make the CR field in the lowest 4 bits of GRC. 1741 BuildMI(MBB, I, DL, get(ShCode), DestReg) 1742 .addReg(DestReg, RegState::Kill) 1743 .addImm(CRNum * 4 + 4) 1744 .addImm(28) 1745 .addImm(31); 1746 return; 1747 } else if (PPC::G8RCRegClass.contains(SrcReg) && 1748 PPC::VSFRCRegClass.contains(DestReg)) { 1749 assert(Subtarget.hasDirectMove() && 1750 "Subtarget doesn't support directmove, don't know how to copy."); 1751 BuildMI(MBB, I, DL, get(PPC::MTVSRD), DestReg).addReg(SrcReg); 1752 NumGPRtoVSRSpill++; 1753 getKillRegState(KillSrc); 1754 return; 1755 } else if (PPC::VSFRCRegClass.contains(SrcReg) && 1756 PPC::G8RCRegClass.contains(DestReg)) { 1757 assert(Subtarget.hasDirectMove() && 1758 "Subtarget doesn't support directmove, don't know how to copy."); 1759 BuildMI(MBB, I, DL, get(PPC::MFVSRD), DestReg).addReg(SrcReg); 1760 getKillRegState(KillSrc); 1761 return; 1762 } else if (PPC::SPERCRegClass.contains(SrcReg) && 1763 PPC::GPRCRegClass.contains(DestReg)) { 1764 BuildMI(MBB, I, DL, get(PPC::EFSCFD), DestReg).addReg(SrcReg); 1765 getKillRegState(KillSrc); 1766 return; 1767 } else if (PPC::GPRCRegClass.contains(SrcReg) && 1768 PPC::SPERCRegClass.contains(DestReg)) { 1769 BuildMI(MBB, I, DL, get(PPC::EFDCFS), DestReg).addReg(SrcReg); 1770 getKillRegState(KillSrc); 1771 return; 1772 } 1773 1774 unsigned Opc; 1775 if (PPC::GPRCRegClass.contains(DestReg, SrcReg)) 1776 Opc = PPC::OR; 1777 else if (PPC::G8RCRegClass.contains(DestReg, SrcReg)) 1778 Opc = PPC::OR8; 1779 else if (PPC::F4RCRegClass.contains(DestReg, SrcReg)) 1780 Opc = PPC::FMR; 1781 else if (PPC::CRRCRegClass.contains(DestReg, SrcReg)) 1782 Opc = PPC::MCRF; 1783 else if (PPC::VRRCRegClass.contains(DestReg, SrcReg)) 1784 Opc = PPC::VOR; 1785 else if (PPC::VSRCRegClass.contains(DestReg, SrcReg)) 1786 // There are two different ways this can be done: 1787 // 1. xxlor : This has lower latency (on the P7), 2 cycles, but can only 1788 // issue in VSU pipeline 0. 1789 // 2. xmovdp/xmovsp: This has higher latency (on the P7), 6 cycles, but 1790 // can go to either pipeline. 1791 // We'll always use xxlor here, because in practically all cases where 1792 // copies are generated, they are close enough to some use that the 1793 // lower-latency form is preferable. 1794 Opc = PPC::XXLOR; 1795 else if (PPC::VSFRCRegClass.contains(DestReg, SrcReg) || 1796 PPC::VSSRCRegClass.contains(DestReg, SrcReg)) 1797 Opc = (Subtarget.hasP9Vector()) ? PPC::XSCPSGNDP : PPC::XXLORf; 1798 else if (Subtarget.pairedVectorMemops() && 1799 PPC::VSRpRCRegClass.contains(DestReg, SrcReg)) { 1800 if (SrcReg > PPC::VSRp15) 1801 SrcReg = PPC::V0 + (SrcReg - PPC::VSRp16) * 2; 1802 else 1803 SrcReg = PPC::VSL0 + (SrcReg - PPC::VSRp0) * 2; 1804 if (DestReg > PPC::VSRp15) 1805 DestReg = PPC::V0 + (DestReg - PPC::VSRp16) * 2; 1806 else 1807 DestReg = PPC::VSL0 + (DestReg - PPC::VSRp0) * 2; 1808 BuildMI(MBB, I, DL, get(PPC::XXLOR), DestReg). 1809 addReg(SrcReg).addReg(SrcReg, getKillRegState(KillSrc)); 1810 BuildMI(MBB, I, DL, get(PPC::XXLOR), DestReg + 1). 1811 addReg(SrcReg + 1).addReg(SrcReg + 1, getKillRegState(KillSrc)); 1812 return; 1813 } 1814 else if (PPC::CRBITRCRegClass.contains(DestReg, SrcReg)) 1815 Opc = PPC::CROR; 1816 else if (PPC::SPERCRegClass.contains(DestReg, SrcReg)) 1817 Opc = PPC::EVOR; 1818 else if ((PPC::ACCRCRegClass.contains(DestReg) || 1819 PPC::UACCRCRegClass.contains(DestReg)) && 1820 (PPC::ACCRCRegClass.contains(SrcReg) || 1821 PPC::UACCRCRegClass.contains(SrcReg))) { 1822 // If primed, de-prime the source register, copy the individual registers 1823 // and prime the destination if needed. The vector subregisters are 1824 // vs[(u)acc * 4] - vs[(u)acc * 4 + 3]. If the copy is not a kill and the 1825 // source is primed, we need to re-prime it after the copy as well. 1826 PPCRegisterInfo::emitAccCopyInfo(MBB, DestReg, SrcReg); 1827 bool DestPrimed = PPC::ACCRCRegClass.contains(DestReg); 1828 bool SrcPrimed = PPC::ACCRCRegClass.contains(SrcReg); 1829 MCRegister VSLSrcReg = 1830 PPC::VSL0 + (SrcReg - (SrcPrimed ? PPC::ACC0 : PPC::UACC0)) * 4; 1831 MCRegister VSLDestReg = 1832 PPC::VSL0 + (DestReg - (DestPrimed ? PPC::ACC0 : PPC::UACC0)) * 4; 1833 if (SrcPrimed) 1834 BuildMI(MBB, I, DL, get(PPC::XXMFACC), SrcReg).addReg(SrcReg); 1835 for (unsigned Idx = 0; Idx < 4; Idx++) 1836 BuildMI(MBB, I, DL, get(PPC::XXLOR), VSLDestReg + Idx) 1837 .addReg(VSLSrcReg + Idx) 1838 .addReg(VSLSrcReg + Idx, getKillRegState(KillSrc)); 1839 if (DestPrimed) 1840 BuildMI(MBB, I, DL, get(PPC::XXMTACC), DestReg).addReg(DestReg); 1841 if (SrcPrimed && !KillSrc) 1842 BuildMI(MBB, I, DL, get(PPC::XXMTACC), SrcReg).addReg(SrcReg); 1843 return; 1844 } else if (PPC::G8pRCRegClass.contains(DestReg) && 1845 PPC::G8pRCRegClass.contains(SrcReg)) { 1846 // TODO: Handle G8RC to G8pRC (and vice versa) copy. 1847 unsigned DestRegIdx = DestReg - PPC::G8p0; 1848 MCRegister DestRegSub0 = PPC::X0 + 2 * DestRegIdx; 1849 MCRegister DestRegSub1 = PPC::X0 + 2 * DestRegIdx + 1; 1850 unsigned SrcRegIdx = SrcReg - PPC::G8p0; 1851 MCRegister SrcRegSub0 = PPC::X0 + 2 * SrcRegIdx; 1852 MCRegister SrcRegSub1 = PPC::X0 + 2 * SrcRegIdx + 1; 1853 BuildMI(MBB, I, DL, get(PPC::OR8), DestRegSub0) 1854 .addReg(SrcRegSub0) 1855 .addReg(SrcRegSub0, getKillRegState(KillSrc)); 1856 BuildMI(MBB, I, DL, get(PPC::OR8), DestRegSub1) 1857 .addReg(SrcRegSub1) 1858 .addReg(SrcRegSub1, getKillRegState(KillSrc)); 1859 return; 1860 } else 1861 llvm_unreachable("Impossible reg-to-reg copy"); 1862 1863 const MCInstrDesc &MCID = get(Opc); 1864 if (MCID.getNumOperands() == 3) 1865 BuildMI(MBB, I, DL, MCID, DestReg) 1866 .addReg(SrcReg).addReg(SrcReg, getKillRegState(KillSrc)); 1867 else 1868 BuildMI(MBB, I, DL, MCID, DestReg).addReg(SrcReg, getKillRegState(KillSrc)); 1869 } 1870 1871 unsigned PPCInstrInfo::getSpillIndex(const TargetRegisterClass *RC) const { 1872 int OpcodeIndex = 0; 1873 1874 if (PPC::GPRCRegClass.hasSubClassEq(RC) || 1875 PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) { 1876 OpcodeIndex = SOK_Int4Spill; 1877 } else if (PPC::G8RCRegClass.hasSubClassEq(RC) || 1878 PPC::G8RC_NOX0RegClass.hasSubClassEq(RC)) { 1879 OpcodeIndex = SOK_Int8Spill; 1880 } else if (PPC::F8RCRegClass.hasSubClassEq(RC)) { 1881 OpcodeIndex = SOK_Float8Spill; 1882 } else if (PPC::F4RCRegClass.hasSubClassEq(RC)) { 1883 OpcodeIndex = SOK_Float4Spill; 1884 } else if (PPC::SPERCRegClass.hasSubClassEq(RC)) { 1885 OpcodeIndex = SOK_SPESpill; 1886 } else if (PPC::CRRCRegClass.hasSubClassEq(RC)) { 1887 OpcodeIndex = SOK_CRSpill; 1888 } else if (PPC::CRBITRCRegClass.hasSubClassEq(RC)) { 1889 OpcodeIndex = SOK_CRBitSpill; 1890 } else if (PPC::VRRCRegClass.hasSubClassEq(RC)) { 1891 OpcodeIndex = SOK_VRVectorSpill; 1892 } else if (PPC::VSRCRegClass.hasSubClassEq(RC)) { 1893 OpcodeIndex = SOK_VSXVectorSpill; 1894 } else if (PPC::VSFRCRegClass.hasSubClassEq(RC)) { 1895 OpcodeIndex = SOK_VectorFloat8Spill; 1896 } else if (PPC::VSSRCRegClass.hasSubClassEq(RC)) { 1897 OpcodeIndex = SOK_VectorFloat4Spill; 1898 } else if (PPC::SPILLTOVSRRCRegClass.hasSubClassEq(RC)) { 1899 OpcodeIndex = SOK_SpillToVSR; 1900 } else if (PPC::ACCRCRegClass.hasSubClassEq(RC)) { 1901 assert(Subtarget.pairedVectorMemops() && 1902 "Register unexpected when paired memops are disabled."); 1903 OpcodeIndex = SOK_AccumulatorSpill; 1904 } else if (PPC::UACCRCRegClass.hasSubClassEq(RC)) { 1905 assert(Subtarget.pairedVectorMemops() && 1906 "Register unexpected when paired memops are disabled."); 1907 OpcodeIndex = SOK_UAccumulatorSpill; 1908 } else if (PPC::VSRpRCRegClass.hasSubClassEq(RC)) { 1909 assert(Subtarget.pairedVectorMemops() && 1910 "Register unexpected when paired memops are disabled."); 1911 OpcodeIndex = SOK_PairedVecSpill; 1912 } else if (PPC::G8pRCRegClass.hasSubClassEq(RC)) { 1913 OpcodeIndex = SOK_PairedG8Spill; 1914 } else { 1915 llvm_unreachable("Unknown regclass!"); 1916 } 1917 return OpcodeIndex; 1918 } 1919 1920 unsigned 1921 PPCInstrInfo::getStoreOpcodeForSpill(const TargetRegisterClass *RC) const { 1922 const unsigned *OpcodesForSpill = getStoreOpcodesForSpillArray(); 1923 return OpcodesForSpill[getSpillIndex(RC)]; 1924 } 1925 1926 unsigned 1927 PPCInstrInfo::getLoadOpcodeForSpill(const TargetRegisterClass *RC) const { 1928 const unsigned *OpcodesForSpill = getLoadOpcodesForSpillArray(); 1929 return OpcodesForSpill[getSpillIndex(RC)]; 1930 } 1931 1932 void PPCInstrInfo::StoreRegToStackSlot( 1933 MachineFunction &MF, unsigned SrcReg, bool isKill, int FrameIdx, 1934 const TargetRegisterClass *RC, 1935 SmallVectorImpl<MachineInstr *> &NewMIs) const { 1936 unsigned Opcode = getStoreOpcodeForSpill(RC); 1937 DebugLoc DL; 1938 1939 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 1940 FuncInfo->setHasSpills(); 1941 1942 NewMIs.push_back(addFrameReference( 1943 BuildMI(MF, DL, get(Opcode)).addReg(SrcReg, getKillRegState(isKill)), 1944 FrameIdx)); 1945 1946 if (PPC::CRRCRegClass.hasSubClassEq(RC) || 1947 PPC::CRBITRCRegClass.hasSubClassEq(RC)) 1948 FuncInfo->setSpillsCR(); 1949 1950 if (isXFormMemOp(Opcode)) 1951 FuncInfo->setHasNonRISpills(); 1952 } 1953 1954 void PPCInstrInfo::storeRegToStackSlotNoUpd( 1955 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned SrcReg, 1956 bool isKill, int FrameIdx, const TargetRegisterClass *RC, 1957 const TargetRegisterInfo *TRI) const { 1958 MachineFunction &MF = *MBB.getParent(); 1959 SmallVector<MachineInstr *, 4> NewMIs; 1960 1961 StoreRegToStackSlot(MF, SrcReg, isKill, FrameIdx, RC, NewMIs); 1962 1963 for (unsigned i = 0, e = NewMIs.size(); i != e; ++i) 1964 MBB.insert(MI, NewMIs[i]); 1965 1966 const MachineFrameInfo &MFI = MF.getFrameInfo(); 1967 MachineMemOperand *MMO = MF.getMachineMemOperand( 1968 MachinePointerInfo::getFixedStack(MF, FrameIdx), 1969 MachineMemOperand::MOStore, MFI.getObjectSize(FrameIdx), 1970 MFI.getObjectAlign(FrameIdx)); 1971 NewMIs.back()->addMemOperand(MF, MMO); 1972 } 1973 1974 void PPCInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB, 1975 MachineBasicBlock::iterator MI, 1976 Register SrcReg, bool isKill, 1977 int FrameIdx, 1978 const TargetRegisterClass *RC, 1979 const TargetRegisterInfo *TRI) const { 1980 // We need to avoid a situation in which the value from a VRRC register is 1981 // spilled using an Altivec instruction and reloaded into a VSRC register 1982 // using a VSX instruction. The issue with this is that the VSX 1983 // load/store instructions swap the doublewords in the vector and the Altivec 1984 // ones don't. The register classes on the spill/reload may be different if 1985 // the register is defined using an Altivec instruction and is then used by a 1986 // VSX instruction. 1987 RC = updatedRC(RC); 1988 storeRegToStackSlotNoUpd(MBB, MI, SrcReg, isKill, FrameIdx, RC, TRI); 1989 } 1990 1991 void PPCInstrInfo::LoadRegFromStackSlot(MachineFunction &MF, const DebugLoc &DL, 1992 unsigned DestReg, int FrameIdx, 1993 const TargetRegisterClass *RC, 1994 SmallVectorImpl<MachineInstr *> &NewMIs) 1995 const { 1996 unsigned Opcode = getLoadOpcodeForSpill(RC); 1997 NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(Opcode), DestReg), 1998 FrameIdx)); 1999 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 2000 2001 if (PPC::CRRCRegClass.hasSubClassEq(RC) || 2002 PPC::CRBITRCRegClass.hasSubClassEq(RC)) 2003 FuncInfo->setSpillsCR(); 2004 2005 if (isXFormMemOp(Opcode)) 2006 FuncInfo->setHasNonRISpills(); 2007 } 2008 2009 void PPCInstrInfo::loadRegFromStackSlotNoUpd( 2010 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned DestReg, 2011 int FrameIdx, const TargetRegisterClass *RC, 2012 const TargetRegisterInfo *TRI) const { 2013 MachineFunction &MF = *MBB.getParent(); 2014 SmallVector<MachineInstr*, 4> NewMIs; 2015 DebugLoc DL; 2016 if (MI != MBB.end()) DL = MI->getDebugLoc(); 2017 2018 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>(); 2019 FuncInfo->setHasSpills(); 2020 2021 LoadRegFromStackSlot(MF, DL, DestReg, FrameIdx, RC, NewMIs); 2022 2023 for (unsigned i = 0, e = NewMIs.size(); i != e; ++i) 2024 MBB.insert(MI, NewMIs[i]); 2025 2026 const MachineFrameInfo &MFI = MF.getFrameInfo(); 2027 MachineMemOperand *MMO = MF.getMachineMemOperand( 2028 MachinePointerInfo::getFixedStack(MF, FrameIdx), 2029 MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIdx), 2030 MFI.getObjectAlign(FrameIdx)); 2031 NewMIs.back()->addMemOperand(MF, MMO); 2032 } 2033 2034 void PPCInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, 2035 MachineBasicBlock::iterator MI, 2036 Register DestReg, int FrameIdx, 2037 const TargetRegisterClass *RC, 2038 const TargetRegisterInfo *TRI) const { 2039 // We need to avoid a situation in which the value from a VRRC register is 2040 // spilled using an Altivec instruction and reloaded into a VSRC register 2041 // using a VSX instruction. The issue with this is that the VSX 2042 // load/store instructions swap the doublewords in the vector and the Altivec 2043 // ones don't. The register classes on the spill/reload may be different if 2044 // the register is defined using an Altivec instruction and is then used by a 2045 // VSX instruction. 2046 RC = updatedRC(RC); 2047 2048 loadRegFromStackSlotNoUpd(MBB, MI, DestReg, FrameIdx, RC, TRI); 2049 } 2050 2051 bool PPCInstrInfo:: 2052 reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const { 2053 assert(Cond.size() == 2 && "Invalid PPC branch opcode!"); 2054 if (Cond[1].getReg() == PPC::CTR8 || Cond[1].getReg() == PPC::CTR) 2055 Cond[0].setImm(Cond[0].getImm() == 0 ? 1 : 0); 2056 else 2057 // Leave the CR# the same, but invert the condition. 2058 Cond[0].setImm(PPC::InvertPredicate((PPC::Predicate)Cond[0].getImm())); 2059 return false; 2060 } 2061 2062 // For some instructions, it is legal to fold ZERO into the RA register field. 2063 // This function performs that fold by replacing the operand with PPC::ZERO, 2064 // it does not consider whether the load immediate zero is no longer in use. 2065 bool PPCInstrInfo::onlyFoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, 2066 Register Reg) const { 2067 // A zero immediate should always be loaded with a single li. 2068 unsigned DefOpc = DefMI.getOpcode(); 2069 if (DefOpc != PPC::LI && DefOpc != PPC::LI8) 2070 return false; 2071 if (!DefMI.getOperand(1).isImm()) 2072 return false; 2073 if (DefMI.getOperand(1).getImm() != 0) 2074 return false; 2075 2076 // Note that we cannot here invert the arguments of an isel in order to fold 2077 // a ZERO into what is presented as the second argument. All we have here 2078 // is the condition bit, and that might come from a CR-logical bit operation. 2079 2080 const MCInstrDesc &UseMCID = UseMI.getDesc(); 2081 2082 // Only fold into real machine instructions. 2083 if (UseMCID.isPseudo()) 2084 return false; 2085 2086 // We need to find which of the User's operands is to be folded, that will be 2087 // the operand that matches the given register ID. 2088 unsigned UseIdx; 2089 for (UseIdx = 0; UseIdx < UseMI.getNumOperands(); ++UseIdx) 2090 if (UseMI.getOperand(UseIdx).isReg() && 2091 UseMI.getOperand(UseIdx).getReg() == Reg) 2092 break; 2093 2094 assert(UseIdx < UseMI.getNumOperands() && "Cannot find Reg in UseMI"); 2095 assert(UseIdx < UseMCID.getNumOperands() && "No operand description for Reg"); 2096 2097 const MCOperandInfo *UseInfo = &UseMCID.OpInfo[UseIdx]; 2098 2099 // We can fold the zero if this register requires a GPRC_NOR0/G8RC_NOX0 2100 // register (which might also be specified as a pointer class kind). 2101 if (UseInfo->isLookupPtrRegClass()) { 2102 if (UseInfo->RegClass /* Kind */ != 1) 2103 return false; 2104 } else { 2105 if (UseInfo->RegClass != PPC::GPRC_NOR0RegClassID && 2106 UseInfo->RegClass != PPC::G8RC_NOX0RegClassID) 2107 return false; 2108 } 2109 2110 // Make sure this is not tied to an output register (or otherwise 2111 // constrained). This is true for ST?UX registers, for example, which 2112 // are tied to their output registers. 2113 if (UseInfo->Constraints != 0) 2114 return false; 2115 2116 MCRegister ZeroReg; 2117 if (UseInfo->isLookupPtrRegClass()) { 2118 bool isPPC64 = Subtarget.isPPC64(); 2119 ZeroReg = isPPC64 ? PPC::ZERO8 : PPC::ZERO; 2120 } else { 2121 ZeroReg = UseInfo->RegClass == PPC::G8RC_NOX0RegClassID ? 2122 PPC::ZERO8 : PPC::ZERO; 2123 } 2124 2125 UseMI.getOperand(UseIdx).setReg(ZeroReg); 2126 return true; 2127 } 2128 2129 // Folds zero into instructions which have a load immediate zero as an operand 2130 // but also recognize zero as immediate zero. If the definition of the load 2131 // has no more users it is deleted. 2132 bool PPCInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, 2133 Register Reg, MachineRegisterInfo *MRI) const { 2134 bool Changed = onlyFoldImmediate(UseMI, DefMI, Reg); 2135 if (MRI->use_nodbg_empty(Reg)) 2136 DefMI.eraseFromParent(); 2137 return Changed; 2138 } 2139 2140 static bool MBBDefinesCTR(MachineBasicBlock &MBB) { 2141 for (MachineInstr &MI : MBB) 2142 if (MI.definesRegister(PPC::CTR) || MI.definesRegister(PPC::CTR8)) 2143 return true; 2144 return false; 2145 } 2146 2147 // We should make sure that, if we're going to predicate both sides of a 2148 // condition (a diamond), that both sides don't define the counter register. We 2149 // can predicate counter-decrement-based branches, but while that predicates 2150 // the branching, it does not predicate the counter decrement. If we tried to 2151 // merge the triangle into one predicated block, we'd decrement the counter 2152 // twice. 2153 bool PPCInstrInfo::isProfitableToIfCvt(MachineBasicBlock &TMBB, 2154 unsigned NumT, unsigned ExtraT, 2155 MachineBasicBlock &FMBB, 2156 unsigned NumF, unsigned ExtraF, 2157 BranchProbability Probability) const { 2158 return !(MBBDefinesCTR(TMBB) && MBBDefinesCTR(FMBB)); 2159 } 2160 2161 2162 bool PPCInstrInfo::isPredicated(const MachineInstr &MI) const { 2163 // The predicated branches are identified by their type, not really by the 2164 // explicit presence of a predicate. Furthermore, some of them can be 2165 // predicated more than once. Because if conversion won't try to predicate 2166 // any instruction which already claims to be predicated (by returning true 2167 // here), always return false. In doing so, we let isPredicable() be the 2168 // final word on whether not the instruction can be (further) predicated. 2169 2170 return false; 2171 } 2172 2173 bool PPCInstrInfo::isSchedulingBoundary(const MachineInstr &MI, 2174 const MachineBasicBlock *MBB, 2175 const MachineFunction &MF) const { 2176 // Set MFFS and MTFSF as scheduling boundary to avoid unexpected code motion 2177 // across them, since some FP operations may change content of FPSCR. 2178 // TODO: Model FPSCR in PPC instruction definitions and remove the workaround 2179 if (MI.getOpcode() == PPC::MFFS || MI.getOpcode() == PPC::MTFSF) 2180 return true; 2181 return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF); 2182 } 2183 2184 bool PPCInstrInfo::PredicateInstruction(MachineInstr &MI, 2185 ArrayRef<MachineOperand> Pred) const { 2186 unsigned OpC = MI.getOpcode(); 2187 if (OpC == PPC::BLR || OpC == PPC::BLR8) { 2188 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) { 2189 bool isPPC64 = Subtarget.isPPC64(); 2190 MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZLR8 : PPC::BDNZLR) 2191 : (isPPC64 ? PPC::BDZLR8 : PPC::BDZLR))); 2192 // Need add Def and Use for CTR implicit operand. 2193 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 2194 .addReg(Pred[1].getReg(), RegState::Implicit) 2195 .addReg(Pred[1].getReg(), RegState::ImplicitDefine); 2196 } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) { 2197 MI.setDesc(get(PPC::BCLR)); 2198 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]); 2199 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) { 2200 MI.setDesc(get(PPC::BCLRn)); 2201 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]); 2202 } else { 2203 MI.setDesc(get(PPC::BCCLR)); 2204 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 2205 .addImm(Pred[0].getImm()) 2206 .add(Pred[1]); 2207 } 2208 2209 return true; 2210 } else if (OpC == PPC::B) { 2211 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) { 2212 bool isPPC64 = Subtarget.isPPC64(); 2213 MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) 2214 : (isPPC64 ? PPC::BDZ8 : PPC::BDZ))); 2215 // Need add Def and Use for CTR implicit operand. 2216 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 2217 .addReg(Pred[1].getReg(), RegState::Implicit) 2218 .addReg(Pred[1].getReg(), RegState::ImplicitDefine); 2219 } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) { 2220 MachineBasicBlock *MBB = MI.getOperand(0).getMBB(); 2221 MI.removeOperand(0); 2222 2223 MI.setDesc(get(PPC::BC)); 2224 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 2225 .add(Pred[1]) 2226 .addMBB(MBB); 2227 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) { 2228 MachineBasicBlock *MBB = MI.getOperand(0).getMBB(); 2229 MI.removeOperand(0); 2230 2231 MI.setDesc(get(PPC::BCn)); 2232 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 2233 .add(Pred[1]) 2234 .addMBB(MBB); 2235 } else { 2236 MachineBasicBlock *MBB = MI.getOperand(0).getMBB(); 2237 MI.removeOperand(0); 2238 2239 MI.setDesc(get(PPC::BCC)); 2240 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 2241 .addImm(Pred[0].getImm()) 2242 .add(Pred[1]) 2243 .addMBB(MBB); 2244 } 2245 2246 return true; 2247 } else if (OpC == PPC::BCTR || OpC == PPC::BCTR8 || OpC == PPC::BCTRL || 2248 OpC == PPC::BCTRL8 || OpC == PPC::BCTRL_RM || 2249 OpC == PPC::BCTRL8_RM) { 2250 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) 2251 llvm_unreachable("Cannot predicate bctr[l] on the ctr register"); 2252 2253 bool setLR = OpC == PPC::BCTRL || OpC == PPC::BCTRL8 || 2254 OpC == PPC::BCTRL_RM || OpC == PPC::BCTRL8_RM; 2255 bool isPPC64 = Subtarget.isPPC64(); 2256 2257 if (Pred[0].getImm() == PPC::PRED_BIT_SET) { 2258 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8 : PPC::BCCTR8) 2259 : (setLR ? PPC::BCCTRL : PPC::BCCTR))); 2260 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]); 2261 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) { 2262 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8n : PPC::BCCTR8n) 2263 : (setLR ? PPC::BCCTRLn : PPC::BCCTRn))); 2264 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]); 2265 } else { 2266 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCCTRL8 : PPC::BCCCTR8) 2267 : (setLR ? PPC::BCCCTRL : PPC::BCCCTR))); 2268 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 2269 .addImm(Pred[0].getImm()) 2270 .add(Pred[1]); 2271 } 2272 2273 // Need add Def and Use for LR implicit operand. 2274 if (setLR) 2275 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 2276 .addReg(isPPC64 ? PPC::LR8 : PPC::LR, RegState::Implicit) 2277 .addReg(isPPC64 ? PPC::LR8 : PPC::LR, RegState::ImplicitDefine); 2278 if (OpC == PPC::BCTRL_RM || OpC == PPC::BCTRL8_RM) 2279 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 2280 .addReg(PPC::RM, RegState::ImplicitDefine); 2281 2282 return true; 2283 } 2284 2285 return false; 2286 } 2287 2288 bool PPCInstrInfo::SubsumesPredicate(ArrayRef<MachineOperand> Pred1, 2289 ArrayRef<MachineOperand> Pred2) const { 2290 assert(Pred1.size() == 2 && "Invalid PPC first predicate"); 2291 assert(Pred2.size() == 2 && "Invalid PPC second predicate"); 2292 2293 if (Pred1[1].getReg() == PPC::CTR8 || Pred1[1].getReg() == PPC::CTR) 2294 return false; 2295 if (Pred2[1].getReg() == PPC::CTR8 || Pred2[1].getReg() == PPC::CTR) 2296 return false; 2297 2298 // P1 can only subsume P2 if they test the same condition register. 2299 if (Pred1[1].getReg() != Pred2[1].getReg()) 2300 return false; 2301 2302 PPC::Predicate P1 = (PPC::Predicate) Pred1[0].getImm(); 2303 PPC::Predicate P2 = (PPC::Predicate) Pred2[0].getImm(); 2304 2305 if (P1 == P2) 2306 return true; 2307 2308 // Does P1 subsume P2, e.g. GE subsumes GT. 2309 if (P1 == PPC::PRED_LE && 2310 (P2 == PPC::PRED_LT || P2 == PPC::PRED_EQ)) 2311 return true; 2312 if (P1 == PPC::PRED_GE && 2313 (P2 == PPC::PRED_GT || P2 == PPC::PRED_EQ)) 2314 return true; 2315 2316 return false; 2317 } 2318 2319 bool PPCInstrInfo::ClobbersPredicate(MachineInstr &MI, 2320 std::vector<MachineOperand> &Pred, 2321 bool SkipDead) const { 2322 // Note: At the present time, the contents of Pred from this function is 2323 // unused by IfConversion. This implementation follows ARM by pushing the 2324 // CR-defining operand. Because the 'DZ' and 'DNZ' count as types of 2325 // predicate, instructions defining CTR or CTR8 are also included as 2326 // predicate-defining instructions. 2327 2328 const TargetRegisterClass *RCs[] = 2329 { &PPC::CRRCRegClass, &PPC::CRBITRCRegClass, 2330 &PPC::CTRRCRegClass, &PPC::CTRRC8RegClass }; 2331 2332 bool Found = false; 2333 for (const MachineOperand &MO : MI.operands()) { 2334 for (unsigned c = 0; c < array_lengthof(RCs) && !Found; ++c) { 2335 const TargetRegisterClass *RC = RCs[c]; 2336 if (MO.isReg()) { 2337 if (MO.isDef() && RC->contains(MO.getReg())) { 2338 Pred.push_back(MO); 2339 Found = true; 2340 } 2341 } else if (MO.isRegMask()) { 2342 for (MCPhysReg R : *RC) 2343 if (MO.clobbersPhysReg(R)) { 2344 Pred.push_back(MO); 2345 Found = true; 2346 } 2347 } 2348 } 2349 } 2350 2351 return Found; 2352 } 2353 2354 bool PPCInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg, 2355 Register &SrcReg2, int64_t &Mask, 2356 int64_t &Value) const { 2357 unsigned Opc = MI.getOpcode(); 2358 2359 switch (Opc) { 2360 default: return false; 2361 case PPC::CMPWI: 2362 case PPC::CMPLWI: 2363 case PPC::CMPDI: 2364 case PPC::CMPLDI: 2365 SrcReg = MI.getOperand(1).getReg(); 2366 SrcReg2 = 0; 2367 Value = MI.getOperand(2).getImm(); 2368 Mask = 0xFFFF; 2369 return true; 2370 case PPC::CMPW: 2371 case PPC::CMPLW: 2372 case PPC::CMPD: 2373 case PPC::CMPLD: 2374 case PPC::FCMPUS: 2375 case PPC::FCMPUD: 2376 SrcReg = MI.getOperand(1).getReg(); 2377 SrcReg2 = MI.getOperand(2).getReg(); 2378 Value = 0; 2379 Mask = 0; 2380 return true; 2381 } 2382 } 2383 2384 bool PPCInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, 2385 Register SrcReg2, int64_t Mask, 2386 int64_t Value, 2387 const MachineRegisterInfo *MRI) const { 2388 if (DisableCmpOpt) 2389 return false; 2390 2391 int OpC = CmpInstr.getOpcode(); 2392 Register CRReg = CmpInstr.getOperand(0).getReg(); 2393 2394 // FP record forms set CR1 based on the exception status bits, not a 2395 // comparison with zero. 2396 if (OpC == PPC::FCMPUS || OpC == PPC::FCMPUD) 2397 return false; 2398 2399 const TargetRegisterInfo *TRI = &getRegisterInfo(); 2400 // The record forms set the condition register based on a signed comparison 2401 // with zero (so says the ISA manual). This is not as straightforward as it 2402 // seems, however, because this is always a 64-bit comparison on PPC64, even 2403 // for instructions that are 32-bit in nature (like slw for example). 2404 // So, on PPC32, for unsigned comparisons, we can use the record forms only 2405 // for equality checks (as those don't depend on the sign). On PPC64, 2406 // we are restricted to equality for unsigned 64-bit comparisons and for 2407 // signed 32-bit comparisons the applicability is more restricted. 2408 bool isPPC64 = Subtarget.isPPC64(); 2409 bool is32BitSignedCompare = OpC == PPC::CMPWI || OpC == PPC::CMPW; 2410 bool is32BitUnsignedCompare = OpC == PPC::CMPLWI || OpC == PPC::CMPLW; 2411 bool is64BitUnsignedCompare = OpC == PPC::CMPLDI || OpC == PPC::CMPLD; 2412 2413 // Look through copies unless that gets us to a physical register. 2414 Register ActualSrc = TRI->lookThruCopyLike(SrcReg, MRI); 2415 if (ActualSrc.isVirtual()) 2416 SrcReg = ActualSrc; 2417 2418 // Get the unique definition of SrcReg. 2419 MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg); 2420 if (!MI) return false; 2421 2422 bool equalityOnly = false; 2423 bool noSub = false; 2424 if (isPPC64) { 2425 if (is32BitSignedCompare) { 2426 // We can perform this optimization only if MI is sign-extending. 2427 if (isSignExtended(*MI)) 2428 noSub = true; 2429 else 2430 return false; 2431 } else if (is32BitUnsignedCompare) { 2432 // We can perform this optimization, equality only, if MI is 2433 // zero-extending. 2434 if (isZeroExtended(*MI)) { 2435 noSub = true; 2436 equalityOnly = true; 2437 } else 2438 return false; 2439 } else 2440 equalityOnly = is64BitUnsignedCompare; 2441 } else 2442 equalityOnly = is32BitUnsignedCompare; 2443 2444 if (equalityOnly) { 2445 // We need to check the uses of the condition register in order to reject 2446 // non-equality comparisons. 2447 for (MachineRegisterInfo::use_instr_iterator 2448 I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end(); 2449 I != IE; ++I) { 2450 MachineInstr *UseMI = &*I; 2451 if (UseMI->getOpcode() == PPC::BCC) { 2452 PPC::Predicate Pred = (PPC::Predicate)UseMI->getOperand(0).getImm(); 2453 unsigned PredCond = PPC::getPredicateCondition(Pred); 2454 // We ignore hint bits when checking for non-equality comparisons. 2455 if (PredCond != PPC::PRED_EQ && PredCond != PPC::PRED_NE) 2456 return false; 2457 } else if (UseMI->getOpcode() == PPC::ISEL || 2458 UseMI->getOpcode() == PPC::ISEL8) { 2459 unsigned SubIdx = UseMI->getOperand(3).getSubReg(); 2460 if (SubIdx != PPC::sub_eq) 2461 return false; 2462 } else 2463 return false; 2464 } 2465 } 2466 2467 MachineBasicBlock::iterator I = CmpInstr; 2468 2469 // Scan forward to find the first use of the compare. 2470 for (MachineBasicBlock::iterator EL = CmpInstr.getParent()->end(); I != EL; 2471 ++I) { 2472 bool FoundUse = false; 2473 for (MachineRegisterInfo::use_instr_iterator 2474 J = MRI->use_instr_begin(CRReg), JE = MRI->use_instr_end(); 2475 J != JE; ++J) 2476 if (&*J == &*I) { 2477 FoundUse = true; 2478 break; 2479 } 2480 2481 if (FoundUse) 2482 break; 2483 } 2484 2485 SmallVector<std::pair<MachineOperand*, PPC::Predicate>, 4> PredsToUpdate; 2486 SmallVector<std::pair<MachineOperand*, unsigned>, 4> SubRegsToUpdate; 2487 2488 // There are two possible candidates which can be changed to set CR[01]. 2489 // One is MI, the other is a SUB instruction. 2490 // For CMPrr(r1,r2), we are looking for SUB(r1,r2) or SUB(r2,r1). 2491 MachineInstr *Sub = nullptr; 2492 if (SrcReg2 != 0) 2493 // MI is not a candidate for CMPrr. 2494 MI = nullptr; 2495 // FIXME: Conservatively refuse to convert an instruction which isn't in the 2496 // same BB as the comparison. This is to allow the check below to avoid calls 2497 // (and other explicit clobbers); instead we should really check for these 2498 // more explicitly (in at least a few predecessors). 2499 else if (MI->getParent() != CmpInstr.getParent()) 2500 return false; 2501 else if (Value != 0) { 2502 // The record-form instructions set CR bit based on signed comparison 2503 // against 0. We try to convert a compare against 1 or -1 into a compare 2504 // against 0 to exploit record-form instructions. For example, we change 2505 // the condition "greater than -1" into "greater than or equal to 0" 2506 // and "less than 1" into "less than or equal to 0". 2507 2508 // Since we optimize comparison based on a specific branch condition, 2509 // we don't optimize if condition code is used by more than once. 2510 if (equalityOnly || !MRI->hasOneUse(CRReg)) 2511 return false; 2512 2513 MachineInstr *UseMI = &*MRI->use_instr_begin(CRReg); 2514 if (UseMI->getOpcode() != PPC::BCC) 2515 return false; 2516 2517 PPC::Predicate Pred = (PPC::Predicate)UseMI->getOperand(0).getImm(); 2518 unsigned PredCond = PPC::getPredicateCondition(Pred); 2519 unsigned PredHint = PPC::getPredicateHint(Pred); 2520 int16_t Immed = (int16_t)Value; 2521 2522 // When modifying the condition in the predicate, we propagate hint bits 2523 // from the original predicate to the new one. 2524 if (Immed == -1 && PredCond == PPC::PRED_GT) 2525 // We convert "greater than -1" into "greater than or equal to 0", 2526 // since we are assuming signed comparison by !equalityOnly 2527 Pred = PPC::getPredicate(PPC::PRED_GE, PredHint); 2528 else if (Immed == -1 && PredCond == PPC::PRED_LE) 2529 // We convert "less than or equal to -1" into "less than 0". 2530 Pred = PPC::getPredicate(PPC::PRED_LT, PredHint); 2531 else if (Immed == 1 && PredCond == PPC::PRED_LT) 2532 // We convert "less than 1" into "less than or equal to 0". 2533 Pred = PPC::getPredicate(PPC::PRED_LE, PredHint); 2534 else if (Immed == 1 && PredCond == PPC::PRED_GE) 2535 // We convert "greater than or equal to 1" into "greater than 0". 2536 Pred = PPC::getPredicate(PPC::PRED_GT, PredHint); 2537 else 2538 return false; 2539 2540 PredsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(0)), Pred)); 2541 } 2542 2543 // Search for Sub. 2544 --I; 2545 2546 // Get ready to iterate backward from CmpInstr. 2547 MachineBasicBlock::iterator E = MI, B = CmpInstr.getParent()->begin(); 2548 2549 for (; I != E && !noSub; --I) { 2550 const MachineInstr &Instr = *I; 2551 unsigned IOpC = Instr.getOpcode(); 2552 2553 if (&*I != &CmpInstr && (Instr.modifiesRegister(PPC::CR0, TRI) || 2554 Instr.readsRegister(PPC::CR0, TRI))) 2555 // This instruction modifies or uses the record condition register after 2556 // the one we want to change. While we could do this transformation, it 2557 // would likely not be profitable. This transformation removes one 2558 // instruction, and so even forcing RA to generate one move probably 2559 // makes it unprofitable. 2560 return false; 2561 2562 // Check whether CmpInstr can be made redundant by the current instruction. 2563 if ((OpC == PPC::CMPW || OpC == PPC::CMPLW || 2564 OpC == PPC::CMPD || OpC == PPC::CMPLD) && 2565 (IOpC == PPC::SUBF || IOpC == PPC::SUBF8) && 2566 ((Instr.getOperand(1).getReg() == SrcReg && 2567 Instr.getOperand(2).getReg() == SrcReg2) || 2568 (Instr.getOperand(1).getReg() == SrcReg2 && 2569 Instr.getOperand(2).getReg() == SrcReg))) { 2570 Sub = &*I; 2571 break; 2572 } 2573 2574 if (I == B) 2575 // The 'and' is below the comparison instruction. 2576 return false; 2577 } 2578 2579 // Return false if no candidates exist. 2580 if (!MI && !Sub) 2581 return false; 2582 2583 // The single candidate is called MI. 2584 if (!MI) MI = Sub; 2585 2586 int NewOpC = -1; 2587 int MIOpC = MI->getOpcode(); 2588 if (MIOpC == PPC::ANDI_rec || MIOpC == PPC::ANDI8_rec || 2589 MIOpC == PPC::ANDIS_rec || MIOpC == PPC::ANDIS8_rec) 2590 NewOpC = MIOpC; 2591 else { 2592 NewOpC = PPC::getRecordFormOpcode(MIOpC); 2593 if (NewOpC == -1 && PPC::getNonRecordFormOpcode(MIOpC) != -1) 2594 NewOpC = MIOpC; 2595 } 2596 2597 // FIXME: On the non-embedded POWER architectures, only some of the record 2598 // forms are fast, and we should use only the fast ones. 2599 2600 // The defining instruction has a record form (or is already a record 2601 // form). It is possible, however, that we'll need to reverse the condition 2602 // code of the users. 2603 if (NewOpC == -1) 2604 return false; 2605 2606 // This transformation should not be performed if `nsw` is missing and is not 2607 // `equalityOnly` comparison. Since if there is overflow, sub_lt, sub_gt in 2608 // CRReg do not reflect correct order. If `equalityOnly` is true, sub_eq in 2609 // CRReg can reflect if compared values are equal, this optz is still valid. 2610 if (!equalityOnly && (NewOpC == PPC::SUBF_rec || NewOpC == PPC::SUBF8_rec) && 2611 Sub && !Sub->getFlag(MachineInstr::NoSWrap)) 2612 return false; 2613 2614 // If we have SUB(r1, r2) and CMP(r2, r1), the condition code based on CMP 2615 // needs to be updated to be based on SUB. Push the condition code 2616 // operands to OperandsToUpdate. If it is safe to remove CmpInstr, the 2617 // condition code of these operands will be modified. 2618 // Here, Value == 0 means we haven't converted comparison against 1 or -1 to 2619 // comparison against 0, which may modify predicate. 2620 bool ShouldSwap = false; 2621 if (Sub && Value == 0) { 2622 ShouldSwap = SrcReg2 != 0 && Sub->getOperand(1).getReg() == SrcReg2 && 2623 Sub->getOperand(2).getReg() == SrcReg; 2624 2625 // The operands to subf are the opposite of sub, so only in the fixed-point 2626 // case, invert the order. 2627 ShouldSwap = !ShouldSwap; 2628 } 2629 2630 if (ShouldSwap) 2631 for (MachineRegisterInfo::use_instr_iterator 2632 I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end(); 2633 I != IE; ++I) { 2634 MachineInstr *UseMI = &*I; 2635 if (UseMI->getOpcode() == PPC::BCC) { 2636 PPC::Predicate Pred = (PPC::Predicate) UseMI->getOperand(0).getImm(); 2637 unsigned PredCond = PPC::getPredicateCondition(Pred); 2638 assert((!equalityOnly || 2639 PredCond == PPC::PRED_EQ || PredCond == PPC::PRED_NE) && 2640 "Invalid predicate for equality-only optimization"); 2641 (void)PredCond; // To suppress warning in release build. 2642 PredsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(0)), 2643 PPC::getSwappedPredicate(Pred))); 2644 } else if (UseMI->getOpcode() == PPC::ISEL || 2645 UseMI->getOpcode() == PPC::ISEL8) { 2646 unsigned NewSubReg = UseMI->getOperand(3).getSubReg(); 2647 assert((!equalityOnly || NewSubReg == PPC::sub_eq) && 2648 "Invalid CR bit for equality-only optimization"); 2649 2650 if (NewSubReg == PPC::sub_lt) 2651 NewSubReg = PPC::sub_gt; 2652 else if (NewSubReg == PPC::sub_gt) 2653 NewSubReg = PPC::sub_lt; 2654 2655 SubRegsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(3)), 2656 NewSubReg)); 2657 } else // We need to abort on a user we don't understand. 2658 return false; 2659 } 2660 assert(!(Value != 0 && ShouldSwap) && 2661 "Non-zero immediate support and ShouldSwap" 2662 "may conflict in updating predicate"); 2663 2664 // Create a new virtual register to hold the value of the CR set by the 2665 // record-form instruction. If the instruction was not previously in 2666 // record form, then set the kill flag on the CR. 2667 CmpInstr.eraseFromParent(); 2668 2669 MachineBasicBlock::iterator MII = MI; 2670 BuildMI(*MI->getParent(), std::next(MII), MI->getDebugLoc(), 2671 get(TargetOpcode::COPY), CRReg) 2672 .addReg(PPC::CR0, MIOpC != NewOpC ? RegState::Kill : 0); 2673 2674 // Even if CR0 register were dead before, it is alive now since the 2675 // instruction we just built uses it. 2676 MI->clearRegisterDeads(PPC::CR0); 2677 2678 if (MIOpC != NewOpC) { 2679 // We need to be careful here: we're replacing one instruction with 2680 // another, and we need to make sure that we get all of the right 2681 // implicit uses and defs. On the other hand, the caller may be holding 2682 // an iterator to this instruction, and so we can't delete it (this is 2683 // specifically the case if this is the instruction directly after the 2684 // compare). 2685 2686 // Rotates are expensive instructions. If we're emitting a record-form 2687 // rotate that can just be an andi/andis, we should just emit that. 2688 if (MIOpC == PPC::RLWINM || MIOpC == PPC::RLWINM8) { 2689 Register GPRRes = MI->getOperand(0).getReg(); 2690 int64_t SH = MI->getOperand(2).getImm(); 2691 int64_t MB = MI->getOperand(3).getImm(); 2692 int64_t ME = MI->getOperand(4).getImm(); 2693 // We can only do this if both the start and end of the mask are in the 2694 // same halfword. 2695 bool MBInLoHWord = MB >= 16; 2696 bool MEInLoHWord = ME >= 16; 2697 uint64_t Mask = ~0LLU; 2698 2699 if (MB <= ME && MBInLoHWord == MEInLoHWord && SH == 0) { 2700 Mask = ((1LLU << (32 - MB)) - 1) & ~((1LLU << (31 - ME)) - 1); 2701 // The mask value needs to shift right 16 if we're emitting andis. 2702 Mask >>= MBInLoHWord ? 0 : 16; 2703 NewOpC = MIOpC == PPC::RLWINM 2704 ? (MBInLoHWord ? PPC::ANDI_rec : PPC::ANDIS_rec) 2705 : (MBInLoHWord ? PPC::ANDI8_rec : PPC::ANDIS8_rec); 2706 } else if (MRI->use_empty(GPRRes) && (ME == 31) && 2707 (ME - MB + 1 == SH) && (MB >= 16)) { 2708 // If we are rotating by the exact number of bits as are in the mask 2709 // and the mask is in the least significant bits of the register, 2710 // that's just an andis. (as long as the GPR result has no uses). 2711 Mask = ((1LLU << 32) - 1) & ~((1LLU << (32 - SH)) - 1); 2712 Mask >>= 16; 2713 NewOpC = MIOpC == PPC::RLWINM ? PPC::ANDIS_rec : PPC::ANDIS8_rec; 2714 } 2715 // If we've set the mask, we can transform. 2716 if (Mask != ~0LLU) { 2717 MI->removeOperand(4); 2718 MI->removeOperand(3); 2719 MI->getOperand(2).setImm(Mask); 2720 NumRcRotatesConvertedToRcAnd++; 2721 } 2722 } else if (MIOpC == PPC::RLDICL && MI->getOperand(2).getImm() == 0) { 2723 int64_t MB = MI->getOperand(3).getImm(); 2724 if (MB >= 48) { 2725 uint64_t Mask = (1LLU << (63 - MB + 1)) - 1; 2726 NewOpC = PPC::ANDI8_rec; 2727 MI->removeOperand(3); 2728 MI->getOperand(2).setImm(Mask); 2729 NumRcRotatesConvertedToRcAnd++; 2730 } 2731 } 2732 2733 const MCInstrDesc &NewDesc = get(NewOpC); 2734 MI->setDesc(NewDesc); 2735 2736 if (NewDesc.ImplicitDefs) 2737 for (const MCPhysReg *ImpDefs = NewDesc.getImplicitDefs(); 2738 *ImpDefs; ++ImpDefs) 2739 if (!MI->definesRegister(*ImpDefs)) 2740 MI->addOperand(*MI->getParent()->getParent(), 2741 MachineOperand::CreateReg(*ImpDefs, true, true)); 2742 if (NewDesc.ImplicitUses) 2743 for (const MCPhysReg *ImpUses = NewDesc.getImplicitUses(); 2744 *ImpUses; ++ImpUses) 2745 if (!MI->readsRegister(*ImpUses)) 2746 MI->addOperand(*MI->getParent()->getParent(), 2747 MachineOperand::CreateReg(*ImpUses, false, true)); 2748 } 2749 assert(MI->definesRegister(PPC::CR0) && 2750 "Record-form instruction does not define cr0?"); 2751 2752 // Modify the condition code of operands in OperandsToUpdate. 2753 // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to 2754 // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc. 2755 for (unsigned i = 0, e = PredsToUpdate.size(); i < e; i++) 2756 PredsToUpdate[i].first->setImm(PredsToUpdate[i].second); 2757 2758 for (unsigned i = 0, e = SubRegsToUpdate.size(); i < e; i++) 2759 SubRegsToUpdate[i].first->setSubReg(SubRegsToUpdate[i].second); 2760 2761 return true; 2762 } 2763 2764 bool PPCInstrInfo::getMemOperandsWithOffsetWidth( 2765 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps, 2766 int64_t &Offset, bool &OffsetIsScalable, unsigned &Width, 2767 const TargetRegisterInfo *TRI) const { 2768 const MachineOperand *BaseOp; 2769 OffsetIsScalable = false; 2770 if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, TRI)) 2771 return false; 2772 BaseOps.push_back(BaseOp); 2773 return true; 2774 } 2775 2776 static bool isLdStSafeToCluster(const MachineInstr &LdSt, 2777 const TargetRegisterInfo *TRI) { 2778 // If this is a volatile load/store, don't mess with it. 2779 if (LdSt.hasOrderedMemoryRef() || LdSt.getNumExplicitOperands() != 3) 2780 return false; 2781 2782 if (LdSt.getOperand(2).isFI()) 2783 return true; 2784 2785 assert(LdSt.getOperand(2).isReg() && "Expected a reg operand."); 2786 // Can't cluster if the instruction modifies the base register 2787 // or it is update form. e.g. ld r2,3(r2) 2788 if (LdSt.modifiesRegister(LdSt.getOperand(2).getReg(), TRI)) 2789 return false; 2790 2791 return true; 2792 } 2793 2794 // Only cluster instruction pair that have the same opcode, and they are 2795 // clusterable according to PowerPC specification. 2796 static bool isClusterableLdStOpcPair(unsigned FirstOpc, unsigned SecondOpc, 2797 const PPCSubtarget &Subtarget) { 2798 switch (FirstOpc) { 2799 default: 2800 return false; 2801 case PPC::STD: 2802 case PPC::STFD: 2803 case PPC::STXSD: 2804 case PPC::DFSTOREf64: 2805 return FirstOpc == SecondOpc; 2806 // PowerPC backend has opcode STW/STW8 for instruction "stw" to deal with 2807 // 32bit and 64bit instruction selection. They are clusterable pair though 2808 // they are different opcode. 2809 case PPC::STW: 2810 case PPC::STW8: 2811 return SecondOpc == PPC::STW || SecondOpc == PPC::STW8; 2812 } 2813 } 2814 2815 bool PPCInstrInfo::shouldClusterMemOps( 2816 ArrayRef<const MachineOperand *> BaseOps1, 2817 ArrayRef<const MachineOperand *> BaseOps2, unsigned NumLoads, 2818 unsigned NumBytes) const { 2819 2820 assert(BaseOps1.size() == 1 && BaseOps2.size() == 1); 2821 const MachineOperand &BaseOp1 = *BaseOps1.front(); 2822 const MachineOperand &BaseOp2 = *BaseOps2.front(); 2823 assert((BaseOp1.isReg() || BaseOp1.isFI()) && 2824 "Only base registers and frame indices are supported."); 2825 2826 // The NumLoads means the number of loads that has been clustered. 2827 // Don't cluster memory op if there are already two ops clustered at least. 2828 if (NumLoads > 2) 2829 return false; 2830 2831 // Cluster the load/store only when they have the same base 2832 // register or FI. 2833 if ((BaseOp1.isReg() != BaseOp2.isReg()) || 2834 (BaseOp1.isReg() && BaseOp1.getReg() != BaseOp2.getReg()) || 2835 (BaseOp1.isFI() && BaseOp1.getIndex() != BaseOp2.getIndex())) 2836 return false; 2837 2838 // Check if the load/store are clusterable according to the PowerPC 2839 // specification. 2840 const MachineInstr &FirstLdSt = *BaseOp1.getParent(); 2841 const MachineInstr &SecondLdSt = *BaseOp2.getParent(); 2842 unsigned FirstOpc = FirstLdSt.getOpcode(); 2843 unsigned SecondOpc = SecondLdSt.getOpcode(); 2844 const TargetRegisterInfo *TRI = &getRegisterInfo(); 2845 // Cluster the load/store only when they have the same opcode, and they are 2846 // clusterable opcode according to PowerPC specification. 2847 if (!isClusterableLdStOpcPair(FirstOpc, SecondOpc, Subtarget)) 2848 return false; 2849 2850 // Can't cluster load/store that have ordered or volatile memory reference. 2851 if (!isLdStSafeToCluster(FirstLdSt, TRI) || 2852 !isLdStSafeToCluster(SecondLdSt, TRI)) 2853 return false; 2854 2855 int64_t Offset1 = 0, Offset2 = 0; 2856 unsigned Width1 = 0, Width2 = 0; 2857 const MachineOperand *Base1 = nullptr, *Base2 = nullptr; 2858 if (!getMemOperandWithOffsetWidth(FirstLdSt, Base1, Offset1, Width1, TRI) || 2859 !getMemOperandWithOffsetWidth(SecondLdSt, Base2, Offset2, Width2, TRI) || 2860 Width1 != Width2) 2861 return false; 2862 2863 assert(Base1 == &BaseOp1 && Base2 == &BaseOp2 && 2864 "getMemOperandWithOffsetWidth return incorrect base op"); 2865 // The caller should already have ordered FirstMemOp/SecondMemOp by offset. 2866 assert(Offset1 <= Offset2 && "Caller should have ordered offsets."); 2867 return Offset1 + Width1 == Offset2; 2868 } 2869 2870 /// GetInstSize - Return the number of bytes of code the specified 2871 /// instruction may be. This returns the maximum number of bytes. 2872 /// 2873 unsigned PPCInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { 2874 unsigned Opcode = MI.getOpcode(); 2875 2876 if (Opcode == PPC::INLINEASM || Opcode == PPC::INLINEASM_BR) { 2877 const MachineFunction *MF = MI.getParent()->getParent(); 2878 const char *AsmStr = MI.getOperand(0).getSymbolName(); 2879 return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo()); 2880 } else if (Opcode == TargetOpcode::STACKMAP) { 2881 StackMapOpers Opers(&MI); 2882 return Opers.getNumPatchBytes(); 2883 } else if (Opcode == TargetOpcode::PATCHPOINT) { 2884 PatchPointOpers Opers(&MI); 2885 return Opers.getNumPatchBytes(); 2886 } else { 2887 return get(Opcode).getSize(); 2888 } 2889 } 2890 2891 std::pair<unsigned, unsigned> 2892 PPCInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const { 2893 const unsigned Mask = PPCII::MO_ACCESS_MASK; 2894 return std::make_pair(TF & Mask, TF & ~Mask); 2895 } 2896 2897 ArrayRef<std::pair<unsigned, const char *>> 2898 PPCInstrInfo::getSerializableDirectMachineOperandTargetFlags() const { 2899 using namespace PPCII; 2900 static const std::pair<unsigned, const char *> TargetFlags[] = { 2901 {MO_LO, "ppc-lo"}, 2902 {MO_HA, "ppc-ha"}, 2903 {MO_TPREL_LO, "ppc-tprel-lo"}, 2904 {MO_TPREL_HA, "ppc-tprel-ha"}, 2905 {MO_DTPREL_LO, "ppc-dtprel-lo"}, 2906 {MO_TLSLD_LO, "ppc-tlsld-lo"}, 2907 {MO_TOC_LO, "ppc-toc-lo"}, 2908 {MO_TLS, "ppc-tls"}}; 2909 return makeArrayRef(TargetFlags); 2910 } 2911 2912 ArrayRef<std::pair<unsigned, const char *>> 2913 PPCInstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const { 2914 using namespace PPCII; 2915 static const std::pair<unsigned, const char *> TargetFlags[] = { 2916 {MO_PLT, "ppc-plt"}, 2917 {MO_PIC_FLAG, "ppc-pic"}, 2918 {MO_PCREL_FLAG, "ppc-pcrel"}, 2919 {MO_GOT_FLAG, "ppc-got"}, 2920 {MO_PCREL_OPT_FLAG, "ppc-opt-pcrel"}, 2921 {MO_TLSGD_FLAG, "ppc-tlsgd"}, 2922 {MO_TLSLD_FLAG, "ppc-tlsld"}, 2923 {MO_TPREL_FLAG, "ppc-tprel"}, 2924 {MO_TLSGDM_FLAG, "ppc-tlsgdm"}, 2925 {MO_GOT_TLSGD_PCREL_FLAG, "ppc-got-tlsgd-pcrel"}, 2926 {MO_GOT_TLSLD_PCREL_FLAG, "ppc-got-tlsld-pcrel"}, 2927 {MO_GOT_TPREL_PCREL_FLAG, "ppc-got-tprel-pcrel"}}; 2928 return makeArrayRef(TargetFlags); 2929 } 2930 2931 // Expand VSX Memory Pseudo instruction to either a VSX or a FP instruction. 2932 // The VSX versions have the advantage of a full 64-register target whereas 2933 // the FP ones have the advantage of lower latency and higher throughput. So 2934 // what we are after is using the faster instructions in low register pressure 2935 // situations and using the larger register file in high register pressure 2936 // situations. 2937 bool PPCInstrInfo::expandVSXMemPseudo(MachineInstr &MI) const { 2938 unsigned UpperOpcode, LowerOpcode; 2939 switch (MI.getOpcode()) { 2940 case PPC::DFLOADf32: 2941 UpperOpcode = PPC::LXSSP; 2942 LowerOpcode = PPC::LFS; 2943 break; 2944 case PPC::DFLOADf64: 2945 UpperOpcode = PPC::LXSD; 2946 LowerOpcode = PPC::LFD; 2947 break; 2948 case PPC::DFSTOREf32: 2949 UpperOpcode = PPC::STXSSP; 2950 LowerOpcode = PPC::STFS; 2951 break; 2952 case PPC::DFSTOREf64: 2953 UpperOpcode = PPC::STXSD; 2954 LowerOpcode = PPC::STFD; 2955 break; 2956 case PPC::XFLOADf32: 2957 UpperOpcode = PPC::LXSSPX; 2958 LowerOpcode = PPC::LFSX; 2959 break; 2960 case PPC::XFLOADf64: 2961 UpperOpcode = PPC::LXSDX; 2962 LowerOpcode = PPC::LFDX; 2963 break; 2964 case PPC::XFSTOREf32: 2965 UpperOpcode = PPC::STXSSPX; 2966 LowerOpcode = PPC::STFSX; 2967 break; 2968 case PPC::XFSTOREf64: 2969 UpperOpcode = PPC::STXSDX; 2970 LowerOpcode = PPC::STFDX; 2971 break; 2972 case PPC::LIWAX: 2973 UpperOpcode = PPC::LXSIWAX; 2974 LowerOpcode = PPC::LFIWAX; 2975 break; 2976 case PPC::LIWZX: 2977 UpperOpcode = PPC::LXSIWZX; 2978 LowerOpcode = PPC::LFIWZX; 2979 break; 2980 case PPC::STIWX: 2981 UpperOpcode = PPC::STXSIWX; 2982 LowerOpcode = PPC::STFIWX; 2983 break; 2984 default: 2985 llvm_unreachable("Unknown Operation!"); 2986 } 2987 2988 Register TargetReg = MI.getOperand(0).getReg(); 2989 unsigned Opcode; 2990 if ((TargetReg >= PPC::F0 && TargetReg <= PPC::F31) || 2991 (TargetReg >= PPC::VSL0 && TargetReg <= PPC::VSL31)) 2992 Opcode = LowerOpcode; 2993 else 2994 Opcode = UpperOpcode; 2995 MI.setDesc(get(Opcode)); 2996 return true; 2997 } 2998 2999 static bool isAnImmediateOperand(const MachineOperand &MO) { 3000 return MO.isCPI() || MO.isGlobal() || MO.isImm(); 3001 } 3002 3003 bool PPCInstrInfo::expandPostRAPseudo(MachineInstr &MI) const { 3004 auto &MBB = *MI.getParent(); 3005 auto DL = MI.getDebugLoc(); 3006 3007 switch (MI.getOpcode()) { 3008 case PPC::BUILD_UACC: { 3009 MCRegister ACC = MI.getOperand(0).getReg(); 3010 MCRegister UACC = MI.getOperand(1).getReg(); 3011 if (ACC - PPC::ACC0 != UACC - PPC::UACC0) { 3012 MCRegister SrcVSR = PPC::VSL0 + (UACC - PPC::UACC0) * 4; 3013 MCRegister DstVSR = PPC::VSL0 + (ACC - PPC::ACC0) * 4; 3014 // FIXME: This can easily be improved to look up to the top of the MBB 3015 // to see if the inputs are XXLOR's. If they are and SrcReg is killed, 3016 // we can just re-target any such XXLOR's to DstVSR + offset. 3017 for (int VecNo = 0; VecNo < 4; VecNo++) 3018 BuildMI(MBB, MI, DL, get(PPC::XXLOR), DstVSR + VecNo) 3019 .addReg(SrcVSR + VecNo) 3020 .addReg(SrcVSR + VecNo); 3021 } 3022 // BUILD_UACC is expanded to 4 copies of the underlying vsx registers. 3023 // So after building the 4 copies, we can replace the BUILD_UACC instruction 3024 // with a NOP. 3025 LLVM_FALLTHROUGH; 3026 } 3027 case PPC::KILL_PAIR: { 3028 MI.setDesc(get(PPC::UNENCODED_NOP)); 3029 MI.removeOperand(1); 3030 MI.removeOperand(0); 3031 return true; 3032 } 3033 case TargetOpcode::LOAD_STACK_GUARD: { 3034 assert(Subtarget.isTargetLinux() && 3035 "Only Linux target is expected to contain LOAD_STACK_GUARD"); 3036 const int64_t Offset = Subtarget.isPPC64() ? -0x7010 : -0x7008; 3037 const unsigned Reg = Subtarget.isPPC64() ? PPC::X13 : PPC::R2; 3038 MI.setDesc(get(Subtarget.isPPC64() ? PPC::LD : PPC::LWZ)); 3039 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 3040 .addImm(Offset) 3041 .addReg(Reg); 3042 return true; 3043 } 3044 case PPC::DFLOADf32: 3045 case PPC::DFLOADf64: 3046 case PPC::DFSTOREf32: 3047 case PPC::DFSTOREf64: { 3048 assert(Subtarget.hasP9Vector() && 3049 "Invalid D-Form Pseudo-ops on Pre-P9 target."); 3050 assert(MI.getOperand(2).isReg() && 3051 isAnImmediateOperand(MI.getOperand(1)) && 3052 "D-form op must have register and immediate operands"); 3053 return expandVSXMemPseudo(MI); 3054 } 3055 case PPC::XFLOADf32: 3056 case PPC::XFSTOREf32: 3057 case PPC::LIWAX: 3058 case PPC::LIWZX: 3059 case PPC::STIWX: { 3060 assert(Subtarget.hasP8Vector() && 3061 "Invalid X-Form Pseudo-ops on Pre-P8 target."); 3062 assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() && 3063 "X-form op must have register and register operands"); 3064 return expandVSXMemPseudo(MI); 3065 } 3066 case PPC::XFLOADf64: 3067 case PPC::XFSTOREf64: { 3068 assert(Subtarget.hasVSX() && 3069 "Invalid X-Form Pseudo-ops on target that has no VSX."); 3070 assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() && 3071 "X-form op must have register and register operands"); 3072 return expandVSXMemPseudo(MI); 3073 } 3074 case PPC::SPILLTOVSR_LD: { 3075 Register TargetReg = MI.getOperand(0).getReg(); 3076 if (PPC::VSFRCRegClass.contains(TargetReg)) { 3077 MI.setDesc(get(PPC::DFLOADf64)); 3078 return expandPostRAPseudo(MI); 3079 } 3080 else 3081 MI.setDesc(get(PPC::LD)); 3082 return true; 3083 } 3084 case PPC::SPILLTOVSR_ST: { 3085 Register SrcReg = MI.getOperand(0).getReg(); 3086 if (PPC::VSFRCRegClass.contains(SrcReg)) { 3087 NumStoreSPILLVSRRCAsVec++; 3088 MI.setDesc(get(PPC::DFSTOREf64)); 3089 return expandPostRAPseudo(MI); 3090 } else { 3091 NumStoreSPILLVSRRCAsGpr++; 3092 MI.setDesc(get(PPC::STD)); 3093 } 3094 return true; 3095 } 3096 case PPC::SPILLTOVSR_LDX: { 3097 Register TargetReg = MI.getOperand(0).getReg(); 3098 if (PPC::VSFRCRegClass.contains(TargetReg)) 3099 MI.setDesc(get(PPC::LXSDX)); 3100 else 3101 MI.setDesc(get(PPC::LDX)); 3102 return true; 3103 } 3104 case PPC::SPILLTOVSR_STX: { 3105 Register SrcReg = MI.getOperand(0).getReg(); 3106 if (PPC::VSFRCRegClass.contains(SrcReg)) { 3107 NumStoreSPILLVSRRCAsVec++; 3108 MI.setDesc(get(PPC::STXSDX)); 3109 } else { 3110 NumStoreSPILLVSRRCAsGpr++; 3111 MI.setDesc(get(PPC::STDX)); 3112 } 3113 return true; 3114 } 3115 3116 // FIXME: Maybe we can expand it in 'PowerPC Expand Atomic' pass. 3117 case PPC::CFENCE8: { 3118 auto Val = MI.getOperand(0).getReg(); 3119 BuildMI(MBB, MI, DL, get(PPC::CMPD), PPC::CR7).addReg(Val).addReg(Val); 3120 BuildMI(MBB, MI, DL, get(PPC::CTRL_DEP)) 3121 .addImm(PPC::PRED_NE_MINUS) 3122 .addReg(PPC::CR7) 3123 .addImm(1); 3124 MI.setDesc(get(PPC::ISYNC)); 3125 MI.removeOperand(0); 3126 return true; 3127 } 3128 } 3129 return false; 3130 } 3131 3132 // Essentially a compile-time implementation of a compare->isel sequence. 3133 // It takes two constants to compare, along with the true/false registers 3134 // and the comparison type (as a subreg to a CR field) and returns one 3135 // of the true/false registers, depending on the comparison results. 3136 static unsigned selectReg(int64_t Imm1, int64_t Imm2, unsigned CompareOpc, 3137 unsigned TrueReg, unsigned FalseReg, 3138 unsigned CRSubReg) { 3139 // Signed comparisons. The immediates are assumed to be sign-extended. 3140 if (CompareOpc == PPC::CMPWI || CompareOpc == PPC::CMPDI) { 3141 switch (CRSubReg) { 3142 default: llvm_unreachable("Unknown integer comparison type."); 3143 case PPC::sub_lt: 3144 return Imm1 < Imm2 ? TrueReg : FalseReg; 3145 case PPC::sub_gt: 3146 return Imm1 > Imm2 ? TrueReg : FalseReg; 3147 case PPC::sub_eq: 3148 return Imm1 == Imm2 ? TrueReg : FalseReg; 3149 } 3150 } 3151 // Unsigned comparisons. 3152 else if (CompareOpc == PPC::CMPLWI || CompareOpc == PPC::CMPLDI) { 3153 switch (CRSubReg) { 3154 default: llvm_unreachable("Unknown integer comparison type."); 3155 case PPC::sub_lt: 3156 return (uint64_t)Imm1 < (uint64_t)Imm2 ? TrueReg : FalseReg; 3157 case PPC::sub_gt: 3158 return (uint64_t)Imm1 > (uint64_t)Imm2 ? TrueReg : FalseReg; 3159 case PPC::sub_eq: 3160 return Imm1 == Imm2 ? TrueReg : FalseReg; 3161 } 3162 } 3163 return PPC::NoRegister; 3164 } 3165 3166 void PPCInstrInfo::replaceInstrOperandWithImm(MachineInstr &MI, 3167 unsigned OpNo, 3168 int64_t Imm) const { 3169 assert(MI.getOperand(OpNo).isReg() && "Operand must be a REG"); 3170 // Replace the REG with the Immediate. 3171 Register InUseReg = MI.getOperand(OpNo).getReg(); 3172 MI.getOperand(OpNo).ChangeToImmediate(Imm); 3173 3174 // We need to make sure that the MI didn't have any implicit use 3175 // of this REG any more. We don't call MI.implicit_operands().empty() to 3176 // return early, since MI's MCID might be changed in calling context, as a 3177 // result its number of explicit operands may be changed, thus the begin of 3178 // implicit operand is changed. 3179 const TargetRegisterInfo *TRI = &getRegisterInfo(); 3180 int UseOpIdx = MI.findRegisterUseOperandIdx(InUseReg, false, TRI); 3181 if (UseOpIdx >= 0) { 3182 MachineOperand &MO = MI.getOperand(UseOpIdx); 3183 if (MO.isImplicit()) 3184 // The operands must always be in the following order: 3185 // - explicit reg defs, 3186 // - other explicit operands (reg uses, immediates, etc.), 3187 // - implicit reg defs 3188 // - implicit reg uses 3189 // Therefore, removing the implicit operand won't change the explicit 3190 // operands layout. 3191 MI.removeOperand(UseOpIdx); 3192 } 3193 } 3194 3195 // Replace an instruction with one that materializes a constant (and sets 3196 // CR0 if the original instruction was a record-form instruction). 3197 void PPCInstrInfo::replaceInstrWithLI(MachineInstr &MI, 3198 const LoadImmediateInfo &LII) const { 3199 // Remove existing operands. 3200 int OperandToKeep = LII.SetCR ? 1 : 0; 3201 for (int i = MI.getNumOperands() - 1; i > OperandToKeep; i--) 3202 MI.removeOperand(i); 3203 3204 // Replace the instruction. 3205 if (LII.SetCR) { 3206 MI.setDesc(get(LII.Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec)); 3207 // Set the immediate. 3208 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 3209 .addImm(LII.Imm).addReg(PPC::CR0, RegState::ImplicitDefine); 3210 return; 3211 } 3212 else 3213 MI.setDesc(get(LII.Is64Bit ? PPC::LI8 : PPC::LI)); 3214 3215 // Set the immediate. 3216 MachineInstrBuilder(*MI.getParent()->getParent(), MI) 3217 .addImm(LII.Imm); 3218 } 3219 3220 MachineInstr *PPCInstrInfo::getDefMIPostRA(unsigned Reg, MachineInstr &MI, 3221 bool &SeenIntermediateUse) const { 3222 assert(!MI.getParent()->getParent()->getRegInfo().isSSA() && 3223 "Should be called after register allocation."); 3224 const TargetRegisterInfo *TRI = &getRegisterInfo(); 3225 MachineBasicBlock::reverse_iterator E = MI.getParent()->rend(), It = MI; 3226 It++; 3227 SeenIntermediateUse = false; 3228 for (; It != E; ++It) { 3229 if (It->modifiesRegister(Reg, TRI)) 3230 return &*It; 3231 if (It->readsRegister(Reg, TRI)) 3232 SeenIntermediateUse = true; 3233 } 3234 return nullptr; 3235 } 3236 3237 void PPCInstrInfo::materializeImmPostRA(MachineBasicBlock &MBB, 3238 MachineBasicBlock::iterator MBBI, 3239 const DebugLoc &DL, Register Reg, 3240 int64_t Imm) const { 3241 assert(!MBB.getParent()->getRegInfo().isSSA() && 3242 "Register should be in non-SSA form after RA"); 3243 bool isPPC64 = Subtarget.isPPC64(); 3244 // FIXME: Materialization here is not optimal. 3245 // For some special bit patterns we can use less instructions. 3246 // See `selectI64ImmDirect` in PPCISelDAGToDAG.cpp. 3247 if (isInt<16>(Imm)) { 3248 BuildMI(MBB, MBBI, DL, get(isPPC64 ? PPC::LI8 : PPC::LI), Reg).addImm(Imm); 3249 } else if (isInt<32>(Imm)) { 3250 BuildMI(MBB, MBBI, DL, get(isPPC64 ? PPC::LIS8 : PPC::LIS), Reg) 3251 .addImm(Imm >> 16); 3252 if (Imm & 0xFFFF) 3253 BuildMI(MBB, MBBI, DL, get(isPPC64 ? PPC::ORI8 : PPC::ORI), Reg) 3254 .addReg(Reg, RegState::Kill) 3255 .addImm(Imm & 0xFFFF); 3256 } else { 3257 assert(isPPC64 && "Materializing 64-bit immediate to single register is " 3258 "only supported in PPC64"); 3259 BuildMI(MBB, MBBI, DL, get(PPC::LIS8), Reg).addImm(Imm >> 48); 3260 if ((Imm >> 32) & 0xFFFF) 3261 BuildMI(MBB, MBBI, DL, get(PPC::ORI8), Reg) 3262 .addReg(Reg, RegState::Kill) 3263 .addImm((Imm >> 32) & 0xFFFF); 3264 BuildMI(MBB, MBBI, DL, get(PPC::RLDICR), Reg) 3265 .addReg(Reg, RegState::Kill) 3266 .addImm(32) 3267 .addImm(31); 3268 BuildMI(MBB, MBBI, DL, get(PPC::ORIS8), Reg) 3269 .addReg(Reg, RegState::Kill) 3270 .addImm((Imm >> 16) & 0xFFFF); 3271 if (Imm & 0xFFFF) 3272 BuildMI(MBB, MBBI, DL, get(PPC::ORI8), Reg) 3273 .addReg(Reg, RegState::Kill) 3274 .addImm(Imm & 0xFFFF); 3275 } 3276 } 3277 3278 MachineInstr *PPCInstrInfo::getForwardingDefMI( 3279 MachineInstr &MI, 3280 unsigned &OpNoForForwarding, 3281 bool &SeenIntermediateUse) const { 3282 OpNoForForwarding = ~0U; 3283 MachineInstr *DefMI = nullptr; 3284 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo(); 3285 const TargetRegisterInfo *TRI = &getRegisterInfo(); 3286 // If we're in SSA, get the defs through the MRI. Otherwise, only look 3287 // within the basic block to see if the register is defined using an 3288 // LI/LI8/ADDI/ADDI8. 3289 if (MRI->isSSA()) { 3290 for (int i = 1, e = MI.getNumOperands(); i < e; i++) { 3291 if (!MI.getOperand(i).isReg()) 3292 continue; 3293 Register Reg = MI.getOperand(i).getReg(); 3294 if (!Register::isVirtualRegister(Reg)) 3295 continue; 3296 Register TrueReg = TRI->lookThruCopyLike(Reg, MRI); 3297 if (Register::isVirtualRegister(TrueReg)) { 3298 DefMI = MRI->getVRegDef(TrueReg); 3299 if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8 || 3300 DefMI->getOpcode() == PPC::ADDI || 3301 DefMI->getOpcode() == PPC::ADDI8) { 3302 OpNoForForwarding = i; 3303 // The ADDI and LI operand maybe exist in one instruction at same 3304 // time. we prefer to fold LI operand as LI only has one Imm operand 3305 // and is more possible to be converted. So if current DefMI is 3306 // ADDI/ADDI8, we continue to find possible LI/LI8. 3307 if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8) 3308 break; 3309 } 3310 } 3311 } 3312 } else { 3313 // Looking back through the definition for each operand could be expensive, 3314 // so exit early if this isn't an instruction that either has an immediate 3315 // form or is already an immediate form that we can handle. 3316 ImmInstrInfo III; 3317 unsigned Opc = MI.getOpcode(); 3318 bool ConvertibleImmForm = 3319 Opc == PPC::CMPWI || Opc == PPC::CMPLWI || Opc == PPC::CMPDI || 3320 Opc == PPC::CMPLDI || Opc == PPC::ADDI || Opc == PPC::ADDI8 || 3321 Opc == PPC::ORI || Opc == PPC::ORI8 || Opc == PPC::XORI || 3322 Opc == PPC::XORI8 || Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec || 3323 Opc == PPC::RLDICL_32 || Opc == PPC::RLDICL_32_64 || 3324 Opc == PPC::RLWINM || Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8 || 3325 Opc == PPC::RLWINM8_rec; 3326 bool IsVFReg = (MI.getNumOperands() && MI.getOperand(0).isReg()) 3327 ? isVFRegister(MI.getOperand(0).getReg()) 3328 : false; 3329 if (!ConvertibleImmForm && !instrHasImmForm(Opc, IsVFReg, III, true)) 3330 return nullptr; 3331 3332 // Don't convert or %X, %Y, %Y since that's just a register move. 3333 if ((Opc == PPC::OR || Opc == PPC::OR8) && 3334 MI.getOperand(1).getReg() == MI.getOperand(2).getReg()) 3335 return nullptr; 3336 for (int i = 1, e = MI.getNumOperands(); i < e; i++) { 3337 MachineOperand &MO = MI.getOperand(i); 3338 SeenIntermediateUse = false; 3339 if (MO.isReg() && MO.isUse() && !MO.isImplicit()) { 3340 Register Reg = MI.getOperand(i).getReg(); 3341 // If we see another use of this reg between the def and the MI, 3342 // we want to flat it so the def isn't deleted. 3343 MachineInstr *DefMI = getDefMIPostRA(Reg, MI, SeenIntermediateUse); 3344 if (DefMI) { 3345 // Is this register defined by some form of add-immediate (including 3346 // load-immediate) within this basic block? 3347 switch (DefMI->getOpcode()) { 3348 default: 3349 break; 3350 case PPC::LI: 3351 case PPC::LI8: 3352 case PPC::ADDItocL: 3353 case PPC::ADDI: 3354 case PPC::ADDI8: 3355 OpNoForForwarding = i; 3356 return DefMI; 3357 } 3358 } 3359 } 3360 } 3361 } 3362 return OpNoForForwarding == ~0U ? nullptr : DefMI; 3363 } 3364 3365 unsigned PPCInstrInfo::getSpillTarget() const { 3366 // With P10, we may need to spill paired vector registers or accumulator 3367 // registers. MMA implies paired vectors, so we can just check that. 3368 bool IsP10Variant = Subtarget.isISA3_1() || Subtarget.pairedVectorMemops(); 3369 return IsP10Variant ? 2 : Subtarget.hasP9Vector() ? 1 : 0; 3370 } 3371 3372 const unsigned *PPCInstrInfo::getStoreOpcodesForSpillArray() const { 3373 return StoreSpillOpcodesArray[getSpillTarget()]; 3374 } 3375 3376 const unsigned *PPCInstrInfo::getLoadOpcodesForSpillArray() const { 3377 return LoadSpillOpcodesArray[getSpillTarget()]; 3378 } 3379 3380 void PPCInstrInfo::fixupIsDeadOrKill(MachineInstr *StartMI, MachineInstr *EndMI, 3381 unsigned RegNo) const { 3382 // Conservatively clear kill flag for the register if the instructions are in 3383 // different basic blocks and in SSA form, because the kill flag may no longer 3384 // be right. There is no need to bother with dead flags since defs with no 3385 // uses will be handled by DCE. 3386 MachineRegisterInfo &MRI = StartMI->getParent()->getParent()->getRegInfo(); 3387 if (MRI.isSSA() && (StartMI->getParent() != EndMI->getParent())) { 3388 MRI.clearKillFlags(RegNo); 3389 return; 3390 } 3391 3392 // Instructions between [StartMI, EndMI] should be in same basic block. 3393 assert((StartMI->getParent() == EndMI->getParent()) && 3394 "Instructions are not in same basic block"); 3395 3396 // If before RA, StartMI may be def through COPY, we need to adjust it to the 3397 // real def. See function getForwardingDefMI. 3398 if (MRI.isSSA()) { 3399 bool Reads, Writes; 3400 std::tie(Reads, Writes) = StartMI->readsWritesVirtualRegister(RegNo); 3401 if (!Reads && !Writes) { 3402 assert(Register::isVirtualRegister(RegNo) && 3403 "Must be a virtual register"); 3404 // Get real def and ignore copies. 3405 StartMI = MRI.getVRegDef(RegNo); 3406 } 3407 } 3408 3409 bool IsKillSet = false; 3410 3411 auto clearOperandKillInfo = [=] (MachineInstr &MI, unsigned Index) { 3412 MachineOperand &MO = MI.getOperand(Index); 3413 if (MO.isReg() && MO.isUse() && MO.isKill() && 3414 getRegisterInfo().regsOverlap(MO.getReg(), RegNo)) 3415 MO.setIsKill(false); 3416 }; 3417 3418 // Set killed flag for EndMI. 3419 // No need to do anything if EndMI defines RegNo. 3420 int UseIndex = 3421 EndMI->findRegisterUseOperandIdx(RegNo, false, &getRegisterInfo()); 3422 if (UseIndex != -1) { 3423 EndMI->getOperand(UseIndex).setIsKill(true); 3424 IsKillSet = true; 3425 // Clear killed flag for other EndMI operands related to RegNo. In some 3426 // upexpected cases, killed may be set multiple times for same register 3427 // operand in same MI. 3428 for (int i = 0, e = EndMI->getNumOperands(); i != e; ++i) 3429 if (i != UseIndex) 3430 clearOperandKillInfo(*EndMI, i); 3431 } 3432 3433 // Walking the inst in reverse order (EndMI -> StartMI]. 3434 MachineBasicBlock::reverse_iterator It = *EndMI; 3435 MachineBasicBlock::reverse_iterator E = EndMI->getParent()->rend(); 3436 // EndMI has been handled above, skip it here. 3437 It++; 3438 MachineOperand *MO = nullptr; 3439 for (; It != E; ++It) { 3440 // Skip insturctions which could not be a def/use of RegNo. 3441 if (It->isDebugInstr() || It->isPosition()) 3442 continue; 3443 3444 // Clear killed flag for all It operands related to RegNo. In some 3445 // upexpected cases, killed may be set multiple times for same register 3446 // operand in same MI. 3447 for (int i = 0, e = It->getNumOperands(); i != e; ++i) 3448 clearOperandKillInfo(*It, i); 3449 3450 // If killed is not set, set killed for its last use or set dead for its def 3451 // if no use found. 3452 if (!IsKillSet) { 3453 if ((MO = It->findRegisterUseOperand(RegNo, false, &getRegisterInfo()))) { 3454 // Use found, set it killed. 3455 IsKillSet = true; 3456 MO->setIsKill(true); 3457 continue; 3458 } else if ((MO = It->findRegisterDefOperand(RegNo, false, true, 3459 &getRegisterInfo()))) { 3460 // No use found, set dead for its def. 3461 assert(&*It == StartMI && "No new def between StartMI and EndMI."); 3462 MO->setIsDead(true); 3463 break; 3464 } 3465 } 3466 3467 if ((&*It) == StartMI) 3468 break; 3469 } 3470 // Ensure RegMo liveness is killed after EndMI. 3471 assert((IsKillSet || (MO && MO->isDead())) && 3472 "RegNo should be killed or dead"); 3473 } 3474 3475 // This opt tries to convert the following imm form to an index form to save an 3476 // add for stack variables. 3477 // Return false if no such pattern found. 3478 // 3479 // ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi 3480 // ADD instr: ToBeDeletedReg = ADD ToBeChangedReg(killed), ScaleReg 3481 // Imm instr: Reg = op OffsetImm, ToBeDeletedReg(killed) 3482 // 3483 // can be converted to: 3484 // 3485 // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, (OffsetAddi + OffsetImm) 3486 // Index instr: Reg = opx ScaleReg, ToBeChangedReg(killed) 3487 // 3488 // In order to eliminate ADD instr, make sure that: 3489 // 1: (OffsetAddi + OffsetImm) must be int16 since this offset will be used in 3490 // new ADDI instr and ADDI can only take int16 Imm. 3491 // 2: ToBeChangedReg must be killed in ADD instr and there is no other use 3492 // between ADDI and ADD instr since its original def in ADDI will be changed 3493 // in new ADDI instr. And also there should be no new def for it between 3494 // ADD and Imm instr as ToBeChangedReg will be used in Index instr. 3495 // 3: ToBeDeletedReg must be killed in Imm instr and there is no other use 3496 // between ADD and Imm instr since ADD instr will be eliminated. 3497 // 4: ScaleReg must not be redefined between ADD and Imm instr since it will be 3498 // moved to Index instr. 3499 bool PPCInstrInfo::foldFrameOffset(MachineInstr &MI) const { 3500 MachineFunction *MF = MI.getParent()->getParent(); 3501 MachineRegisterInfo *MRI = &MF->getRegInfo(); 3502 bool PostRA = !MRI->isSSA(); 3503 // Do this opt after PEI which is after RA. The reason is stack slot expansion 3504 // in PEI may expose such opportunities since in PEI, stack slot offsets to 3505 // frame base(OffsetAddi) are determined. 3506 if (!PostRA) 3507 return false; 3508 unsigned ToBeDeletedReg = 0; 3509 int64_t OffsetImm = 0; 3510 unsigned XFormOpcode = 0; 3511 ImmInstrInfo III; 3512 3513 // Check if Imm instr meets requirement. 3514 if (!isImmInstrEligibleForFolding(MI, ToBeDeletedReg, XFormOpcode, OffsetImm, 3515 III)) 3516 return false; 3517 3518 bool OtherIntermediateUse = false; 3519 MachineInstr *ADDMI = getDefMIPostRA(ToBeDeletedReg, MI, OtherIntermediateUse); 3520 3521 // Exit if there is other use between ADD and Imm instr or no def found. 3522 if (OtherIntermediateUse || !ADDMI) 3523 return false; 3524 3525 // Check if ADD instr meets requirement. 3526 if (!isADDInstrEligibleForFolding(*ADDMI)) 3527 return false; 3528 3529 unsigned ScaleRegIdx = 0; 3530 int64_t OffsetAddi = 0; 3531 MachineInstr *ADDIMI = nullptr; 3532 3533 // Check if there is a valid ToBeChangedReg in ADDMI. 3534 // 1: It must be killed. 3535 // 2: Its definition must be a valid ADDIMI. 3536 // 3: It must satify int16 offset requirement. 3537 if (isValidToBeChangedReg(ADDMI, 1, ADDIMI, OffsetAddi, OffsetImm)) 3538 ScaleRegIdx = 2; 3539 else if (isValidToBeChangedReg(ADDMI, 2, ADDIMI, OffsetAddi, OffsetImm)) 3540 ScaleRegIdx = 1; 3541 else 3542 return false; 3543 3544 assert(ADDIMI && "There should be ADDIMI for valid ToBeChangedReg."); 3545 Register ToBeChangedReg = ADDIMI->getOperand(0).getReg(); 3546 Register ScaleReg = ADDMI->getOperand(ScaleRegIdx).getReg(); 3547 auto NewDefFor = [&](unsigned Reg, MachineBasicBlock::iterator Start, 3548 MachineBasicBlock::iterator End) { 3549 for (auto It = ++Start; It != End; It++) 3550 if (It->modifiesRegister(Reg, &getRegisterInfo())) 3551 return true; 3552 return false; 3553 }; 3554 3555 // We are trying to replace the ImmOpNo with ScaleReg. Give up if it is 3556 // treated as special zero when ScaleReg is R0/X0 register. 3557 if (III.ZeroIsSpecialOrig == III.ImmOpNo && 3558 (ScaleReg == PPC::R0 || ScaleReg == PPC::X0)) 3559 return false; 3560 3561 // Make sure no other def for ToBeChangedReg and ScaleReg between ADD Instr 3562 // and Imm Instr. 3563 if (NewDefFor(ToBeChangedReg, *ADDMI, MI) || NewDefFor(ScaleReg, *ADDMI, MI)) 3564 return false; 3565 3566 // Now start to do the transformation. 3567 LLVM_DEBUG(dbgs() << "Replace instruction: " 3568 << "\n"); 3569 LLVM_DEBUG(ADDIMI->dump()); 3570 LLVM_DEBUG(ADDMI->dump()); 3571 LLVM_DEBUG(MI.dump()); 3572 LLVM_DEBUG(dbgs() << "with: " 3573 << "\n"); 3574 3575 // Update ADDI instr. 3576 ADDIMI->getOperand(2).setImm(OffsetAddi + OffsetImm); 3577 3578 // Update Imm instr. 3579 MI.setDesc(get(XFormOpcode)); 3580 MI.getOperand(III.ImmOpNo) 3581 .ChangeToRegister(ScaleReg, false, false, 3582 ADDMI->getOperand(ScaleRegIdx).isKill()); 3583 3584 MI.getOperand(III.OpNoForForwarding) 3585 .ChangeToRegister(ToBeChangedReg, false, false, true); 3586 3587 // Eliminate ADD instr. 3588 ADDMI->eraseFromParent(); 3589 3590 LLVM_DEBUG(ADDIMI->dump()); 3591 LLVM_DEBUG(MI.dump()); 3592 3593 return true; 3594 } 3595 3596 bool PPCInstrInfo::isADDIInstrEligibleForFolding(MachineInstr &ADDIMI, 3597 int64_t &Imm) const { 3598 unsigned Opc = ADDIMI.getOpcode(); 3599 3600 // Exit if the instruction is not ADDI. 3601 if (Opc != PPC::ADDI && Opc != PPC::ADDI8) 3602 return false; 3603 3604 // The operand may not necessarily be an immediate - it could be a relocation. 3605 if (!ADDIMI.getOperand(2).isImm()) 3606 return false; 3607 3608 Imm = ADDIMI.getOperand(2).getImm(); 3609 3610 return true; 3611 } 3612 3613 bool PPCInstrInfo::isADDInstrEligibleForFolding(MachineInstr &ADDMI) const { 3614 unsigned Opc = ADDMI.getOpcode(); 3615 3616 // Exit if the instruction is not ADD. 3617 return Opc == PPC::ADD4 || Opc == PPC::ADD8; 3618 } 3619 3620 bool PPCInstrInfo::isImmInstrEligibleForFolding(MachineInstr &MI, 3621 unsigned &ToBeDeletedReg, 3622 unsigned &XFormOpcode, 3623 int64_t &OffsetImm, 3624 ImmInstrInfo &III) const { 3625 // Only handle load/store. 3626 if (!MI.mayLoadOrStore()) 3627 return false; 3628 3629 unsigned Opc = MI.getOpcode(); 3630 3631 XFormOpcode = RI.getMappedIdxOpcForImmOpc(Opc); 3632 3633 // Exit if instruction has no index form. 3634 if (XFormOpcode == PPC::INSTRUCTION_LIST_END) 3635 return false; 3636 3637 // TODO: sync the logic between instrHasImmForm() and ImmToIdxMap. 3638 if (!instrHasImmForm(XFormOpcode, isVFRegister(MI.getOperand(0).getReg()), 3639 III, true)) 3640 return false; 3641 3642 if (!III.IsSummingOperands) 3643 return false; 3644 3645 MachineOperand ImmOperand = MI.getOperand(III.ImmOpNo); 3646 MachineOperand RegOperand = MI.getOperand(III.OpNoForForwarding); 3647 // Only support imm operands, not relocation slots or others. 3648 if (!ImmOperand.isImm()) 3649 return false; 3650 3651 assert(RegOperand.isReg() && "Instruction format is not right"); 3652 3653 // There are other use for ToBeDeletedReg after Imm instr, can not delete it. 3654 if (!RegOperand.isKill()) 3655 return false; 3656 3657 ToBeDeletedReg = RegOperand.getReg(); 3658 OffsetImm = ImmOperand.getImm(); 3659 3660 return true; 3661 } 3662 3663 bool PPCInstrInfo::isValidToBeChangedReg(MachineInstr *ADDMI, unsigned Index, 3664 MachineInstr *&ADDIMI, 3665 int64_t &OffsetAddi, 3666 int64_t OffsetImm) const { 3667 assert((Index == 1 || Index == 2) && "Invalid operand index for add."); 3668 MachineOperand &MO = ADDMI->getOperand(Index); 3669 3670 if (!MO.isKill()) 3671 return false; 3672 3673 bool OtherIntermediateUse = false; 3674 3675 ADDIMI = getDefMIPostRA(MO.getReg(), *ADDMI, OtherIntermediateUse); 3676 // Currently handle only one "add + Imminstr" pair case, exit if other 3677 // intermediate use for ToBeChangedReg found. 3678 // TODO: handle the cases where there are other "add + Imminstr" pairs 3679 // with same offset in Imminstr which is like: 3680 // 3681 // ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi 3682 // ADD instr1: ToBeDeletedReg1 = ADD ToBeChangedReg, ScaleReg1 3683 // Imm instr1: Reg1 = op1 OffsetImm, ToBeDeletedReg1(killed) 3684 // ADD instr2: ToBeDeletedReg2 = ADD ToBeChangedReg(killed), ScaleReg2 3685 // Imm instr2: Reg2 = op2 OffsetImm, ToBeDeletedReg2(killed) 3686 // 3687 // can be converted to: 3688 // 3689 // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, 3690 // (OffsetAddi + OffsetImm) 3691 // Index instr1: Reg1 = opx1 ScaleReg1, ToBeChangedReg 3692 // Index instr2: Reg2 = opx2 ScaleReg2, ToBeChangedReg(killed) 3693 3694 if (OtherIntermediateUse || !ADDIMI) 3695 return false; 3696 // Check if ADDI instr meets requirement. 3697 if (!isADDIInstrEligibleForFolding(*ADDIMI, OffsetAddi)) 3698 return false; 3699 3700 if (isInt<16>(OffsetAddi + OffsetImm)) 3701 return true; 3702 return false; 3703 } 3704 3705 // If this instruction has an immediate form and one of its operands is a 3706 // result of a load-immediate or an add-immediate, convert it to 3707 // the immediate form if the constant is in range. 3708 bool PPCInstrInfo::convertToImmediateForm(MachineInstr &MI, 3709 MachineInstr **KilledDef) const { 3710 MachineFunction *MF = MI.getParent()->getParent(); 3711 MachineRegisterInfo *MRI = &MF->getRegInfo(); 3712 bool PostRA = !MRI->isSSA(); 3713 bool SeenIntermediateUse = true; 3714 unsigned ForwardingOperand = ~0U; 3715 MachineInstr *DefMI = getForwardingDefMI(MI, ForwardingOperand, 3716 SeenIntermediateUse); 3717 if (!DefMI) 3718 return false; 3719 assert(ForwardingOperand < MI.getNumOperands() && 3720 "The forwarding operand needs to be valid at this point"); 3721 bool IsForwardingOperandKilled = MI.getOperand(ForwardingOperand).isKill(); 3722 bool KillFwdDefMI = !SeenIntermediateUse && IsForwardingOperandKilled; 3723 if (KilledDef && KillFwdDefMI) 3724 *KilledDef = DefMI; 3725 3726 // If this is a imm instruction and its register operands is produced by ADDI, 3727 // put the imm into imm inst directly. 3728 if (RI.getMappedIdxOpcForImmOpc(MI.getOpcode()) != 3729 PPC::INSTRUCTION_LIST_END && 3730 transformToNewImmFormFedByAdd(MI, *DefMI, ForwardingOperand)) 3731 return true; 3732 3733 ImmInstrInfo III; 3734 bool IsVFReg = MI.getOperand(0).isReg() 3735 ? isVFRegister(MI.getOperand(0).getReg()) 3736 : false; 3737 bool HasImmForm = instrHasImmForm(MI.getOpcode(), IsVFReg, III, PostRA); 3738 // If this is a reg+reg instruction that has a reg+imm form, 3739 // and one of the operands is produced by an add-immediate, 3740 // try to convert it. 3741 if (HasImmForm && 3742 transformToImmFormFedByAdd(MI, III, ForwardingOperand, *DefMI, 3743 KillFwdDefMI)) 3744 return true; 3745 3746 // If this is a reg+reg instruction that has a reg+imm form, 3747 // and one of the operands is produced by LI, convert it now. 3748 if (HasImmForm && 3749 transformToImmFormFedByLI(MI, III, ForwardingOperand, *DefMI)) 3750 return true; 3751 3752 // If this is not a reg+reg, but the DefMI is LI/LI8, check if its user MI 3753 // can be simpified to LI. 3754 if (!HasImmForm && simplifyToLI(MI, *DefMI, ForwardingOperand, KilledDef)) 3755 return true; 3756 3757 return false; 3758 } 3759 3760 bool PPCInstrInfo::combineRLWINM(MachineInstr &MI, 3761 MachineInstr **ToErase) const { 3762 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo(); 3763 Register FoldingReg = MI.getOperand(1).getReg(); 3764 if (!Register::isVirtualRegister(FoldingReg)) 3765 return false; 3766 MachineInstr *SrcMI = MRI->getVRegDef(FoldingReg); 3767 if (SrcMI->getOpcode() != PPC::RLWINM && 3768 SrcMI->getOpcode() != PPC::RLWINM_rec && 3769 SrcMI->getOpcode() != PPC::RLWINM8 && 3770 SrcMI->getOpcode() != PPC::RLWINM8_rec) 3771 return false; 3772 assert((MI.getOperand(2).isImm() && MI.getOperand(3).isImm() && 3773 MI.getOperand(4).isImm() && SrcMI->getOperand(2).isImm() && 3774 SrcMI->getOperand(3).isImm() && SrcMI->getOperand(4).isImm()) && 3775 "Invalid PPC::RLWINM Instruction!"); 3776 uint64_t SHSrc = SrcMI->getOperand(2).getImm(); 3777 uint64_t SHMI = MI.getOperand(2).getImm(); 3778 uint64_t MBSrc = SrcMI->getOperand(3).getImm(); 3779 uint64_t MBMI = MI.getOperand(3).getImm(); 3780 uint64_t MESrc = SrcMI->getOperand(4).getImm(); 3781 uint64_t MEMI = MI.getOperand(4).getImm(); 3782 3783 assert((MEMI < 32 && MESrc < 32 && MBMI < 32 && MBSrc < 32) && 3784 "Invalid PPC::RLWINM Instruction!"); 3785 // If MBMI is bigger than MEMI, we always can not get run of ones. 3786 // RotatedSrcMask non-wrap: 3787 // 0........31|32........63 3788 // RotatedSrcMask: B---E B---E 3789 // MaskMI: -----------|--E B------ 3790 // Result: ----- --- (Bad candidate) 3791 // 3792 // RotatedSrcMask wrap: 3793 // 0........31|32........63 3794 // RotatedSrcMask: --E B----|--E B---- 3795 // MaskMI: -----------|--E B------ 3796 // Result: --- -----|--- ----- (Bad candidate) 3797 // 3798 // One special case is RotatedSrcMask is a full set mask. 3799 // RotatedSrcMask full: 3800 // 0........31|32........63 3801 // RotatedSrcMask: ------EB---|-------EB--- 3802 // MaskMI: -----------|--E B------ 3803 // Result: -----------|--- ------- (Good candidate) 3804 3805 // Mark special case. 3806 bool SrcMaskFull = (MBSrc - MESrc == 1) || (MBSrc == 0 && MESrc == 31); 3807 3808 // For other MBMI > MEMI cases, just return. 3809 if ((MBMI > MEMI) && !SrcMaskFull) 3810 return false; 3811 3812 // Handle MBMI <= MEMI cases. 3813 APInt MaskMI = APInt::getBitsSetWithWrap(32, 32 - MEMI - 1, 32 - MBMI); 3814 // In MI, we only need low 32 bits of SrcMI, just consider about low 32 3815 // bit of SrcMI mask. Note that in APInt, lowerest bit is at index 0, 3816 // while in PowerPC ISA, lowerest bit is at index 63. 3817 APInt MaskSrc = APInt::getBitsSetWithWrap(32, 32 - MESrc - 1, 32 - MBSrc); 3818 3819 APInt RotatedSrcMask = MaskSrc.rotl(SHMI); 3820 APInt FinalMask = RotatedSrcMask & MaskMI; 3821 uint32_t NewMB, NewME; 3822 bool Simplified = false; 3823 3824 // If final mask is 0, MI result should be 0 too. 3825 if (FinalMask.isZero()) { 3826 bool Is64Bit = 3827 (MI.getOpcode() == PPC::RLWINM8 || MI.getOpcode() == PPC::RLWINM8_rec); 3828 Simplified = true; 3829 LLVM_DEBUG(dbgs() << "Replace Instr: "); 3830 LLVM_DEBUG(MI.dump()); 3831 3832 if (MI.getOpcode() == PPC::RLWINM || MI.getOpcode() == PPC::RLWINM8) { 3833 // Replace MI with "LI 0" 3834 MI.removeOperand(4); 3835 MI.removeOperand(3); 3836 MI.removeOperand(2); 3837 MI.getOperand(1).ChangeToImmediate(0); 3838 MI.setDesc(get(Is64Bit ? PPC::LI8 : PPC::LI)); 3839 } else { 3840 // Replace MI with "ANDI_rec reg, 0" 3841 MI.removeOperand(4); 3842 MI.removeOperand(3); 3843 MI.getOperand(2).setImm(0); 3844 MI.setDesc(get(Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec)); 3845 MI.getOperand(1).setReg(SrcMI->getOperand(1).getReg()); 3846 if (SrcMI->getOperand(1).isKill()) { 3847 MI.getOperand(1).setIsKill(true); 3848 SrcMI->getOperand(1).setIsKill(false); 3849 } else 3850 // About to replace MI.getOperand(1), clear its kill flag. 3851 MI.getOperand(1).setIsKill(false); 3852 } 3853 3854 LLVM_DEBUG(dbgs() << "With: "); 3855 LLVM_DEBUG(MI.dump()); 3856 3857 } else if ((isRunOfOnes((unsigned)(FinalMask.getZExtValue()), NewMB, NewME) && 3858 NewMB <= NewME) || 3859 SrcMaskFull) { 3860 // Here we only handle MBMI <= MEMI case, so NewMB must be no bigger 3861 // than NewME. Otherwise we get a 64 bit value after folding, but MI 3862 // return a 32 bit value. 3863 Simplified = true; 3864 LLVM_DEBUG(dbgs() << "Converting Instr: "); 3865 LLVM_DEBUG(MI.dump()); 3866 3867 uint16_t NewSH = (SHSrc + SHMI) % 32; 3868 MI.getOperand(2).setImm(NewSH); 3869 // If SrcMI mask is full, no need to update MBMI and MEMI. 3870 if (!SrcMaskFull) { 3871 MI.getOperand(3).setImm(NewMB); 3872 MI.getOperand(4).setImm(NewME); 3873 } 3874 MI.getOperand(1).setReg(SrcMI->getOperand(1).getReg()); 3875 if (SrcMI->getOperand(1).isKill()) { 3876 MI.getOperand(1).setIsKill(true); 3877 SrcMI->getOperand(1).setIsKill(false); 3878 } else 3879 // About to replace MI.getOperand(1), clear its kill flag. 3880 MI.getOperand(1).setIsKill(false); 3881 3882 LLVM_DEBUG(dbgs() << "To: "); 3883 LLVM_DEBUG(MI.dump()); 3884 } 3885 if (Simplified & MRI->use_nodbg_empty(FoldingReg) && 3886 !SrcMI->hasImplicitDef()) { 3887 // If FoldingReg has no non-debug use and it has no implicit def (it 3888 // is not RLWINMO or RLWINM8o), it's safe to delete its def SrcMI. 3889 // Otherwise keep it. 3890 *ToErase = SrcMI; 3891 LLVM_DEBUG(dbgs() << "Delete dead instruction: "); 3892 LLVM_DEBUG(SrcMI->dump()); 3893 } 3894 return Simplified; 3895 } 3896 3897 bool PPCInstrInfo::instrHasImmForm(unsigned Opc, bool IsVFReg, 3898 ImmInstrInfo &III, bool PostRA) const { 3899 // The vast majority of the instructions would need their operand 2 replaced 3900 // with an immediate when switching to the reg+imm form. A marked exception 3901 // are the update form loads/stores for which a constant operand 2 would need 3902 // to turn into a displacement and move operand 1 to the operand 2 position. 3903 III.ImmOpNo = 2; 3904 III.OpNoForForwarding = 2; 3905 III.ImmWidth = 16; 3906 III.ImmMustBeMultipleOf = 1; 3907 III.TruncateImmTo = 0; 3908 III.IsSummingOperands = false; 3909 switch (Opc) { 3910 default: return false; 3911 case PPC::ADD4: 3912 case PPC::ADD8: 3913 III.SignedImm = true; 3914 III.ZeroIsSpecialOrig = 0; 3915 III.ZeroIsSpecialNew = 1; 3916 III.IsCommutative = true; 3917 III.IsSummingOperands = true; 3918 III.ImmOpcode = Opc == PPC::ADD4 ? PPC::ADDI : PPC::ADDI8; 3919 break; 3920 case PPC::ADDC: 3921 case PPC::ADDC8: 3922 III.SignedImm = true; 3923 III.ZeroIsSpecialOrig = 0; 3924 III.ZeroIsSpecialNew = 0; 3925 III.IsCommutative = true; 3926 III.IsSummingOperands = true; 3927 III.ImmOpcode = Opc == PPC::ADDC ? PPC::ADDIC : PPC::ADDIC8; 3928 break; 3929 case PPC::ADDC_rec: 3930 III.SignedImm = true; 3931 III.ZeroIsSpecialOrig = 0; 3932 III.ZeroIsSpecialNew = 0; 3933 III.IsCommutative = true; 3934 III.IsSummingOperands = true; 3935 III.ImmOpcode = PPC::ADDIC_rec; 3936 break; 3937 case PPC::SUBFC: 3938 case PPC::SUBFC8: 3939 III.SignedImm = true; 3940 III.ZeroIsSpecialOrig = 0; 3941 III.ZeroIsSpecialNew = 0; 3942 III.IsCommutative = false; 3943 III.ImmOpcode = Opc == PPC::SUBFC ? PPC::SUBFIC : PPC::SUBFIC8; 3944 break; 3945 case PPC::CMPW: 3946 case PPC::CMPD: 3947 III.SignedImm = true; 3948 III.ZeroIsSpecialOrig = 0; 3949 III.ZeroIsSpecialNew = 0; 3950 III.IsCommutative = false; 3951 III.ImmOpcode = Opc == PPC::CMPW ? PPC::CMPWI : PPC::CMPDI; 3952 break; 3953 case PPC::CMPLW: 3954 case PPC::CMPLD: 3955 III.SignedImm = false; 3956 III.ZeroIsSpecialOrig = 0; 3957 III.ZeroIsSpecialNew = 0; 3958 III.IsCommutative = false; 3959 III.ImmOpcode = Opc == PPC::CMPLW ? PPC::CMPLWI : PPC::CMPLDI; 3960 break; 3961 case PPC::AND_rec: 3962 case PPC::AND8_rec: 3963 case PPC::OR: 3964 case PPC::OR8: 3965 case PPC::XOR: 3966 case PPC::XOR8: 3967 III.SignedImm = false; 3968 III.ZeroIsSpecialOrig = 0; 3969 III.ZeroIsSpecialNew = 0; 3970 III.IsCommutative = true; 3971 switch(Opc) { 3972 default: llvm_unreachable("Unknown opcode"); 3973 case PPC::AND_rec: 3974 III.ImmOpcode = PPC::ANDI_rec; 3975 break; 3976 case PPC::AND8_rec: 3977 III.ImmOpcode = PPC::ANDI8_rec; 3978 break; 3979 case PPC::OR: III.ImmOpcode = PPC::ORI; break; 3980 case PPC::OR8: III.ImmOpcode = PPC::ORI8; break; 3981 case PPC::XOR: III.ImmOpcode = PPC::XORI; break; 3982 case PPC::XOR8: III.ImmOpcode = PPC::XORI8; break; 3983 } 3984 break; 3985 case PPC::RLWNM: 3986 case PPC::RLWNM8: 3987 case PPC::RLWNM_rec: 3988 case PPC::RLWNM8_rec: 3989 case PPC::SLW: 3990 case PPC::SLW8: 3991 case PPC::SLW_rec: 3992 case PPC::SLW8_rec: 3993 case PPC::SRW: 3994 case PPC::SRW8: 3995 case PPC::SRW_rec: 3996 case PPC::SRW8_rec: 3997 case PPC::SRAW: 3998 case PPC::SRAW_rec: 3999 III.SignedImm = false; 4000 III.ZeroIsSpecialOrig = 0; 4001 III.ZeroIsSpecialNew = 0; 4002 III.IsCommutative = false; 4003 // This isn't actually true, but the instructions ignore any of the 4004 // upper bits, so any immediate loaded with an LI is acceptable. 4005 // This does not apply to shift right algebraic because a value 4006 // out of range will produce a -1/0. 4007 III.ImmWidth = 16; 4008 if (Opc == PPC::RLWNM || Opc == PPC::RLWNM8 || Opc == PPC::RLWNM_rec || 4009 Opc == PPC::RLWNM8_rec) 4010 III.TruncateImmTo = 5; 4011 else 4012 III.TruncateImmTo = 6; 4013 switch(Opc) { 4014 default: llvm_unreachable("Unknown opcode"); 4015 case PPC::RLWNM: III.ImmOpcode = PPC::RLWINM; break; 4016 case PPC::RLWNM8: III.ImmOpcode = PPC::RLWINM8; break; 4017 case PPC::RLWNM_rec: 4018 III.ImmOpcode = PPC::RLWINM_rec; 4019 break; 4020 case PPC::RLWNM8_rec: 4021 III.ImmOpcode = PPC::RLWINM8_rec; 4022 break; 4023 case PPC::SLW: III.ImmOpcode = PPC::RLWINM; break; 4024 case PPC::SLW8: III.ImmOpcode = PPC::RLWINM8; break; 4025 case PPC::SLW_rec: 4026 III.ImmOpcode = PPC::RLWINM_rec; 4027 break; 4028 case PPC::SLW8_rec: 4029 III.ImmOpcode = PPC::RLWINM8_rec; 4030 break; 4031 case PPC::SRW: III.ImmOpcode = PPC::RLWINM; break; 4032 case PPC::SRW8: III.ImmOpcode = PPC::RLWINM8; break; 4033 case PPC::SRW_rec: 4034 III.ImmOpcode = PPC::RLWINM_rec; 4035 break; 4036 case PPC::SRW8_rec: 4037 III.ImmOpcode = PPC::RLWINM8_rec; 4038 break; 4039 case PPC::SRAW: 4040 III.ImmWidth = 5; 4041 III.TruncateImmTo = 0; 4042 III.ImmOpcode = PPC::SRAWI; 4043 break; 4044 case PPC::SRAW_rec: 4045 III.ImmWidth = 5; 4046 III.TruncateImmTo = 0; 4047 III.ImmOpcode = PPC::SRAWI_rec; 4048 break; 4049 } 4050 break; 4051 case PPC::RLDCL: 4052 case PPC::RLDCL_rec: 4053 case PPC::RLDCR: 4054 case PPC::RLDCR_rec: 4055 case PPC::SLD: 4056 case PPC::SLD_rec: 4057 case PPC::SRD: 4058 case PPC::SRD_rec: 4059 case PPC::SRAD: 4060 case PPC::SRAD_rec: 4061 III.SignedImm = false; 4062 III.ZeroIsSpecialOrig = 0; 4063 III.ZeroIsSpecialNew = 0; 4064 III.IsCommutative = false; 4065 // This isn't actually true, but the instructions ignore any of the 4066 // upper bits, so any immediate loaded with an LI is acceptable. 4067 // This does not apply to shift right algebraic because a value 4068 // out of range will produce a -1/0. 4069 III.ImmWidth = 16; 4070 if (Opc == PPC::RLDCL || Opc == PPC::RLDCL_rec || Opc == PPC::RLDCR || 4071 Opc == PPC::RLDCR_rec) 4072 III.TruncateImmTo = 6; 4073 else 4074 III.TruncateImmTo = 7; 4075 switch(Opc) { 4076 default: llvm_unreachable("Unknown opcode"); 4077 case PPC::RLDCL: III.ImmOpcode = PPC::RLDICL; break; 4078 case PPC::RLDCL_rec: 4079 III.ImmOpcode = PPC::RLDICL_rec; 4080 break; 4081 case PPC::RLDCR: III.ImmOpcode = PPC::RLDICR; break; 4082 case PPC::RLDCR_rec: 4083 III.ImmOpcode = PPC::RLDICR_rec; 4084 break; 4085 case PPC::SLD: III.ImmOpcode = PPC::RLDICR; break; 4086 case PPC::SLD_rec: 4087 III.ImmOpcode = PPC::RLDICR_rec; 4088 break; 4089 case PPC::SRD: III.ImmOpcode = PPC::RLDICL; break; 4090 case PPC::SRD_rec: 4091 III.ImmOpcode = PPC::RLDICL_rec; 4092 break; 4093 case PPC::SRAD: 4094 III.ImmWidth = 6; 4095 III.TruncateImmTo = 0; 4096 III.ImmOpcode = PPC::SRADI; 4097 break; 4098 case PPC::SRAD_rec: 4099 III.ImmWidth = 6; 4100 III.TruncateImmTo = 0; 4101 III.ImmOpcode = PPC::SRADI_rec; 4102 break; 4103 } 4104 break; 4105 // Loads and stores: 4106 case PPC::LBZX: 4107 case PPC::LBZX8: 4108 case PPC::LHZX: 4109 case PPC::LHZX8: 4110 case PPC::LHAX: 4111 case PPC::LHAX8: 4112 case PPC::LWZX: 4113 case PPC::LWZX8: 4114 case PPC::LWAX: 4115 case PPC::LDX: 4116 case PPC::LFSX: 4117 case PPC::LFDX: 4118 case PPC::STBX: 4119 case PPC::STBX8: 4120 case PPC::STHX: 4121 case PPC::STHX8: 4122 case PPC::STWX: 4123 case PPC::STWX8: 4124 case PPC::STDX: 4125 case PPC::STFSX: 4126 case PPC::STFDX: 4127 III.SignedImm = true; 4128 III.ZeroIsSpecialOrig = 1; 4129 III.ZeroIsSpecialNew = 2; 4130 III.IsCommutative = true; 4131 III.IsSummingOperands = true; 4132 III.ImmOpNo = 1; 4133 III.OpNoForForwarding = 2; 4134 switch(Opc) { 4135 default: llvm_unreachable("Unknown opcode"); 4136 case PPC::LBZX: III.ImmOpcode = PPC::LBZ; break; 4137 case PPC::LBZX8: III.ImmOpcode = PPC::LBZ8; break; 4138 case PPC::LHZX: III.ImmOpcode = PPC::LHZ; break; 4139 case PPC::LHZX8: III.ImmOpcode = PPC::LHZ8; break; 4140 case PPC::LHAX: III.ImmOpcode = PPC::LHA; break; 4141 case PPC::LHAX8: III.ImmOpcode = PPC::LHA8; break; 4142 case PPC::LWZX: III.ImmOpcode = PPC::LWZ; break; 4143 case PPC::LWZX8: III.ImmOpcode = PPC::LWZ8; break; 4144 case PPC::LWAX: 4145 III.ImmOpcode = PPC::LWA; 4146 III.ImmMustBeMultipleOf = 4; 4147 break; 4148 case PPC::LDX: III.ImmOpcode = PPC::LD; III.ImmMustBeMultipleOf = 4; break; 4149 case PPC::LFSX: III.ImmOpcode = PPC::LFS; break; 4150 case PPC::LFDX: III.ImmOpcode = PPC::LFD; break; 4151 case PPC::STBX: III.ImmOpcode = PPC::STB; break; 4152 case PPC::STBX8: III.ImmOpcode = PPC::STB8; break; 4153 case PPC::STHX: III.ImmOpcode = PPC::STH; break; 4154 case PPC::STHX8: III.ImmOpcode = PPC::STH8; break; 4155 case PPC::STWX: III.ImmOpcode = PPC::STW; break; 4156 case PPC::STWX8: III.ImmOpcode = PPC::STW8; break; 4157 case PPC::STDX: 4158 III.ImmOpcode = PPC::STD; 4159 III.ImmMustBeMultipleOf = 4; 4160 break; 4161 case PPC::STFSX: III.ImmOpcode = PPC::STFS; break; 4162 case PPC::STFDX: III.ImmOpcode = PPC::STFD; break; 4163 } 4164 break; 4165 case PPC::LBZUX: 4166 case PPC::LBZUX8: 4167 case PPC::LHZUX: 4168 case PPC::LHZUX8: 4169 case PPC::LHAUX: 4170 case PPC::LHAUX8: 4171 case PPC::LWZUX: 4172 case PPC::LWZUX8: 4173 case PPC::LDUX: 4174 case PPC::LFSUX: 4175 case PPC::LFDUX: 4176 case PPC::STBUX: 4177 case PPC::STBUX8: 4178 case PPC::STHUX: 4179 case PPC::STHUX8: 4180 case PPC::STWUX: 4181 case PPC::STWUX8: 4182 case PPC::STDUX: 4183 case PPC::STFSUX: 4184 case PPC::STFDUX: 4185 III.SignedImm = true; 4186 III.ZeroIsSpecialOrig = 2; 4187 III.ZeroIsSpecialNew = 3; 4188 III.IsCommutative = false; 4189 III.IsSummingOperands = true; 4190 III.ImmOpNo = 2; 4191 III.OpNoForForwarding = 3; 4192 switch(Opc) { 4193 default: llvm_unreachable("Unknown opcode"); 4194 case PPC::LBZUX: III.ImmOpcode = PPC::LBZU; break; 4195 case PPC::LBZUX8: III.ImmOpcode = PPC::LBZU8; break; 4196 case PPC::LHZUX: III.ImmOpcode = PPC::LHZU; break; 4197 case PPC::LHZUX8: III.ImmOpcode = PPC::LHZU8; break; 4198 case PPC::LHAUX: III.ImmOpcode = PPC::LHAU; break; 4199 case PPC::LHAUX8: III.ImmOpcode = PPC::LHAU8; break; 4200 case PPC::LWZUX: III.ImmOpcode = PPC::LWZU; break; 4201 case PPC::LWZUX8: III.ImmOpcode = PPC::LWZU8; break; 4202 case PPC::LDUX: 4203 III.ImmOpcode = PPC::LDU; 4204 III.ImmMustBeMultipleOf = 4; 4205 break; 4206 case PPC::LFSUX: III.ImmOpcode = PPC::LFSU; break; 4207 case PPC::LFDUX: III.ImmOpcode = PPC::LFDU; break; 4208 case PPC::STBUX: III.ImmOpcode = PPC::STBU; break; 4209 case PPC::STBUX8: III.ImmOpcode = PPC::STBU8; break; 4210 case PPC::STHUX: III.ImmOpcode = PPC::STHU; break; 4211 case PPC::STHUX8: III.ImmOpcode = PPC::STHU8; break; 4212 case PPC::STWUX: III.ImmOpcode = PPC::STWU; break; 4213 case PPC::STWUX8: III.ImmOpcode = PPC::STWU8; break; 4214 case PPC::STDUX: 4215 III.ImmOpcode = PPC::STDU; 4216 III.ImmMustBeMultipleOf = 4; 4217 break; 4218 case PPC::STFSUX: III.ImmOpcode = PPC::STFSU; break; 4219 case PPC::STFDUX: III.ImmOpcode = PPC::STFDU; break; 4220 } 4221 break; 4222 // Power9 and up only. For some of these, the X-Form version has access to all 4223 // 64 VSR's whereas the D-Form only has access to the VR's. We replace those 4224 // with pseudo-ops pre-ra and for post-ra, we check that the register loaded 4225 // into or stored from is one of the VR registers. 4226 case PPC::LXVX: 4227 case PPC::LXSSPX: 4228 case PPC::LXSDX: 4229 case PPC::STXVX: 4230 case PPC::STXSSPX: 4231 case PPC::STXSDX: 4232 case PPC::XFLOADf32: 4233 case PPC::XFLOADf64: 4234 case PPC::XFSTOREf32: 4235 case PPC::XFSTOREf64: 4236 if (!Subtarget.hasP9Vector()) 4237 return false; 4238 III.SignedImm = true; 4239 III.ZeroIsSpecialOrig = 1; 4240 III.ZeroIsSpecialNew = 2; 4241 III.IsCommutative = true; 4242 III.IsSummingOperands = true; 4243 III.ImmOpNo = 1; 4244 III.OpNoForForwarding = 2; 4245 III.ImmMustBeMultipleOf = 4; 4246 switch(Opc) { 4247 default: llvm_unreachable("Unknown opcode"); 4248 case PPC::LXVX: 4249 III.ImmOpcode = PPC::LXV; 4250 III.ImmMustBeMultipleOf = 16; 4251 break; 4252 case PPC::LXSSPX: 4253 if (PostRA) { 4254 if (IsVFReg) 4255 III.ImmOpcode = PPC::LXSSP; 4256 else { 4257 III.ImmOpcode = PPC::LFS; 4258 III.ImmMustBeMultipleOf = 1; 4259 } 4260 break; 4261 } 4262 LLVM_FALLTHROUGH; 4263 case PPC::XFLOADf32: 4264 III.ImmOpcode = PPC::DFLOADf32; 4265 break; 4266 case PPC::LXSDX: 4267 if (PostRA) { 4268 if (IsVFReg) 4269 III.ImmOpcode = PPC::LXSD; 4270 else { 4271 III.ImmOpcode = PPC::LFD; 4272 III.ImmMustBeMultipleOf = 1; 4273 } 4274 break; 4275 } 4276 LLVM_FALLTHROUGH; 4277 case PPC::XFLOADf64: 4278 III.ImmOpcode = PPC::DFLOADf64; 4279 break; 4280 case PPC::STXVX: 4281 III.ImmOpcode = PPC::STXV; 4282 III.ImmMustBeMultipleOf = 16; 4283 break; 4284 case PPC::STXSSPX: 4285 if (PostRA) { 4286 if (IsVFReg) 4287 III.ImmOpcode = PPC::STXSSP; 4288 else { 4289 III.ImmOpcode = PPC::STFS; 4290 III.ImmMustBeMultipleOf = 1; 4291 } 4292 break; 4293 } 4294 LLVM_FALLTHROUGH; 4295 case PPC::XFSTOREf32: 4296 III.ImmOpcode = PPC::DFSTOREf32; 4297 break; 4298 case PPC::STXSDX: 4299 if (PostRA) { 4300 if (IsVFReg) 4301 III.ImmOpcode = PPC::STXSD; 4302 else { 4303 III.ImmOpcode = PPC::STFD; 4304 III.ImmMustBeMultipleOf = 1; 4305 } 4306 break; 4307 } 4308 LLVM_FALLTHROUGH; 4309 case PPC::XFSTOREf64: 4310 III.ImmOpcode = PPC::DFSTOREf64; 4311 break; 4312 } 4313 break; 4314 } 4315 return true; 4316 } 4317 4318 // Utility function for swaping two arbitrary operands of an instruction. 4319 static void swapMIOperands(MachineInstr &MI, unsigned Op1, unsigned Op2) { 4320 assert(Op1 != Op2 && "Cannot swap operand with itself."); 4321 4322 unsigned MaxOp = std::max(Op1, Op2); 4323 unsigned MinOp = std::min(Op1, Op2); 4324 MachineOperand MOp1 = MI.getOperand(MinOp); 4325 MachineOperand MOp2 = MI.getOperand(MaxOp); 4326 MI.removeOperand(std::max(Op1, Op2)); 4327 MI.removeOperand(std::min(Op1, Op2)); 4328 4329 // If the operands we are swapping are the two at the end (the common case) 4330 // we can just remove both and add them in the opposite order. 4331 if (MaxOp - MinOp == 1 && MI.getNumOperands() == MinOp) { 4332 MI.addOperand(MOp2); 4333 MI.addOperand(MOp1); 4334 } else { 4335 // Store all operands in a temporary vector, remove them and re-add in the 4336 // right order. 4337 SmallVector<MachineOperand, 2> MOps; 4338 unsigned TotalOps = MI.getNumOperands() + 2; // We've already removed 2 ops. 4339 for (unsigned i = MI.getNumOperands() - 1; i >= MinOp; i--) { 4340 MOps.push_back(MI.getOperand(i)); 4341 MI.removeOperand(i); 4342 } 4343 // MOp2 needs to be added next. 4344 MI.addOperand(MOp2); 4345 // Now add the rest. 4346 for (unsigned i = MI.getNumOperands(); i < TotalOps; i++) { 4347 if (i == MaxOp) 4348 MI.addOperand(MOp1); 4349 else { 4350 MI.addOperand(MOps.back()); 4351 MOps.pop_back(); 4352 } 4353 } 4354 } 4355 } 4356 4357 // Check if the 'MI' that has the index OpNoForForwarding 4358 // meets the requirement described in the ImmInstrInfo. 4359 bool PPCInstrInfo::isUseMIElgibleForForwarding(MachineInstr &MI, 4360 const ImmInstrInfo &III, 4361 unsigned OpNoForForwarding 4362 ) const { 4363 // As the algorithm of checking for PPC::ZERO/PPC::ZERO8 4364 // would not work pre-RA, we can only do the check post RA. 4365 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 4366 if (MRI.isSSA()) 4367 return false; 4368 4369 // Cannot do the transform if MI isn't summing the operands. 4370 if (!III.IsSummingOperands) 4371 return false; 4372 4373 // The instruction we are trying to replace must have the ZeroIsSpecialOrig set. 4374 if (!III.ZeroIsSpecialOrig) 4375 return false; 4376 4377 // We cannot do the transform if the operand we are trying to replace 4378 // isn't the same as the operand the instruction allows. 4379 if (OpNoForForwarding != III.OpNoForForwarding) 4380 return false; 4381 4382 // Check if the instruction we are trying to transform really has 4383 // the special zero register as its operand. 4384 if (MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO && 4385 MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO8) 4386 return false; 4387 4388 // This machine instruction is convertible if it is, 4389 // 1. summing the operands. 4390 // 2. one of the operands is special zero register. 4391 // 3. the operand we are trying to replace is allowed by the MI. 4392 return true; 4393 } 4394 4395 // Check if the DefMI is the add inst and set the ImmMO and RegMO 4396 // accordingly. 4397 bool PPCInstrInfo::isDefMIElgibleForForwarding(MachineInstr &DefMI, 4398 const ImmInstrInfo &III, 4399 MachineOperand *&ImmMO, 4400 MachineOperand *&RegMO) const { 4401 unsigned Opc = DefMI.getOpcode(); 4402 if (Opc != PPC::ADDItocL && Opc != PPC::ADDI && Opc != PPC::ADDI8) 4403 return false; 4404 4405 assert(DefMI.getNumOperands() >= 3 && 4406 "Add inst must have at least three operands"); 4407 RegMO = &DefMI.getOperand(1); 4408 ImmMO = &DefMI.getOperand(2); 4409 4410 // Before RA, ADDI first operand could be a frame index. 4411 if (!RegMO->isReg()) 4412 return false; 4413 4414 // This DefMI is elgible for forwarding if it is: 4415 // 1. add inst 4416 // 2. one of the operands is Imm/CPI/Global. 4417 return isAnImmediateOperand(*ImmMO); 4418 } 4419 4420 bool PPCInstrInfo::isRegElgibleForForwarding( 4421 const MachineOperand &RegMO, const MachineInstr &DefMI, 4422 const MachineInstr &MI, bool KillDefMI, 4423 bool &IsFwdFeederRegKilled) const { 4424 // x = addi y, imm 4425 // ... 4426 // z = lfdx 0, x -> z = lfd imm(y) 4427 // The Reg "y" can be forwarded to the MI(z) only when there is no DEF 4428 // of "y" between the DEF of "x" and "z". 4429 // The query is only valid post RA. 4430 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 4431 if (MRI.isSSA()) 4432 return false; 4433 4434 Register Reg = RegMO.getReg(); 4435 4436 // Walking the inst in reverse(MI-->DefMI) to get the last DEF of the Reg. 4437 MachineBasicBlock::const_reverse_iterator It = MI; 4438 MachineBasicBlock::const_reverse_iterator E = MI.getParent()->rend(); 4439 It++; 4440 for (; It != E; ++It) { 4441 if (It->modifiesRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI) 4442 return false; 4443 else if (It->killsRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI) 4444 IsFwdFeederRegKilled = true; 4445 // Made it to DefMI without encountering a clobber. 4446 if ((&*It) == &DefMI) 4447 break; 4448 } 4449 assert((&*It) == &DefMI && "DefMI is missing"); 4450 4451 // If DefMI also defines the register to be forwarded, we can only forward it 4452 // if DefMI is being erased. 4453 if (DefMI.modifiesRegister(Reg, &getRegisterInfo())) 4454 return KillDefMI; 4455 4456 return true; 4457 } 4458 4459 bool PPCInstrInfo::isImmElgibleForForwarding(const MachineOperand &ImmMO, 4460 const MachineInstr &DefMI, 4461 const ImmInstrInfo &III, 4462 int64_t &Imm, 4463 int64_t BaseImm) const { 4464 assert(isAnImmediateOperand(ImmMO) && "ImmMO is NOT an immediate"); 4465 if (DefMI.getOpcode() == PPC::ADDItocL) { 4466 // The operand for ADDItocL is CPI, which isn't imm at compiling time, 4467 // However, we know that, it is 16-bit width, and has the alignment of 4. 4468 // Check if the instruction met the requirement. 4469 if (III.ImmMustBeMultipleOf > 4 || 4470 III.TruncateImmTo || III.ImmWidth != 16) 4471 return false; 4472 4473 // Going from XForm to DForm loads means that the displacement needs to be 4474 // not just an immediate but also a multiple of 4, or 16 depending on the 4475 // load. A DForm load cannot be represented if it is a multiple of say 2. 4476 // XForm loads do not have this restriction. 4477 if (ImmMO.isGlobal()) { 4478 const DataLayout &DL = ImmMO.getGlobal()->getParent()->getDataLayout(); 4479 if (ImmMO.getGlobal()->getPointerAlignment(DL) < III.ImmMustBeMultipleOf) 4480 return false; 4481 } 4482 4483 return true; 4484 } 4485 4486 if (ImmMO.isImm()) { 4487 // It is Imm, we need to check if the Imm fit the range. 4488 // Sign-extend to 64-bits. 4489 // DefMI may be folded with another imm form instruction, the result Imm is 4490 // the sum of Imm of DefMI and BaseImm which is from imm form instruction. 4491 APInt ActualValue(64, ImmMO.getImm() + BaseImm, true); 4492 if (III.SignedImm && !ActualValue.isSignedIntN(III.ImmWidth)) 4493 return false; 4494 if (!III.SignedImm && !ActualValue.isIntN(III.ImmWidth)) 4495 return false; 4496 Imm = SignExtend64<16>(ImmMO.getImm() + BaseImm); 4497 4498 if (Imm % III.ImmMustBeMultipleOf) 4499 return false; 4500 if (III.TruncateImmTo) 4501 Imm &= ((1 << III.TruncateImmTo) - 1); 4502 } 4503 else 4504 return false; 4505 4506 // This ImmMO is forwarded if it meets the requriement describle 4507 // in ImmInstrInfo 4508 return true; 4509 } 4510 4511 bool PPCInstrInfo::simplifyToLI(MachineInstr &MI, MachineInstr &DefMI, 4512 unsigned OpNoForForwarding, 4513 MachineInstr **KilledDef) const { 4514 if ((DefMI.getOpcode() != PPC::LI && DefMI.getOpcode() != PPC::LI8) || 4515 !DefMI.getOperand(1).isImm()) 4516 return false; 4517 4518 MachineFunction *MF = MI.getParent()->getParent(); 4519 MachineRegisterInfo *MRI = &MF->getRegInfo(); 4520 bool PostRA = !MRI->isSSA(); 4521 4522 int64_t Immediate = DefMI.getOperand(1).getImm(); 4523 // Sign-extend to 64-bits. 4524 int64_t SExtImm = SignExtend64<16>(Immediate); 4525 4526 bool IsForwardingOperandKilled = MI.getOperand(OpNoForForwarding).isKill(); 4527 Register ForwardingOperandReg = MI.getOperand(OpNoForForwarding).getReg(); 4528 4529 bool ReplaceWithLI = false; 4530 bool Is64BitLI = false; 4531 int64_t NewImm = 0; 4532 bool SetCR = false; 4533 unsigned Opc = MI.getOpcode(); 4534 switch (Opc) { 4535 default: 4536 return false; 4537 4538 // FIXME: Any branches conditional on such a comparison can be made 4539 // unconditional. At this time, this happens too infrequently to be worth 4540 // the implementation effort, but if that ever changes, we could convert 4541 // such a pattern here. 4542 case PPC::CMPWI: 4543 case PPC::CMPLWI: 4544 case PPC::CMPDI: 4545 case PPC::CMPLDI: { 4546 // Doing this post-RA would require dataflow analysis to reliably find uses 4547 // of the CR register set by the compare. 4548 // No need to fixup killed/dead flag since this transformation is only valid 4549 // before RA. 4550 if (PostRA) 4551 return false; 4552 // If a compare-immediate is fed by an immediate and is itself an input of 4553 // an ISEL (the most common case) into a COPY of the correct register. 4554 bool Changed = false; 4555 Register DefReg = MI.getOperand(0).getReg(); 4556 int64_t Comparand = MI.getOperand(2).getImm(); 4557 int64_t SExtComparand = ((uint64_t)Comparand & ~0x7FFFuLL) != 0 4558 ? (Comparand | 0xFFFFFFFFFFFF0000) 4559 : Comparand; 4560 4561 for (auto &CompareUseMI : MRI->use_instructions(DefReg)) { 4562 unsigned UseOpc = CompareUseMI.getOpcode(); 4563 if (UseOpc != PPC::ISEL && UseOpc != PPC::ISEL8) 4564 continue; 4565 unsigned CRSubReg = CompareUseMI.getOperand(3).getSubReg(); 4566 Register TrueReg = CompareUseMI.getOperand(1).getReg(); 4567 Register FalseReg = CompareUseMI.getOperand(2).getReg(); 4568 unsigned RegToCopy = 4569 selectReg(SExtImm, SExtComparand, Opc, TrueReg, FalseReg, CRSubReg); 4570 if (RegToCopy == PPC::NoRegister) 4571 continue; 4572 // Can't use PPC::COPY to copy PPC::ZERO[8]. Convert it to LI[8] 0. 4573 if (RegToCopy == PPC::ZERO || RegToCopy == PPC::ZERO8) { 4574 CompareUseMI.setDesc(get(UseOpc == PPC::ISEL8 ? PPC::LI8 : PPC::LI)); 4575 replaceInstrOperandWithImm(CompareUseMI, 1, 0); 4576 CompareUseMI.removeOperand(3); 4577 CompareUseMI.removeOperand(2); 4578 continue; 4579 } 4580 LLVM_DEBUG( 4581 dbgs() << "Found LI -> CMPI -> ISEL, replacing with a copy.\n"); 4582 LLVM_DEBUG(DefMI.dump(); MI.dump(); CompareUseMI.dump()); 4583 LLVM_DEBUG(dbgs() << "Is converted to:\n"); 4584 // Convert to copy and remove unneeded operands. 4585 CompareUseMI.setDesc(get(PPC::COPY)); 4586 CompareUseMI.removeOperand(3); 4587 CompareUseMI.removeOperand(RegToCopy == TrueReg ? 2 : 1); 4588 CmpIselsConverted++; 4589 Changed = true; 4590 LLVM_DEBUG(CompareUseMI.dump()); 4591 } 4592 if (Changed) 4593 return true; 4594 // This may end up incremented multiple times since this function is called 4595 // during a fixed-point transformation, but it is only meant to indicate the 4596 // presence of this opportunity. 4597 MissedConvertibleImmediateInstrs++; 4598 return false; 4599 } 4600 4601 // Immediate forms - may simply be convertable to an LI. 4602 case PPC::ADDI: 4603 case PPC::ADDI8: { 4604 // Does the sum fit in a 16-bit signed field? 4605 int64_t Addend = MI.getOperand(2).getImm(); 4606 if (isInt<16>(Addend + SExtImm)) { 4607 ReplaceWithLI = true; 4608 Is64BitLI = Opc == PPC::ADDI8; 4609 NewImm = Addend + SExtImm; 4610 break; 4611 } 4612 return false; 4613 } 4614 case PPC::SUBFIC: 4615 case PPC::SUBFIC8: { 4616 // Only transform this if the CARRY implicit operand is dead. 4617 if (MI.getNumOperands() > 3 && !MI.getOperand(3).isDead()) 4618 return false; 4619 int64_t Minuend = MI.getOperand(2).getImm(); 4620 if (isInt<16>(Minuend - SExtImm)) { 4621 ReplaceWithLI = true; 4622 Is64BitLI = Opc == PPC::SUBFIC8; 4623 NewImm = Minuend - SExtImm; 4624 break; 4625 } 4626 return false; 4627 } 4628 case PPC::RLDICL: 4629 case PPC::RLDICL_rec: 4630 case PPC::RLDICL_32: 4631 case PPC::RLDICL_32_64: { 4632 // Use APInt's rotate function. 4633 int64_t SH = MI.getOperand(2).getImm(); 4634 int64_t MB = MI.getOperand(3).getImm(); 4635 APInt InVal((Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec) ? 64 : 32, 4636 SExtImm, true); 4637 InVal = InVal.rotl(SH); 4638 uint64_t Mask = MB == 0 ? -1LLU : (1LLU << (63 - MB + 1)) - 1; 4639 InVal &= Mask; 4640 // Can't replace negative values with an LI as that will sign-extend 4641 // and not clear the left bits. If we're setting the CR bit, we will use 4642 // ANDI_rec which won't sign extend, so that's safe. 4643 if (isUInt<15>(InVal.getSExtValue()) || 4644 (Opc == PPC::RLDICL_rec && isUInt<16>(InVal.getSExtValue()))) { 4645 ReplaceWithLI = true; 4646 Is64BitLI = Opc != PPC::RLDICL_32; 4647 NewImm = InVal.getSExtValue(); 4648 SetCR = Opc == PPC::RLDICL_rec; 4649 break; 4650 } 4651 return false; 4652 } 4653 case PPC::RLWINM: 4654 case PPC::RLWINM8: 4655 case PPC::RLWINM_rec: 4656 case PPC::RLWINM8_rec: { 4657 int64_t SH = MI.getOperand(2).getImm(); 4658 int64_t MB = MI.getOperand(3).getImm(); 4659 int64_t ME = MI.getOperand(4).getImm(); 4660 APInt InVal(32, SExtImm, true); 4661 InVal = InVal.rotl(SH); 4662 APInt Mask = APInt::getBitsSetWithWrap(32, 32 - ME - 1, 32 - MB); 4663 InVal &= Mask; 4664 // Can't replace negative values with an LI as that will sign-extend 4665 // and not clear the left bits. If we're setting the CR bit, we will use 4666 // ANDI_rec which won't sign extend, so that's safe. 4667 bool ValueFits = isUInt<15>(InVal.getSExtValue()); 4668 ValueFits |= ((Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec) && 4669 isUInt<16>(InVal.getSExtValue())); 4670 if (ValueFits) { 4671 ReplaceWithLI = true; 4672 Is64BitLI = Opc == PPC::RLWINM8 || Opc == PPC::RLWINM8_rec; 4673 NewImm = InVal.getSExtValue(); 4674 SetCR = Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec; 4675 break; 4676 } 4677 return false; 4678 } 4679 case PPC::ORI: 4680 case PPC::ORI8: 4681 case PPC::XORI: 4682 case PPC::XORI8: { 4683 int64_t LogicalImm = MI.getOperand(2).getImm(); 4684 int64_t Result = 0; 4685 if (Opc == PPC::ORI || Opc == PPC::ORI8) 4686 Result = LogicalImm | SExtImm; 4687 else 4688 Result = LogicalImm ^ SExtImm; 4689 if (isInt<16>(Result)) { 4690 ReplaceWithLI = true; 4691 Is64BitLI = Opc == PPC::ORI8 || Opc == PPC::XORI8; 4692 NewImm = Result; 4693 break; 4694 } 4695 return false; 4696 } 4697 } 4698 4699 if (ReplaceWithLI) { 4700 // We need to be careful with CR-setting instructions we're replacing. 4701 if (SetCR) { 4702 // We don't know anything about uses when we're out of SSA, so only 4703 // replace if the new immediate will be reproduced. 4704 bool ImmChanged = (SExtImm & NewImm) != NewImm; 4705 if (PostRA && ImmChanged) 4706 return false; 4707 4708 if (!PostRA) { 4709 // If the defining load-immediate has no other uses, we can just replace 4710 // the immediate with the new immediate. 4711 if (MRI->hasOneUse(DefMI.getOperand(0).getReg())) 4712 DefMI.getOperand(1).setImm(NewImm); 4713 4714 // If we're not using the GPR result of the CR-setting instruction, we 4715 // just need to and with zero/non-zero depending on the new immediate. 4716 else if (MRI->use_empty(MI.getOperand(0).getReg())) { 4717 if (NewImm) { 4718 assert(Immediate && "Transformation converted zero to non-zero?"); 4719 NewImm = Immediate; 4720 } 4721 } else if (ImmChanged) 4722 return false; 4723 } 4724 } 4725 4726 LLVM_DEBUG(dbgs() << "Replacing instruction:\n"); 4727 LLVM_DEBUG(MI.dump()); 4728 LLVM_DEBUG(dbgs() << "Fed by:\n"); 4729 LLVM_DEBUG(DefMI.dump()); 4730 LoadImmediateInfo LII; 4731 LII.Imm = NewImm; 4732 LII.Is64Bit = Is64BitLI; 4733 LII.SetCR = SetCR; 4734 // If we're setting the CR, the original load-immediate must be kept (as an 4735 // operand to ANDI_rec/ANDI8_rec). 4736 if (KilledDef && SetCR) 4737 *KilledDef = nullptr; 4738 replaceInstrWithLI(MI, LII); 4739 4740 // Fixup killed/dead flag after transformation. 4741 // Pattern: 4742 // ForwardingOperandReg = LI imm1 4743 // y = op2 imm2, ForwardingOperandReg(killed) 4744 if (IsForwardingOperandKilled) 4745 fixupIsDeadOrKill(&DefMI, &MI, ForwardingOperandReg); 4746 4747 LLVM_DEBUG(dbgs() << "With:\n"); 4748 LLVM_DEBUG(MI.dump()); 4749 return true; 4750 } 4751 return false; 4752 } 4753 4754 bool PPCInstrInfo::transformToNewImmFormFedByAdd( 4755 MachineInstr &MI, MachineInstr &DefMI, unsigned OpNoForForwarding) const { 4756 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo(); 4757 bool PostRA = !MRI->isSSA(); 4758 // FIXME: extend this to post-ra. Need to do some change in getForwardingDefMI 4759 // for post-ra. 4760 if (PostRA) 4761 return false; 4762 4763 // Only handle load/store. 4764 if (!MI.mayLoadOrStore()) 4765 return false; 4766 4767 unsigned XFormOpcode = RI.getMappedIdxOpcForImmOpc(MI.getOpcode()); 4768 4769 assert((XFormOpcode != PPC::INSTRUCTION_LIST_END) && 4770 "MI must have x-form opcode"); 4771 4772 // get Imm Form info. 4773 ImmInstrInfo III; 4774 bool IsVFReg = MI.getOperand(0).isReg() 4775 ? isVFRegister(MI.getOperand(0).getReg()) 4776 : false; 4777 4778 if (!instrHasImmForm(XFormOpcode, IsVFReg, III, PostRA)) 4779 return false; 4780 4781 if (!III.IsSummingOperands) 4782 return false; 4783 4784 if (OpNoForForwarding != III.OpNoForForwarding) 4785 return false; 4786 4787 MachineOperand ImmOperandMI = MI.getOperand(III.ImmOpNo); 4788 if (!ImmOperandMI.isImm()) 4789 return false; 4790 4791 // Check DefMI. 4792 MachineOperand *ImmMO = nullptr; 4793 MachineOperand *RegMO = nullptr; 4794 if (!isDefMIElgibleForForwarding(DefMI, III, ImmMO, RegMO)) 4795 return false; 4796 assert(ImmMO && RegMO && "Imm and Reg operand must have been set"); 4797 4798 // Check Imm. 4799 // Set ImmBase from imm instruction as base and get new Imm inside 4800 // isImmElgibleForForwarding. 4801 int64_t ImmBase = ImmOperandMI.getImm(); 4802 int64_t Imm = 0; 4803 if (!isImmElgibleForForwarding(*ImmMO, DefMI, III, Imm, ImmBase)) 4804 return false; 4805 4806 // Get killed info in case fixup needed after transformation. 4807 unsigned ForwardKilledOperandReg = ~0U; 4808 if (MI.getOperand(III.OpNoForForwarding).isKill()) 4809 ForwardKilledOperandReg = MI.getOperand(III.OpNoForForwarding).getReg(); 4810 4811 // Do the transform 4812 LLVM_DEBUG(dbgs() << "Replacing instruction:\n"); 4813 LLVM_DEBUG(MI.dump()); 4814 LLVM_DEBUG(dbgs() << "Fed by:\n"); 4815 LLVM_DEBUG(DefMI.dump()); 4816 4817 MI.getOperand(III.OpNoForForwarding).setReg(RegMO->getReg()); 4818 if (RegMO->isKill()) { 4819 MI.getOperand(III.OpNoForForwarding).setIsKill(true); 4820 // Clear the killed flag in RegMO. Doing this here can handle some cases 4821 // that DefMI and MI are not in same basic block. 4822 RegMO->setIsKill(false); 4823 } 4824 MI.getOperand(III.ImmOpNo).setImm(Imm); 4825 4826 // FIXME: fix kill/dead flag if MI and DefMI are not in same basic block. 4827 if (DefMI.getParent() == MI.getParent()) { 4828 // Check if reg is killed between MI and DefMI. 4829 auto IsKilledFor = [&](unsigned Reg) { 4830 MachineBasicBlock::const_reverse_iterator It = MI; 4831 MachineBasicBlock::const_reverse_iterator E = DefMI; 4832 It++; 4833 for (; It != E; ++It) { 4834 if (It->killsRegister(Reg)) 4835 return true; 4836 } 4837 return false; 4838 }; 4839 4840 // Update kill flag 4841 if (RegMO->isKill() || IsKilledFor(RegMO->getReg())) 4842 fixupIsDeadOrKill(&DefMI, &MI, RegMO->getReg()); 4843 if (ForwardKilledOperandReg != ~0U) 4844 fixupIsDeadOrKill(&DefMI, &MI, ForwardKilledOperandReg); 4845 } 4846 4847 LLVM_DEBUG(dbgs() << "With:\n"); 4848 LLVM_DEBUG(MI.dump()); 4849 return true; 4850 } 4851 4852 // If an X-Form instruction is fed by an add-immediate and one of its operands 4853 // is the literal zero, attempt to forward the source of the add-immediate to 4854 // the corresponding D-Form instruction with the displacement coming from 4855 // the immediate being added. 4856 bool PPCInstrInfo::transformToImmFormFedByAdd( 4857 MachineInstr &MI, const ImmInstrInfo &III, unsigned OpNoForForwarding, 4858 MachineInstr &DefMI, bool KillDefMI) const { 4859 // RegMO ImmMO 4860 // | | 4861 // x = addi reg, imm <----- DefMI 4862 // y = op 0 , x <----- MI 4863 // | 4864 // OpNoForForwarding 4865 // Check if the MI meet the requirement described in the III. 4866 if (!isUseMIElgibleForForwarding(MI, III, OpNoForForwarding)) 4867 return false; 4868 4869 // Check if the DefMI meet the requirement 4870 // described in the III. If yes, set the ImmMO and RegMO accordingly. 4871 MachineOperand *ImmMO = nullptr; 4872 MachineOperand *RegMO = nullptr; 4873 if (!isDefMIElgibleForForwarding(DefMI, III, ImmMO, RegMO)) 4874 return false; 4875 assert(ImmMO && RegMO && "Imm and Reg operand must have been set"); 4876 4877 // As we get the Imm operand now, we need to check if the ImmMO meet 4878 // the requirement described in the III. If yes set the Imm. 4879 int64_t Imm = 0; 4880 if (!isImmElgibleForForwarding(*ImmMO, DefMI, III, Imm)) 4881 return false; 4882 4883 bool IsFwdFeederRegKilled = false; 4884 // Check if the RegMO can be forwarded to MI. 4885 if (!isRegElgibleForForwarding(*RegMO, DefMI, MI, KillDefMI, 4886 IsFwdFeederRegKilled)) 4887 return false; 4888 4889 // Get killed info in case fixup needed after transformation. 4890 unsigned ForwardKilledOperandReg = ~0U; 4891 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 4892 bool PostRA = !MRI.isSSA(); 4893 if (PostRA && MI.getOperand(OpNoForForwarding).isKill()) 4894 ForwardKilledOperandReg = MI.getOperand(OpNoForForwarding).getReg(); 4895 4896 // We know that, the MI and DefMI both meet the pattern, and 4897 // the Imm also meet the requirement with the new Imm-form. 4898 // It is safe to do the transformation now. 4899 LLVM_DEBUG(dbgs() << "Replacing instruction:\n"); 4900 LLVM_DEBUG(MI.dump()); 4901 LLVM_DEBUG(dbgs() << "Fed by:\n"); 4902 LLVM_DEBUG(DefMI.dump()); 4903 4904 // Update the base reg first. 4905 MI.getOperand(III.OpNoForForwarding).ChangeToRegister(RegMO->getReg(), 4906 false, false, 4907 RegMO->isKill()); 4908 4909 // Then, update the imm. 4910 if (ImmMO->isImm()) { 4911 // If the ImmMO is Imm, change the operand that has ZERO to that Imm 4912 // directly. 4913 replaceInstrOperandWithImm(MI, III.ZeroIsSpecialOrig, Imm); 4914 } 4915 else { 4916 // Otherwise, it is Constant Pool Index(CPI) or Global, 4917 // which is relocation in fact. We need to replace the special zero 4918 // register with ImmMO. 4919 // Before that, we need to fixup the target flags for imm. 4920 // For some reason, we miss to set the flag for the ImmMO if it is CPI. 4921 if (DefMI.getOpcode() == PPC::ADDItocL) 4922 ImmMO->setTargetFlags(PPCII::MO_TOC_LO); 4923 4924 // MI didn't have the interface such as MI.setOperand(i) though 4925 // it has MI.getOperand(i). To repalce the ZERO MachineOperand with 4926 // ImmMO, we need to remove ZERO operand and all the operands behind it, 4927 // and, add the ImmMO, then, move back all the operands behind ZERO. 4928 SmallVector<MachineOperand, 2> MOps; 4929 for (unsigned i = MI.getNumOperands() - 1; i >= III.ZeroIsSpecialOrig; i--) { 4930 MOps.push_back(MI.getOperand(i)); 4931 MI.removeOperand(i); 4932 } 4933 4934 // Remove the last MO in the list, which is ZERO operand in fact. 4935 MOps.pop_back(); 4936 // Add the imm operand. 4937 MI.addOperand(*ImmMO); 4938 // Now add the rest back. 4939 for (auto &MO : MOps) 4940 MI.addOperand(MO); 4941 } 4942 4943 // Update the opcode. 4944 MI.setDesc(get(III.ImmOpcode)); 4945 4946 // Fix up killed/dead flag after transformation. 4947 // Pattern 1: 4948 // x = ADD KilledFwdFeederReg, imm 4949 // n = opn KilledFwdFeederReg(killed), regn 4950 // y = XOP 0, x 4951 // Pattern 2: 4952 // x = ADD reg(killed), imm 4953 // y = XOP 0, x 4954 if (IsFwdFeederRegKilled || RegMO->isKill()) 4955 fixupIsDeadOrKill(&DefMI, &MI, RegMO->getReg()); 4956 // Pattern 3: 4957 // ForwardKilledOperandReg = ADD reg, imm 4958 // y = XOP 0, ForwardKilledOperandReg(killed) 4959 if (ForwardKilledOperandReg != ~0U) 4960 fixupIsDeadOrKill(&DefMI, &MI, ForwardKilledOperandReg); 4961 4962 LLVM_DEBUG(dbgs() << "With:\n"); 4963 LLVM_DEBUG(MI.dump()); 4964 4965 return true; 4966 } 4967 4968 bool PPCInstrInfo::transformToImmFormFedByLI(MachineInstr &MI, 4969 const ImmInstrInfo &III, 4970 unsigned ConstantOpNo, 4971 MachineInstr &DefMI) const { 4972 // DefMI must be LI or LI8. 4973 if ((DefMI.getOpcode() != PPC::LI && DefMI.getOpcode() != PPC::LI8) || 4974 !DefMI.getOperand(1).isImm()) 4975 return false; 4976 4977 // Get Imm operand and Sign-extend to 64-bits. 4978 int64_t Imm = SignExtend64<16>(DefMI.getOperand(1).getImm()); 4979 4980 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 4981 bool PostRA = !MRI.isSSA(); 4982 // Exit early if we can't convert this. 4983 if ((ConstantOpNo != III.OpNoForForwarding) && !III.IsCommutative) 4984 return false; 4985 if (Imm % III.ImmMustBeMultipleOf) 4986 return false; 4987 if (III.TruncateImmTo) 4988 Imm &= ((1 << III.TruncateImmTo) - 1); 4989 if (III.SignedImm) { 4990 APInt ActualValue(64, Imm, true); 4991 if (!ActualValue.isSignedIntN(III.ImmWidth)) 4992 return false; 4993 } else { 4994 uint64_t UnsignedMax = (1 << III.ImmWidth) - 1; 4995 if ((uint64_t)Imm > UnsignedMax) 4996 return false; 4997 } 4998 4999 // If we're post-RA, the instructions don't agree on whether register zero is 5000 // special, we can transform this as long as the register operand that will 5001 // end up in the location where zero is special isn't R0. 5002 if (PostRA && III.ZeroIsSpecialOrig != III.ZeroIsSpecialNew) { 5003 unsigned PosForOrigZero = III.ZeroIsSpecialOrig ? III.ZeroIsSpecialOrig : 5004 III.ZeroIsSpecialNew + 1; 5005 Register OrigZeroReg = MI.getOperand(PosForOrigZero).getReg(); 5006 Register NewZeroReg = MI.getOperand(III.ZeroIsSpecialNew).getReg(); 5007 // If R0 is in the operand where zero is special for the new instruction, 5008 // it is unsafe to transform if the constant operand isn't that operand. 5009 if ((NewZeroReg == PPC::R0 || NewZeroReg == PPC::X0) && 5010 ConstantOpNo != III.ZeroIsSpecialNew) 5011 return false; 5012 if ((OrigZeroReg == PPC::R0 || OrigZeroReg == PPC::X0) && 5013 ConstantOpNo != PosForOrigZero) 5014 return false; 5015 } 5016 5017 // Get killed info in case fixup needed after transformation. 5018 unsigned ForwardKilledOperandReg = ~0U; 5019 if (PostRA && MI.getOperand(ConstantOpNo).isKill()) 5020 ForwardKilledOperandReg = MI.getOperand(ConstantOpNo).getReg(); 5021 5022 unsigned Opc = MI.getOpcode(); 5023 bool SpecialShift32 = Opc == PPC::SLW || Opc == PPC::SLW_rec || 5024 Opc == PPC::SRW || Opc == PPC::SRW_rec || 5025 Opc == PPC::SLW8 || Opc == PPC::SLW8_rec || 5026 Opc == PPC::SRW8 || Opc == PPC::SRW8_rec; 5027 bool SpecialShift64 = Opc == PPC::SLD || Opc == PPC::SLD_rec || 5028 Opc == PPC::SRD || Opc == PPC::SRD_rec; 5029 bool SetCR = Opc == PPC::SLW_rec || Opc == PPC::SRW_rec || 5030 Opc == PPC::SLD_rec || Opc == PPC::SRD_rec; 5031 bool RightShift = Opc == PPC::SRW || Opc == PPC::SRW_rec || Opc == PPC::SRD || 5032 Opc == PPC::SRD_rec; 5033 5034 MI.setDesc(get(III.ImmOpcode)); 5035 if (ConstantOpNo == III.OpNoForForwarding) { 5036 // Converting shifts to immediate form is a bit tricky since they may do 5037 // one of three things: 5038 // 1. If the shift amount is between OpSize and 2*OpSize, the result is zero 5039 // 2. If the shift amount is zero, the result is unchanged (save for maybe 5040 // setting CR0) 5041 // 3. If the shift amount is in [1, OpSize), it's just a shift 5042 if (SpecialShift32 || SpecialShift64) { 5043 LoadImmediateInfo LII; 5044 LII.Imm = 0; 5045 LII.SetCR = SetCR; 5046 LII.Is64Bit = SpecialShift64; 5047 uint64_t ShAmt = Imm & (SpecialShift32 ? 0x1F : 0x3F); 5048 if (Imm & (SpecialShift32 ? 0x20 : 0x40)) 5049 replaceInstrWithLI(MI, LII); 5050 // Shifts by zero don't change the value. If we don't need to set CR0, 5051 // just convert this to a COPY. Can't do this post-RA since we've already 5052 // cleaned up the copies. 5053 else if (!SetCR && ShAmt == 0 && !PostRA) { 5054 MI.removeOperand(2); 5055 MI.setDesc(get(PPC::COPY)); 5056 } else { 5057 // The 32 bit and 64 bit instructions are quite different. 5058 if (SpecialShift32) { 5059 // Left shifts use (N, 0, 31-N). 5060 // Right shifts use (32-N, N, 31) if 0 < N < 32. 5061 // use (0, 0, 31) if N == 0. 5062 uint64_t SH = ShAmt == 0 ? 0 : RightShift ? 32 - ShAmt : ShAmt; 5063 uint64_t MB = RightShift ? ShAmt : 0; 5064 uint64_t ME = RightShift ? 31 : 31 - ShAmt; 5065 replaceInstrOperandWithImm(MI, III.OpNoForForwarding, SH); 5066 MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(MB) 5067 .addImm(ME); 5068 } else { 5069 // Left shifts use (N, 63-N). 5070 // Right shifts use (64-N, N) if 0 < N < 64. 5071 // use (0, 0) if N == 0. 5072 uint64_t SH = ShAmt == 0 ? 0 : RightShift ? 64 - ShAmt : ShAmt; 5073 uint64_t ME = RightShift ? ShAmt : 63 - ShAmt; 5074 replaceInstrOperandWithImm(MI, III.OpNoForForwarding, SH); 5075 MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(ME); 5076 } 5077 } 5078 } else 5079 replaceInstrOperandWithImm(MI, ConstantOpNo, Imm); 5080 } 5081 // Convert commutative instructions (switch the operands and convert the 5082 // desired one to an immediate. 5083 else if (III.IsCommutative) { 5084 replaceInstrOperandWithImm(MI, ConstantOpNo, Imm); 5085 swapMIOperands(MI, ConstantOpNo, III.OpNoForForwarding); 5086 } else 5087 llvm_unreachable("Should have exited early!"); 5088 5089 // For instructions for which the constant register replaces a different 5090 // operand than where the immediate goes, we need to swap them. 5091 if (III.OpNoForForwarding != III.ImmOpNo) 5092 swapMIOperands(MI, III.OpNoForForwarding, III.ImmOpNo); 5093 5094 // If the special R0/X0 register index are different for original instruction 5095 // and new instruction, we need to fix up the register class in new 5096 // instruction. 5097 if (!PostRA && III.ZeroIsSpecialOrig != III.ZeroIsSpecialNew) { 5098 if (III.ZeroIsSpecialNew) { 5099 // If operand at III.ZeroIsSpecialNew is physical reg(eg: ZERO/ZERO8), no 5100 // need to fix up register class. 5101 Register RegToModify = MI.getOperand(III.ZeroIsSpecialNew).getReg(); 5102 if (Register::isVirtualRegister(RegToModify)) { 5103 const TargetRegisterClass *NewRC = 5104 MRI.getRegClass(RegToModify)->hasSuperClassEq(&PPC::GPRCRegClass) ? 5105 &PPC::GPRC_and_GPRC_NOR0RegClass : &PPC::G8RC_and_G8RC_NOX0RegClass; 5106 MRI.setRegClass(RegToModify, NewRC); 5107 } 5108 } 5109 } 5110 5111 // Fix up killed/dead flag after transformation. 5112 // Pattern: 5113 // ForwardKilledOperandReg = LI imm 5114 // y = XOP reg, ForwardKilledOperandReg(killed) 5115 if (ForwardKilledOperandReg != ~0U) 5116 fixupIsDeadOrKill(&DefMI, &MI, ForwardKilledOperandReg); 5117 return true; 5118 } 5119 5120 const TargetRegisterClass * 5121 PPCInstrInfo::updatedRC(const TargetRegisterClass *RC) const { 5122 if (Subtarget.hasVSX() && RC == &PPC::VRRCRegClass) 5123 return &PPC::VSRCRegClass; 5124 return RC; 5125 } 5126 5127 int PPCInstrInfo::getRecordFormOpcode(unsigned Opcode) { 5128 return PPC::getRecordFormOpcode(Opcode); 5129 } 5130 5131 // This function returns true if the machine instruction 5132 // always outputs a value by sign-extending a 32 bit value, 5133 // i.e. 0 to 31-th bits are same as 32-th bit. 5134 static bool isSignExtendingOp(const MachineInstr &MI) { 5135 int Opcode = MI.getOpcode(); 5136 if (Opcode == PPC::LI || Opcode == PPC::LI8 || Opcode == PPC::LIS || 5137 Opcode == PPC::LIS8 || Opcode == PPC::SRAW || Opcode == PPC::SRAW_rec || 5138 Opcode == PPC::SRAWI || Opcode == PPC::SRAWI_rec || Opcode == PPC::LWA || 5139 Opcode == PPC::LWAX || Opcode == PPC::LWA_32 || Opcode == PPC::LWAX_32 || 5140 Opcode == PPC::LHA || Opcode == PPC::LHAX || Opcode == PPC::LHA8 || 5141 Opcode == PPC::LHAX8 || Opcode == PPC::LBZ || Opcode == PPC::LBZX || 5142 Opcode == PPC::LBZ8 || Opcode == PPC::LBZX8 || Opcode == PPC::LBZU || 5143 Opcode == PPC::LBZUX || Opcode == PPC::LBZU8 || Opcode == PPC::LBZUX8 || 5144 Opcode == PPC::LHZ || Opcode == PPC::LHZX || Opcode == PPC::LHZ8 || 5145 Opcode == PPC::LHZX8 || Opcode == PPC::LHZU || Opcode == PPC::LHZUX || 5146 Opcode == PPC::LHZU8 || Opcode == PPC::LHZUX8 || Opcode == PPC::EXTSB || 5147 Opcode == PPC::EXTSB_rec || Opcode == PPC::EXTSH || 5148 Opcode == PPC::EXTSH_rec || Opcode == PPC::EXTSB8 || 5149 Opcode == PPC::EXTSH8 || Opcode == PPC::EXTSW || 5150 Opcode == PPC::EXTSW_rec || Opcode == PPC::SETB || Opcode == PPC::SETB8 || 5151 Opcode == PPC::EXTSH8_32_64 || Opcode == PPC::EXTSW_32_64 || 5152 Opcode == PPC::EXTSB8_32_64) 5153 return true; 5154 5155 if (Opcode == PPC::RLDICL && MI.getOperand(3).getImm() >= 33) 5156 return true; 5157 5158 if ((Opcode == PPC::RLWINM || Opcode == PPC::RLWINM_rec || 5159 Opcode == PPC::RLWNM || Opcode == PPC::RLWNM_rec) && 5160 MI.getOperand(3).getImm() > 0 && 5161 MI.getOperand(3).getImm() <= MI.getOperand(4).getImm()) 5162 return true; 5163 5164 return false; 5165 } 5166 5167 // This function returns true if the machine instruction 5168 // always outputs zeros in higher 32 bits. 5169 static bool isZeroExtendingOp(const MachineInstr &MI) { 5170 int Opcode = MI.getOpcode(); 5171 // The 16-bit immediate is sign-extended in li/lis. 5172 // If the most significant bit is zero, all higher bits are zero. 5173 if (Opcode == PPC::LI || Opcode == PPC::LI8 || 5174 Opcode == PPC::LIS || Opcode == PPC::LIS8) { 5175 int64_t Imm = MI.getOperand(1).getImm(); 5176 if (((uint64_t)Imm & ~0x7FFFuLL) == 0) 5177 return true; 5178 } 5179 5180 // We have some variations of rotate-and-mask instructions 5181 // that clear higher 32-bits. 5182 if ((Opcode == PPC::RLDICL || Opcode == PPC::RLDICL_rec || 5183 Opcode == PPC::RLDCL || Opcode == PPC::RLDCL_rec || 5184 Opcode == PPC::RLDICL_32_64) && 5185 MI.getOperand(3).getImm() >= 32) 5186 return true; 5187 5188 if ((Opcode == PPC::RLDIC || Opcode == PPC::RLDIC_rec) && 5189 MI.getOperand(3).getImm() >= 32 && 5190 MI.getOperand(3).getImm() <= 63 - MI.getOperand(2).getImm()) 5191 return true; 5192 5193 if ((Opcode == PPC::RLWINM || Opcode == PPC::RLWINM_rec || 5194 Opcode == PPC::RLWNM || Opcode == PPC::RLWNM_rec || 5195 Opcode == PPC::RLWINM8 || Opcode == PPC::RLWNM8) && 5196 MI.getOperand(3).getImm() <= MI.getOperand(4).getImm()) 5197 return true; 5198 5199 // There are other instructions that clear higher 32-bits. 5200 if (Opcode == PPC::CNTLZW || Opcode == PPC::CNTLZW_rec || 5201 Opcode == PPC::CNTTZW || Opcode == PPC::CNTTZW_rec || 5202 Opcode == PPC::CNTLZW8 || Opcode == PPC::CNTTZW8 || 5203 Opcode == PPC::CNTLZD || Opcode == PPC::CNTLZD_rec || 5204 Opcode == PPC::CNTTZD || Opcode == PPC::CNTTZD_rec || 5205 Opcode == PPC::POPCNTD || Opcode == PPC::POPCNTW || Opcode == PPC::SLW || 5206 Opcode == PPC::SLW_rec || Opcode == PPC::SRW || Opcode == PPC::SRW_rec || 5207 Opcode == PPC::SLW8 || Opcode == PPC::SRW8 || Opcode == PPC::SLWI || 5208 Opcode == PPC::SLWI_rec || Opcode == PPC::SRWI || 5209 Opcode == PPC::SRWI_rec || Opcode == PPC::LWZ || Opcode == PPC::LWZX || 5210 Opcode == PPC::LWZU || Opcode == PPC::LWZUX || Opcode == PPC::LWBRX || 5211 Opcode == PPC::LHBRX || Opcode == PPC::LHZ || Opcode == PPC::LHZX || 5212 Opcode == PPC::LHZU || Opcode == PPC::LHZUX || Opcode == PPC::LBZ || 5213 Opcode == PPC::LBZX || Opcode == PPC::LBZU || Opcode == PPC::LBZUX || 5214 Opcode == PPC::LWZ8 || Opcode == PPC::LWZX8 || Opcode == PPC::LWZU8 || 5215 Opcode == PPC::LWZUX8 || Opcode == PPC::LWBRX8 || Opcode == PPC::LHBRX8 || 5216 Opcode == PPC::LHZ8 || Opcode == PPC::LHZX8 || Opcode == PPC::LHZU8 || 5217 Opcode == PPC::LHZUX8 || Opcode == PPC::LBZ8 || Opcode == PPC::LBZX8 || 5218 Opcode == PPC::LBZU8 || Opcode == PPC::LBZUX8 || 5219 Opcode == PPC::ANDI_rec || Opcode == PPC::ANDIS_rec || 5220 Opcode == PPC::ROTRWI || Opcode == PPC::ROTRWI_rec || 5221 Opcode == PPC::EXTLWI || Opcode == PPC::EXTLWI_rec || 5222 Opcode == PPC::MFVSRWZ) 5223 return true; 5224 5225 return false; 5226 } 5227 5228 // This function returns true if the input MachineInstr is a TOC save 5229 // instruction. 5230 bool PPCInstrInfo::isTOCSaveMI(const MachineInstr &MI) const { 5231 if (!MI.getOperand(1).isImm() || !MI.getOperand(2).isReg()) 5232 return false; 5233 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset(); 5234 unsigned StackOffset = MI.getOperand(1).getImm(); 5235 Register StackReg = MI.getOperand(2).getReg(); 5236 Register SPReg = Subtarget.isPPC64() ? PPC::X1 : PPC::R1; 5237 if (StackReg == SPReg && StackOffset == TOCSaveOffset) 5238 return true; 5239 5240 return false; 5241 } 5242 5243 // We limit the max depth to track incoming values of PHIs or binary ops 5244 // (e.g. AND) to avoid excessive cost. 5245 const unsigned MAX_DEPTH = 1; 5246 5247 bool 5248 PPCInstrInfo::isSignOrZeroExtended(const MachineInstr &MI, bool SignExt, 5249 const unsigned Depth) const { 5250 const MachineFunction *MF = MI.getParent()->getParent(); 5251 const MachineRegisterInfo *MRI = &MF->getRegInfo(); 5252 5253 // If we know this instruction returns sign- or zero-extended result, 5254 // return true. 5255 if (SignExt ? isSignExtendingOp(MI): 5256 isZeroExtendingOp(MI)) 5257 return true; 5258 5259 switch (MI.getOpcode()) { 5260 case PPC::COPY: { 5261 Register SrcReg = MI.getOperand(1).getReg(); 5262 5263 // In both ELFv1 and v2 ABI, method parameters and the return value 5264 // are sign- or zero-extended. 5265 if (MF->getSubtarget<PPCSubtarget>().isSVR4ABI()) { 5266 const PPCFunctionInfo *FuncInfo = MF->getInfo<PPCFunctionInfo>(); 5267 // We check the ZExt/SExt flags for a method parameter. 5268 if (MI.getParent()->getBasicBlock() == 5269 &MF->getFunction().getEntryBlock()) { 5270 Register VReg = MI.getOperand(0).getReg(); 5271 if (MF->getRegInfo().isLiveIn(VReg)) 5272 return SignExt ? FuncInfo->isLiveInSExt(VReg) : 5273 FuncInfo->isLiveInZExt(VReg); 5274 } 5275 5276 // For a method return value, we check the ZExt/SExt flags in attribute. 5277 // We assume the following code sequence for method call. 5278 // ADJCALLSTACKDOWN 32, implicit dead %r1, implicit %r1 5279 // BL8_NOP @func,... 5280 // ADJCALLSTACKUP 32, 0, implicit dead %r1, implicit %r1 5281 // %5 = COPY %x3; G8RC:%5 5282 if (SrcReg == PPC::X3) { 5283 const MachineBasicBlock *MBB = MI.getParent(); 5284 MachineBasicBlock::const_instr_iterator II = 5285 MachineBasicBlock::const_instr_iterator(&MI); 5286 if (II != MBB->instr_begin() && 5287 (--II)->getOpcode() == PPC::ADJCALLSTACKUP) { 5288 const MachineInstr &CallMI = *(--II); 5289 if (CallMI.isCall() && CallMI.getOperand(0).isGlobal()) { 5290 const Function *CalleeFn = 5291 dyn_cast<Function>(CallMI.getOperand(0).getGlobal()); 5292 if (!CalleeFn) 5293 return false; 5294 const IntegerType *IntTy = 5295 dyn_cast<IntegerType>(CalleeFn->getReturnType()); 5296 const AttributeSet &Attrs = CalleeFn->getAttributes().getRetAttrs(); 5297 if (IntTy && IntTy->getBitWidth() <= 32) 5298 return Attrs.hasAttribute(SignExt ? Attribute::SExt : 5299 Attribute::ZExt); 5300 } 5301 } 5302 } 5303 } 5304 5305 // If this is a copy from another register, we recursively check source. 5306 if (!Register::isVirtualRegister(SrcReg)) 5307 return false; 5308 const MachineInstr *SrcMI = MRI->getVRegDef(SrcReg); 5309 if (SrcMI != nullptr) 5310 return isSignOrZeroExtended(*SrcMI, SignExt, Depth); 5311 5312 return false; 5313 } 5314 5315 case PPC::ANDI_rec: 5316 case PPC::ANDIS_rec: 5317 case PPC::ORI: 5318 case PPC::ORIS: 5319 case PPC::XORI: 5320 case PPC::XORIS: 5321 case PPC::ANDI8_rec: 5322 case PPC::ANDIS8_rec: 5323 case PPC::ORI8: 5324 case PPC::ORIS8: 5325 case PPC::XORI8: 5326 case PPC::XORIS8: { 5327 // logical operation with 16-bit immediate does not change the upper bits. 5328 // So, we track the operand register as we do for register copy. 5329 Register SrcReg = MI.getOperand(1).getReg(); 5330 if (!Register::isVirtualRegister(SrcReg)) 5331 return false; 5332 const MachineInstr *SrcMI = MRI->getVRegDef(SrcReg); 5333 if (SrcMI != nullptr) 5334 return isSignOrZeroExtended(*SrcMI, SignExt, Depth); 5335 5336 return false; 5337 } 5338 5339 // If all incoming values are sign-/zero-extended, 5340 // the output of OR, ISEL or PHI is also sign-/zero-extended. 5341 case PPC::OR: 5342 case PPC::OR8: 5343 case PPC::ISEL: 5344 case PPC::PHI: { 5345 if (Depth >= MAX_DEPTH) 5346 return false; 5347 5348 // The input registers for PHI are operand 1, 3, ... 5349 // The input registers for others are operand 1 and 2. 5350 unsigned E = 3, D = 1; 5351 if (MI.getOpcode() == PPC::PHI) { 5352 E = MI.getNumOperands(); 5353 D = 2; 5354 } 5355 5356 for (unsigned I = 1; I != E; I += D) { 5357 if (MI.getOperand(I).isReg()) { 5358 Register SrcReg = MI.getOperand(I).getReg(); 5359 if (!Register::isVirtualRegister(SrcReg)) 5360 return false; 5361 const MachineInstr *SrcMI = MRI->getVRegDef(SrcReg); 5362 if (SrcMI == nullptr || 5363 !isSignOrZeroExtended(*SrcMI, SignExt, Depth + 1)) 5364 return false; 5365 } 5366 else 5367 return false; 5368 } 5369 return true; 5370 } 5371 5372 // If at least one of the incoming values of an AND is zero extended 5373 // then the output is also zero-extended. If both of the incoming values 5374 // are sign-extended then the output is also sign extended. 5375 case PPC::AND: 5376 case PPC::AND8: { 5377 if (Depth >= MAX_DEPTH) 5378 return false; 5379 5380 assert(MI.getOperand(1).isReg() && MI.getOperand(2).isReg()); 5381 5382 Register SrcReg1 = MI.getOperand(1).getReg(); 5383 Register SrcReg2 = MI.getOperand(2).getReg(); 5384 5385 if (!Register::isVirtualRegister(SrcReg1) || 5386 !Register::isVirtualRegister(SrcReg2)) 5387 return false; 5388 5389 const MachineInstr *MISrc1 = MRI->getVRegDef(SrcReg1); 5390 const MachineInstr *MISrc2 = MRI->getVRegDef(SrcReg2); 5391 if (!MISrc1 || !MISrc2) 5392 return false; 5393 5394 if(SignExt) 5395 return isSignOrZeroExtended(*MISrc1, SignExt, Depth+1) && 5396 isSignOrZeroExtended(*MISrc2, SignExt, Depth+1); 5397 else 5398 return isSignOrZeroExtended(*MISrc1, SignExt, Depth+1) || 5399 isSignOrZeroExtended(*MISrc2, SignExt, Depth+1); 5400 } 5401 5402 default: 5403 break; 5404 } 5405 return false; 5406 } 5407 5408 bool PPCInstrInfo::isBDNZ(unsigned Opcode) const { 5409 return (Opcode == (Subtarget.isPPC64() ? PPC::BDNZ8 : PPC::BDNZ)); 5410 } 5411 5412 namespace { 5413 class PPCPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo { 5414 MachineInstr *Loop, *EndLoop, *LoopCount; 5415 MachineFunction *MF; 5416 const TargetInstrInfo *TII; 5417 int64_t TripCount; 5418 5419 public: 5420 PPCPipelinerLoopInfo(MachineInstr *Loop, MachineInstr *EndLoop, 5421 MachineInstr *LoopCount) 5422 : Loop(Loop), EndLoop(EndLoop), LoopCount(LoopCount), 5423 MF(Loop->getParent()->getParent()), 5424 TII(MF->getSubtarget().getInstrInfo()) { 5425 // Inspect the Loop instruction up-front, as it may be deleted when we call 5426 // createTripCountGreaterCondition. 5427 if (LoopCount->getOpcode() == PPC::LI8 || LoopCount->getOpcode() == PPC::LI) 5428 TripCount = LoopCount->getOperand(1).getImm(); 5429 else 5430 TripCount = -1; 5431 } 5432 5433 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override { 5434 // Only ignore the terminator. 5435 return MI == EndLoop; 5436 } 5437 5438 Optional<bool> 5439 createTripCountGreaterCondition(int TC, MachineBasicBlock &MBB, 5440 SmallVectorImpl<MachineOperand> &Cond) override { 5441 if (TripCount == -1) { 5442 // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1, 5443 // so we don't need to generate any thing here. 5444 Cond.push_back(MachineOperand::CreateImm(0)); 5445 Cond.push_back(MachineOperand::CreateReg( 5446 MF->getSubtarget<PPCSubtarget>().isPPC64() ? PPC::CTR8 : PPC::CTR, 5447 true)); 5448 return {}; 5449 } 5450 5451 return TripCount > TC; 5452 } 5453 5454 void setPreheader(MachineBasicBlock *NewPreheader) override { 5455 // Do nothing. We want the LOOP setup instruction to stay in the *old* 5456 // preheader, so we can use BDZ in the prologs to adapt the loop trip count. 5457 } 5458 5459 void adjustTripCount(int TripCountAdjust) override { 5460 // If the loop trip count is a compile-time value, then just change the 5461 // value. 5462 if (LoopCount->getOpcode() == PPC::LI8 || 5463 LoopCount->getOpcode() == PPC::LI) { 5464 int64_t TripCount = LoopCount->getOperand(1).getImm() + TripCountAdjust; 5465 LoopCount->getOperand(1).setImm(TripCount); 5466 return; 5467 } 5468 5469 // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1, 5470 // so we don't need to generate any thing here. 5471 } 5472 5473 void disposed() override { 5474 Loop->eraseFromParent(); 5475 // Ensure the loop setup instruction is deleted too. 5476 LoopCount->eraseFromParent(); 5477 } 5478 }; 5479 } // namespace 5480 5481 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo> 5482 PPCInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const { 5483 // We really "analyze" only hardware loops right now. 5484 MachineBasicBlock::iterator I = LoopBB->getFirstTerminator(); 5485 MachineBasicBlock *Preheader = *LoopBB->pred_begin(); 5486 if (Preheader == LoopBB) 5487 Preheader = *std::next(LoopBB->pred_begin()); 5488 MachineFunction *MF = Preheader->getParent(); 5489 5490 if (I != LoopBB->end() && isBDNZ(I->getOpcode())) { 5491 SmallPtrSet<MachineBasicBlock *, 8> Visited; 5492 if (MachineInstr *LoopInst = findLoopInstr(*Preheader, Visited)) { 5493 Register LoopCountReg = LoopInst->getOperand(0).getReg(); 5494 MachineRegisterInfo &MRI = MF->getRegInfo(); 5495 MachineInstr *LoopCount = MRI.getUniqueVRegDef(LoopCountReg); 5496 return std::make_unique<PPCPipelinerLoopInfo>(LoopInst, &*I, LoopCount); 5497 } 5498 } 5499 return nullptr; 5500 } 5501 5502 MachineInstr *PPCInstrInfo::findLoopInstr( 5503 MachineBasicBlock &PreHeader, 5504 SmallPtrSet<MachineBasicBlock *, 8> &Visited) const { 5505 5506 unsigned LOOPi = (Subtarget.isPPC64() ? PPC::MTCTR8loop : PPC::MTCTRloop); 5507 5508 // The loop set-up instruction should be in preheader 5509 for (auto &I : PreHeader.instrs()) 5510 if (I.getOpcode() == LOOPi) 5511 return &I; 5512 return nullptr; 5513 } 5514 5515 // Return true if get the base operand, byte offset of an instruction and the 5516 // memory width. Width is the size of memory that is being loaded/stored. 5517 bool PPCInstrInfo::getMemOperandWithOffsetWidth( 5518 const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset, 5519 unsigned &Width, const TargetRegisterInfo *TRI) const { 5520 if (!LdSt.mayLoadOrStore() || LdSt.getNumExplicitOperands() != 3) 5521 return false; 5522 5523 // Handle only loads/stores with base register followed by immediate offset. 5524 if (!LdSt.getOperand(1).isImm() || 5525 (!LdSt.getOperand(2).isReg() && !LdSt.getOperand(2).isFI())) 5526 return false; 5527 if (!LdSt.getOperand(1).isImm() || 5528 (!LdSt.getOperand(2).isReg() && !LdSt.getOperand(2).isFI())) 5529 return false; 5530 5531 if (!LdSt.hasOneMemOperand()) 5532 return false; 5533 5534 Width = (*LdSt.memoperands_begin())->getSize(); 5535 Offset = LdSt.getOperand(1).getImm(); 5536 BaseReg = &LdSt.getOperand(2); 5537 return true; 5538 } 5539 5540 bool PPCInstrInfo::areMemAccessesTriviallyDisjoint( 5541 const MachineInstr &MIa, const MachineInstr &MIb) const { 5542 assert(MIa.mayLoadOrStore() && "MIa must be a load or store."); 5543 assert(MIb.mayLoadOrStore() && "MIb must be a load or store."); 5544 5545 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() || 5546 MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef()) 5547 return false; 5548 5549 // Retrieve the base register, offset from the base register and width. Width 5550 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4). If 5551 // base registers are identical, and the offset of a lower memory access + 5552 // the width doesn't overlap the offset of a higher memory access, 5553 // then the memory accesses are different. 5554 const TargetRegisterInfo *TRI = &getRegisterInfo(); 5555 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr; 5556 int64_t OffsetA = 0, OffsetB = 0; 5557 unsigned int WidthA = 0, WidthB = 0; 5558 if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) && 5559 getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) { 5560 if (BaseOpA->isIdenticalTo(*BaseOpB)) { 5561 int LowOffset = std::min(OffsetA, OffsetB); 5562 int HighOffset = std::max(OffsetA, OffsetB); 5563 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB; 5564 if (LowOffset + LowWidth <= HighOffset) 5565 return true; 5566 } 5567 } 5568 return false; 5569 } 5570