1 //===-- SIFoldOperands.cpp - Fold operands --- ----------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 /// \file 8 //===----------------------------------------------------------------------===// 9 // 10 11 #include "AMDGPU.h" 12 #include "GCNSubtarget.h" 13 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 14 #include "SIMachineFunctionInfo.h" 15 #include "llvm/ADT/DepthFirstIterator.h" 16 #include "llvm/CodeGen/MachineFunctionPass.h" 17 #include "llvm/CodeGen/MachineOperand.h" 18 19 #define DEBUG_TYPE "si-fold-operands" 20 using namespace llvm; 21 22 namespace { 23 24 struct FoldCandidate { 25 MachineInstr *UseMI; 26 union { 27 MachineOperand *OpToFold; 28 uint64_t ImmToFold; 29 int FrameIndexToFold; 30 }; 31 int ShrinkOpcode; 32 unsigned UseOpNo; 33 MachineOperand::MachineOperandType Kind; 34 bool Commuted; 35 36 FoldCandidate(MachineInstr *MI, unsigned OpNo, MachineOperand *FoldOp, 37 bool Commuted_ = false, 38 int ShrinkOp = -1) : 39 UseMI(MI), OpToFold(nullptr), ShrinkOpcode(ShrinkOp), UseOpNo(OpNo), 40 Kind(FoldOp->getType()), 41 Commuted(Commuted_) { 42 if (FoldOp->isImm()) { 43 ImmToFold = FoldOp->getImm(); 44 } else if (FoldOp->isFI()) { 45 FrameIndexToFold = FoldOp->getIndex(); 46 } else { 47 assert(FoldOp->isReg() || FoldOp->isGlobal()); 48 OpToFold = FoldOp; 49 } 50 } 51 52 bool isFI() const { 53 return Kind == MachineOperand::MO_FrameIndex; 54 } 55 56 bool isImm() const { 57 return Kind == MachineOperand::MO_Immediate; 58 } 59 60 bool isReg() const { 61 return Kind == MachineOperand::MO_Register; 62 } 63 64 bool isGlobal() const { return Kind == MachineOperand::MO_GlobalAddress; } 65 66 bool needsShrink() const { return ShrinkOpcode != -1; } 67 }; 68 69 class SIFoldOperands : public MachineFunctionPass { 70 public: 71 static char ID; 72 MachineRegisterInfo *MRI; 73 const SIInstrInfo *TII; 74 const SIRegisterInfo *TRI; 75 const GCNSubtarget *ST; 76 const SIMachineFunctionInfo *MFI; 77 78 bool frameIndexMayFold(const MachineInstr &UseMI, int OpNo, 79 const MachineOperand &OpToFold) const; 80 81 bool updateOperand(FoldCandidate &Fold) const; 82 83 bool tryAddToFoldList(SmallVectorImpl<FoldCandidate> &FoldList, 84 MachineInstr *MI, unsigned OpNo, 85 MachineOperand *OpToFold) const; 86 bool isUseSafeToFold(const MachineInstr &MI, 87 const MachineOperand &UseMO) const; 88 bool 89 getRegSeqInit(SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs, 90 Register UseReg, uint8_t OpTy) const; 91 bool tryToFoldACImm(const MachineOperand &OpToFold, MachineInstr *UseMI, 92 unsigned UseOpIdx, 93 SmallVectorImpl<FoldCandidate> &FoldList) const; 94 void foldOperand(MachineOperand &OpToFold, 95 MachineInstr *UseMI, 96 int UseOpIdx, 97 SmallVectorImpl<FoldCandidate> &FoldList, 98 SmallVectorImpl<MachineInstr *> &CopiesToReplace) const; 99 100 MachineOperand *getImmOrMaterializedImm(MachineOperand &Op) const; 101 bool tryConstantFoldOp(MachineInstr *MI) const; 102 bool tryFoldCndMask(MachineInstr &MI) const; 103 bool tryFoldZeroHighBits(MachineInstr &MI) const; 104 bool foldInstOperand(MachineInstr &MI, MachineOperand &OpToFold) const; 105 bool tryFoldFoldableCopy(MachineInstr &MI, 106 MachineOperand *&CurrentKnownM0Val) const; 107 108 const MachineOperand *isClamp(const MachineInstr &MI) const; 109 bool tryFoldClamp(MachineInstr &MI); 110 111 std::pair<const MachineOperand *, int> isOMod(const MachineInstr &MI) const; 112 bool tryFoldOMod(MachineInstr &MI); 113 bool tryFoldRegSequence(MachineInstr &MI); 114 bool tryFoldPhiAGPR(MachineInstr &MI); 115 bool tryFoldLoad(MachineInstr &MI); 116 117 bool tryOptimizeAGPRPhis(MachineBasicBlock &MBB); 118 119 public: 120 SIFoldOperands() : MachineFunctionPass(ID) { 121 initializeSIFoldOperandsPass(*PassRegistry::getPassRegistry()); 122 } 123 124 bool runOnMachineFunction(MachineFunction &MF) override; 125 126 StringRef getPassName() const override { return "SI Fold Operands"; } 127 128 void getAnalysisUsage(AnalysisUsage &AU) const override { 129 AU.setPreservesCFG(); 130 MachineFunctionPass::getAnalysisUsage(AU); 131 } 132 }; 133 134 } // End anonymous namespace. 135 136 INITIALIZE_PASS(SIFoldOperands, DEBUG_TYPE, 137 "SI Fold Operands", false, false) 138 139 char SIFoldOperands::ID = 0; 140 141 char &llvm::SIFoldOperandsID = SIFoldOperands::ID; 142 143 static const TargetRegisterClass *getRegOpRC(const MachineRegisterInfo &MRI, 144 const TargetRegisterInfo &TRI, 145 const MachineOperand &MO) { 146 const TargetRegisterClass *RC = MRI.getRegClass(MO.getReg()); 147 if (const TargetRegisterClass *SubRC = 148 TRI.getSubRegisterClass(RC, MO.getSubReg())) 149 RC = SubRC; 150 return RC; 151 } 152 153 // Map multiply-accumulate opcode to corresponding multiply-add opcode if any. 154 static unsigned macToMad(unsigned Opc) { 155 switch (Opc) { 156 case AMDGPU::V_MAC_F32_e64: 157 return AMDGPU::V_MAD_F32_e64; 158 case AMDGPU::V_MAC_F16_e64: 159 return AMDGPU::V_MAD_F16_e64; 160 case AMDGPU::V_FMAC_F32_e64: 161 return AMDGPU::V_FMA_F32_e64; 162 case AMDGPU::V_FMAC_F16_e64: 163 return AMDGPU::V_FMA_F16_gfx9_e64; 164 case AMDGPU::V_FMAC_F16_t16_e64: 165 return AMDGPU::V_FMA_F16_gfx9_e64; 166 case AMDGPU::V_FMAC_LEGACY_F32_e64: 167 return AMDGPU::V_FMA_LEGACY_F32_e64; 168 case AMDGPU::V_FMAC_F64_e64: 169 return AMDGPU::V_FMA_F64_e64; 170 } 171 return AMDGPU::INSTRUCTION_LIST_END; 172 } 173 174 // TODO: Add heuristic that the frame index might not fit in the addressing mode 175 // immediate offset to avoid materializing in loops. 176 bool SIFoldOperands::frameIndexMayFold(const MachineInstr &UseMI, int OpNo, 177 const MachineOperand &OpToFold) const { 178 if (!OpToFold.isFI()) 179 return false; 180 181 const unsigned Opc = UseMI.getOpcode(); 182 if (TII->isMUBUF(UseMI)) 183 return OpNo == AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr); 184 if (!TII->isFLATScratch(UseMI)) 185 return false; 186 187 int SIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr); 188 if (OpNo == SIdx) 189 return true; 190 191 int VIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr); 192 return OpNo == VIdx && SIdx == -1; 193 } 194 195 FunctionPass *llvm::createSIFoldOperandsPass() { 196 return new SIFoldOperands(); 197 } 198 199 bool SIFoldOperands::updateOperand(FoldCandidate &Fold) const { 200 MachineInstr *MI = Fold.UseMI; 201 MachineOperand &Old = MI->getOperand(Fold.UseOpNo); 202 assert(Old.isReg()); 203 204 205 const uint64_t TSFlags = MI->getDesc().TSFlags; 206 if (Fold.isImm()) { 207 if (TSFlags & SIInstrFlags::IsPacked && !(TSFlags & SIInstrFlags::IsMAI) && 208 (!ST->hasDOTOpSelHazard() || !(TSFlags & SIInstrFlags::IsDOT)) && 209 AMDGPU::isFoldableLiteralV216(Fold.ImmToFold, 210 ST->hasInv2PiInlineImm())) { 211 // Set op_sel/op_sel_hi on this operand or bail out if op_sel is 212 // already set. 213 unsigned Opcode = MI->getOpcode(); 214 int OpNo = MI->getOperandNo(&Old); 215 int ModIdx = -1; 216 if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0)) 217 ModIdx = AMDGPU::OpName::src0_modifiers; 218 else if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1)) 219 ModIdx = AMDGPU::OpName::src1_modifiers; 220 else if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2)) 221 ModIdx = AMDGPU::OpName::src2_modifiers; 222 assert(ModIdx != -1); 223 ModIdx = AMDGPU::getNamedOperandIdx(Opcode, ModIdx); 224 MachineOperand &Mod = MI->getOperand(ModIdx); 225 unsigned Val = Mod.getImm(); 226 if (!(Val & SISrcMods::OP_SEL_0) && (Val & SISrcMods::OP_SEL_1)) { 227 // Only apply the following transformation if that operand requires 228 // a packed immediate. 229 switch (TII->get(Opcode).operands()[OpNo].OperandType) { 230 case AMDGPU::OPERAND_REG_IMM_V2FP16: 231 case AMDGPU::OPERAND_REG_IMM_V2INT16: 232 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: 233 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: 234 // If upper part is all zero we do not need op_sel_hi. 235 if (!isUInt<16>(Fold.ImmToFold)) { 236 if (!(Fold.ImmToFold & 0xffff)) { 237 Mod.setImm(Mod.getImm() | SISrcMods::OP_SEL_0); 238 Mod.setImm(Mod.getImm() & ~SISrcMods::OP_SEL_1); 239 Old.ChangeToImmediate((Fold.ImmToFold >> 16) & 0xffff); 240 return true; 241 } 242 Mod.setImm(Mod.getImm() & ~SISrcMods::OP_SEL_1); 243 Old.ChangeToImmediate(Fold.ImmToFold & 0xffff); 244 return true; 245 } 246 break; 247 default: 248 break; 249 } 250 } 251 } 252 } 253 254 if ((Fold.isImm() || Fold.isFI() || Fold.isGlobal()) && Fold.needsShrink()) { 255 MachineBasicBlock *MBB = MI->getParent(); 256 auto Liveness = MBB->computeRegisterLiveness(TRI, AMDGPU::VCC, MI, 16); 257 if (Liveness != MachineBasicBlock::LQR_Dead) { 258 LLVM_DEBUG(dbgs() << "Not shrinking " << MI << " due to vcc liveness\n"); 259 return false; 260 } 261 262 int Op32 = Fold.ShrinkOpcode; 263 MachineOperand &Dst0 = MI->getOperand(0); 264 MachineOperand &Dst1 = MI->getOperand(1); 265 assert(Dst0.isDef() && Dst1.isDef()); 266 267 bool HaveNonDbgCarryUse = !MRI->use_nodbg_empty(Dst1.getReg()); 268 269 const TargetRegisterClass *Dst0RC = MRI->getRegClass(Dst0.getReg()); 270 Register NewReg0 = MRI->createVirtualRegister(Dst0RC); 271 272 MachineInstr *Inst32 = TII->buildShrunkInst(*MI, Op32); 273 274 if (HaveNonDbgCarryUse) { 275 BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(AMDGPU::COPY), 276 Dst1.getReg()) 277 .addReg(AMDGPU::VCC, RegState::Kill); 278 } 279 280 // Keep the old instruction around to avoid breaking iterators, but 281 // replace it with a dummy instruction to remove uses. 282 // 283 // FIXME: We should not invert how this pass looks at operands to avoid 284 // this. Should track set of foldable movs instead of looking for uses 285 // when looking at a use. 286 Dst0.setReg(NewReg0); 287 for (unsigned I = MI->getNumOperands() - 1; I > 0; --I) 288 MI->removeOperand(I); 289 MI->setDesc(TII->get(AMDGPU::IMPLICIT_DEF)); 290 291 if (Fold.Commuted) 292 TII->commuteInstruction(*Inst32, false); 293 return true; 294 } 295 296 assert(!Fold.needsShrink() && "not handled"); 297 298 if (Fold.isImm()) { 299 if (Old.isTied()) { 300 int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(MI->getOpcode()); 301 if (NewMFMAOpc == -1) 302 return false; 303 MI->setDesc(TII->get(NewMFMAOpc)); 304 MI->untieRegOperand(0); 305 } 306 Old.ChangeToImmediate(Fold.ImmToFold); 307 return true; 308 } 309 310 if (Fold.isGlobal()) { 311 Old.ChangeToGA(Fold.OpToFold->getGlobal(), Fold.OpToFold->getOffset(), 312 Fold.OpToFold->getTargetFlags()); 313 return true; 314 } 315 316 if (Fold.isFI()) { 317 Old.ChangeToFrameIndex(Fold.FrameIndexToFold); 318 return true; 319 } 320 321 MachineOperand *New = Fold.OpToFold; 322 Old.substVirtReg(New->getReg(), New->getSubReg(), *TRI); 323 Old.setIsUndef(New->isUndef()); 324 return true; 325 } 326 327 static bool isUseMIInFoldList(ArrayRef<FoldCandidate> FoldList, 328 const MachineInstr *MI) { 329 return any_of(FoldList, [&](const auto &C) { return C.UseMI == MI; }); 330 } 331 332 static void appendFoldCandidate(SmallVectorImpl<FoldCandidate> &FoldList, 333 MachineInstr *MI, unsigned OpNo, 334 MachineOperand *FoldOp, bool Commuted = false, 335 int ShrinkOp = -1) { 336 // Skip additional folding on the same operand. 337 for (FoldCandidate &Fold : FoldList) 338 if (Fold.UseMI == MI && Fold.UseOpNo == OpNo) 339 return; 340 LLVM_DEBUG(dbgs() << "Append " << (Commuted ? "commuted" : "normal") 341 << " operand " << OpNo << "\n " << *MI); 342 FoldList.emplace_back(MI, OpNo, FoldOp, Commuted, ShrinkOp); 343 } 344 345 bool SIFoldOperands::tryAddToFoldList(SmallVectorImpl<FoldCandidate> &FoldList, 346 MachineInstr *MI, unsigned OpNo, 347 MachineOperand *OpToFold) const { 348 if (!TII->isOperandLegal(*MI, OpNo, OpToFold)) { 349 // Special case for v_mac_{f16, f32}_e64 if we are trying to fold into src2 350 unsigned Opc = MI->getOpcode(); 351 unsigned NewOpc = macToMad(Opc); 352 if (NewOpc != AMDGPU::INSTRUCTION_LIST_END) { 353 // Check if changing this to a v_mad_{f16, f32} instruction will allow us 354 // to fold the operand. 355 MI->setDesc(TII->get(NewOpc)); 356 bool AddOpSel = !AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::op_sel) && 357 AMDGPU::hasNamedOperand(NewOpc, AMDGPU::OpName::op_sel); 358 if (AddOpSel) 359 MI->addOperand(MachineOperand::CreateImm(0)); 360 bool FoldAsMAD = tryAddToFoldList(FoldList, MI, OpNo, OpToFold); 361 if (FoldAsMAD) { 362 MI->untieRegOperand(OpNo); 363 return true; 364 } 365 if (AddOpSel) 366 MI->removeOperand(MI->getNumExplicitOperands() - 1); 367 MI->setDesc(TII->get(Opc)); 368 } 369 370 // Special case for s_setreg_b32 371 if (OpToFold->isImm()) { 372 unsigned ImmOpc = 0; 373 if (Opc == AMDGPU::S_SETREG_B32) 374 ImmOpc = AMDGPU::S_SETREG_IMM32_B32; 375 else if (Opc == AMDGPU::S_SETREG_B32_mode) 376 ImmOpc = AMDGPU::S_SETREG_IMM32_B32_mode; 377 if (ImmOpc) { 378 MI->setDesc(TII->get(ImmOpc)); 379 appendFoldCandidate(FoldList, MI, OpNo, OpToFold); 380 return true; 381 } 382 } 383 384 // If we are already folding into another operand of MI, then 385 // we can't commute the instruction, otherwise we risk making the 386 // other fold illegal. 387 if (isUseMIInFoldList(FoldList, MI)) 388 return false; 389 390 unsigned CommuteOpNo = OpNo; 391 392 // Operand is not legal, so try to commute the instruction to 393 // see if this makes it possible to fold. 394 unsigned CommuteIdx0 = TargetInstrInfo::CommuteAnyOperandIndex; 395 unsigned CommuteIdx1 = TargetInstrInfo::CommuteAnyOperandIndex; 396 bool CanCommute = TII->findCommutedOpIndices(*MI, CommuteIdx0, CommuteIdx1); 397 398 if (CanCommute) { 399 if (CommuteIdx0 == OpNo) 400 CommuteOpNo = CommuteIdx1; 401 else if (CommuteIdx1 == OpNo) 402 CommuteOpNo = CommuteIdx0; 403 } 404 405 406 // One of operands might be an Imm operand, and OpNo may refer to it after 407 // the call of commuteInstruction() below. Such situations are avoided 408 // here explicitly as OpNo must be a register operand to be a candidate 409 // for memory folding. 410 if (CanCommute && (!MI->getOperand(CommuteIdx0).isReg() || 411 !MI->getOperand(CommuteIdx1).isReg())) 412 return false; 413 414 if (!CanCommute || 415 !TII->commuteInstruction(*MI, false, CommuteIdx0, CommuteIdx1)) 416 return false; 417 418 if (!TII->isOperandLegal(*MI, CommuteOpNo, OpToFold)) { 419 if ((Opc == AMDGPU::V_ADD_CO_U32_e64 || 420 Opc == AMDGPU::V_SUB_CO_U32_e64 || 421 Opc == AMDGPU::V_SUBREV_CO_U32_e64) && // FIXME 422 (OpToFold->isImm() || OpToFold->isFI() || OpToFold->isGlobal())) { 423 424 // Verify the other operand is a VGPR, otherwise we would violate the 425 // constant bus restriction. 426 unsigned OtherIdx = CommuteOpNo == CommuteIdx0 ? CommuteIdx1 : CommuteIdx0; 427 MachineOperand &OtherOp = MI->getOperand(OtherIdx); 428 if (!OtherOp.isReg() || 429 !TII->getRegisterInfo().isVGPR(*MRI, OtherOp.getReg())) 430 return false; 431 432 assert(MI->getOperand(1).isDef()); 433 434 // Make sure to get the 32-bit version of the commuted opcode. 435 unsigned MaybeCommutedOpc = MI->getOpcode(); 436 int Op32 = AMDGPU::getVOPe32(MaybeCommutedOpc); 437 438 appendFoldCandidate(FoldList, MI, CommuteOpNo, OpToFold, true, Op32); 439 return true; 440 } 441 442 TII->commuteInstruction(*MI, false, CommuteIdx0, CommuteIdx1); 443 return false; 444 } 445 446 appendFoldCandidate(FoldList, MI, CommuteOpNo, OpToFold, true); 447 return true; 448 } 449 450 // Check the case where we might introduce a second constant operand to a 451 // scalar instruction 452 if (TII->isSALU(MI->getOpcode())) { 453 const MCInstrDesc &InstDesc = MI->getDesc(); 454 const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo]; 455 456 // Fine if the operand can be encoded as an inline constant 457 if (!OpToFold->isReg() && !TII->isInlineConstant(*OpToFold, OpInfo)) { 458 // Otherwise check for another constant 459 for (unsigned i = 0, e = InstDesc.getNumOperands(); i != e; ++i) { 460 auto &Op = MI->getOperand(i); 461 if (OpNo != i && !Op.isReg() && !TII->isInlineConstant(Op, OpInfo)) 462 return false; 463 } 464 } 465 } 466 467 appendFoldCandidate(FoldList, MI, OpNo, OpToFold); 468 return true; 469 } 470 471 bool SIFoldOperands::isUseSafeToFold(const MachineInstr &MI, 472 const MachineOperand &UseMO) const { 473 // Operands of SDWA instructions must be registers. 474 return !TII->isSDWA(MI); 475 } 476 477 // Find a def of the UseReg, check if it is a reg_sequence and find initializers 478 // for each subreg, tracking it to foldable inline immediate if possible. 479 // Returns true on success. 480 bool SIFoldOperands::getRegSeqInit( 481 SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs, 482 Register UseReg, uint8_t OpTy) const { 483 MachineInstr *Def = MRI->getVRegDef(UseReg); 484 if (!Def || !Def->isRegSequence()) 485 return false; 486 487 for (unsigned I = 1, E = Def->getNumExplicitOperands(); I < E; I += 2) { 488 MachineOperand *Sub = &Def->getOperand(I); 489 assert(Sub->isReg()); 490 491 for (MachineInstr *SubDef = MRI->getVRegDef(Sub->getReg()); 492 SubDef && Sub->isReg() && Sub->getReg().isVirtual() && 493 !Sub->getSubReg() && TII->isFoldableCopy(*SubDef); 494 SubDef = MRI->getVRegDef(Sub->getReg())) { 495 MachineOperand *Op = &SubDef->getOperand(1); 496 if (Op->isImm()) { 497 if (TII->isInlineConstant(*Op, OpTy)) 498 Sub = Op; 499 break; 500 } 501 if (!Op->isReg() || Op->getReg().isPhysical()) 502 break; 503 Sub = Op; 504 } 505 506 Defs.emplace_back(Sub, Def->getOperand(I + 1).getImm()); 507 } 508 509 return true; 510 } 511 512 bool SIFoldOperands::tryToFoldACImm( 513 const MachineOperand &OpToFold, MachineInstr *UseMI, unsigned UseOpIdx, 514 SmallVectorImpl<FoldCandidate> &FoldList) const { 515 const MCInstrDesc &Desc = UseMI->getDesc(); 516 if (UseOpIdx >= Desc.getNumOperands()) 517 return false; 518 519 uint8_t OpTy = Desc.operands()[UseOpIdx].OperandType; 520 if ((OpTy < AMDGPU::OPERAND_REG_INLINE_AC_FIRST || 521 OpTy > AMDGPU::OPERAND_REG_INLINE_AC_LAST) && 522 (OpTy < AMDGPU::OPERAND_REG_INLINE_C_FIRST || 523 OpTy > AMDGPU::OPERAND_REG_INLINE_C_LAST)) 524 return false; 525 526 if (OpToFold.isImm() && TII->isInlineConstant(OpToFold, OpTy) && 527 TII->isOperandLegal(*UseMI, UseOpIdx, &OpToFold)) { 528 UseMI->getOperand(UseOpIdx).ChangeToImmediate(OpToFold.getImm()); 529 return true; 530 } 531 532 if (!OpToFold.isReg()) 533 return false; 534 535 Register UseReg = OpToFold.getReg(); 536 if (!UseReg.isVirtual()) 537 return false; 538 539 if (isUseMIInFoldList(FoldList, UseMI)) 540 return false; 541 542 // Maybe it is just a COPY of an immediate itself. 543 MachineInstr *Def = MRI->getVRegDef(UseReg); 544 MachineOperand &UseOp = UseMI->getOperand(UseOpIdx); 545 if (!UseOp.getSubReg() && Def && TII->isFoldableCopy(*Def)) { 546 MachineOperand &DefOp = Def->getOperand(1); 547 if (DefOp.isImm() && TII->isInlineConstant(DefOp, OpTy) && 548 TII->isOperandLegal(*UseMI, UseOpIdx, &DefOp)) { 549 UseMI->getOperand(UseOpIdx).ChangeToImmediate(DefOp.getImm()); 550 return true; 551 } 552 } 553 554 SmallVector<std::pair<MachineOperand*, unsigned>, 32> Defs; 555 if (!getRegSeqInit(Defs, UseReg, OpTy)) 556 return false; 557 558 int32_t Imm; 559 for (unsigned I = 0, E = Defs.size(); I != E; ++I) { 560 const MachineOperand *Op = Defs[I].first; 561 if (!Op->isImm()) 562 return false; 563 564 auto SubImm = Op->getImm(); 565 if (!I) { 566 Imm = SubImm; 567 if (!TII->isInlineConstant(*Op, OpTy) || 568 !TII->isOperandLegal(*UseMI, UseOpIdx, Op)) 569 return false; 570 571 continue; 572 } 573 if (Imm != SubImm) 574 return false; // Can only fold splat constants 575 } 576 577 appendFoldCandidate(FoldList, UseMI, UseOpIdx, Defs[0].first); 578 return true; 579 } 580 581 void SIFoldOperands::foldOperand( 582 MachineOperand &OpToFold, 583 MachineInstr *UseMI, 584 int UseOpIdx, 585 SmallVectorImpl<FoldCandidate> &FoldList, 586 SmallVectorImpl<MachineInstr *> &CopiesToReplace) const { 587 const MachineOperand &UseOp = UseMI->getOperand(UseOpIdx); 588 589 if (!isUseSafeToFold(*UseMI, UseOp)) 590 return; 591 592 // FIXME: Fold operands with subregs. 593 if (UseOp.isReg() && OpToFold.isReg() && 594 (UseOp.isImplicit() || UseOp.getSubReg() != AMDGPU::NoSubRegister)) 595 return; 596 597 // Special case for REG_SEQUENCE: We can't fold literals into 598 // REG_SEQUENCE instructions, so we have to fold them into the 599 // uses of REG_SEQUENCE. 600 if (UseMI->isRegSequence()) { 601 Register RegSeqDstReg = UseMI->getOperand(0).getReg(); 602 unsigned RegSeqDstSubReg = UseMI->getOperand(UseOpIdx + 1).getImm(); 603 604 for (auto &RSUse : make_early_inc_range(MRI->use_nodbg_operands(RegSeqDstReg))) { 605 MachineInstr *RSUseMI = RSUse.getParent(); 606 607 if (tryToFoldACImm(UseMI->getOperand(0), RSUseMI, 608 RSUseMI->getOperandNo(&RSUse), FoldList)) 609 continue; 610 611 if (RSUse.getSubReg() != RegSeqDstSubReg) 612 continue; 613 614 foldOperand(OpToFold, RSUseMI, RSUseMI->getOperandNo(&RSUse), FoldList, 615 CopiesToReplace); 616 } 617 618 return; 619 } 620 621 if (tryToFoldACImm(OpToFold, UseMI, UseOpIdx, FoldList)) 622 return; 623 624 if (frameIndexMayFold(*UseMI, UseOpIdx, OpToFold)) { 625 // Verify that this is a stack access. 626 // FIXME: Should probably use stack pseudos before frame lowering. 627 628 if (TII->isMUBUF(*UseMI)) { 629 if (TII->getNamedOperand(*UseMI, AMDGPU::OpName::srsrc)->getReg() != 630 MFI->getScratchRSrcReg()) 631 return; 632 633 // Ensure this is either relative to the current frame or the current 634 // wave. 635 MachineOperand &SOff = 636 *TII->getNamedOperand(*UseMI, AMDGPU::OpName::soffset); 637 if (!SOff.isImm() || SOff.getImm() != 0) 638 return; 639 } 640 641 // A frame index will resolve to a positive constant, so it should always be 642 // safe to fold the addressing mode, even pre-GFX9. 643 UseMI->getOperand(UseOpIdx).ChangeToFrameIndex(OpToFold.getIndex()); 644 645 const unsigned Opc = UseMI->getOpcode(); 646 if (TII->isFLATScratch(*UseMI) && 647 AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::vaddr) && 648 !AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::saddr)) { 649 unsigned NewOpc = AMDGPU::getFlatScratchInstSSfromSV(Opc); 650 UseMI->setDesc(TII->get(NewOpc)); 651 } 652 653 return; 654 } 655 656 bool FoldingImmLike = 657 OpToFold.isImm() || OpToFold.isFI() || OpToFold.isGlobal(); 658 659 if (FoldingImmLike && UseMI->isCopy()) { 660 Register DestReg = UseMI->getOperand(0).getReg(); 661 Register SrcReg = UseMI->getOperand(1).getReg(); 662 assert(SrcReg.isVirtual()); 663 664 const TargetRegisterClass *SrcRC = MRI->getRegClass(SrcReg); 665 666 // Don't fold into a copy to a physical register with the same class. Doing 667 // so would interfere with the register coalescer's logic which would avoid 668 // redundant initializations. 669 if (DestReg.isPhysical() && SrcRC->contains(DestReg)) 670 return; 671 672 const TargetRegisterClass *DestRC = TRI->getRegClassForReg(*MRI, DestReg); 673 if (!DestReg.isPhysical()) { 674 if (TRI->isSGPRClass(SrcRC) && TRI->hasVectorRegisters(DestRC)) { 675 SmallVector<FoldCandidate, 4> CopyUses; 676 for (auto &Use : MRI->use_nodbg_operands(DestReg)) { 677 // There's no point trying to fold into an implicit operand. 678 if (Use.isImplicit()) 679 continue; 680 681 CopyUses.emplace_back(Use.getParent(), 682 Use.getParent()->getOperandNo(&Use), 683 &UseMI->getOperand(1)); 684 } 685 686 for (auto &F : CopyUses) { 687 foldOperand(*F.OpToFold, F.UseMI, F.UseOpNo, FoldList, 688 CopiesToReplace); 689 } 690 } 691 692 if (DestRC == &AMDGPU::AGPR_32RegClass && 693 TII->isInlineConstant(OpToFold, AMDGPU::OPERAND_REG_INLINE_C_INT32)) { 694 UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64)); 695 UseMI->getOperand(1).ChangeToImmediate(OpToFold.getImm()); 696 CopiesToReplace.push_back(UseMI); 697 return; 698 } 699 } 700 701 // In order to fold immediates into copies, we need to change the 702 // copy to a MOV. 703 704 unsigned MovOp = TII->getMovOpcode(DestRC); 705 if (MovOp == AMDGPU::COPY) 706 return; 707 708 UseMI->setDesc(TII->get(MovOp)); 709 MachineInstr::mop_iterator ImpOpI = UseMI->implicit_operands().begin(); 710 MachineInstr::mop_iterator ImpOpE = UseMI->implicit_operands().end(); 711 while (ImpOpI != ImpOpE) { 712 MachineInstr::mop_iterator Tmp = ImpOpI; 713 ImpOpI++; 714 UseMI->removeOperand(UseMI->getOperandNo(Tmp)); 715 } 716 CopiesToReplace.push_back(UseMI); 717 } else { 718 if (UseMI->isCopy() && OpToFold.isReg() && 719 UseMI->getOperand(0).getReg().isVirtual() && 720 !UseMI->getOperand(1).getSubReg()) { 721 LLVM_DEBUG(dbgs() << "Folding " << OpToFold << "\n into " << *UseMI); 722 unsigned Size = TII->getOpSize(*UseMI, 1); 723 Register UseReg = OpToFold.getReg(); 724 UseMI->getOperand(1).setReg(UseReg); 725 UseMI->getOperand(1).setSubReg(OpToFold.getSubReg()); 726 UseMI->getOperand(1).setIsKill(false); 727 CopiesToReplace.push_back(UseMI); 728 OpToFold.setIsKill(false); 729 730 // Remove kill flags as kills may now be out of order with uses. 731 MRI->clearKillFlags(OpToFold.getReg()); 732 733 // That is very tricky to store a value into an AGPR. v_accvgpr_write_b32 734 // can only accept VGPR or inline immediate. Recreate a reg_sequence with 735 // its initializers right here, so we will rematerialize immediates and 736 // avoid copies via different reg classes. 737 SmallVector<std::pair<MachineOperand*, unsigned>, 32> Defs; 738 if (Size > 4 && TRI->isAGPR(*MRI, UseMI->getOperand(0).getReg()) && 739 getRegSeqInit(Defs, UseReg, AMDGPU::OPERAND_REG_INLINE_C_INT32)) { 740 const DebugLoc &DL = UseMI->getDebugLoc(); 741 MachineBasicBlock &MBB = *UseMI->getParent(); 742 743 UseMI->setDesc(TII->get(AMDGPU::REG_SEQUENCE)); 744 for (unsigned I = UseMI->getNumOperands() - 1; I > 0; --I) 745 UseMI->removeOperand(I); 746 747 MachineInstrBuilder B(*MBB.getParent(), UseMI); 748 DenseMap<TargetInstrInfo::RegSubRegPair, Register> VGPRCopies; 749 SmallSetVector<TargetInstrInfo::RegSubRegPair, 32> SeenAGPRs; 750 for (unsigned I = 0; I < Size / 4; ++I) { 751 MachineOperand *Def = Defs[I].first; 752 TargetInstrInfo::RegSubRegPair CopyToVGPR; 753 if (Def->isImm() && 754 TII->isInlineConstant(*Def, AMDGPU::OPERAND_REG_INLINE_C_INT32)) { 755 int64_t Imm = Def->getImm(); 756 757 auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass); 758 BuildMI(MBB, UseMI, DL, 759 TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), Tmp).addImm(Imm); 760 B.addReg(Tmp); 761 } else if (Def->isReg() && TRI->isAGPR(*MRI, Def->getReg())) { 762 auto Src = getRegSubRegPair(*Def); 763 Def->setIsKill(false); 764 if (!SeenAGPRs.insert(Src)) { 765 // We cannot build a reg_sequence out of the same registers, they 766 // must be copied. Better do it here before copyPhysReg() created 767 // several reads to do the AGPR->VGPR->AGPR copy. 768 CopyToVGPR = Src; 769 } else { 770 B.addReg(Src.Reg, Def->isUndef() ? RegState::Undef : 0, 771 Src.SubReg); 772 } 773 } else { 774 assert(Def->isReg()); 775 Def->setIsKill(false); 776 auto Src = getRegSubRegPair(*Def); 777 778 // Direct copy from SGPR to AGPR is not possible. To avoid creation 779 // of exploded copies SGPR->VGPR->AGPR in the copyPhysReg() later, 780 // create a copy here and track if we already have such a copy. 781 if (TRI->isSGPRReg(*MRI, Src.Reg)) { 782 CopyToVGPR = Src; 783 } else { 784 auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass); 785 BuildMI(MBB, UseMI, DL, TII->get(AMDGPU::COPY), Tmp).add(*Def); 786 B.addReg(Tmp); 787 } 788 } 789 790 if (CopyToVGPR.Reg) { 791 Register Vgpr; 792 if (VGPRCopies.count(CopyToVGPR)) { 793 Vgpr = VGPRCopies[CopyToVGPR]; 794 } else { 795 Vgpr = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); 796 BuildMI(MBB, UseMI, DL, TII->get(AMDGPU::COPY), Vgpr).add(*Def); 797 VGPRCopies[CopyToVGPR] = Vgpr; 798 } 799 auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass); 800 BuildMI(MBB, UseMI, DL, 801 TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), Tmp).addReg(Vgpr); 802 B.addReg(Tmp); 803 } 804 805 B.addImm(Defs[I].second); 806 } 807 LLVM_DEBUG(dbgs() << "Folded " << *UseMI); 808 return; 809 } 810 811 if (Size != 4) 812 return; 813 814 Register Reg0 = UseMI->getOperand(0).getReg(); 815 Register Reg1 = UseMI->getOperand(1).getReg(); 816 if (TRI->isAGPR(*MRI, Reg0) && TRI->isVGPR(*MRI, Reg1)) 817 UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64)); 818 else if (TRI->isVGPR(*MRI, Reg0) && TRI->isAGPR(*MRI, Reg1)) 819 UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_READ_B32_e64)); 820 else if (ST->hasGFX90AInsts() && TRI->isAGPR(*MRI, Reg0) && 821 TRI->isAGPR(*MRI, Reg1)) 822 UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_MOV_B32)); 823 return; 824 } 825 826 unsigned UseOpc = UseMI->getOpcode(); 827 if (UseOpc == AMDGPU::V_READFIRSTLANE_B32 || 828 (UseOpc == AMDGPU::V_READLANE_B32 && 829 (int)UseOpIdx == 830 AMDGPU::getNamedOperandIdx(UseOpc, AMDGPU::OpName::src0))) { 831 // %vgpr = V_MOV_B32 imm 832 // %sgpr = V_READFIRSTLANE_B32 %vgpr 833 // => 834 // %sgpr = S_MOV_B32 imm 835 if (FoldingImmLike) { 836 if (execMayBeModifiedBeforeUse(*MRI, 837 UseMI->getOperand(UseOpIdx).getReg(), 838 *OpToFold.getParent(), 839 *UseMI)) 840 return; 841 842 UseMI->setDesc(TII->get(AMDGPU::S_MOV_B32)); 843 844 if (OpToFold.isImm()) 845 UseMI->getOperand(1).ChangeToImmediate(OpToFold.getImm()); 846 else 847 UseMI->getOperand(1).ChangeToFrameIndex(OpToFold.getIndex()); 848 UseMI->removeOperand(2); // Remove exec read (or src1 for readlane) 849 return; 850 } 851 852 if (OpToFold.isReg() && TRI->isSGPRReg(*MRI, OpToFold.getReg())) { 853 if (execMayBeModifiedBeforeUse(*MRI, 854 UseMI->getOperand(UseOpIdx).getReg(), 855 *OpToFold.getParent(), 856 *UseMI)) 857 return; 858 859 // %vgpr = COPY %sgpr0 860 // %sgpr1 = V_READFIRSTLANE_B32 %vgpr 861 // => 862 // %sgpr1 = COPY %sgpr0 863 UseMI->setDesc(TII->get(AMDGPU::COPY)); 864 UseMI->getOperand(1).setReg(OpToFold.getReg()); 865 UseMI->getOperand(1).setSubReg(OpToFold.getSubReg()); 866 UseMI->getOperand(1).setIsKill(false); 867 UseMI->removeOperand(2); // Remove exec read (or src1 for readlane) 868 return; 869 } 870 } 871 872 const MCInstrDesc &UseDesc = UseMI->getDesc(); 873 874 // Don't fold into target independent nodes. Target independent opcodes 875 // don't have defined register classes. 876 if (UseDesc.isVariadic() || UseOp.isImplicit() || 877 UseDesc.operands()[UseOpIdx].RegClass == -1) 878 return; 879 } 880 881 if (!FoldingImmLike) { 882 if (OpToFold.isReg() && ST->needsAlignedVGPRs()) { 883 // Don't fold if OpToFold doesn't hold an aligned register. 884 const TargetRegisterClass *RC = 885 TRI->getRegClassForReg(*MRI, OpToFold.getReg()); 886 if (TRI->hasVectorRegisters(RC) && OpToFold.getSubReg()) { 887 unsigned SubReg = OpToFold.getSubReg(); 888 if (const TargetRegisterClass *SubRC = 889 TRI->getSubRegisterClass(RC, SubReg)) 890 RC = SubRC; 891 } 892 893 if (!RC || !TRI->isProperlyAlignedRC(*RC)) 894 return; 895 } 896 897 tryAddToFoldList(FoldList, UseMI, UseOpIdx, &OpToFold); 898 899 // FIXME: We could try to change the instruction from 64-bit to 32-bit 900 // to enable more folding opportunities. The shrink operands pass 901 // already does this. 902 return; 903 } 904 905 906 const MCInstrDesc &FoldDesc = OpToFold.getParent()->getDesc(); 907 const TargetRegisterClass *FoldRC = 908 TRI->getRegClass(FoldDesc.operands()[0].RegClass); 909 910 // Split 64-bit constants into 32-bits for folding. 911 if (UseOp.getSubReg() && AMDGPU::getRegBitWidth(*FoldRC) == 64) { 912 Register UseReg = UseOp.getReg(); 913 const TargetRegisterClass *UseRC = MRI->getRegClass(UseReg); 914 if (AMDGPU::getRegBitWidth(*UseRC) != 64) 915 return; 916 917 APInt Imm(64, OpToFold.getImm()); 918 if (UseOp.getSubReg() == AMDGPU::sub0) { 919 Imm = Imm.getLoBits(32); 920 } else { 921 assert(UseOp.getSubReg() == AMDGPU::sub1); 922 Imm = Imm.getHiBits(32); 923 } 924 925 MachineOperand ImmOp = MachineOperand::CreateImm(Imm.getSExtValue()); 926 tryAddToFoldList(FoldList, UseMI, UseOpIdx, &ImmOp); 927 return; 928 } 929 930 tryAddToFoldList(FoldList, UseMI, UseOpIdx, &OpToFold); 931 } 932 933 static bool evalBinaryInstruction(unsigned Opcode, int32_t &Result, 934 uint32_t LHS, uint32_t RHS) { 935 switch (Opcode) { 936 case AMDGPU::V_AND_B32_e64: 937 case AMDGPU::V_AND_B32_e32: 938 case AMDGPU::S_AND_B32: 939 Result = LHS & RHS; 940 return true; 941 case AMDGPU::V_OR_B32_e64: 942 case AMDGPU::V_OR_B32_e32: 943 case AMDGPU::S_OR_B32: 944 Result = LHS | RHS; 945 return true; 946 case AMDGPU::V_XOR_B32_e64: 947 case AMDGPU::V_XOR_B32_e32: 948 case AMDGPU::S_XOR_B32: 949 Result = LHS ^ RHS; 950 return true; 951 case AMDGPU::S_XNOR_B32: 952 Result = ~(LHS ^ RHS); 953 return true; 954 case AMDGPU::S_NAND_B32: 955 Result = ~(LHS & RHS); 956 return true; 957 case AMDGPU::S_NOR_B32: 958 Result = ~(LHS | RHS); 959 return true; 960 case AMDGPU::S_ANDN2_B32: 961 Result = LHS & ~RHS; 962 return true; 963 case AMDGPU::S_ORN2_B32: 964 Result = LHS | ~RHS; 965 return true; 966 case AMDGPU::V_LSHL_B32_e64: 967 case AMDGPU::V_LSHL_B32_e32: 968 case AMDGPU::S_LSHL_B32: 969 // The instruction ignores the high bits for out of bounds shifts. 970 Result = LHS << (RHS & 31); 971 return true; 972 case AMDGPU::V_LSHLREV_B32_e64: 973 case AMDGPU::V_LSHLREV_B32_e32: 974 Result = RHS << (LHS & 31); 975 return true; 976 case AMDGPU::V_LSHR_B32_e64: 977 case AMDGPU::V_LSHR_B32_e32: 978 case AMDGPU::S_LSHR_B32: 979 Result = LHS >> (RHS & 31); 980 return true; 981 case AMDGPU::V_LSHRREV_B32_e64: 982 case AMDGPU::V_LSHRREV_B32_e32: 983 Result = RHS >> (LHS & 31); 984 return true; 985 case AMDGPU::V_ASHR_I32_e64: 986 case AMDGPU::V_ASHR_I32_e32: 987 case AMDGPU::S_ASHR_I32: 988 Result = static_cast<int32_t>(LHS) >> (RHS & 31); 989 return true; 990 case AMDGPU::V_ASHRREV_I32_e64: 991 case AMDGPU::V_ASHRREV_I32_e32: 992 Result = static_cast<int32_t>(RHS) >> (LHS & 31); 993 return true; 994 default: 995 return false; 996 } 997 } 998 999 static unsigned getMovOpc(bool IsScalar) { 1000 return IsScalar ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; 1001 } 1002 1003 static void mutateCopyOp(MachineInstr &MI, const MCInstrDesc &NewDesc) { 1004 MI.setDesc(NewDesc); 1005 1006 // Remove any leftover implicit operands from mutating the instruction. e.g. 1007 // if we replace an s_and_b32 with a copy, we don't need the implicit scc def 1008 // anymore. 1009 const MCInstrDesc &Desc = MI.getDesc(); 1010 unsigned NumOps = Desc.getNumOperands() + Desc.implicit_uses().size() + 1011 Desc.implicit_defs().size(); 1012 1013 for (unsigned I = MI.getNumOperands() - 1; I >= NumOps; --I) 1014 MI.removeOperand(I); 1015 } 1016 1017 MachineOperand * 1018 SIFoldOperands::getImmOrMaterializedImm(MachineOperand &Op) const { 1019 // If this has a subregister, it obviously is a register source. 1020 if (!Op.isReg() || Op.getSubReg() != AMDGPU::NoSubRegister || 1021 !Op.getReg().isVirtual()) 1022 return &Op; 1023 1024 MachineInstr *Def = MRI->getVRegDef(Op.getReg()); 1025 if (Def && Def->isMoveImmediate()) { 1026 MachineOperand &ImmSrc = Def->getOperand(1); 1027 if (ImmSrc.isImm()) 1028 return &ImmSrc; 1029 } 1030 1031 return &Op; 1032 } 1033 1034 // Try to simplify operations with a constant that may appear after instruction 1035 // selection. 1036 // TODO: See if a frame index with a fixed offset can fold. 1037 bool SIFoldOperands::tryConstantFoldOp(MachineInstr *MI) const { 1038 unsigned Opc = MI->getOpcode(); 1039 1040 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); 1041 if (Src0Idx == -1) 1042 return false; 1043 MachineOperand *Src0 = getImmOrMaterializedImm(MI->getOperand(Src0Idx)); 1044 1045 if ((Opc == AMDGPU::V_NOT_B32_e64 || Opc == AMDGPU::V_NOT_B32_e32 || 1046 Opc == AMDGPU::S_NOT_B32) && 1047 Src0->isImm()) { 1048 MI->getOperand(1).ChangeToImmediate(~Src0->getImm()); 1049 mutateCopyOp(*MI, TII->get(getMovOpc(Opc == AMDGPU::S_NOT_B32))); 1050 return true; 1051 } 1052 1053 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); 1054 if (Src1Idx == -1) 1055 return false; 1056 MachineOperand *Src1 = getImmOrMaterializedImm(MI->getOperand(Src1Idx)); 1057 1058 if (!Src0->isImm() && !Src1->isImm()) 1059 return false; 1060 1061 // and k0, k1 -> v_mov_b32 (k0 & k1) 1062 // or k0, k1 -> v_mov_b32 (k0 | k1) 1063 // xor k0, k1 -> v_mov_b32 (k0 ^ k1) 1064 if (Src0->isImm() && Src1->isImm()) { 1065 int32_t NewImm; 1066 if (!evalBinaryInstruction(Opc, NewImm, Src0->getImm(), Src1->getImm())) 1067 return false; 1068 1069 bool IsSGPR = TRI->isSGPRReg(*MRI, MI->getOperand(0).getReg()); 1070 1071 // Be careful to change the right operand, src0 may belong to a different 1072 // instruction. 1073 MI->getOperand(Src0Idx).ChangeToImmediate(NewImm); 1074 MI->removeOperand(Src1Idx); 1075 mutateCopyOp(*MI, TII->get(getMovOpc(IsSGPR))); 1076 return true; 1077 } 1078 1079 if (!MI->isCommutable()) 1080 return false; 1081 1082 if (Src0->isImm() && !Src1->isImm()) { 1083 std::swap(Src0, Src1); 1084 std::swap(Src0Idx, Src1Idx); 1085 } 1086 1087 int32_t Src1Val = static_cast<int32_t>(Src1->getImm()); 1088 if (Opc == AMDGPU::V_OR_B32_e64 || 1089 Opc == AMDGPU::V_OR_B32_e32 || 1090 Opc == AMDGPU::S_OR_B32) { 1091 if (Src1Val == 0) { 1092 // y = or x, 0 => y = copy x 1093 MI->removeOperand(Src1Idx); 1094 mutateCopyOp(*MI, TII->get(AMDGPU::COPY)); 1095 } else if (Src1Val == -1) { 1096 // y = or x, -1 => y = v_mov_b32 -1 1097 MI->removeOperand(Src1Idx); 1098 mutateCopyOp(*MI, TII->get(getMovOpc(Opc == AMDGPU::S_OR_B32))); 1099 } else 1100 return false; 1101 1102 return true; 1103 } 1104 1105 if (Opc == AMDGPU::V_AND_B32_e64 || Opc == AMDGPU::V_AND_B32_e32 || 1106 Opc == AMDGPU::S_AND_B32) { 1107 if (Src1Val == 0) { 1108 // y = and x, 0 => y = v_mov_b32 0 1109 MI->removeOperand(Src0Idx); 1110 mutateCopyOp(*MI, TII->get(getMovOpc(Opc == AMDGPU::S_AND_B32))); 1111 } else if (Src1Val == -1) { 1112 // y = and x, -1 => y = copy x 1113 MI->removeOperand(Src1Idx); 1114 mutateCopyOp(*MI, TII->get(AMDGPU::COPY)); 1115 } else 1116 return false; 1117 1118 return true; 1119 } 1120 1121 if (Opc == AMDGPU::V_XOR_B32_e64 || Opc == AMDGPU::V_XOR_B32_e32 || 1122 Opc == AMDGPU::S_XOR_B32) { 1123 if (Src1Val == 0) { 1124 // y = xor x, 0 => y = copy x 1125 MI->removeOperand(Src1Idx); 1126 mutateCopyOp(*MI, TII->get(AMDGPU::COPY)); 1127 return true; 1128 } 1129 } 1130 1131 return false; 1132 } 1133 1134 // Try to fold an instruction into a simpler one 1135 bool SIFoldOperands::tryFoldCndMask(MachineInstr &MI) const { 1136 unsigned Opc = MI.getOpcode(); 1137 if (Opc != AMDGPU::V_CNDMASK_B32_e32 && Opc != AMDGPU::V_CNDMASK_B32_e64 && 1138 Opc != AMDGPU::V_CNDMASK_B64_PSEUDO) 1139 return false; 1140 1141 MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); 1142 MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); 1143 if (!Src1->isIdenticalTo(*Src0)) { 1144 auto *Src0Imm = getImmOrMaterializedImm(*Src0); 1145 auto *Src1Imm = getImmOrMaterializedImm(*Src1); 1146 if (!Src1Imm->isIdenticalTo(*Src0Imm)) 1147 return false; 1148 } 1149 1150 int Src1ModIdx = 1151 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1_modifiers); 1152 int Src0ModIdx = 1153 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers); 1154 if ((Src1ModIdx != -1 && MI.getOperand(Src1ModIdx).getImm() != 0) || 1155 (Src0ModIdx != -1 && MI.getOperand(Src0ModIdx).getImm() != 0)) 1156 return false; 1157 1158 LLVM_DEBUG(dbgs() << "Folded " << MI << " into "); 1159 auto &NewDesc = 1160 TII->get(Src0->isReg() ? (unsigned)AMDGPU::COPY : getMovOpc(false)); 1161 int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); 1162 if (Src2Idx != -1) 1163 MI.removeOperand(Src2Idx); 1164 MI.removeOperand(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1)); 1165 if (Src1ModIdx != -1) 1166 MI.removeOperand(Src1ModIdx); 1167 if (Src0ModIdx != -1) 1168 MI.removeOperand(Src0ModIdx); 1169 mutateCopyOp(MI, NewDesc); 1170 LLVM_DEBUG(dbgs() << MI); 1171 return true; 1172 } 1173 1174 bool SIFoldOperands::tryFoldZeroHighBits(MachineInstr &MI) const { 1175 if (MI.getOpcode() != AMDGPU::V_AND_B32_e64 && 1176 MI.getOpcode() != AMDGPU::V_AND_B32_e32) 1177 return false; 1178 1179 MachineOperand *Src0 = getImmOrMaterializedImm(MI.getOperand(1)); 1180 if (!Src0->isImm() || Src0->getImm() != 0xffff) 1181 return false; 1182 1183 Register Src1 = MI.getOperand(2).getReg(); 1184 MachineInstr *SrcDef = MRI->getVRegDef(Src1); 1185 if (!ST->zeroesHigh16BitsOfDest(SrcDef->getOpcode())) 1186 return false; 1187 1188 Register Dst = MI.getOperand(0).getReg(); 1189 MRI->replaceRegWith(Dst, SrcDef->getOperand(0).getReg()); 1190 MI.eraseFromParent(); 1191 return true; 1192 } 1193 1194 bool SIFoldOperands::foldInstOperand(MachineInstr &MI, 1195 MachineOperand &OpToFold) const { 1196 // We need mutate the operands of new mov instructions to add implicit 1197 // uses of EXEC, but adding them invalidates the use_iterator, so defer 1198 // this. 1199 SmallVector<MachineInstr *, 4> CopiesToReplace; 1200 SmallVector<FoldCandidate, 4> FoldList; 1201 MachineOperand &Dst = MI.getOperand(0); 1202 bool Changed = false; 1203 1204 if (OpToFold.isImm()) { 1205 for (auto &UseMI : 1206 make_early_inc_range(MRI->use_nodbg_instructions(Dst.getReg()))) { 1207 // Folding the immediate may reveal operations that can be constant 1208 // folded or replaced with a copy. This can happen for example after 1209 // frame indices are lowered to constants or from splitting 64-bit 1210 // constants. 1211 // 1212 // We may also encounter cases where one or both operands are 1213 // immediates materialized into a register, which would ordinarily not 1214 // be folded due to multiple uses or operand constraints. 1215 if (tryConstantFoldOp(&UseMI)) { 1216 LLVM_DEBUG(dbgs() << "Constant folded " << UseMI); 1217 Changed = true; 1218 } 1219 } 1220 } 1221 1222 SmallVector<MachineOperand *, 4> UsesToProcess; 1223 for (auto &Use : MRI->use_nodbg_operands(Dst.getReg())) 1224 UsesToProcess.push_back(&Use); 1225 for (auto *U : UsesToProcess) { 1226 MachineInstr *UseMI = U->getParent(); 1227 foldOperand(OpToFold, UseMI, UseMI->getOperandNo(U), FoldList, 1228 CopiesToReplace); 1229 } 1230 1231 if (CopiesToReplace.empty() && FoldList.empty()) 1232 return Changed; 1233 1234 MachineFunction *MF = MI.getParent()->getParent(); 1235 // Make sure we add EXEC uses to any new v_mov instructions created. 1236 for (MachineInstr *Copy : CopiesToReplace) 1237 Copy->addImplicitDefUseOperands(*MF); 1238 1239 for (FoldCandidate &Fold : FoldList) { 1240 assert(!Fold.isReg() || Fold.OpToFold); 1241 if (Fold.isReg() && Fold.OpToFold->getReg().isVirtual()) { 1242 Register Reg = Fold.OpToFold->getReg(); 1243 MachineInstr *DefMI = Fold.OpToFold->getParent(); 1244 if (DefMI->readsRegister(AMDGPU::EXEC, TRI) && 1245 execMayBeModifiedBeforeUse(*MRI, Reg, *DefMI, *Fold.UseMI)) 1246 continue; 1247 } 1248 if (updateOperand(Fold)) { 1249 // Clear kill flags. 1250 if (Fold.isReg()) { 1251 assert(Fold.OpToFold && Fold.OpToFold->isReg()); 1252 // FIXME: Probably shouldn't bother trying to fold if not an 1253 // SGPR. PeepholeOptimizer can eliminate redundant VGPR->VGPR 1254 // copies. 1255 MRI->clearKillFlags(Fold.OpToFold->getReg()); 1256 } 1257 LLVM_DEBUG(dbgs() << "Folded source from " << MI << " into OpNo " 1258 << static_cast<int>(Fold.UseOpNo) << " of " 1259 << *Fold.UseMI); 1260 } else if (Fold.Commuted) { 1261 // Restoring instruction's original operand order if fold has failed. 1262 TII->commuteInstruction(*Fold.UseMI, false); 1263 } 1264 } 1265 return true; 1266 } 1267 1268 bool SIFoldOperands::tryFoldFoldableCopy( 1269 MachineInstr &MI, MachineOperand *&CurrentKnownM0Val) const { 1270 // Specially track simple redefs of m0 to the same value in a block, so we 1271 // can erase the later ones. 1272 if (MI.getOperand(0).getReg() == AMDGPU::M0) { 1273 MachineOperand &NewM0Val = MI.getOperand(1); 1274 if (CurrentKnownM0Val && CurrentKnownM0Val->isIdenticalTo(NewM0Val)) { 1275 MI.eraseFromParent(); 1276 return true; 1277 } 1278 1279 // We aren't tracking other physical registers 1280 CurrentKnownM0Val = (NewM0Val.isReg() && NewM0Val.getReg().isPhysical()) 1281 ? nullptr 1282 : &NewM0Val; 1283 return false; 1284 } 1285 1286 MachineOperand &OpToFold = MI.getOperand(1); 1287 bool FoldingImm = OpToFold.isImm() || OpToFold.isFI() || OpToFold.isGlobal(); 1288 1289 // FIXME: We could also be folding things like TargetIndexes. 1290 if (!FoldingImm && !OpToFold.isReg()) 1291 return false; 1292 1293 if (OpToFold.isReg() && !OpToFold.getReg().isVirtual()) 1294 return false; 1295 1296 // Prevent folding operands backwards in the function. For example, 1297 // the COPY opcode must not be replaced by 1 in this example: 1298 // 1299 // %3 = COPY %vgpr0; VGPR_32:%3 1300 // ... 1301 // %vgpr0 = V_MOV_B32_e32 1, implicit %exec 1302 if (!MI.getOperand(0).getReg().isVirtual()) 1303 return false; 1304 1305 bool Changed = foldInstOperand(MI, OpToFold); 1306 1307 // If we managed to fold all uses of this copy then we might as well 1308 // delete it now. 1309 // The only reason we need to follow chains of copies here is that 1310 // tryFoldRegSequence looks forward through copies before folding a 1311 // REG_SEQUENCE into its eventual users. 1312 auto *InstToErase = &MI; 1313 while (MRI->use_nodbg_empty(InstToErase->getOperand(0).getReg())) { 1314 auto &SrcOp = InstToErase->getOperand(1); 1315 auto SrcReg = SrcOp.isReg() ? SrcOp.getReg() : Register(); 1316 InstToErase->eraseFromParent(); 1317 Changed = true; 1318 InstToErase = nullptr; 1319 if (!SrcReg || SrcReg.isPhysical()) 1320 break; 1321 InstToErase = MRI->getVRegDef(SrcReg); 1322 if (!InstToErase || !TII->isFoldableCopy(*InstToErase)) 1323 break; 1324 } 1325 1326 if (InstToErase && InstToErase->isRegSequence() && 1327 MRI->use_nodbg_empty(InstToErase->getOperand(0).getReg())) { 1328 InstToErase->eraseFromParent(); 1329 Changed = true; 1330 } 1331 1332 return Changed; 1333 } 1334 1335 // Clamp patterns are canonically selected to v_max_* instructions, so only 1336 // handle them. 1337 const MachineOperand *SIFoldOperands::isClamp(const MachineInstr &MI) const { 1338 unsigned Op = MI.getOpcode(); 1339 switch (Op) { 1340 case AMDGPU::V_MAX_F32_e64: 1341 case AMDGPU::V_MAX_F16_e64: 1342 case AMDGPU::V_MAX_F16_t16_e64: 1343 case AMDGPU::V_MAX_F64_e64: 1344 case AMDGPU::V_PK_MAX_F16: { 1345 if (!TII->getNamedOperand(MI, AMDGPU::OpName::clamp)->getImm()) 1346 return nullptr; 1347 1348 // Make sure sources are identical. 1349 const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); 1350 const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); 1351 if (!Src0->isReg() || !Src1->isReg() || 1352 Src0->getReg() != Src1->getReg() || 1353 Src0->getSubReg() != Src1->getSubReg() || 1354 Src0->getSubReg() != AMDGPU::NoSubRegister) 1355 return nullptr; 1356 1357 // Can't fold up if we have modifiers. 1358 if (TII->hasModifiersSet(MI, AMDGPU::OpName::omod)) 1359 return nullptr; 1360 1361 unsigned Src0Mods 1362 = TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm(); 1363 unsigned Src1Mods 1364 = TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm(); 1365 1366 // Having a 0 op_sel_hi would require swizzling the output in the source 1367 // instruction, which we can't do. 1368 unsigned UnsetMods = (Op == AMDGPU::V_PK_MAX_F16) ? SISrcMods::OP_SEL_1 1369 : 0u; 1370 if (Src0Mods != UnsetMods && Src1Mods != UnsetMods) 1371 return nullptr; 1372 return Src0; 1373 } 1374 default: 1375 return nullptr; 1376 } 1377 } 1378 1379 // FIXME: Clamp for v_mad_mixhi_f16 handled during isel. 1380 bool SIFoldOperands::tryFoldClamp(MachineInstr &MI) { 1381 const MachineOperand *ClampSrc = isClamp(MI); 1382 if (!ClampSrc || !MRI->hasOneNonDBGUser(ClampSrc->getReg())) 1383 return false; 1384 1385 MachineInstr *Def = MRI->getVRegDef(ClampSrc->getReg()); 1386 1387 // The type of clamp must be compatible. 1388 if (TII->getClampMask(*Def) != TII->getClampMask(MI)) 1389 return false; 1390 1391 MachineOperand *DefClamp = TII->getNamedOperand(*Def, AMDGPU::OpName::clamp); 1392 if (!DefClamp) 1393 return false; 1394 1395 LLVM_DEBUG(dbgs() << "Folding clamp " << *DefClamp << " into " << *Def); 1396 1397 // Clamp is applied after omod, so it is OK if omod is set. 1398 DefClamp->setImm(1); 1399 MRI->replaceRegWith(MI.getOperand(0).getReg(), Def->getOperand(0).getReg()); 1400 MI.eraseFromParent(); 1401 1402 // Use of output modifiers forces VOP3 encoding for a VOP2 mac/fmac 1403 // instruction, so we might as well convert it to the more flexible VOP3-only 1404 // mad/fma form. 1405 if (TII->convertToThreeAddress(*Def, nullptr, nullptr)) 1406 Def->eraseFromParent(); 1407 1408 return true; 1409 } 1410 1411 static int getOModValue(unsigned Opc, int64_t Val) { 1412 switch (Opc) { 1413 case AMDGPU::V_MUL_F64_e64: { 1414 switch (Val) { 1415 case 0x3fe0000000000000: // 0.5 1416 return SIOutMods::DIV2; 1417 case 0x4000000000000000: // 2.0 1418 return SIOutMods::MUL2; 1419 case 0x4010000000000000: // 4.0 1420 return SIOutMods::MUL4; 1421 default: 1422 return SIOutMods::NONE; 1423 } 1424 } 1425 case AMDGPU::V_MUL_F32_e64: { 1426 switch (static_cast<uint32_t>(Val)) { 1427 case 0x3f000000: // 0.5 1428 return SIOutMods::DIV2; 1429 case 0x40000000: // 2.0 1430 return SIOutMods::MUL2; 1431 case 0x40800000: // 4.0 1432 return SIOutMods::MUL4; 1433 default: 1434 return SIOutMods::NONE; 1435 } 1436 } 1437 case AMDGPU::V_MUL_F16_e64: 1438 case AMDGPU::V_MUL_F16_t16_e64: { 1439 switch (static_cast<uint16_t>(Val)) { 1440 case 0x3800: // 0.5 1441 return SIOutMods::DIV2; 1442 case 0x4000: // 2.0 1443 return SIOutMods::MUL2; 1444 case 0x4400: // 4.0 1445 return SIOutMods::MUL4; 1446 default: 1447 return SIOutMods::NONE; 1448 } 1449 } 1450 default: 1451 llvm_unreachable("invalid mul opcode"); 1452 } 1453 } 1454 1455 // FIXME: Does this really not support denormals with f16? 1456 // FIXME: Does this need to check IEEE mode bit? SNaNs are generally not 1457 // handled, so will anything other than that break? 1458 std::pair<const MachineOperand *, int> 1459 SIFoldOperands::isOMod(const MachineInstr &MI) const { 1460 unsigned Op = MI.getOpcode(); 1461 switch (Op) { 1462 case AMDGPU::V_MUL_F64_e64: 1463 case AMDGPU::V_MUL_F32_e64: 1464 case AMDGPU::V_MUL_F16_t16_e64: 1465 case AMDGPU::V_MUL_F16_e64: { 1466 // If output denormals are enabled, omod is ignored. 1467 if ((Op == AMDGPU::V_MUL_F32_e64 && 1468 MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign) || 1469 ((Op == AMDGPU::V_MUL_F64_e64 || Op == AMDGPU::V_MUL_F16_e64 || 1470 Op == AMDGPU::V_MUL_F16_t16_e64) && 1471 MFI->getMode().FP64FP16Denormals.Output != DenormalMode::PreserveSign)) 1472 return std::pair(nullptr, SIOutMods::NONE); 1473 1474 const MachineOperand *RegOp = nullptr; 1475 const MachineOperand *ImmOp = nullptr; 1476 const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); 1477 const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); 1478 if (Src0->isImm()) { 1479 ImmOp = Src0; 1480 RegOp = Src1; 1481 } else if (Src1->isImm()) { 1482 ImmOp = Src1; 1483 RegOp = Src0; 1484 } else 1485 return std::pair(nullptr, SIOutMods::NONE); 1486 1487 int OMod = getOModValue(Op, ImmOp->getImm()); 1488 if (OMod == SIOutMods::NONE || 1489 TII->hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) || 1490 TII->hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) || 1491 TII->hasModifiersSet(MI, AMDGPU::OpName::omod) || 1492 TII->hasModifiersSet(MI, AMDGPU::OpName::clamp)) 1493 return std::pair(nullptr, SIOutMods::NONE); 1494 1495 return std::pair(RegOp, OMod); 1496 } 1497 case AMDGPU::V_ADD_F64_e64: 1498 case AMDGPU::V_ADD_F32_e64: 1499 case AMDGPU::V_ADD_F16_e64: 1500 case AMDGPU::V_ADD_F16_t16_e64: { 1501 // If output denormals are enabled, omod is ignored. 1502 if ((Op == AMDGPU::V_ADD_F32_e64 && 1503 MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign) || 1504 ((Op == AMDGPU::V_ADD_F64_e64 || Op == AMDGPU::V_ADD_F16_e64 || 1505 Op == AMDGPU::V_ADD_F16_t16_e64) && 1506 MFI->getMode().FP64FP16Denormals.Output != DenormalMode::PreserveSign)) 1507 return std::pair(nullptr, SIOutMods::NONE); 1508 1509 // Look through the DAGCombiner canonicalization fmul x, 2 -> fadd x, x 1510 const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0); 1511 const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1); 1512 1513 if (Src0->isReg() && Src1->isReg() && Src0->getReg() == Src1->getReg() && 1514 Src0->getSubReg() == Src1->getSubReg() && 1515 !TII->hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) && 1516 !TII->hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) && 1517 !TII->hasModifiersSet(MI, AMDGPU::OpName::clamp) && 1518 !TII->hasModifiersSet(MI, AMDGPU::OpName::omod)) 1519 return std::pair(Src0, SIOutMods::MUL2); 1520 1521 return std::pair(nullptr, SIOutMods::NONE); 1522 } 1523 default: 1524 return std::pair(nullptr, SIOutMods::NONE); 1525 } 1526 } 1527 1528 // FIXME: Does this need to check IEEE bit on function? 1529 bool SIFoldOperands::tryFoldOMod(MachineInstr &MI) { 1530 const MachineOperand *RegOp; 1531 int OMod; 1532 std::tie(RegOp, OMod) = isOMod(MI); 1533 if (OMod == SIOutMods::NONE || !RegOp->isReg() || 1534 RegOp->getSubReg() != AMDGPU::NoSubRegister || 1535 !MRI->hasOneNonDBGUser(RegOp->getReg())) 1536 return false; 1537 1538 MachineInstr *Def = MRI->getVRegDef(RegOp->getReg()); 1539 MachineOperand *DefOMod = TII->getNamedOperand(*Def, AMDGPU::OpName::omod); 1540 if (!DefOMod || DefOMod->getImm() != SIOutMods::NONE) 1541 return false; 1542 1543 // Clamp is applied after omod. If the source already has clamp set, don't 1544 // fold it. 1545 if (TII->hasModifiersSet(*Def, AMDGPU::OpName::clamp)) 1546 return false; 1547 1548 LLVM_DEBUG(dbgs() << "Folding omod " << MI << " into " << *Def); 1549 1550 DefOMod->setImm(OMod); 1551 MRI->replaceRegWith(MI.getOperand(0).getReg(), Def->getOperand(0).getReg()); 1552 MI.eraseFromParent(); 1553 1554 // Use of output modifiers forces VOP3 encoding for a VOP2 mac/fmac 1555 // instruction, so we might as well convert it to the more flexible VOP3-only 1556 // mad/fma form. 1557 if (TII->convertToThreeAddress(*Def, nullptr, nullptr)) 1558 Def->eraseFromParent(); 1559 1560 return true; 1561 } 1562 1563 // Try to fold a reg_sequence with vgpr output and agpr inputs into an 1564 // instruction which can take an agpr. So far that means a store. 1565 bool SIFoldOperands::tryFoldRegSequence(MachineInstr &MI) { 1566 assert(MI.isRegSequence()); 1567 auto Reg = MI.getOperand(0).getReg(); 1568 1569 if (!ST->hasGFX90AInsts() || !TRI->isVGPR(*MRI, Reg) || 1570 !MRI->hasOneNonDBGUse(Reg)) 1571 return false; 1572 1573 SmallVector<std::pair<MachineOperand*, unsigned>, 32> Defs; 1574 if (!getRegSeqInit(Defs, Reg, MCOI::OPERAND_REGISTER)) 1575 return false; 1576 1577 for (auto &Def : Defs) { 1578 const auto *Op = Def.first; 1579 if (!Op->isReg()) 1580 return false; 1581 if (TRI->isAGPR(*MRI, Op->getReg())) 1582 continue; 1583 // Maybe this is a COPY from AREG 1584 const MachineInstr *SubDef = MRI->getVRegDef(Op->getReg()); 1585 if (!SubDef || !SubDef->isCopy() || SubDef->getOperand(1).getSubReg()) 1586 return false; 1587 if (!TRI->isAGPR(*MRI, SubDef->getOperand(1).getReg())) 1588 return false; 1589 } 1590 1591 MachineOperand *Op = &*MRI->use_nodbg_begin(Reg); 1592 MachineInstr *UseMI = Op->getParent(); 1593 while (UseMI->isCopy() && !Op->getSubReg()) { 1594 Reg = UseMI->getOperand(0).getReg(); 1595 if (!TRI->isVGPR(*MRI, Reg) || !MRI->hasOneNonDBGUse(Reg)) 1596 return false; 1597 Op = &*MRI->use_nodbg_begin(Reg); 1598 UseMI = Op->getParent(); 1599 } 1600 1601 if (Op->getSubReg()) 1602 return false; 1603 1604 unsigned OpIdx = Op - &UseMI->getOperand(0); 1605 const MCInstrDesc &InstDesc = UseMI->getDesc(); 1606 const TargetRegisterClass *OpRC = 1607 TII->getRegClass(InstDesc, OpIdx, TRI, *MI.getMF()); 1608 if (!OpRC || !TRI->isVectorSuperClass(OpRC)) 1609 return false; 1610 1611 const auto *NewDstRC = TRI->getEquivalentAGPRClass(MRI->getRegClass(Reg)); 1612 auto Dst = MRI->createVirtualRegister(NewDstRC); 1613 auto RS = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), 1614 TII->get(AMDGPU::REG_SEQUENCE), Dst); 1615 1616 for (unsigned I = 0; I < Defs.size(); ++I) { 1617 MachineOperand *Def = Defs[I].first; 1618 Def->setIsKill(false); 1619 if (TRI->isAGPR(*MRI, Def->getReg())) { 1620 RS.add(*Def); 1621 } else { // This is a copy 1622 MachineInstr *SubDef = MRI->getVRegDef(Def->getReg()); 1623 SubDef->getOperand(1).setIsKill(false); 1624 RS.addReg(SubDef->getOperand(1).getReg(), 0, Def->getSubReg()); 1625 } 1626 RS.addImm(Defs[I].second); 1627 } 1628 1629 Op->setReg(Dst); 1630 if (!TII->isOperandLegal(*UseMI, OpIdx, Op)) { 1631 Op->setReg(Reg); 1632 RS->eraseFromParent(); 1633 return false; 1634 } 1635 1636 LLVM_DEBUG(dbgs() << "Folded " << *RS << " into " << *UseMI); 1637 1638 // Erase the REG_SEQUENCE eagerly, unless we followed a chain of COPY users, 1639 // in which case we can erase them all later in runOnMachineFunction. 1640 if (MRI->use_nodbg_empty(MI.getOperand(0).getReg())) 1641 MI.eraseFromParent(); 1642 return true; 1643 } 1644 1645 /// Checks whether \p Copy is a AGPR -> VGPR copy. Returns `true` on success and 1646 /// stores the AGPR register in \p OutReg and the subreg in \p OutSubReg 1647 static bool isAGPRCopy(const SIRegisterInfo &TRI, 1648 const MachineRegisterInfo &MRI, const MachineInstr &Copy, 1649 Register &OutReg, unsigned &OutSubReg) { 1650 assert(Copy.isCopy()); 1651 1652 const MachineOperand &CopySrc = Copy.getOperand(1); 1653 Register CopySrcReg = CopySrc.getReg(); 1654 if (!CopySrcReg.isVirtual()) 1655 return false; 1656 1657 // Common case: copy from AGPR directly, e.g. 1658 // %1:vgpr_32 = COPY %0:agpr_32 1659 if (TRI.isAGPR(MRI, CopySrcReg)) { 1660 OutReg = CopySrcReg; 1661 OutSubReg = CopySrc.getSubReg(); 1662 return true; 1663 } 1664 1665 // Sometimes it can also involve two copies, e.g. 1666 // %1:vgpr_256 = COPY %0:agpr_256 1667 // %2:vgpr_32 = COPY %1:vgpr_256.sub0 1668 const MachineInstr *CopySrcDef = MRI.getVRegDef(CopySrcReg); 1669 if (!CopySrcDef || !CopySrcDef->isCopy()) 1670 return false; 1671 1672 const MachineOperand &OtherCopySrc = CopySrcDef->getOperand(1); 1673 Register OtherCopySrcReg = OtherCopySrc.getReg(); 1674 if (!OtherCopySrcReg.isVirtual() || 1675 CopySrcDef->getOperand(0).getSubReg() != AMDGPU::NoSubRegister || 1676 OtherCopySrc.getSubReg() != AMDGPU::NoSubRegister || 1677 !TRI.isAGPR(MRI, OtherCopySrcReg)) 1678 return false; 1679 1680 OutReg = OtherCopySrcReg; 1681 OutSubReg = CopySrc.getSubReg(); 1682 return true; 1683 } 1684 1685 // Try to hoist an AGPR to VGPR copy across a PHI. 1686 // This should allow folding of an AGPR into a consumer which may support it. 1687 // 1688 // Example 1: LCSSA PHI 1689 // loop: 1690 // %1:vreg = COPY %0:areg 1691 // exit: 1692 // %2:vreg = PHI %1:vreg, %loop 1693 // => 1694 // loop: 1695 // exit: 1696 // %1:areg = PHI %0:areg, %loop 1697 // %2:vreg = COPY %1:areg 1698 // 1699 // Example 2: PHI with multiple incoming values: 1700 // entry: 1701 // %1:vreg = GLOBAL_LOAD(..) 1702 // loop: 1703 // %2:vreg = PHI %1:vreg, %entry, %5:vreg, %loop 1704 // %3:areg = COPY %2:vreg 1705 // %4:areg = (instr using %3:areg) 1706 // %5:vreg = COPY %4:areg 1707 // => 1708 // entry: 1709 // %1:vreg = GLOBAL_LOAD(..) 1710 // %2:areg = COPY %1:vreg 1711 // loop: 1712 // %3:areg = PHI %2:areg, %entry, %X:areg, 1713 // %4:areg = (instr using %3:areg) 1714 bool SIFoldOperands::tryFoldPhiAGPR(MachineInstr &PHI) { 1715 assert(PHI.isPHI()); 1716 1717 Register PhiOut = PHI.getOperand(0).getReg(); 1718 if (!TRI->isVGPR(*MRI, PhiOut)) 1719 return false; 1720 1721 // Iterate once over all incoming values of the PHI to check if this PHI is 1722 // eligible, and determine the exact AGPR RC we'll target. 1723 const TargetRegisterClass *ARC = nullptr; 1724 for (unsigned K = 1; K < PHI.getNumExplicitOperands(); K += 2) { 1725 MachineOperand &MO = PHI.getOperand(K); 1726 MachineInstr *Copy = MRI->getVRegDef(MO.getReg()); 1727 if (!Copy || !Copy->isCopy()) 1728 continue; 1729 1730 Register AGPRSrc; 1731 unsigned AGPRRegMask = AMDGPU::NoSubRegister; 1732 if (!isAGPRCopy(*TRI, *MRI, *Copy, AGPRSrc, AGPRRegMask)) 1733 continue; 1734 1735 const TargetRegisterClass *CopyInRC = MRI->getRegClass(AGPRSrc); 1736 if (const auto *SubRC = TRI->getSubRegisterClass(CopyInRC, AGPRRegMask)) 1737 CopyInRC = SubRC; 1738 1739 if (ARC && !ARC->hasSubClassEq(CopyInRC)) 1740 return false; 1741 ARC = CopyInRC; 1742 } 1743 1744 if (!ARC) 1745 return false; 1746 1747 bool IsAGPR32 = (ARC == &AMDGPU::AGPR_32RegClass); 1748 1749 // Rewrite the PHI's incoming values to ARC. 1750 LLVM_DEBUG(dbgs() << "Folding AGPR copies into: " << PHI); 1751 for (unsigned K = 1; K < PHI.getNumExplicitOperands(); K += 2) { 1752 MachineOperand &MO = PHI.getOperand(K); 1753 Register Reg = MO.getReg(); 1754 1755 MachineBasicBlock::iterator InsertPt; 1756 MachineBasicBlock *InsertMBB = nullptr; 1757 1758 // Look at the def of Reg, ignoring all copies. 1759 unsigned CopyOpc = AMDGPU::COPY; 1760 if (MachineInstr *Def = MRI->getVRegDef(Reg)) { 1761 1762 // Look at pre-existing COPY instructions from ARC: Steal the operand. If 1763 // the copy was single-use, it will be removed by DCE later. 1764 if (Def->isCopy()) { 1765 Register AGPRSrc; 1766 unsigned AGPRSubReg = AMDGPU::NoSubRegister; 1767 if (isAGPRCopy(*TRI, *MRI, *Def, AGPRSrc, AGPRSubReg)) { 1768 MO.setReg(AGPRSrc); 1769 MO.setSubReg(AGPRSubReg); 1770 continue; 1771 } 1772 1773 // If this is a multi-use SGPR -> VGPR copy, use V_ACCVGPR_WRITE on 1774 // GFX908 directly instead of a COPY. Otherwise, SIFoldOperand may try 1775 // to fold the sgpr -> vgpr -> agpr copy into a sgpr -> agpr copy which 1776 // is unlikely to be profitable. 1777 // 1778 // Note that V_ACCVGPR_WRITE is only used for AGPR_32. 1779 MachineOperand &CopyIn = Def->getOperand(1); 1780 if (IsAGPR32 && !ST->hasGFX90AInsts() && !MRI->hasOneNonDBGUse(Reg) && 1781 TRI->isSGPRReg(*MRI, CopyIn.getReg())) 1782 CopyOpc = AMDGPU::V_ACCVGPR_WRITE_B32_e64; 1783 } 1784 1785 InsertMBB = Def->getParent(); 1786 InsertPt = InsertMBB->SkipPHIsLabelsAndDebug(++Def->getIterator()); 1787 } else { 1788 InsertMBB = PHI.getOperand(MO.getOperandNo() + 1).getMBB(); 1789 InsertPt = InsertMBB->getFirstTerminator(); 1790 } 1791 1792 Register NewReg = MRI->createVirtualRegister(ARC); 1793 MachineInstr *MI = BuildMI(*InsertMBB, InsertPt, PHI.getDebugLoc(), 1794 TII->get(CopyOpc), NewReg) 1795 .addReg(Reg); 1796 MO.setReg(NewReg); 1797 1798 (void)MI; 1799 LLVM_DEBUG(dbgs() << " Created COPY: " << *MI); 1800 } 1801 1802 // Replace the PHI's result with a new register. 1803 Register NewReg = MRI->createVirtualRegister(ARC); 1804 PHI.getOperand(0).setReg(NewReg); 1805 1806 // COPY that new register back to the original PhiOut register. This COPY will 1807 // usually be folded out later. 1808 MachineBasicBlock *MBB = PHI.getParent(); 1809 BuildMI(*MBB, MBB->getFirstNonPHI(), PHI.getDebugLoc(), 1810 TII->get(AMDGPU::COPY), PhiOut) 1811 .addReg(NewReg); 1812 1813 LLVM_DEBUG(dbgs() << " Done: Folded " << PHI); 1814 return true; 1815 } 1816 1817 // Attempt to convert VGPR load to an AGPR load. 1818 bool SIFoldOperands::tryFoldLoad(MachineInstr &MI) { 1819 assert(MI.mayLoad()); 1820 if (!ST->hasGFX90AInsts() || MI.getNumExplicitDefs() != 1) 1821 return false; 1822 1823 MachineOperand &Def = MI.getOperand(0); 1824 if (!Def.isDef()) 1825 return false; 1826 1827 Register DefReg = Def.getReg(); 1828 1829 if (DefReg.isPhysical() || !TRI->isVGPR(*MRI, DefReg)) 1830 return false; 1831 1832 SmallVector<const MachineInstr*, 8> Users; 1833 SmallVector<Register, 8> MoveRegs; 1834 for (const MachineInstr &I : MRI->use_nodbg_instructions(DefReg)) 1835 Users.push_back(&I); 1836 1837 if (Users.empty()) 1838 return false; 1839 1840 // Check that all uses a copy to an agpr or a reg_sequence producing an agpr. 1841 while (!Users.empty()) { 1842 const MachineInstr *I = Users.pop_back_val(); 1843 if (!I->isCopy() && !I->isRegSequence()) 1844 return false; 1845 Register DstReg = I->getOperand(0).getReg(); 1846 // Physical registers may have more than one instruction definitions 1847 if (DstReg.isPhysical()) 1848 return false; 1849 if (TRI->isAGPR(*MRI, DstReg)) 1850 continue; 1851 MoveRegs.push_back(DstReg); 1852 for (const MachineInstr &U : MRI->use_nodbg_instructions(DstReg)) 1853 Users.push_back(&U); 1854 } 1855 1856 const TargetRegisterClass *RC = MRI->getRegClass(DefReg); 1857 MRI->setRegClass(DefReg, TRI->getEquivalentAGPRClass(RC)); 1858 if (!TII->isOperandLegal(MI, 0, &Def)) { 1859 MRI->setRegClass(DefReg, RC); 1860 return false; 1861 } 1862 1863 while (!MoveRegs.empty()) { 1864 Register Reg = MoveRegs.pop_back_val(); 1865 MRI->setRegClass(Reg, TRI->getEquivalentAGPRClass(MRI->getRegClass(Reg))); 1866 } 1867 1868 LLVM_DEBUG(dbgs() << "Folded " << MI); 1869 1870 return true; 1871 } 1872 1873 // tryFoldPhiAGPR will aggressively try to create AGPR PHIs. 1874 // For GFX90A and later, this is pretty much always a good thing, but for GFX908 1875 // there's cases where it can create a lot more AGPR-AGPR copies, which are 1876 // expensive on this architecture due to the lack of V_ACCVGPR_MOV. 1877 // 1878 // This function looks at all AGPR PHIs in a basic block and collects their 1879 // operands. Then, it checks for register that are used more than once across 1880 // all PHIs and caches them in a VGPR. This prevents ExpandPostRAPseudo from 1881 // having to create one VGPR temporary per use, which can get very messy if 1882 // these PHIs come from a broken-up large PHI (e.g. 32 AGPR phis, one per vector 1883 // element). 1884 // 1885 // Example 1886 // a: 1887 // %in:agpr_256 = COPY %foo:vgpr_256 1888 // c: 1889 // %x:agpr_32 = .. 1890 // b: 1891 // %0:areg = PHI %in.sub0:agpr_32, %a, %x, %c 1892 // %1:areg = PHI %in.sub0:agpr_32, %a, %y, %c 1893 // %2:areg = PHI %in.sub0:agpr_32, %a, %z, %c 1894 // => 1895 // a: 1896 // %in:agpr_256 = COPY %foo:vgpr_256 1897 // %tmp:vgpr_32 = V_ACCVGPR_READ_B32_e64 %in.sub0:agpr_32 1898 // %tmp_agpr:agpr_32 = COPY %tmp 1899 // c: 1900 // %x:agpr_32 = .. 1901 // b: 1902 // %0:areg = PHI %tmp_agpr, %a, %x, %c 1903 // %1:areg = PHI %tmp_agpr, %a, %y, %c 1904 // %2:areg = PHI %tmp_agpr, %a, %z, %c 1905 bool SIFoldOperands::tryOptimizeAGPRPhis(MachineBasicBlock &MBB) { 1906 // This is only really needed on GFX908 where AGPR-AGPR copies are 1907 // unreasonably difficult. 1908 if (ST->hasGFX90AInsts()) 1909 return false; 1910 1911 // Look at all AGPR Phis and collect the register + subregister used. 1912 DenseMap<std::pair<Register, unsigned>, std::vector<MachineOperand *>> 1913 RegToMO; 1914 1915 for (auto &MI : MBB) { 1916 if (!MI.isPHI()) 1917 break; 1918 1919 if (!TRI->isAGPR(*MRI, MI.getOperand(0).getReg())) 1920 continue; 1921 1922 for (unsigned K = 1; K < MI.getNumOperands(); K += 2) { 1923 MachineOperand &PhiMO = MI.getOperand(K); 1924 RegToMO[{PhiMO.getReg(), PhiMO.getSubReg()}].push_back(&PhiMO); 1925 } 1926 } 1927 1928 // For all (Reg, SubReg) pair that are used more than once, cache the value in 1929 // a VGPR. 1930 bool Changed = false; 1931 for (const auto &[Entry, MOs] : RegToMO) { 1932 if (MOs.size() == 1) 1933 continue; 1934 1935 const auto [Reg, SubReg] = Entry; 1936 MachineInstr *Def = MRI->getVRegDef(Reg); 1937 MachineBasicBlock *DefMBB = Def->getParent(); 1938 1939 // Create a copy in a VGPR using V_ACCVGPR_READ_B32_e64 so it's not folded 1940 // out. 1941 const TargetRegisterClass *ARC = getRegOpRC(*MRI, *TRI, *MOs.front()); 1942 Register TempVGPR = 1943 MRI->createVirtualRegister(TRI->getEquivalentVGPRClass(ARC)); 1944 MachineInstr *VGPRCopy = 1945 BuildMI(*DefMBB, ++Def->getIterator(), Def->getDebugLoc(), 1946 TII->get(AMDGPU::V_ACCVGPR_READ_B32_e64), TempVGPR) 1947 .addReg(Reg, /* flags */ 0, SubReg); 1948 1949 // Copy back to an AGPR and use that instead of the AGPR subreg in all MOs. 1950 Register TempAGPR = MRI->createVirtualRegister(ARC); 1951 BuildMI(*DefMBB, ++VGPRCopy->getIterator(), Def->getDebugLoc(), 1952 TII->get(AMDGPU::COPY), TempAGPR) 1953 .addReg(TempVGPR); 1954 1955 LLVM_DEBUG(dbgs() << "Caching AGPR into VGPR: " << *VGPRCopy); 1956 for (MachineOperand *MO : MOs) { 1957 MO->setReg(TempAGPR); 1958 MO->setSubReg(AMDGPU::NoSubRegister); 1959 LLVM_DEBUG(dbgs() << " Changed PHI Operand: " << *MO << "\n"); 1960 } 1961 1962 Changed = true; 1963 } 1964 1965 return Changed; 1966 } 1967 1968 bool SIFoldOperands::runOnMachineFunction(MachineFunction &MF) { 1969 if (skipFunction(MF.getFunction())) 1970 return false; 1971 1972 MRI = &MF.getRegInfo(); 1973 ST = &MF.getSubtarget<GCNSubtarget>(); 1974 TII = ST->getInstrInfo(); 1975 TRI = &TII->getRegisterInfo(); 1976 MFI = MF.getInfo<SIMachineFunctionInfo>(); 1977 1978 // omod is ignored by hardware if IEEE bit is enabled. omod also does not 1979 // correctly handle signed zeros. 1980 // 1981 // FIXME: Also need to check strictfp 1982 bool IsIEEEMode = MFI->getMode().IEEE; 1983 bool HasNSZ = MFI->hasNoSignedZerosFPMath(); 1984 1985 bool Changed = false; 1986 for (MachineBasicBlock *MBB : depth_first(&MF)) { 1987 MachineOperand *CurrentKnownM0Val = nullptr; 1988 for (auto &MI : make_early_inc_range(*MBB)) { 1989 Changed |= tryFoldCndMask(MI); 1990 1991 if (tryFoldZeroHighBits(MI)) { 1992 Changed = true; 1993 continue; 1994 } 1995 1996 if (MI.isRegSequence() && tryFoldRegSequence(MI)) { 1997 Changed = true; 1998 continue; 1999 } 2000 2001 if (MI.isPHI() && tryFoldPhiAGPR(MI)) { 2002 Changed = true; 2003 continue; 2004 } 2005 2006 if (MI.mayLoad() && tryFoldLoad(MI)) { 2007 Changed = true; 2008 continue; 2009 } 2010 2011 if (TII->isFoldableCopy(MI)) { 2012 Changed |= tryFoldFoldableCopy(MI, CurrentKnownM0Val); 2013 continue; 2014 } 2015 2016 // Saw an unknown clobber of m0, so we no longer know what it is. 2017 if (CurrentKnownM0Val && MI.modifiesRegister(AMDGPU::M0, TRI)) 2018 CurrentKnownM0Val = nullptr; 2019 2020 // TODO: Omod might be OK if there is NSZ only on the source 2021 // instruction, and not the omod multiply. 2022 if (IsIEEEMode || (!HasNSZ && !MI.getFlag(MachineInstr::FmNsz)) || 2023 !tryFoldOMod(MI)) 2024 Changed |= tryFoldClamp(MI); 2025 } 2026 2027 Changed |= tryOptimizeAGPRPhis(*MBB); 2028 } 2029 2030 return Changed; 2031 } 2032