1 //=======- GCNDPPCombine.cpp - optimization for DPP instructions ---==========// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // The pass combines V_MOV_B32_dpp instruction with its VALU uses as a DPP src0 9 // operand. If any of the use instruction cannot be combined with the mov the 10 // whole sequence is reverted. 11 // 12 // $old = ... 13 // $dpp_value = V_MOV_B32_dpp $old, $vgpr_to_be_read_from_other_lane, 14 // dpp_controls..., $row_mask, $bank_mask, $bound_ctrl 15 // $res = VALU $dpp_value [, src1] 16 // 17 // to 18 // 19 // $res = VALU_DPP $combined_old, $vgpr_to_be_read_from_other_lane, [src1,] 20 // dpp_controls..., $row_mask, $bank_mask, $combined_bound_ctrl 21 // 22 // Combining rules : 23 // 24 // if $row_mask and $bank_mask are fully enabled (0xF) and 25 // $bound_ctrl==DPP_BOUND_ZERO or $old==0 26 // -> $combined_old = undef, 27 // $combined_bound_ctrl = DPP_BOUND_ZERO 28 // 29 // if the VALU op is binary and 30 // $bound_ctrl==DPP_BOUND_OFF and 31 // $old==identity value (immediate) for the VALU op 32 // -> $combined_old = src1, 33 // $combined_bound_ctrl = DPP_BOUND_OFF 34 // 35 // Otherwise cancel. 36 // 37 // The mov_dpp instruction should reside in the same BB as all its uses 38 //===----------------------------------------------------------------------===// 39 40 #include "AMDGPU.h" 41 #include "GCNSubtarget.h" 42 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 43 #include "llvm/ADT/Statistic.h" 44 #include "llvm/CodeGen/MachineFunctionPass.h" 45 46 using namespace llvm; 47 48 #define DEBUG_TYPE "gcn-dpp-combine" 49 50 STATISTIC(NumDPPMovsCombined, "Number of DPP moves combined."); 51 52 namespace { 53 54 class GCNDPPCombine : public MachineFunctionPass { 55 MachineRegisterInfo *MRI; 56 const SIInstrInfo *TII; 57 const GCNSubtarget *ST; 58 59 using RegSubRegPair = TargetInstrInfo::RegSubRegPair; 60 61 MachineOperand *getOldOpndValue(MachineOperand &OldOpnd) const; 62 63 MachineInstr *createDPPInst(MachineInstr &OrigMI, MachineInstr &MovMI, 64 RegSubRegPair CombOldVGPR, 65 MachineOperand *OldOpnd, bool CombBCZ, 66 bool IsShrinkable) const; 67 68 MachineInstr *createDPPInst(MachineInstr &OrigMI, MachineInstr &MovMI, 69 RegSubRegPair CombOldVGPR, bool CombBCZ, 70 bool IsShrinkable) const; 71 72 bool hasNoImmOrEqual(MachineInstr &MI, 73 unsigned OpndName, 74 int64_t Value, 75 int64_t Mask = -1) const; 76 77 bool combineDPPMov(MachineInstr &MI) const; 78 79 public: 80 static char ID; 81 82 GCNDPPCombine() : MachineFunctionPass(ID) { 83 initializeGCNDPPCombinePass(*PassRegistry::getPassRegistry()); 84 } 85 86 bool runOnMachineFunction(MachineFunction &MF) override; 87 88 StringRef getPassName() const override { return "GCN DPP Combine"; } 89 90 void getAnalysisUsage(AnalysisUsage &AU) const override { 91 AU.setPreservesCFG(); 92 MachineFunctionPass::getAnalysisUsage(AU); 93 } 94 95 MachineFunctionProperties getRequiredProperties() const override { 96 return MachineFunctionProperties() 97 .set(MachineFunctionProperties::Property::IsSSA); 98 } 99 100 private: 101 int getDPPOp(unsigned Op, bool IsShrinkable) const; 102 bool isShrinkable(MachineInstr &MI) const; 103 }; 104 105 } // end anonymous namespace 106 107 INITIALIZE_PASS(GCNDPPCombine, DEBUG_TYPE, "GCN DPP Combine", false, false) 108 109 char GCNDPPCombine::ID = 0; 110 111 char &llvm::GCNDPPCombineID = GCNDPPCombine::ID; 112 113 FunctionPass *llvm::createGCNDPPCombinePass() { 114 return new GCNDPPCombine(); 115 } 116 117 bool GCNDPPCombine::isShrinkable(MachineInstr &MI) const { 118 unsigned Op = MI.getOpcode(); 119 if (!TII->isVOP3(Op)) { 120 return false; 121 } 122 if (!TII->hasVALU32BitEncoding(Op)) { 123 LLVM_DEBUG(dbgs() << " Inst hasn't e32 equivalent\n"); 124 return false; 125 } 126 if (const auto *SDst = TII->getNamedOperand(MI, AMDGPU::OpName::sdst)) { 127 // Give up if there are any uses of the carry-out from instructions like 128 // V_ADD_CO_U32. The shrunken form of the instruction would write it to vcc 129 // instead of to a virtual register. 130 if (!MRI->use_nodbg_empty(SDst->getReg())) 131 return false; 132 } 133 // check if other than abs|neg modifiers are set (opsel for example) 134 const int64_t Mask = ~(SISrcMods::ABS | SISrcMods::NEG); 135 if (!hasNoImmOrEqual(MI, AMDGPU::OpName::src0_modifiers, 0, Mask) || 136 !hasNoImmOrEqual(MI, AMDGPU::OpName::src1_modifiers, 0, Mask) || 137 !hasNoImmOrEqual(MI, AMDGPU::OpName::clamp, 0) || 138 !hasNoImmOrEqual(MI, AMDGPU::OpName::omod, 0)) { 139 LLVM_DEBUG(dbgs() << " Inst has non-default modifiers\n"); 140 return false; 141 } 142 return true; 143 } 144 145 int GCNDPPCombine::getDPPOp(unsigned Op, bool IsShrinkable) const { 146 int DPP32 = AMDGPU::getDPPOp32(Op); 147 if (IsShrinkable) { 148 assert(DPP32 == -1); 149 int E32 = AMDGPU::getVOPe32(Op); 150 DPP32 = (E32 == -1) ? -1 : AMDGPU::getDPPOp32(E32); 151 } 152 if (DPP32 != -1 && TII->pseudoToMCOpcode(DPP32) != -1) 153 return DPP32; 154 int DPP64 = -1; 155 if (ST->hasVOP3DPP()) 156 DPP64 = AMDGPU::getDPPOp64(Op); 157 if (DPP64 != -1 && TII->pseudoToMCOpcode(DPP64) != -1) 158 return DPP64; 159 return -1; 160 } 161 162 // tracks the register operand definition and returns: 163 // 1. immediate operand used to initialize the register if found 164 // 2. nullptr if the register operand is undef 165 // 3. the operand itself otherwise 166 MachineOperand *GCNDPPCombine::getOldOpndValue(MachineOperand &OldOpnd) const { 167 auto *Def = getVRegSubRegDef(getRegSubRegPair(OldOpnd), *MRI); 168 if (!Def) 169 return nullptr; 170 171 switch(Def->getOpcode()) { 172 default: break; 173 case AMDGPU::IMPLICIT_DEF: 174 return nullptr; 175 case AMDGPU::COPY: 176 case AMDGPU::V_MOV_B32_e32: 177 case AMDGPU::V_MOV_B64_PSEUDO: 178 case AMDGPU::V_MOV_B64_e32: 179 case AMDGPU::V_MOV_B64_e64: { 180 auto &Op1 = Def->getOperand(1); 181 if (Op1.isImm()) 182 return &Op1; 183 break; 184 } 185 } 186 return &OldOpnd; 187 } 188 189 MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI, 190 MachineInstr &MovMI, 191 RegSubRegPair CombOldVGPR, 192 bool CombBCZ, 193 bool IsShrinkable) const { 194 assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp || 195 MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp || 196 MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO); 197 198 bool HasVOP3DPP = ST->hasVOP3DPP(); 199 auto OrigOp = OrigMI.getOpcode(); 200 auto DPPOp = getDPPOp(OrigOp, IsShrinkable); 201 if (DPPOp == -1) { 202 LLVM_DEBUG(dbgs() << " failed: no DPP opcode\n"); 203 return nullptr; 204 } 205 int OrigOpE32 = AMDGPU::getVOPe32(OrigOp); 206 // Prior checks cover Mask with VOPC condition, but not on purpose 207 auto *RowMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::row_mask); 208 assert(RowMaskOpnd && RowMaskOpnd->isImm()); 209 auto *BankMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::bank_mask); 210 assert(BankMaskOpnd && BankMaskOpnd->isImm()); 211 const bool MaskAllLanes = 212 RowMaskOpnd->getImm() == 0xF && BankMaskOpnd->getImm() == 0xF; 213 (void)MaskAllLanes; 214 assert(MaskAllLanes || 215 !(TII->isVOPC(DPPOp) || 216 (TII->isVOP3(DPPOp) && OrigOpE32 != -1 && TII->isVOPC(OrigOpE32))) && 217 "VOPC cannot form DPP unless mask is full"); 218 219 auto DPPInst = BuildMI(*OrigMI.getParent(), OrigMI, 220 OrigMI.getDebugLoc(), TII->get(DPPOp)) 221 .setMIFlags(OrigMI.getFlags()); 222 223 bool Fail = false; 224 do { 225 int NumOperands = 0; 226 if (auto *Dst = TII->getNamedOperand(OrigMI, AMDGPU::OpName::vdst)) { 227 DPPInst.add(*Dst); 228 ++NumOperands; 229 } 230 if (auto *SDst = TII->getNamedOperand(OrigMI, AMDGPU::OpName::sdst)) { 231 if (TII->isOperandLegal(*DPPInst.getInstr(), NumOperands, SDst)) { 232 DPPInst.add(*SDst); 233 ++NumOperands; 234 } 235 // If we shrunk a 64bit vop3b to 32bits, just ignore the sdst 236 } 237 238 const int OldIdx = AMDGPU::getNamedOperandIdx(DPPOp, AMDGPU::OpName::old); 239 if (OldIdx != -1) { 240 assert(OldIdx == NumOperands); 241 assert(isOfRegClass( 242 CombOldVGPR, 243 *MRI->getRegClass( 244 TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst)->getReg()), 245 *MRI)); 246 auto *Def = getVRegSubRegDef(CombOldVGPR, *MRI); 247 DPPInst.addReg(CombOldVGPR.Reg, Def ? 0 : RegState::Undef, 248 CombOldVGPR.SubReg); 249 ++NumOperands; 250 } else if (TII->isVOPC(DPPOp) || (TII->isVOP3(DPPOp) && OrigOpE32 != -1 && 251 TII->isVOPC(OrigOpE32))) { 252 // VOPC DPP and VOPC promoted to VOP3 DPP do not have an old operand 253 // because they write to SGPRs not VGPRs 254 } else { 255 // TODO: this discards MAC/FMA instructions for now, let's add it later 256 LLVM_DEBUG(dbgs() << " failed: no old operand in DPP instruction," 257 " TBD\n"); 258 Fail = true; 259 break; 260 } 261 262 if (auto *Mod0 = TII->getNamedOperand(OrigMI, 263 AMDGPU::OpName::src0_modifiers)) { 264 assert(NumOperands == AMDGPU::getNamedOperandIdx(DPPOp, 265 AMDGPU::OpName::src0_modifiers)); 266 assert(HasVOP3DPP || 267 (0LL == (Mod0->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG)))); 268 DPPInst.addImm(Mod0->getImm()); 269 ++NumOperands; 270 } else if (AMDGPU::getNamedOperandIdx(DPPOp, 271 AMDGPU::OpName::src0_modifiers) != -1) { 272 DPPInst.addImm(0); 273 ++NumOperands; 274 } 275 auto *Src0 = TII->getNamedOperand(MovMI, AMDGPU::OpName::src0); 276 assert(Src0); 277 if (!TII->isOperandLegal(*DPPInst.getInstr(), NumOperands, Src0)) { 278 LLVM_DEBUG(dbgs() << " failed: src0 is illegal\n"); 279 Fail = true; 280 break; 281 } 282 DPPInst.add(*Src0); 283 DPPInst->getOperand(NumOperands).setIsKill(false); 284 ++NumOperands; 285 286 if (auto *Mod1 = TII->getNamedOperand(OrigMI, 287 AMDGPU::OpName::src1_modifiers)) { 288 assert(NumOperands == AMDGPU::getNamedOperandIdx(DPPOp, 289 AMDGPU::OpName::src1_modifiers)); 290 assert(HasVOP3DPP || 291 (0LL == (Mod1->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG)))); 292 DPPInst.addImm(Mod1->getImm()); 293 ++NumOperands; 294 } else if (AMDGPU::getNamedOperandIdx(DPPOp, 295 AMDGPU::OpName::src1_modifiers) != -1) { 296 DPPInst.addImm(0); 297 ++NumOperands; 298 } 299 auto *Src1 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1); 300 if (Src1) { 301 if (!TII->isOperandLegal(*DPPInst.getInstr(), NumOperands, Src1)) { 302 LLVM_DEBUG(dbgs() << " failed: src1 is illegal\n"); 303 Fail = true; 304 break; 305 } 306 DPPInst.add(*Src1); 307 ++NumOperands; 308 } 309 if (auto *Mod2 = 310 TII->getNamedOperand(OrigMI, AMDGPU::OpName::src2_modifiers)) { 311 assert(NumOperands == 312 AMDGPU::getNamedOperandIdx(DPPOp, AMDGPU::OpName::src2_modifiers)); 313 assert(HasVOP3DPP || 314 (0LL == (Mod2->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG)))); 315 DPPInst.addImm(Mod2->getImm()); 316 ++NumOperands; 317 } 318 auto *Src2 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src2); 319 if (Src2) { 320 if (!TII->getNamedOperand(*DPPInst.getInstr(), AMDGPU::OpName::src2) || 321 !TII->isOperandLegal(*DPPInst.getInstr(), NumOperands, Src2)) { 322 LLVM_DEBUG(dbgs() << " failed: src2 is illegal\n"); 323 Fail = true; 324 break; 325 } 326 DPPInst.add(*Src2); 327 ++NumOperands; 328 } 329 if (HasVOP3DPP) { 330 auto *ClampOpr = TII->getNamedOperand(OrigMI, AMDGPU::OpName::clamp); 331 if (ClampOpr && 332 AMDGPU::getNamedOperandIdx(DPPOp, AMDGPU::OpName::clamp) != -1) { 333 DPPInst.addImm(ClampOpr->getImm()); 334 } 335 auto *VdstInOpr = TII->getNamedOperand(OrigMI, AMDGPU::OpName::vdst_in); 336 if (VdstInOpr && 337 AMDGPU::getNamedOperandIdx(DPPOp, AMDGPU::OpName::vdst_in) != -1) { 338 DPPInst.add(*VdstInOpr); 339 } 340 auto *OmodOpr = TII->getNamedOperand(OrigMI, AMDGPU::OpName::omod); 341 if (OmodOpr && 342 AMDGPU::getNamedOperandIdx(DPPOp, AMDGPU::OpName::omod) != -1) { 343 DPPInst.addImm(OmodOpr->getImm()); 344 } 345 // Validate OP_SEL has to be set to all 0 and OP_SEL_HI has to be set to 346 // all 1. 347 if (auto *OpSelOpr = 348 TII->getNamedOperand(OrigMI, AMDGPU::OpName::op_sel)) { 349 auto OpSel = OpSelOpr->getImm(); 350 if (OpSel != 0) { 351 LLVM_DEBUG(dbgs() << " failed: op_sel must be zero\n"); 352 Fail = true; 353 break; 354 } 355 if (AMDGPU::getNamedOperandIdx(DPPOp, AMDGPU::OpName::op_sel) != -1) 356 DPPInst.addImm(OpSel); 357 } 358 if (auto *OpSelHiOpr = 359 TII->getNamedOperand(OrigMI, AMDGPU::OpName::op_sel_hi)) { 360 auto OpSelHi = OpSelHiOpr->getImm(); 361 // Only vop3p has op_sel_hi, and all vop3p have 3 operands, so check 362 // the bitmask for 3 op_sel_hi bits set 363 assert(Src2 && "Expected vop3p with 3 operands"); 364 if (OpSelHi != 7) { 365 LLVM_DEBUG(dbgs() << " failed: op_sel_hi must be all set to one\n"); 366 Fail = true; 367 break; 368 } 369 if (AMDGPU::getNamedOperandIdx(DPPOp, AMDGPU::OpName::op_sel_hi) != -1) 370 DPPInst.addImm(OpSelHi); 371 } 372 auto *NegOpr = TII->getNamedOperand(OrigMI, AMDGPU::OpName::neg_lo); 373 if (NegOpr && 374 AMDGPU::getNamedOperandIdx(DPPOp, AMDGPU::OpName::neg_lo) != -1) { 375 DPPInst.addImm(NegOpr->getImm()); 376 } 377 auto *NegHiOpr = TII->getNamedOperand(OrigMI, AMDGPU::OpName::neg_hi); 378 if (NegHiOpr && 379 AMDGPU::getNamedOperandIdx(DPPOp, AMDGPU::OpName::neg_hi) != -1) { 380 DPPInst.addImm(NegHiOpr->getImm()); 381 } 382 } 383 DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::dpp_ctrl)); 384 DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::row_mask)); 385 DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::bank_mask)); 386 DPPInst.addImm(CombBCZ ? 1 : 0); 387 } while (false); 388 389 if (Fail) { 390 DPPInst.getInstr()->eraseFromParent(); 391 return nullptr; 392 } 393 LLVM_DEBUG(dbgs() << " combined: " << *DPPInst.getInstr()); 394 return DPPInst.getInstr(); 395 } 396 397 static bool isIdentityValue(unsigned OrigMIOp, MachineOperand *OldOpnd) { 398 assert(OldOpnd->isImm()); 399 switch (OrigMIOp) { 400 default: break; 401 case AMDGPU::V_ADD_U32_e32: 402 case AMDGPU::V_ADD_U32_e64: 403 case AMDGPU::V_ADD_CO_U32_e32: 404 case AMDGPU::V_ADD_CO_U32_e64: 405 case AMDGPU::V_OR_B32_e32: 406 case AMDGPU::V_OR_B32_e64: 407 case AMDGPU::V_SUBREV_U32_e32: 408 case AMDGPU::V_SUBREV_U32_e64: 409 case AMDGPU::V_SUBREV_CO_U32_e32: 410 case AMDGPU::V_SUBREV_CO_U32_e64: 411 case AMDGPU::V_MAX_U32_e32: 412 case AMDGPU::V_MAX_U32_e64: 413 case AMDGPU::V_XOR_B32_e32: 414 case AMDGPU::V_XOR_B32_e64: 415 if (OldOpnd->getImm() == 0) 416 return true; 417 break; 418 case AMDGPU::V_AND_B32_e32: 419 case AMDGPU::V_AND_B32_e64: 420 case AMDGPU::V_MIN_U32_e32: 421 case AMDGPU::V_MIN_U32_e64: 422 if (static_cast<uint32_t>(OldOpnd->getImm()) == 423 std::numeric_limits<uint32_t>::max()) 424 return true; 425 break; 426 case AMDGPU::V_MIN_I32_e32: 427 case AMDGPU::V_MIN_I32_e64: 428 if (static_cast<int32_t>(OldOpnd->getImm()) == 429 std::numeric_limits<int32_t>::max()) 430 return true; 431 break; 432 case AMDGPU::V_MAX_I32_e32: 433 case AMDGPU::V_MAX_I32_e64: 434 if (static_cast<int32_t>(OldOpnd->getImm()) == 435 std::numeric_limits<int32_t>::min()) 436 return true; 437 break; 438 case AMDGPU::V_MUL_I32_I24_e32: 439 case AMDGPU::V_MUL_I32_I24_e64: 440 case AMDGPU::V_MUL_U32_U24_e32: 441 case AMDGPU::V_MUL_U32_U24_e64: 442 if (OldOpnd->getImm() == 1) 443 return true; 444 break; 445 } 446 return false; 447 } 448 449 MachineInstr *GCNDPPCombine::createDPPInst( 450 MachineInstr &OrigMI, MachineInstr &MovMI, RegSubRegPair CombOldVGPR, 451 MachineOperand *OldOpndValue, bool CombBCZ, bool IsShrinkable) const { 452 assert(CombOldVGPR.Reg); 453 if (!CombBCZ && OldOpndValue && OldOpndValue->isImm()) { 454 auto *Src1 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1); 455 if (!Src1 || !Src1->isReg()) { 456 LLVM_DEBUG(dbgs() << " failed: no src1 or it isn't a register\n"); 457 return nullptr; 458 } 459 if (!isIdentityValue(OrigMI.getOpcode(), OldOpndValue)) { 460 LLVM_DEBUG(dbgs() << " failed: old immediate isn't an identity\n"); 461 return nullptr; 462 } 463 CombOldVGPR = getRegSubRegPair(*Src1); 464 auto MovDst = TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst); 465 const TargetRegisterClass *RC = MRI->getRegClass(MovDst->getReg()); 466 if (!isOfRegClass(CombOldVGPR, *RC, *MRI)) { 467 LLVM_DEBUG(dbgs() << " failed: src1 has wrong register class\n"); 468 return nullptr; 469 } 470 } 471 return createDPPInst(OrigMI, MovMI, CombOldVGPR, CombBCZ, IsShrinkable); 472 } 473 474 // returns true if MI doesn't have OpndName immediate operand or the 475 // operand has Value 476 bool GCNDPPCombine::hasNoImmOrEqual(MachineInstr &MI, unsigned OpndName, 477 int64_t Value, int64_t Mask) const { 478 auto *Imm = TII->getNamedOperand(MI, OpndName); 479 if (!Imm) 480 return true; 481 482 assert(Imm->isImm()); 483 return (Imm->getImm() & Mask) == Value; 484 } 485 486 bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const { 487 assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp || 488 MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp || 489 MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO); 490 LLVM_DEBUG(dbgs() << "\nDPP combine: " << MovMI); 491 492 auto *DstOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst); 493 assert(DstOpnd && DstOpnd->isReg()); 494 auto DPPMovReg = DstOpnd->getReg(); 495 if (DPPMovReg.isPhysical()) { 496 LLVM_DEBUG(dbgs() << " failed: dpp move writes physreg\n"); 497 return false; 498 } 499 if (execMayBeModifiedBeforeAnyUse(*MRI, DPPMovReg, MovMI)) { 500 LLVM_DEBUG(dbgs() << " failed: EXEC mask should remain the same" 501 " for all uses\n"); 502 return false; 503 } 504 505 if (MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO || 506 MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp) { 507 auto *DppCtrl = TII->getNamedOperand(MovMI, AMDGPU::OpName::dpp_ctrl); 508 assert(DppCtrl && DppCtrl->isImm()); 509 if (!AMDGPU::isLegal64BitDPPControl(DppCtrl->getImm())) { 510 LLVM_DEBUG(dbgs() << " failed: 64 bit dpp move uses unsupported" 511 " control value\n"); 512 // Let it split, then control may become legal. 513 return false; 514 } 515 } 516 517 auto *RowMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::row_mask); 518 assert(RowMaskOpnd && RowMaskOpnd->isImm()); 519 auto *BankMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::bank_mask); 520 assert(BankMaskOpnd && BankMaskOpnd->isImm()); 521 const bool MaskAllLanes = RowMaskOpnd->getImm() == 0xF && 522 BankMaskOpnd->getImm() == 0xF; 523 524 auto *BCZOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::bound_ctrl); 525 assert(BCZOpnd && BCZOpnd->isImm()); 526 bool BoundCtrlZero = BCZOpnd->getImm(); 527 528 auto *OldOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::old); 529 auto *SrcOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::src0); 530 assert(OldOpnd && OldOpnd->isReg()); 531 assert(SrcOpnd && SrcOpnd->isReg()); 532 if (OldOpnd->getReg().isPhysical() || SrcOpnd->getReg().isPhysical()) { 533 LLVM_DEBUG(dbgs() << " failed: dpp move reads physreg\n"); 534 return false; 535 } 536 537 auto * const OldOpndValue = getOldOpndValue(*OldOpnd); 538 // OldOpndValue is either undef (IMPLICIT_DEF) or immediate or something else 539 // We could use: assert(!OldOpndValue || OldOpndValue->isImm()) 540 // but the third option is used to distinguish undef from non-immediate 541 // to reuse IMPLICIT_DEF instruction later 542 assert(!OldOpndValue || OldOpndValue->isImm() || OldOpndValue == OldOpnd); 543 544 bool CombBCZ = false; 545 546 if (MaskAllLanes && BoundCtrlZero) { // [1] 547 CombBCZ = true; 548 } else { 549 if (!OldOpndValue || !OldOpndValue->isImm()) { 550 LLVM_DEBUG(dbgs() << " failed: the DPP mov isn't combinable\n"); 551 return false; 552 } 553 554 if (OldOpndValue->getImm() == 0) { 555 if (MaskAllLanes) { 556 assert(!BoundCtrlZero); // by check [1] 557 CombBCZ = true; 558 } 559 } else if (BoundCtrlZero) { 560 assert(!MaskAllLanes); // by check [1] 561 LLVM_DEBUG(dbgs() << 562 " failed: old!=0 and bctrl:0 and not all lanes isn't combinable\n"); 563 return false; 564 } 565 } 566 567 LLVM_DEBUG(dbgs() << " old="; 568 if (!OldOpndValue) 569 dbgs() << "undef"; 570 else 571 dbgs() << *OldOpndValue; 572 dbgs() << ", bound_ctrl=" << CombBCZ << '\n'); 573 574 SmallVector<MachineInstr*, 4> OrigMIs, DPPMIs; 575 DenseMap<MachineInstr*, SmallVector<unsigned, 4>> RegSeqWithOpNos; 576 auto CombOldVGPR = getRegSubRegPair(*OldOpnd); 577 // try to reuse previous old reg if its undefined (IMPLICIT_DEF) 578 if (CombBCZ && OldOpndValue) { // CombOldVGPR should be undef 579 const TargetRegisterClass *RC = MRI->getRegClass(DPPMovReg); 580 CombOldVGPR = RegSubRegPair( 581 MRI->createVirtualRegister(RC)); 582 auto UndefInst = BuildMI(*MovMI.getParent(), MovMI, MovMI.getDebugLoc(), 583 TII->get(AMDGPU::IMPLICIT_DEF), CombOldVGPR.Reg); 584 DPPMIs.push_back(UndefInst.getInstr()); 585 } 586 587 OrigMIs.push_back(&MovMI); 588 bool Rollback = true; 589 SmallVector<MachineOperand*, 16> Uses; 590 591 for (auto &Use : MRI->use_nodbg_operands(DPPMovReg)) { 592 Uses.push_back(&Use); 593 } 594 595 while (!Uses.empty()) { 596 MachineOperand *Use = Uses.pop_back_val(); 597 Rollback = true; 598 599 auto &OrigMI = *Use->getParent(); 600 LLVM_DEBUG(dbgs() << " try: " << OrigMI); 601 602 auto OrigOp = OrigMI.getOpcode(); 603 if (OrigOp == AMDGPU::REG_SEQUENCE) { 604 Register FwdReg = OrigMI.getOperand(0).getReg(); 605 unsigned FwdSubReg = 0; 606 607 if (execMayBeModifiedBeforeAnyUse(*MRI, FwdReg, OrigMI)) { 608 LLVM_DEBUG(dbgs() << " failed: EXEC mask should remain the same" 609 " for all uses\n"); 610 break; 611 } 612 613 unsigned OpNo, E = OrigMI.getNumOperands(); 614 for (OpNo = 1; OpNo < E; OpNo += 2) { 615 if (OrigMI.getOperand(OpNo).getReg() == DPPMovReg) { 616 FwdSubReg = OrigMI.getOperand(OpNo + 1).getImm(); 617 break; 618 } 619 } 620 621 if (!FwdSubReg) 622 break; 623 624 for (auto &Op : MRI->use_nodbg_operands(FwdReg)) { 625 if (Op.getSubReg() == FwdSubReg) 626 Uses.push_back(&Op); 627 } 628 RegSeqWithOpNos[&OrigMI].push_back(OpNo); 629 continue; 630 } 631 632 bool IsShrinkable = isShrinkable(OrigMI); 633 if (!(IsShrinkable || 634 ((TII->isVOP3P(OrigOp) || TII->isVOPC(OrigOp) || 635 TII->isVOP3(OrigOp)) && 636 ST->hasVOP3DPP()) || 637 TII->isVOP1(OrigOp) || TII->isVOP2(OrigOp))) { 638 LLVM_DEBUG(dbgs() << " failed: not VOP1/2/3/3P/C\n"); 639 break; 640 } 641 if (OrigMI.modifiesRegister(AMDGPU::EXEC, ST->getRegisterInfo())) { 642 LLVM_DEBUG(dbgs() << " failed: can't combine v_cmpx\n"); 643 break; 644 } 645 646 auto *Src0 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src0); 647 auto *Src1 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1); 648 if (Use != Src0 && !(Use == Src1 && OrigMI.isCommutable())) { // [1] 649 LLVM_DEBUG(dbgs() << " failed: no suitable operands\n"); 650 break; 651 } 652 653 auto *Src2 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src2); 654 assert(Src0 && "Src1 without Src0?"); 655 if ((Use == Src0 && ((Src1 && Src1->isIdenticalTo(*Src0)) || 656 (Src2 && Src2->isIdenticalTo(*Src0)))) || 657 (Use == Src1 && (Src1->isIdenticalTo(*Src0) || 658 (Src2 && Src2->isIdenticalTo(*Src1))))) { 659 LLVM_DEBUG( 660 dbgs() 661 << " " << OrigMI 662 << " failed: DPP register is used more than once per instruction\n"); 663 break; 664 } 665 666 LLVM_DEBUG(dbgs() << " combining: " << OrigMI); 667 if (Use == Src0) { 668 if (auto *DPPInst = createDPPInst(OrigMI, MovMI, CombOldVGPR, 669 OldOpndValue, CombBCZ, IsShrinkable)) { 670 DPPMIs.push_back(DPPInst); 671 Rollback = false; 672 } 673 } else { 674 assert(Use == Src1 && OrigMI.isCommutable()); // by check [1] 675 auto *BB = OrigMI.getParent(); 676 auto *NewMI = BB->getParent()->CloneMachineInstr(&OrigMI); 677 BB->insert(OrigMI, NewMI); 678 if (TII->commuteInstruction(*NewMI)) { 679 LLVM_DEBUG(dbgs() << " commuted: " << *NewMI); 680 if (auto *DPPInst = 681 createDPPInst(*NewMI, MovMI, CombOldVGPR, OldOpndValue, CombBCZ, 682 IsShrinkable)) { 683 DPPMIs.push_back(DPPInst); 684 Rollback = false; 685 } 686 } else 687 LLVM_DEBUG(dbgs() << " failed: cannot be commuted\n"); 688 NewMI->eraseFromParent(); 689 } 690 if (Rollback) 691 break; 692 OrigMIs.push_back(&OrigMI); 693 } 694 695 Rollback |= !Uses.empty(); 696 697 for (auto *MI : *(Rollback? &DPPMIs : &OrigMIs)) 698 MI->eraseFromParent(); 699 700 if (!Rollback) { 701 for (auto &S : RegSeqWithOpNos) { 702 if (MRI->use_nodbg_empty(S.first->getOperand(0).getReg())) { 703 S.first->eraseFromParent(); 704 continue; 705 } 706 while (!S.second.empty()) 707 S.first->getOperand(S.second.pop_back_val()).setIsUndef(true); 708 } 709 } 710 711 return !Rollback; 712 } 713 714 bool GCNDPPCombine::runOnMachineFunction(MachineFunction &MF) { 715 ST = &MF.getSubtarget<GCNSubtarget>(); 716 if (!ST->hasDPP() || skipFunction(MF.getFunction())) 717 return false; 718 719 MRI = &MF.getRegInfo(); 720 TII = ST->getInstrInfo(); 721 722 bool Changed = false; 723 for (auto &MBB : MF) { 724 for (MachineInstr &MI : llvm::make_early_inc_range(llvm::reverse(MBB))) { 725 if (MI.getOpcode() == AMDGPU::V_MOV_B32_dpp && combineDPPMov(MI)) { 726 Changed = true; 727 ++NumDPPMovsCombined; 728 } else if (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO || 729 MI.getOpcode() == AMDGPU::V_MOV_B64_dpp) { 730 if (ST->has64BitDPP() && combineDPPMov(MI)) { 731 Changed = true; 732 ++NumDPPMovsCombined; 733 } else { 734 auto Split = TII->expandMovDPP64(MI); 735 for (auto M : { Split.first, Split.second }) { 736 if (M && combineDPPMov(*M)) 737 ++NumDPPMovsCombined; 738 } 739 Changed = true; 740 } 741 } 742 } 743 } 744 return Changed; 745 } 746