1 //=======- GCNDPPCombine.cpp - optimization for DPP instructions ---==========// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // The pass combines V_MOV_B32_dpp instruction with its VALU uses as a DPP src0 9 // operand. If any of the use instruction cannot be combined with the mov the 10 // whole sequence is reverted. 11 // 12 // $old = ... 13 // $dpp_value = V_MOV_B32_dpp $old, $vgpr_to_be_read_from_other_lane, 14 // dpp_controls..., $row_mask, $bank_mask, $bound_ctrl 15 // $res = VALU $dpp_value [, src1] 16 // 17 // to 18 // 19 // $res = VALU_DPP $combined_old, $vgpr_to_be_read_from_other_lane, [src1,] 20 // dpp_controls..., $row_mask, $bank_mask, $combined_bound_ctrl 21 // 22 // Combining rules : 23 // 24 // if $row_mask and $bank_mask are fully enabled (0xF) and 25 // $bound_ctrl==DPP_BOUND_ZERO or $old==0 26 // -> $combined_old = undef, 27 // $combined_bound_ctrl = DPP_BOUND_ZERO 28 // 29 // if the VALU op is binary and 30 // $bound_ctrl==DPP_BOUND_OFF and 31 // $old==identity value (immediate) for the VALU op 32 // -> $combined_old = src1, 33 // $combined_bound_ctrl = DPP_BOUND_OFF 34 // 35 // Otherwise cancel. 36 // 37 // The mov_dpp instruction should reside in the same BB as all its uses 38 //===----------------------------------------------------------------------===// 39 40 #include "AMDGPU.h" 41 #include "AMDGPUSubtarget.h" 42 #include "SIInstrInfo.h" 43 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 44 #include "llvm/ADT/SmallVector.h" 45 #include "llvm/ADT/Statistic.h" 46 #include "llvm/CodeGen/MachineBasicBlock.h" 47 #include "llvm/CodeGen/MachineFunction.h" 48 #include "llvm/CodeGen/MachineFunctionPass.h" 49 #include "llvm/CodeGen/MachineInstr.h" 50 #include "llvm/CodeGen/MachineInstrBuilder.h" 51 #include "llvm/CodeGen/MachineOperand.h" 52 #include "llvm/CodeGen/MachineRegisterInfo.h" 53 #include "llvm/CodeGen/TargetRegisterInfo.h" 54 #include "llvm/Pass.h" 55 #include <cassert> 56 57 using namespace llvm; 58 59 #define DEBUG_TYPE "gcn-dpp-combine" 60 61 STATISTIC(NumDPPMovsCombined, "Number of DPP moves combined."); 62 63 namespace { 64 65 class GCNDPPCombine : public MachineFunctionPass { 66 MachineRegisterInfo *MRI; 67 const SIInstrInfo *TII; 68 69 using RegSubRegPair = TargetInstrInfo::RegSubRegPair; 70 71 MachineOperand *getOldOpndValue(MachineOperand &OldOpnd) const; 72 73 MachineInstr *createDPPInst(MachineInstr &OrigMI, 74 MachineInstr &MovMI, 75 RegSubRegPair CombOldVGPR, 76 MachineOperand *OldOpnd, 77 bool CombBCZ) const; 78 79 MachineInstr *createDPPInst(MachineInstr &OrigMI, 80 MachineInstr &MovMI, 81 RegSubRegPair CombOldVGPR, 82 bool CombBCZ) const; 83 84 bool hasNoImmOrEqual(MachineInstr &MI, 85 unsigned OpndName, 86 int64_t Value, 87 int64_t Mask = -1) const; 88 89 bool combineDPPMov(MachineInstr &MI) const; 90 91 public: 92 static char ID; 93 94 GCNDPPCombine() : MachineFunctionPass(ID) { 95 initializeGCNDPPCombinePass(*PassRegistry::getPassRegistry()); 96 } 97 98 bool runOnMachineFunction(MachineFunction &MF) override; 99 100 StringRef getPassName() const override { return "GCN DPP Combine"; } 101 102 void getAnalysisUsage(AnalysisUsage &AU) const override { 103 AU.setPreservesCFG(); 104 MachineFunctionPass::getAnalysisUsage(AU); 105 } 106 }; 107 108 } // end anonymous namespace 109 110 INITIALIZE_PASS(GCNDPPCombine, DEBUG_TYPE, "GCN DPP Combine", false, false) 111 112 char GCNDPPCombine::ID = 0; 113 114 char &llvm::GCNDPPCombineID = GCNDPPCombine::ID; 115 116 FunctionPass *llvm::createGCNDPPCombinePass() { 117 return new GCNDPPCombine(); 118 } 119 120 static int getDPPOp(unsigned Op) { 121 auto DPP32 = AMDGPU::getDPPOp32(Op); 122 if (DPP32 != -1) 123 return DPP32; 124 125 auto E32 = AMDGPU::getVOPe32(Op); 126 return E32 != -1 ? AMDGPU::getDPPOp32(E32) : -1; 127 } 128 129 // tracks the register operand definition and returns: 130 // 1. immediate operand used to initialize the register if found 131 // 2. nullptr if the register operand is undef 132 // 3. the operand itself otherwise 133 MachineOperand *GCNDPPCombine::getOldOpndValue(MachineOperand &OldOpnd) const { 134 auto *Def = getVRegSubRegDef(getRegSubRegPair(OldOpnd), *MRI); 135 if (!Def) 136 return nullptr; 137 138 switch(Def->getOpcode()) { 139 default: break; 140 case AMDGPU::IMPLICIT_DEF: 141 return nullptr; 142 case AMDGPU::COPY: 143 case AMDGPU::V_MOV_B32_e32: { 144 auto &Op1 = Def->getOperand(1); 145 if (Op1.isImm()) 146 return &Op1; 147 break; 148 } 149 } 150 return &OldOpnd; 151 } 152 153 MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI, 154 MachineInstr &MovMI, 155 RegSubRegPair CombOldVGPR, 156 bool CombBCZ) const { 157 assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp); 158 assert(TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst)->getReg() == 159 TII->getNamedOperand(OrigMI, AMDGPU::OpName::src0)->getReg()); 160 161 auto OrigOp = OrigMI.getOpcode(); 162 auto DPPOp = getDPPOp(OrigOp); 163 if (DPPOp == -1) { 164 LLVM_DEBUG(dbgs() << " failed: no DPP opcode\n"); 165 return nullptr; 166 } 167 168 auto DPPInst = BuildMI(*OrigMI.getParent(), OrigMI, 169 OrigMI.getDebugLoc(), TII->get(DPPOp)); 170 bool Fail = false; 171 do { 172 auto *Dst = TII->getNamedOperand(OrigMI, AMDGPU::OpName::vdst); 173 assert(Dst); 174 DPPInst.add(*Dst); 175 int NumOperands = 1; 176 177 const int OldIdx = AMDGPU::getNamedOperandIdx(DPPOp, AMDGPU::OpName::old); 178 if (OldIdx != -1) { 179 assert(OldIdx == NumOperands); 180 assert(isOfRegClass(CombOldVGPR, AMDGPU::VGPR_32RegClass, *MRI)); 181 DPPInst.addReg(CombOldVGPR.Reg, 0, CombOldVGPR.SubReg); 182 ++NumOperands; 183 } else { 184 // TODO: this discards MAC/FMA instructions for now, let's add it later 185 LLVM_DEBUG(dbgs() << " failed: no old operand in DPP instruction," 186 " TBD\n"); 187 Fail = true; 188 break; 189 } 190 191 if (auto *Mod0 = TII->getNamedOperand(OrigMI, 192 AMDGPU::OpName::src0_modifiers)) { 193 assert(NumOperands == AMDGPU::getNamedOperandIdx(DPPOp, 194 AMDGPU::OpName::src0_modifiers)); 195 assert(0LL == (Mod0->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG))); 196 DPPInst.addImm(Mod0->getImm()); 197 ++NumOperands; 198 } 199 auto *Src0 = TII->getNamedOperand(MovMI, AMDGPU::OpName::src0); 200 assert(Src0); 201 if (!TII->isOperandLegal(*DPPInst.getInstr(), NumOperands, Src0)) { 202 LLVM_DEBUG(dbgs() << " failed: src0 is illegal\n"); 203 Fail = true; 204 break; 205 } 206 DPPInst.add(*Src0); 207 DPPInst->getOperand(NumOperands).setIsKill(false); 208 ++NumOperands; 209 210 if (auto *Mod1 = TII->getNamedOperand(OrigMI, 211 AMDGPU::OpName::src1_modifiers)) { 212 assert(NumOperands == AMDGPU::getNamedOperandIdx(DPPOp, 213 AMDGPU::OpName::src1_modifiers)); 214 assert(0LL == (Mod1->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG))); 215 DPPInst.addImm(Mod1->getImm()); 216 ++NumOperands; 217 } 218 if (auto *Src1 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1)) { 219 if (!TII->isOperandLegal(*DPPInst.getInstr(), NumOperands, Src1)) { 220 LLVM_DEBUG(dbgs() << " failed: src1 is illegal\n"); 221 Fail = true; 222 break; 223 } 224 DPPInst.add(*Src1); 225 ++NumOperands; 226 } 227 228 if (auto *Src2 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src2)) { 229 if (!TII->isOperandLegal(*DPPInst.getInstr(), NumOperands, Src2)) { 230 LLVM_DEBUG(dbgs() << " failed: src2 is illegal\n"); 231 Fail = true; 232 break; 233 } 234 DPPInst.add(*Src2); 235 } 236 237 DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::dpp_ctrl)); 238 DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::row_mask)); 239 DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::bank_mask)); 240 DPPInst.addImm(CombBCZ ? 1 : 0); 241 } while (false); 242 243 if (Fail) { 244 DPPInst.getInstr()->eraseFromParent(); 245 return nullptr; 246 } 247 LLVM_DEBUG(dbgs() << " combined: " << *DPPInst.getInstr()); 248 return DPPInst.getInstr(); 249 } 250 251 static bool isIdentityValue(unsigned OrigMIOp, MachineOperand *OldOpnd) { 252 assert(OldOpnd->isImm()); 253 switch (OrigMIOp) { 254 default: break; 255 case AMDGPU::V_ADD_U32_e32: 256 case AMDGPU::V_ADD_U32_e64: 257 case AMDGPU::V_ADD_I32_e32: 258 case AMDGPU::V_ADD_I32_e64: 259 case AMDGPU::V_OR_B32_e32: 260 case AMDGPU::V_OR_B32_e64: 261 case AMDGPU::V_SUBREV_U32_e32: 262 case AMDGPU::V_SUBREV_U32_e64: 263 case AMDGPU::V_SUBREV_I32_e32: 264 case AMDGPU::V_SUBREV_I32_e64: 265 case AMDGPU::V_MAX_U32_e32: 266 case AMDGPU::V_MAX_U32_e64: 267 case AMDGPU::V_XOR_B32_e32: 268 case AMDGPU::V_XOR_B32_e64: 269 if (OldOpnd->getImm() == 0) 270 return true; 271 break; 272 case AMDGPU::V_AND_B32_e32: 273 case AMDGPU::V_AND_B32_e64: 274 case AMDGPU::V_MIN_U32_e32: 275 case AMDGPU::V_MIN_U32_e64: 276 if (static_cast<uint32_t>(OldOpnd->getImm()) == 277 std::numeric_limits<uint32_t>::max()) 278 return true; 279 break; 280 case AMDGPU::V_MIN_I32_e32: 281 case AMDGPU::V_MIN_I32_e64: 282 if (static_cast<int32_t>(OldOpnd->getImm()) == 283 std::numeric_limits<int32_t>::max()) 284 return true; 285 break; 286 case AMDGPU::V_MAX_I32_e32: 287 case AMDGPU::V_MAX_I32_e64: 288 if (static_cast<int32_t>(OldOpnd->getImm()) == 289 std::numeric_limits<int32_t>::min()) 290 return true; 291 break; 292 case AMDGPU::V_MUL_I32_I24_e32: 293 case AMDGPU::V_MUL_I32_I24_e64: 294 case AMDGPU::V_MUL_U32_U24_e32: 295 case AMDGPU::V_MUL_U32_U24_e64: 296 if (OldOpnd->getImm() == 1) 297 return true; 298 break; 299 } 300 return false; 301 } 302 303 MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI, 304 MachineInstr &MovMI, 305 RegSubRegPair CombOldVGPR, 306 MachineOperand *OldOpndValue, 307 bool CombBCZ) const { 308 assert(CombOldVGPR.Reg); 309 if (!CombBCZ && OldOpndValue && OldOpndValue->isImm()) { 310 auto *Src1 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1); 311 if (!Src1 || !Src1->isReg()) { 312 LLVM_DEBUG(dbgs() << " failed: no src1 or it isn't a register\n"); 313 return nullptr; 314 } 315 if (!isIdentityValue(OrigMI.getOpcode(), OldOpndValue)) { 316 LLVM_DEBUG(dbgs() << " failed: old immediate isn't an identity\n"); 317 return nullptr; 318 } 319 CombOldVGPR = getRegSubRegPair(*Src1); 320 if (!isOfRegClass(CombOldVGPR, AMDGPU::VGPR_32RegClass, *MRI)) { 321 LLVM_DEBUG(dbgs() << " failed: src1 isn't a VGPR32 register\n"); 322 return nullptr; 323 } 324 } 325 return createDPPInst(OrigMI, MovMI, CombOldVGPR, CombBCZ); 326 } 327 328 // returns true if MI doesn't have OpndName immediate operand or the 329 // operand has Value 330 bool GCNDPPCombine::hasNoImmOrEqual(MachineInstr &MI, unsigned OpndName, 331 int64_t Value, int64_t Mask) const { 332 auto *Imm = TII->getNamedOperand(MI, OpndName); 333 if (!Imm) 334 return true; 335 336 assert(Imm->isImm()); 337 return (Imm->getImm() & Mask) == Value; 338 } 339 340 bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const { 341 assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp); 342 LLVM_DEBUG(dbgs() << "\nDPP combine: " << MovMI); 343 344 auto *DstOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst); 345 assert(DstOpnd && DstOpnd->isReg()); 346 auto DPPMovReg = DstOpnd->getReg(); 347 if (execMayBeModifiedBeforeAnyUse(*MRI, DPPMovReg, MovMI)) { 348 LLVM_DEBUG(dbgs() << " failed: EXEC mask should remain the same" 349 " for all uses\n"); 350 return false; 351 } 352 353 auto *RowMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::row_mask); 354 assert(RowMaskOpnd && RowMaskOpnd->isImm()); 355 auto *BankMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::bank_mask); 356 assert(BankMaskOpnd && BankMaskOpnd->isImm()); 357 const bool MaskAllLanes = RowMaskOpnd->getImm() == 0xF && 358 BankMaskOpnd->getImm() == 0xF; 359 360 auto *BCZOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::bound_ctrl); 361 assert(BCZOpnd && BCZOpnd->isImm()); 362 bool BoundCtrlZero = BCZOpnd->getImm(); 363 364 auto *OldOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::old); 365 assert(OldOpnd && OldOpnd->isReg()); 366 367 auto * const OldOpndValue = getOldOpndValue(*OldOpnd); 368 // OldOpndValue is either undef (IMPLICIT_DEF) or immediate or something else 369 // We could use: assert(!OldOpndValue || OldOpndValue->isImm()) 370 // but the third option is used to distinguish undef from non-immediate 371 // to reuse IMPLICIT_DEF instruction later 372 assert(!OldOpndValue || OldOpndValue->isImm() || OldOpndValue == OldOpnd); 373 374 bool CombBCZ = false; 375 376 if (MaskAllLanes && BoundCtrlZero) { // [1] 377 CombBCZ = true; 378 } else { 379 if (!OldOpndValue || !OldOpndValue->isImm()) { 380 LLVM_DEBUG(dbgs() << " failed: the DPP mov isn't combinable\n"); 381 return false; 382 } 383 384 if (OldOpndValue->getParent()->getParent() != MovMI.getParent()) { 385 LLVM_DEBUG(dbgs() << 386 " failed: old reg def and mov should be in the same BB\n"); 387 return false; 388 } 389 390 if (OldOpndValue->getImm() == 0) { 391 if (MaskAllLanes) { 392 assert(!BoundCtrlZero); // by check [1] 393 CombBCZ = true; 394 } 395 } else if (BoundCtrlZero) { 396 assert(!MaskAllLanes); // by check [1] 397 LLVM_DEBUG(dbgs() << 398 " failed: old!=0 and bctrl:0 and not all lanes isn't combinable\n"); 399 return false; 400 } 401 } 402 403 LLVM_DEBUG(dbgs() << " old="; 404 if (!OldOpndValue) 405 dbgs() << "undef"; 406 else 407 dbgs() << *OldOpndValue; 408 dbgs() << ", bound_ctrl=" << CombBCZ << '\n'); 409 410 SmallVector<MachineInstr*, 4> OrigMIs, DPPMIs; 411 auto CombOldVGPR = getRegSubRegPair(*OldOpnd); 412 // try to reuse previous old reg if its undefined (IMPLICIT_DEF) 413 if (CombBCZ && OldOpndValue) { // CombOldVGPR should be undef 414 CombOldVGPR = RegSubRegPair( 415 MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass)); 416 auto UndefInst = BuildMI(*MovMI.getParent(), MovMI, MovMI.getDebugLoc(), 417 TII->get(AMDGPU::IMPLICIT_DEF), CombOldVGPR.Reg); 418 DPPMIs.push_back(UndefInst.getInstr()); 419 } 420 421 OrigMIs.push_back(&MovMI); 422 bool Rollback = true; 423 for (auto &Use : MRI->use_nodbg_operands(DPPMovReg)) { 424 Rollback = true; 425 426 auto &OrigMI = *Use.getParent(); 427 LLVM_DEBUG(dbgs() << " try: " << OrigMI); 428 429 auto OrigOp = OrigMI.getOpcode(); 430 if (TII->isVOP3(OrigOp)) { 431 if (!TII->hasVALU32BitEncoding(OrigOp)) { 432 LLVM_DEBUG(dbgs() << " failed: VOP3 hasn't e32 equivalent\n"); 433 break; 434 } 435 // check if other than abs|neg modifiers are set (opsel for example) 436 const int64_t Mask = ~(SISrcMods::ABS | SISrcMods::NEG); 437 if (!hasNoImmOrEqual(OrigMI, AMDGPU::OpName::src0_modifiers, 0, Mask) || 438 !hasNoImmOrEqual(OrigMI, AMDGPU::OpName::src1_modifiers, 0, Mask) || 439 !hasNoImmOrEqual(OrigMI, AMDGPU::OpName::clamp, 0) || 440 !hasNoImmOrEqual(OrigMI, AMDGPU::OpName::omod, 0)) { 441 LLVM_DEBUG(dbgs() << " failed: VOP3 has non-default modifiers\n"); 442 break; 443 } 444 } else if (!TII->isVOP1(OrigOp) && !TII->isVOP2(OrigOp)) { 445 LLVM_DEBUG(dbgs() << " failed: not VOP1/2/3\n"); 446 break; 447 } 448 449 LLVM_DEBUG(dbgs() << " combining: " << OrigMI); 450 if (&Use == TII->getNamedOperand(OrigMI, AMDGPU::OpName::src0)) { 451 if (auto *DPPInst = createDPPInst(OrigMI, MovMI, CombOldVGPR, 452 OldOpndValue, CombBCZ)) { 453 DPPMIs.push_back(DPPInst); 454 Rollback = false; 455 } 456 } else if (OrigMI.isCommutable() && 457 &Use == TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1)) { 458 auto *BB = OrigMI.getParent(); 459 auto *NewMI = BB->getParent()->CloneMachineInstr(&OrigMI); 460 BB->insert(OrigMI, NewMI); 461 if (TII->commuteInstruction(*NewMI)) { 462 LLVM_DEBUG(dbgs() << " commuted: " << *NewMI); 463 if (auto *DPPInst = createDPPInst(*NewMI, MovMI, CombOldVGPR, 464 OldOpndValue, CombBCZ)) { 465 DPPMIs.push_back(DPPInst); 466 Rollback = false; 467 } 468 } else 469 LLVM_DEBUG(dbgs() << " failed: cannot be commuted\n"); 470 NewMI->eraseFromParent(); 471 } else 472 LLVM_DEBUG(dbgs() << " failed: no suitable operands\n"); 473 if (Rollback) 474 break; 475 OrigMIs.push_back(&OrigMI); 476 } 477 478 for (auto *MI : *(Rollback? &DPPMIs : &OrigMIs)) 479 MI->eraseFromParent(); 480 481 return !Rollback; 482 } 483 484 bool GCNDPPCombine::runOnMachineFunction(MachineFunction &MF) { 485 auto &ST = MF.getSubtarget<GCNSubtarget>(); 486 if (!ST.hasDPP() || skipFunction(MF.getFunction())) 487 return false; 488 489 MRI = &MF.getRegInfo(); 490 TII = ST.getInstrInfo(); 491 492 assert(MRI->isSSA() && "Must be run on SSA"); 493 494 bool Changed = false; 495 for (auto &MBB : MF) { 496 for (auto I = MBB.rbegin(), E = MBB.rend(); I != E;) { 497 auto &MI = *I++; 498 if (MI.getOpcode() == AMDGPU::V_MOV_B32_dpp && combineDPPMov(MI)) { 499 Changed = true; 500 ++NumDPPMovsCombined; 501 } 502 } 503 } 504 return Changed; 505 } 506