1 //===- SILoadStoreOptimizer.cpp -------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This pass tries to fuse DS instructions with close by immediate offsets. 10 // This will fuse operations such as 11 // ds_read_b32 v0, v2 offset:16 12 // ds_read_b32 v1, v2 offset:32 13 // ==> 14 // ds_read2_b32 v[0:1], v2, offset0:4 offset1:8 15 // 16 // The same is done for certain SMEM and VMEM opcodes, e.g.: 17 // s_buffer_load_dword s4, s[0:3], 4 18 // s_buffer_load_dword s5, s[0:3], 8 19 // ==> 20 // s_buffer_load_dwordx2 s[4:5], s[0:3], 4 21 // 22 // This pass also tries to promote constant offset to the immediate by 23 // adjusting the base. It tries to use a base from the nearby instructions that 24 // allows it to have a 13bit constant offset and then promotes the 13bit offset 25 // to the immediate. 26 // E.g. 27 // s_movk_i32 s0, 0x1800 28 // v_add_co_u32_e32 v0, vcc, s0, v2 29 // v_addc_co_u32_e32 v1, vcc, 0, v6, vcc 30 // 31 // s_movk_i32 s0, 0x1000 32 // v_add_co_u32_e32 v5, vcc, s0, v2 33 // v_addc_co_u32_e32 v6, vcc, 0, v6, vcc 34 // global_load_dwordx2 v[5:6], v[5:6], off 35 // global_load_dwordx2 v[0:1], v[0:1], off 36 // => 37 // s_movk_i32 s0, 0x1000 38 // v_add_co_u32_e32 v5, vcc, s0, v2 39 // v_addc_co_u32_e32 v6, vcc, 0, v6, vcc 40 // global_load_dwordx2 v[5:6], v[5:6], off 41 // global_load_dwordx2 v[0:1], v[5:6], off offset:2048 42 // 43 // Future improvements: 44 // 45 // - This is currently missing stores of constants because loading 46 // the constant into the data register is placed between the stores, although 47 // this is arguably a scheduling problem. 48 // 49 // - Live interval recomputing seems inefficient. This currently only matches 50 // one pair, and recomputes live intervals and moves on to the next pair. It 51 // would be better to compute a list of all merges that need to occur. 52 // 53 // - With a list of instructions to process, we can also merge more. If a 54 // cluster of loads have offsets that are too large to fit in the 8-bit 55 // offsets, but are close enough to fit in the 8 bits, we can add to the base 56 // pointer and use the new reduced offsets. 57 // 58 //===----------------------------------------------------------------------===// 59 60 #include "AMDGPU.h" 61 #include "GCNSubtarget.h" 62 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 63 #include "llvm/Analysis/AliasAnalysis.h" 64 #include "llvm/CodeGen/MachineFunctionPass.h" 65 #include "llvm/InitializePasses.h" 66 67 using namespace llvm; 68 69 #define DEBUG_TYPE "si-load-store-opt" 70 71 namespace { 72 enum InstClassEnum { 73 UNKNOWN, 74 DS_READ, 75 DS_WRITE, 76 S_BUFFER_LOAD_IMM, 77 BUFFER_LOAD, 78 BUFFER_STORE, 79 MIMG, 80 TBUFFER_LOAD, 81 TBUFFER_STORE, 82 }; 83 84 struct AddressRegs { 85 unsigned char NumVAddrs = 0; 86 bool SBase = false; 87 bool SRsrc = false; 88 bool SOffset = false; 89 bool VAddr = false; 90 bool Addr = false; 91 bool SSamp = false; 92 }; 93 94 // GFX10 image_sample instructions can have 12 vaddrs + srsrc + ssamp. 95 const unsigned MaxAddressRegs = 12 + 1 + 1; 96 97 class SILoadStoreOptimizer : public MachineFunctionPass { 98 struct CombineInfo { 99 MachineBasicBlock::iterator I; 100 unsigned EltSize; 101 unsigned Offset; 102 unsigned Width; 103 unsigned Format; 104 unsigned BaseOff; 105 unsigned DMask; 106 InstClassEnum InstClass; 107 bool GLC; 108 bool SLC; 109 bool DLC; 110 bool UseST64; 111 int AddrIdx[MaxAddressRegs]; 112 const MachineOperand *AddrReg[MaxAddressRegs]; 113 unsigned NumAddresses; 114 unsigned Order; 115 116 bool hasSameBaseAddress(const MachineInstr &MI) { 117 for (unsigned i = 0; i < NumAddresses; i++) { 118 const MachineOperand &AddrRegNext = MI.getOperand(AddrIdx[i]); 119 120 if (AddrReg[i]->isImm() || AddrRegNext.isImm()) { 121 if (AddrReg[i]->isImm() != AddrRegNext.isImm() || 122 AddrReg[i]->getImm() != AddrRegNext.getImm()) { 123 return false; 124 } 125 continue; 126 } 127 128 // Check same base pointer. Be careful of subregisters, which can occur 129 // with vectors of pointers. 130 if (AddrReg[i]->getReg() != AddrRegNext.getReg() || 131 AddrReg[i]->getSubReg() != AddrRegNext.getSubReg()) { 132 return false; 133 } 134 } 135 return true; 136 } 137 138 bool hasMergeableAddress(const MachineRegisterInfo &MRI) { 139 for (unsigned i = 0; i < NumAddresses; ++i) { 140 const MachineOperand *AddrOp = AddrReg[i]; 141 // Immediates are always OK. 142 if (AddrOp->isImm()) 143 continue; 144 145 // Don't try to merge addresses that aren't either immediates or registers. 146 // TODO: Should be possible to merge FrameIndexes and maybe some other 147 // non-register 148 if (!AddrOp->isReg()) 149 return false; 150 151 // TODO: We should be able to merge physical reg addreses. 152 if (AddrOp->getReg().isPhysical()) 153 return false; 154 155 // If an address has only one use then there will be on other 156 // instructions with the same address, so we can't merge this one. 157 if (MRI.hasOneNonDBGUse(AddrOp->getReg())) 158 return false; 159 } 160 return true; 161 } 162 163 void setMI(MachineBasicBlock::iterator MI, const SIInstrInfo &TII, 164 const GCNSubtarget &STM); 165 }; 166 167 struct BaseRegisters { 168 Register LoReg; 169 Register HiReg; 170 171 unsigned LoSubReg = 0; 172 unsigned HiSubReg = 0; 173 }; 174 175 struct MemAddress { 176 BaseRegisters Base; 177 int64_t Offset = 0; 178 }; 179 180 using MemInfoMap = DenseMap<MachineInstr *, MemAddress>; 181 182 private: 183 const GCNSubtarget *STM = nullptr; 184 const SIInstrInfo *TII = nullptr; 185 const SIRegisterInfo *TRI = nullptr; 186 MachineRegisterInfo *MRI = nullptr; 187 AliasAnalysis *AA = nullptr; 188 bool OptimizeAgain; 189 190 static bool dmasksCanBeCombined(const CombineInfo &CI, 191 const SIInstrInfo &TII, 192 const CombineInfo &Paired); 193 static bool offsetsCanBeCombined(CombineInfo &CI, const GCNSubtarget &STI, 194 CombineInfo &Paired, bool Modify = false); 195 static bool widthsFit(const GCNSubtarget &STI, const CombineInfo &CI, 196 const CombineInfo &Paired); 197 static unsigned getNewOpcode(const CombineInfo &CI, const CombineInfo &Paired); 198 static std::pair<unsigned, unsigned> getSubRegIdxs(const CombineInfo &CI, 199 const CombineInfo &Paired); 200 const TargetRegisterClass *getTargetRegisterClass(const CombineInfo &CI, 201 const CombineInfo &Paired); 202 203 bool checkAndPrepareMerge(CombineInfo &CI, CombineInfo &Paired, 204 SmallVectorImpl<MachineInstr *> &InstsToMove); 205 206 unsigned read2Opcode(unsigned EltSize) const; 207 unsigned read2ST64Opcode(unsigned EltSize) const; 208 MachineBasicBlock::iterator mergeRead2Pair(CombineInfo &CI, 209 CombineInfo &Paired, 210 const SmallVectorImpl<MachineInstr *> &InstsToMove); 211 212 unsigned write2Opcode(unsigned EltSize) const; 213 unsigned write2ST64Opcode(unsigned EltSize) const; 214 MachineBasicBlock::iterator 215 mergeWrite2Pair(CombineInfo &CI, CombineInfo &Paired, 216 const SmallVectorImpl<MachineInstr *> &InstsToMove); 217 MachineBasicBlock::iterator 218 mergeImagePair(CombineInfo &CI, CombineInfo &Paired, 219 const SmallVectorImpl<MachineInstr *> &InstsToMove); 220 MachineBasicBlock::iterator 221 mergeSBufferLoadImmPair(CombineInfo &CI, CombineInfo &Paired, 222 const SmallVectorImpl<MachineInstr *> &InstsToMove); 223 MachineBasicBlock::iterator 224 mergeBufferLoadPair(CombineInfo &CI, CombineInfo &Paired, 225 const SmallVectorImpl<MachineInstr *> &InstsToMove); 226 MachineBasicBlock::iterator 227 mergeBufferStorePair(CombineInfo &CI, CombineInfo &Paired, 228 const SmallVectorImpl<MachineInstr *> &InstsToMove); 229 MachineBasicBlock::iterator 230 mergeTBufferLoadPair(CombineInfo &CI, CombineInfo &Paired, 231 const SmallVectorImpl<MachineInstr *> &InstsToMove); 232 MachineBasicBlock::iterator 233 mergeTBufferStorePair(CombineInfo &CI, CombineInfo &Paired, 234 const SmallVectorImpl<MachineInstr *> &InstsToMove); 235 236 void updateBaseAndOffset(MachineInstr &I, Register NewBase, 237 int32_t NewOffset) const; 238 Register computeBase(MachineInstr &MI, const MemAddress &Addr) const; 239 MachineOperand createRegOrImm(int32_t Val, MachineInstr &MI) const; 240 Optional<int32_t> extractConstOffset(const MachineOperand &Op) const; 241 void processBaseWithConstOffset(const MachineOperand &Base, MemAddress &Addr) const; 242 /// Promotes constant offset to the immediate by adjusting the base. It 243 /// tries to use a base from the nearby instructions that allows it to have 244 /// a 13bit constant offset which gets promoted to the immediate. 245 bool promoteConstantOffsetToImm(MachineInstr &CI, 246 MemInfoMap &Visited, 247 SmallPtrSet<MachineInstr *, 4> &Promoted) const; 248 void addInstToMergeableList(const CombineInfo &CI, 249 std::list<std::list<CombineInfo> > &MergeableInsts) const; 250 251 std::pair<MachineBasicBlock::iterator, bool> collectMergeableInsts( 252 MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End, 253 MemInfoMap &Visited, SmallPtrSet<MachineInstr *, 4> &AnchorList, 254 std::list<std::list<CombineInfo>> &MergeableInsts) const; 255 256 public: 257 static char ID; 258 259 SILoadStoreOptimizer() : MachineFunctionPass(ID) { 260 initializeSILoadStoreOptimizerPass(*PassRegistry::getPassRegistry()); 261 } 262 263 bool optimizeInstsWithSameBaseAddr(std::list<CombineInfo> &MergeList, 264 bool &OptimizeListAgain); 265 bool optimizeBlock(std::list<std::list<CombineInfo> > &MergeableInsts); 266 267 bool runOnMachineFunction(MachineFunction &MF) override; 268 269 StringRef getPassName() const override { return "SI Load Store Optimizer"; } 270 271 void getAnalysisUsage(AnalysisUsage &AU) const override { 272 AU.setPreservesCFG(); 273 AU.addRequired<AAResultsWrapperPass>(); 274 275 MachineFunctionPass::getAnalysisUsage(AU); 276 } 277 278 MachineFunctionProperties getRequiredProperties() const override { 279 return MachineFunctionProperties() 280 .set(MachineFunctionProperties::Property::IsSSA); 281 } 282 }; 283 284 static unsigned getOpcodeWidth(const MachineInstr &MI, const SIInstrInfo &TII) { 285 const unsigned Opc = MI.getOpcode(); 286 287 if (TII.isMUBUF(Opc)) { 288 // FIXME: Handle d16 correctly 289 return AMDGPU::getMUBUFElements(Opc); 290 } 291 if (TII.isMIMG(MI)) { 292 uint64_t DMaskImm = 293 TII.getNamedOperand(MI, AMDGPU::OpName::dmask)->getImm(); 294 return countPopulation(DMaskImm); 295 } 296 if (TII.isMTBUF(Opc)) { 297 return AMDGPU::getMTBUFElements(Opc); 298 } 299 300 switch (Opc) { 301 case AMDGPU::S_BUFFER_LOAD_DWORD_IMM: 302 return 1; 303 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM: 304 return 2; 305 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM: 306 return 4; 307 default: 308 return 0; 309 } 310 } 311 312 /// Maps instruction opcode to enum InstClassEnum. 313 static InstClassEnum getInstClass(unsigned Opc, const SIInstrInfo &TII) { 314 switch (Opc) { 315 default: 316 if (TII.isMUBUF(Opc)) { 317 switch (AMDGPU::getMUBUFBaseOpcode(Opc)) { 318 default: 319 return UNKNOWN; 320 case AMDGPU::BUFFER_LOAD_DWORD_OFFEN: 321 case AMDGPU::BUFFER_LOAD_DWORD_OFFEN_exact: 322 case AMDGPU::BUFFER_LOAD_DWORD_OFFSET: 323 case AMDGPU::BUFFER_LOAD_DWORD_OFFSET_exact: 324 return BUFFER_LOAD; 325 case AMDGPU::BUFFER_STORE_DWORD_OFFEN: 326 case AMDGPU::BUFFER_STORE_DWORD_OFFEN_exact: 327 case AMDGPU::BUFFER_STORE_DWORD_OFFSET: 328 case AMDGPU::BUFFER_STORE_DWORD_OFFSET_exact: 329 return BUFFER_STORE; 330 } 331 } 332 if (TII.isMIMG(Opc)) { 333 // Ignore instructions encoded without vaddr. 334 if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr) == -1 && 335 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0) == -1) 336 return UNKNOWN; 337 // TODO: Support IMAGE_GET_RESINFO and IMAGE_GET_LOD. 338 if (TII.get(Opc).mayStore() || !TII.get(Opc).mayLoad() || 339 TII.isGather4(Opc)) 340 return UNKNOWN; 341 return MIMG; 342 } 343 if (TII.isMTBUF(Opc)) { 344 switch (AMDGPU::getMTBUFBaseOpcode(Opc)) { 345 default: 346 return UNKNOWN; 347 case AMDGPU::TBUFFER_LOAD_FORMAT_X_OFFEN: 348 case AMDGPU::TBUFFER_LOAD_FORMAT_X_OFFEN_exact: 349 case AMDGPU::TBUFFER_LOAD_FORMAT_X_OFFSET: 350 case AMDGPU::TBUFFER_LOAD_FORMAT_X_OFFSET_exact: 351 return TBUFFER_LOAD; 352 case AMDGPU::TBUFFER_STORE_FORMAT_X_OFFEN: 353 case AMDGPU::TBUFFER_STORE_FORMAT_X_OFFEN_exact: 354 case AMDGPU::TBUFFER_STORE_FORMAT_X_OFFSET: 355 case AMDGPU::TBUFFER_STORE_FORMAT_X_OFFSET_exact: 356 return TBUFFER_STORE; 357 } 358 } 359 return UNKNOWN; 360 case AMDGPU::S_BUFFER_LOAD_DWORD_IMM: 361 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM: 362 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM: 363 return S_BUFFER_LOAD_IMM; 364 case AMDGPU::DS_READ_B32: 365 case AMDGPU::DS_READ_B32_gfx9: 366 case AMDGPU::DS_READ_B64: 367 case AMDGPU::DS_READ_B64_gfx9: 368 return DS_READ; 369 case AMDGPU::DS_WRITE_B32: 370 case AMDGPU::DS_WRITE_B32_gfx9: 371 case AMDGPU::DS_WRITE_B64: 372 case AMDGPU::DS_WRITE_B64_gfx9: 373 return DS_WRITE; 374 case AMDGPU::IMAGE_BVH_INTERSECT_RAY_sa: 375 case AMDGPU::IMAGE_BVH64_INTERSECT_RAY_sa: 376 case AMDGPU::IMAGE_BVH_INTERSECT_RAY_a16_sa: 377 case AMDGPU::IMAGE_BVH64_INTERSECT_RAY_a16_sa: 378 case AMDGPU::IMAGE_BVH_INTERSECT_RAY_nsa: 379 case AMDGPU::IMAGE_BVH64_INTERSECT_RAY_nsa: 380 case AMDGPU::IMAGE_BVH_INTERSECT_RAY_a16_nsa: 381 case AMDGPU::IMAGE_BVH64_INTERSECT_RAY_a16_nsa: 382 return UNKNOWN; 383 } 384 } 385 386 /// Determines instruction subclass from opcode. Only instructions 387 /// of the same subclass can be merged together. 388 static unsigned getInstSubclass(unsigned Opc, const SIInstrInfo &TII) { 389 switch (Opc) { 390 default: 391 if (TII.isMUBUF(Opc)) 392 return AMDGPU::getMUBUFBaseOpcode(Opc); 393 if (TII.isMIMG(Opc)) { 394 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc); 395 assert(Info); 396 return Info->BaseOpcode; 397 } 398 if (TII.isMTBUF(Opc)) 399 return AMDGPU::getMTBUFBaseOpcode(Opc); 400 return -1; 401 case AMDGPU::DS_READ_B32: 402 case AMDGPU::DS_READ_B32_gfx9: 403 case AMDGPU::DS_READ_B64: 404 case AMDGPU::DS_READ_B64_gfx9: 405 case AMDGPU::DS_WRITE_B32: 406 case AMDGPU::DS_WRITE_B32_gfx9: 407 case AMDGPU::DS_WRITE_B64: 408 case AMDGPU::DS_WRITE_B64_gfx9: 409 return Opc; 410 case AMDGPU::S_BUFFER_LOAD_DWORD_IMM: 411 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM: 412 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM: 413 return AMDGPU::S_BUFFER_LOAD_DWORD_IMM; 414 } 415 } 416 417 static AddressRegs getRegs(unsigned Opc, const SIInstrInfo &TII) { 418 AddressRegs Result; 419 420 if (TII.isMUBUF(Opc)) { 421 if (AMDGPU::getMUBUFHasVAddr(Opc)) 422 Result.VAddr = true; 423 if (AMDGPU::getMUBUFHasSrsrc(Opc)) 424 Result.SRsrc = true; 425 if (AMDGPU::getMUBUFHasSoffset(Opc)) 426 Result.SOffset = true; 427 428 return Result; 429 } 430 431 if (TII.isMIMG(Opc)) { 432 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0); 433 if (VAddr0Idx >= 0) { 434 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc); 435 Result.NumVAddrs = SRsrcIdx - VAddr0Idx; 436 } else { 437 Result.VAddr = true; 438 } 439 Result.SRsrc = true; 440 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc); 441 if (Info && AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode)->Sampler) 442 Result.SSamp = true; 443 444 return Result; 445 } 446 if (TII.isMTBUF(Opc)) { 447 if (AMDGPU::getMTBUFHasVAddr(Opc)) 448 Result.VAddr = true; 449 if (AMDGPU::getMTBUFHasSrsrc(Opc)) 450 Result.SRsrc = true; 451 if (AMDGPU::getMTBUFHasSoffset(Opc)) 452 Result.SOffset = true; 453 454 return Result; 455 } 456 457 switch (Opc) { 458 default: 459 return Result; 460 case AMDGPU::S_BUFFER_LOAD_DWORD_IMM: 461 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM: 462 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM: 463 Result.SBase = true; 464 return Result; 465 case AMDGPU::DS_READ_B32: 466 case AMDGPU::DS_READ_B64: 467 case AMDGPU::DS_READ_B32_gfx9: 468 case AMDGPU::DS_READ_B64_gfx9: 469 case AMDGPU::DS_WRITE_B32: 470 case AMDGPU::DS_WRITE_B64: 471 case AMDGPU::DS_WRITE_B32_gfx9: 472 case AMDGPU::DS_WRITE_B64_gfx9: 473 Result.Addr = true; 474 return Result; 475 } 476 } 477 478 void SILoadStoreOptimizer::CombineInfo::setMI(MachineBasicBlock::iterator MI, 479 const SIInstrInfo &TII, 480 const GCNSubtarget &STM) { 481 I = MI; 482 unsigned Opc = MI->getOpcode(); 483 InstClass = getInstClass(Opc, TII); 484 485 if (InstClass == UNKNOWN) 486 return; 487 488 switch (InstClass) { 489 case DS_READ: 490 EltSize = 491 (Opc == AMDGPU::DS_READ_B64 || Opc == AMDGPU::DS_READ_B64_gfx9) ? 8 492 : 4; 493 break; 494 case DS_WRITE: 495 EltSize = 496 (Opc == AMDGPU::DS_WRITE_B64 || Opc == AMDGPU::DS_WRITE_B64_gfx9) ? 8 497 : 4; 498 break; 499 case S_BUFFER_LOAD_IMM: 500 EltSize = AMDGPU::convertSMRDOffsetUnits(STM, 4); 501 break; 502 default: 503 EltSize = 4; 504 break; 505 } 506 507 if (InstClass == MIMG) { 508 DMask = TII.getNamedOperand(*I, AMDGPU::OpName::dmask)->getImm(); 509 // Offset is not considered for MIMG instructions. 510 Offset = 0; 511 } else { 512 int OffsetIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::offset); 513 Offset = I->getOperand(OffsetIdx).getImm(); 514 } 515 516 if (InstClass == TBUFFER_LOAD || InstClass == TBUFFER_STORE) 517 Format = TII.getNamedOperand(*I, AMDGPU::OpName::format)->getImm(); 518 519 Width = getOpcodeWidth(*I, TII); 520 521 if ((InstClass == DS_READ) || (InstClass == DS_WRITE)) { 522 Offset &= 0xffff; 523 } else if (InstClass != MIMG) { 524 GLC = TII.getNamedOperand(*I, AMDGPU::OpName::glc)->getImm(); 525 if (InstClass != S_BUFFER_LOAD_IMM) { 526 SLC = TII.getNamedOperand(*I, AMDGPU::OpName::slc)->getImm(); 527 } 528 DLC = TII.getNamedOperand(*I, AMDGPU::OpName::dlc)->getImm(); 529 } 530 531 AddressRegs Regs = getRegs(Opc, TII); 532 533 NumAddresses = 0; 534 for (unsigned J = 0; J < Regs.NumVAddrs; J++) 535 AddrIdx[NumAddresses++] = 536 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0) + J; 537 if (Regs.Addr) 538 AddrIdx[NumAddresses++] = 539 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::addr); 540 if (Regs.SBase) 541 AddrIdx[NumAddresses++] = 542 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::sbase); 543 if (Regs.SRsrc) 544 AddrIdx[NumAddresses++] = 545 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc); 546 if (Regs.SOffset) 547 AddrIdx[NumAddresses++] = 548 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::soffset); 549 if (Regs.VAddr) 550 AddrIdx[NumAddresses++] = 551 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr); 552 if (Regs.SSamp) 553 AddrIdx[NumAddresses++] = 554 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::ssamp); 555 assert(NumAddresses <= MaxAddressRegs); 556 557 for (unsigned J = 0; J < NumAddresses; J++) 558 AddrReg[J] = &I->getOperand(AddrIdx[J]); 559 } 560 561 } // end anonymous namespace. 562 563 INITIALIZE_PASS_BEGIN(SILoadStoreOptimizer, DEBUG_TYPE, 564 "SI Load Store Optimizer", false, false) 565 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) 566 INITIALIZE_PASS_END(SILoadStoreOptimizer, DEBUG_TYPE, "SI Load Store Optimizer", 567 false, false) 568 569 char SILoadStoreOptimizer::ID = 0; 570 571 char &llvm::SILoadStoreOptimizerID = SILoadStoreOptimizer::ID; 572 573 FunctionPass *llvm::createSILoadStoreOptimizerPass() { 574 return new SILoadStoreOptimizer(); 575 } 576 577 static void moveInstsAfter(MachineBasicBlock::iterator I, 578 ArrayRef<MachineInstr *> InstsToMove) { 579 MachineBasicBlock *MBB = I->getParent(); 580 ++I; 581 for (MachineInstr *MI : InstsToMove) { 582 MI->removeFromParent(); 583 MBB->insert(I, MI); 584 } 585 } 586 587 static void addDefsUsesToList(const MachineInstr &MI, 588 DenseSet<Register> &RegDefs, 589 DenseSet<Register> &PhysRegUses) { 590 for (const MachineOperand &Op : MI.operands()) { 591 if (Op.isReg()) { 592 if (Op.isDef()) 593 RegDefs.insert(Op.getReg()); 594 else if (Op.readsReg() && Op.getReg().isPhysical()) 595 PhysRegUses.insert(Op.getReg()); 596 } 597 } 598 } 599 600 static bool memAccessesCanBeReordered(MachineBasicBlock::iterator A, 601 MachineBasicBlock::iterator B, 602 AliasAnalysis *AA) { 603 // RAW or WAR - cannot reorder 604 // WAW - cannot reorder 605 // RAR - safe to reorder 606 return !(A->mayStore() || B->mayStore()) || !A->mayAlias(AA, *B, true); 607 } 608 609 // Add MI and its defs to the lists if MI reads one of the defs that are 610 // already in the list. Returns true in that case. 611 static bool addToListsIfDependent(MachineInstr &MI, DenseSet<Register> &RegDefs, 612 DenseSet<Register> &PhysRegUses, 613 SmallVectorImpl<MachineInstr *> &Insts) { 614 for (MachineOperand &Use : MI.operands()) { 615 // If one of the defs is read, then there is a use of Def between I and the 616 // instruction that I will potentially be merged with. We will need to move 617 // this instruction after the merged instructions. 618 // 619 // Similarly, if there is a def which is read by an instruction that is to 620 // be moved for merging, then we need to move the def-instruction as well. 621 // This can only happen for physical registers such as M0; virtual 622 // registers are in SSA form. 623 if (Use.isReg() && ((Use.readsReg() && RegDefs.count(Use.getReg())) || 624 (Use.isDef() && RegDefs.count(Use.getReg())) || 625 (Use.isDef() && Use.getReg().isPhysical() && 626 PhysRegUses.count(Use.getReg())))) { 627 Insts.push_back(&MI); 628 addDefsUsesToList(MI, RegDefs, PhysRegUses); 629 return true; 630 } 631 } 632 633 return false; 634 } 635 636 static bool canMoveInstsAcrossMemOp(MachineInstr &MemOp, 637 ArrayRef<MachineInstr *> InstsToMove, 638 AliasAnalysis *AA) { 639 assert(MemOp.mayLoadOrStore()); 640 641 for (MachineInstr *InstToMove : InstsToMove) { 642 if (!InstToMove->mayLoadOrStore()) 643 continue; 644 if (!memAccessesCanBeReordered(MemOp, *InstToMove, AA)) 645 return false; 646 } 647 return true; 648 } 649 650 // This function assumes that \p A and \p B have are identical except for 651 // size and offset, and they referecne adjacent memory. 652 static MachineMemOperand *combineKnownAdjacentMMOs(MachineFunction &MF, 653 const MachineMemOperand *A, 654 const MachineMemOperand *B) { 655 unsigned MinOffset = std::min(A->getOffset(), B->getOffset()); 656 unsigned Size = A->getSize() + B->getSize(); 657 // This function adds the offset parameter to the existing offset for A, 658 // so we pass 0 here as the offset and then manually set it to the correct 659 // value after the call. 660 MachineMemOperand *MMO = MF.getMachineMemOperand(A, 0, Size); 661 MMO->setOffset(MinOffset); 662 return MMO; 663 } 664 665 bool SILoadStoreOptimizer::dmasksCanBeCombined(const CombineInfo &CI, 666 const SIInstrInfo &TII, 667 const CombineInfo &Paired) { 668 assert(CI.InstClass == MIMG); 669 670 // Ignore instructions with tfe/lwe set. 671 const auto *TFEOp = TII.getNamedOperand(*CI.I, AMDGPU::OpName::tfe); 672 const auto *LWEOp = TII.getNamedOperand(*CI.I, AMDGPU::OpName::lwe); 673 674 if ((TFEOp && TFEOp->getImm()) || (LWEOp && LWEOp->getImm())) 675 return false; 676 677 // Check other optional immediate operands for equality. 678 unsigned OperandsToMatch[] = {AMDGPU::OpName::glc, AMDGPU::OpName::slc, 679 AMDGPU::OpName::d16, AMDGPU::OpName::unorm, 680 AMDGPU::OpName::da, AMDGPU::OpName::r128, 681 AMDGPU::OpName::a16, AMDGPU::OpName::dlc}; 682 683 for (auto op : OperandsToMatch) { 684 int Idx = AMDGPU::getNamedOperandIdx(CI.I->getOpcode(), op); 685 if (AMDGPU::getNamedOperandIdx(Paired.I->getOpcode(), op) != Idx) 686 return false; 687 if (Idx != -1 && 688 CI.I->getOperand(Idx).getImm() != Paired.I->getOperand(Idx).getImm()) 689 return false; 690 } 691 692 // Check DMask for overlaps. 693 unsigned MaxMask = std::max(CI.DMask, Paired.DMask); 694 unsigned MinMask = std::min(CI.DMask, Paired.DMask); 695 696 unsigned AllowedBitsForMin = llvm::countTrailingZeros(MaxMask); 697 if ((1u << AllowedBitsForMin) <= MinMask) 698 return false; 699 700 return true; 701 } 702 703 static unsigned getBufferFormatWithCompCount(unsigned OldFormat, 704 unsigned ComponentCount, 705 const GCNSubtarget &STI) { 706 if (ComponentCount > 4) 707 return 0; 708 709 const llvm::AMDGPU::GcnBufferFormatInfo *OldFormatInfo = 710 llvm::AMDGPU::getGcnBufferFormatInfo(OldFormat, STI); 711 if (!OldFormatInfo) 712 return 0; 713 714 const llvm::AMDGPU::GcnBufferFormatInfo *NewFormatInfo = 715 llvm::AMDGPU::getGcnBufferFormatInfo(OldFormatInfo->BitsPerComp, 716 ComponentCount, 717 OldFormatInfo->NumFormat, STI); 718 719 if (!NewFormatInfo) 720 return 0; 721 722 assert(NewFormatInfo->NumFormat == OldFormatInfo->NumFormat && 723 NewFormatInfo->BitsPerComp == OldFormatInfo->BitsPerComp); 724 725 return NewFormatInfo->Format; 726 } 727 728 bool SILoadStoreOptimizer::offsetsCanBeCombined(CombineInfo &CI, 729 const GCNSubtarget &STI, 730 CombineInfo &Paired, 731 bool Modify) { 732 assert(CI.InstClass != MIMG); 733 734 // XXX - Would the same offset be OK? Is there any reason this would happen or 735 // be useful? 736 if (CI.Offset == Paired.Offset) 737 return false; 738 739 // This won't be valid if the offset isn't aligned. 740 if ((CI.Offset % CI.EltSize != 0) || (Paired.Offset % CI.EltSize != 0)) 741 return false; 742 743 if (CI.InstClass == TBUFFER_LOAD || CI.InstClass == TBUFFER_STORE) { 744 745 const llvm::AMDGPU::GcnBufferFormatInfo *Info0 = 746 llvm::AMDGPU::getGcnBufferFormatInfo(CI.Format, STI); 747 if (!Info0) 748 return false; 749 const llvm::AMDGPU::GcnBufferFormatInfo *Info1 = 750 llvm::AMDGPU::getGcnBufferFormatInfo(Paired.Format, STI); 751 if (!Info1) 752 return false; 753 754 if (Info0->BitsPerComp != Info1->BitsPerComp || 755 Info0->NumFormat != Info1->NumFormat) 756 return false; 757 758 // TODO: Should be possible to support more formats, but if format loads 759 // are not dword-aligned, the merged load might not be valid. 760 if (Info0->BitsPerComp != 32) 761 return false; 762 763 if (getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, STI) == 0) 764 return false; 765 } 766 767 unsigned EltOffset0 = CI.Offset / CI.EltSize; 768 unsigned EltOffset1 = Paired.Offset / CI.EltSize; 769 CI.UseST64 = false; 770 CI.BaseOff = 0; 771 772 // Handle DS instructions. 773 if ((CI.InstClass != DS_READ) && (CI.InstClass != DS_WRITE)) { 774 return (EltOffset0 + CI.Width == EltOffset1 || 775 EltOffset1 + Paired.Width == EltOffset0) && 776 CI.GLC == Paired.GLC && CI.DLC == Paired.DLC && 777 (CI.InstClass == S_BUFFER_LOAD_IMM || CI.SLC == Paired.SLC); 778 } 779 780 // Handle SMEM and VMEM instructions. 781 // If the offset in elements doesn't fit in 8-bits, we might be able to use 782 // the stride 64 versions. 783 if ((EltOffset0 % 64 == 0) && (EltOffset1 % 64) == 0 && 784 isUInt<8>(EltOffset0 / 64) && isUInt<8>(EltOffset1 / 64)) { 785 if (Modify) { 786 CI.Offset = EltOffset0 / 64; 787 Paired.Offset = EltOffset1 / 64; 788 CI.UseST64 = true; 789 } 790 return true; 791 } 792 793 // Check if the new offsets fit in the reduced 8-bit range. 794 if (isUInt<8>(EltOffset0) && isUInt<8>(EltOffset1)) { 795 if (Modify) { 796 CI.Offset = EltOffset0; 797 Paired.Offset = EltOffset1; 798 } 799 return true; 800 } 801 802 // Try to shift base address to decrease offsets. 803 unsigned OffsetDiff = std::abs((int)EltOffset1 - (int)EltOffset0); 804 CI.BaseOff = std::min(CI.Offset, Paired.Offset); 805 806 if ((OffsetDiff % 64 == 0) && isUInt<8>(OffsetDiff / 64)) { 807 if (Modify) { 808 CI.Offset = (EltOffset0 - CI.BaseOff / CI.EltSize) / 64; 809 Paired.Offset = (EltOffset1 - CI.BaseOff / CI.EltSize) / 64; 810 CI.UseST64 = true; 811 } 812 return true; 813 } 814 815 if (isUInt<8>(OffsetDiff)) { 816 if (Modify) { 817 CI.Offset = EltOffset0 - CI.BaseOff / CI.EltSize; 818 Paired.Offset = EltOffset1 - CI.BaseOff / CI.EltSize; 819 } 820 return true; 821 } 822 823 return false; 824 } 825 826 bool SILoadStoreOptimizer::widthsFit(const GCNSubtarget &STM, 827 const CombineInfo &CI, 828 const CombineInfo &Paired) { 829 const unsigned Width = (CI.Width + Paired.Width); 830 switch (CI.InstClass) { 831 default: 832 return (Width <= 4) && (STM.hasDwordx3LoadStores() || (Width != 3)); 833 case S_BUFFER_LOAD_IMM: 834 switch (Width) { 835 default: 836 return false; 837 case 2: 838 case 4: 839 return true; 840 } 841 } 842 } 843 844 /// This function assumes that CI comes before Paired in a basic block. 845 bool SILoadStoreOptimizer::checkAndPrepareMerge( 846 CombineInfo &CI, CombineInfo &Paired, 847 SmallVectorImpl<MachineInstr *> &InstsToMove) { 848 849 // Check both offsets (or masks for MIMG) can be combined and fit in the 850 // reduced range. 851 if (CI.InstClass == MIMG && !dmasksCanBeCombined(CI, *TII, Paired)) 852 return false; 853 854 if (CI.InstClass != MIMG && 855 (!widthsFit(*STM, CI, Paired) || !offsetsCanBeCombined(CI, *STM, Paired))) 856 return false; 857 858 const unsigned Opc = CI.I->getOpcode(); 859 const InstClassEnum InstClass = getInstClass(Opc, *TII); 860 861 if (InstClass == UNKNOWN) { 862 return false; 863 } 864 const unsigned InstSubclass = getInstSubclass(Opc, *TII); 865 866 // Do not merge VMEM buffer instructions with "swizzled" bit set. 867 int Swizzled = 868 AMDGPU::getNamedOperandIdx(CI.I->getOpcode(), AMDGPU::OpName::swz); 869 if (Swizzled != -1 && CI.I->getOperand(Swizzled).getImm()) 870 return false; 871 872 DenseSet<Register> RegDefsToMove; 873 DenseSet<Register> PhysRegUsesToMove; 874 addDefsUsesToList(*CI.I, RegDefsToMove, PhysRegUsesToMove); 875 876 MachineBasicBlock::iterator E = std::next(Paired.I); 877 MachineBasicBlock::iterator MBBI = std::next(CI.I); 878 MachineBasicBlock::iterator MBBE = CI.I->getParent()->end(); 879 for (; MBBI != E; ++MBBI) { 880 881 if (MBBI == MBBE) { 882 // CombineInfo::Order is a hint on the instruction ordering within the 883 // basic block. This hint suggests that CI precedes Paired, which is 884 // true most of the time. However, moveInstsAfter() processing a 885 // previous list may have changed this order in a situation when it 886 // moves an instruction which exists in some other merge list. 887 // In this case it must be dependent. 888 return false; 889 } 890 891 if ((getInstClass(MBBI->getOpcode(), *TII) != InstClass) || 892 (getInstSubclass(MBBI->getOpcode(), *TII) != InstSubclass)) { 893 // This is not a matching instruction, but we can keep looking as 894 // long as one of these conditions are met: 895 // 1. It is safe to move I down past MBBI. 896 // 2. It is safe to move MBBI down past the instruction that I will 897 // be merged into. 898 899 if (MBBI->hasUnmodeledSideEffects()) { 900 // We can't re-order this instruction with respect to other memory 901 // operations, so we fail both conditions mentioned above. 902 return false; 903 } 904 905 if (MBBI->mayLoadOrStore() && 906 (!memAccessesCanBeReordered(*CI.I, *MBBI, AA) || 907 !canMoveInstsAcrossMemOp(*MBBI, InstsToMove, AA))) { 908 // We fail condition #1, but we may still be able to satisfy condition 909 // #2. Add this instruction to the move list and then we will check 910 // if condition #2 holds once we have selected the matching instruction. 911 InstsToMove.push_back(&*MBBI); 912 addDefsUsesToList(*MBBI, RegDefsToMove, PhysRegUsesToMove); 913 continue; 914 } 915 916 // When we match I with another DS instruction we will be moving I down 917 // to the location of the matched instruction any uses of I will need to 918 // be moved down as well. 919 addToListsIfDependent(*MBBI, RegDefsToMove, PhysRegUsesToMove, 920 InstsToMove); 921 continue; 922 } 923 924 // Don't merge volatiles. 925 if (MBBI->hasOrderedMemoryRef()) 926 return false; 927 928 int Swizzled = 929 AMDGPU::getNamedOperandIdx(MBBI->getOpcode(), AMDGPU::OpName::swz); 930 if (Swizzled != -1 && MBBI->getOperand(Swizzled).getImm()) 931 return false; 932 933 // Handle a case like 934 // DS_WRITE_B32 addr, v, idx0 935 // w = DS_READ_B32 addr, idx0 936 // DS_WRITE_B32 addr, f(w), idx1 937 // where the DS_READ_B32 ends up in InstsToMove and therefore prevents 938 // merging of the two writes. 939 if (addToListsIfDependent(*MBBI, RegDefsToMove, PhysRegUsesToMove, 940 InstsToMove)) 941 continue; 942 943 if (&*MBBI == &*Paired.I) { 944 // We need to go through the list of instructions that we plan to 945 // move and make sure they are all safe to move down past the merged 946 // instruction. 947 if (canMoveInstsAcrossMemOp(*MBBI, InstsToMove, AA)) { 948 949 // Call offsetsCanBeCombined with modify = true so that the offsets are 950 // correct for the new instruction. This should return true, because 951 // this function should only be called on CombineInfo objects that 952 // have already been confirmed to be mergeable. 953 if (CI.InstClass != MIMG) 954 offsetsCanBeCombined(CI, *STM, Paired, true); 955 return true; 956 } 957 return false; 958 } 959 960 // We've found a load/store that we couldn't merge for some reason. 961 // We could potentially keep looking, but we'd need to make sure that 962 // it was safe to move I and also all the instruction in InstsToMove 963 // down past this instruction. 964 // check if we can move I across MBBI and if we can move all I's users 965 if (!memAccessesCanBeReordered(*CI.I, *MBBI, AA) || 966 !canMoveInstsAcrossMemOp(*MBBI, InstsToMove, AA)) 967 break; 968 } 969 return false; 970 } 971 972 unsigned SILoadStoreOptimizer::read2Opcode(unsigned EltSize) const { 973 if (STM->ldsRequiresM0Init()) 974 return (EltSize == 4) ? AMDGPU::DS_READ2_B32 : AMDGPU::DS_READ2_B64; 975 return (EltSize == 4) ? AMDGPU::DS_READ2_B32_gfx9 : AMDGPU::DS_READ2_B64_gfx9; 976 } 977 978 unsigned SILoadStoreOptimizer::read2ST64Opcode(unsigned EltSize) const { 979 if (STM->ldsRequiresM0Init()) 980 return (EltSize == 4) ? AMDGPU::DS_READ2ST64_B32 : AMDGPU::DS_READ2ST64_B64; 981 982 return (EltSize == 4) ? AMDGPU::DS_READ2ST64_B32_gfx9 983 : AMDGPU::DS_READ2ST64_B64_gfx9; 984 } 985 986 MachineBasicBlock::iterator 987 SILoadStoreOptimizer::mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired, 988 const SmallVectorImpl<MachineInstr *> &InstsToMove) { 989 MachineBasicBlock *MBB = CI.I->getParent(); 990 991 // Be careful, since the addresses could be subregisters themselves in weird 992 // cases, like vectors of pointers. 993 const auto *AddrReg = TII->getNamedOperand(*CI.I, AMDGPU::OpName::addr); 994 995 const auto *Dest0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdst); 996 const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdst); 997 998 unsigned NewOffset0 = CI.Offset; 999 unsigned NewOffset1 = Paired.Offset; 1000 unsigned Opc = 1001 CI.UseST64 ? read2ST64Opcode(CI.EltSize) : read2Opcode(CI.EltSize); 1002 1003 unsigned SubRegIdx0 = (CI.EltSize == 4) ? AMDGPU::sub0 : AMDGPU::sub0_sub1; 1004 unsigned SubRegIdx1 = (CI.EltSize == 4) ? AMDGPU::sub1 : AMDGPU::sub2_sub3; 1005 1006 if (NewOffset0 > NewOffset1) { 1007 // Canonicalize the merged instruction so the smaller offset comes first. 1008 std::swap(NewOffset0, NewOffset1); 1009 std::swap(SubRegIdx0, SubRegIdx1); 1010 } 1011 1012 assert((isUInt<8>(NewOffset0) && isUInt<8>(NewOffset1)) && 1013 (NewOffset0 != NewOffset1) && "Computed offset doesn't fit"); 1014 1015 const MCInstrDesc &Read2Desc = TII->get(Opc); 1016 1017 const TargetRegisterClass *SuperRC = 1018 (CI.EltSize == 4) ? &AMDGPU::VReg_64RegClass : &AMDGPU::VReg_128RegClass; 1019 Register DestReg = MRI->createVirtualRegister(SuperRC); 1020 1021 DebugLoc DL = CI.I->getDebugLoc(); 1022 1023 Register BaseReg = AddrReg->getReg(); 1024 unsigned BaseSubReg = AddrReg->getSubReg(); 1025 unsigned BaseRegFlags = 0; 1026 if (CI.BaseOff) { 1027 Register ImmReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); 1028 BuildMI(*MBB, Paired.I, DL, TII->get(AMDGPU::S_MOV_B32), ImmReg) 1029 .addImm(CI.BaseOff); 1030 1031 BaseReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); 1032 BaseRegFlags = RegState::Kill; 1033 1034 TII->getAddNoCarry(*MBB, Paired.I, DL, BaseReg) 1035 .addReg(ImmReg) 1036 .addReg(AddrReg->getReg(), 0, BaseSubReg) 1037 .addImm(0); // clamp bit 1038 BaseSubReg = 0; 1039 } 1040 1041 MachineInstrBuilder Read2 = 1042 BuildMI(*MBB, Paired.I, DL, Read2Desc, DestReg) 1043 .addReg(BaseReg, BaseRegFlags, BaseSubReg) // addr 1044 .addImm(NewOffset0) // offset0 1045 .addImm(NewOffset1) // offset1 1046 .addImm(0) // gds 1047 .cloneMergedMemRefs({&*CI.I, &*Paired.I}); 1048 1049 (void)Read2; 1050 1051 const MCInstrDesc &CopyDesc = TII->get(TargetOpcode::COPY); 1052 1053 // Copy to the old destination registers. 1054 BuildMI(*MBB, Paired.I, DL, CopyDesc) 1055 .add(*Dest0) // Copy to same destination including flags and sub reg. 1056 .addReg(DestReg, 0, SubRegIdx0); 1057 MachineInstr *Copy1 = BuildMI(*MBB, Paired.I, DL, CopyDesc) 1058 .add(*Dest1) 1059 .addReg(DestReg, RegState::Kill, SubRegIdx1); 1060 1061 moveInstsAfter(Copy1, InstsToMove); 1062 1063 CI.I->eraseFromParent(); 1064 Paired.I->eraseFromParent(); 1065 1066 LLVM_DEBUG(dbgs() << "Inserted read2: " << *Read2 << '\n'); 1067 return Read2; 1068 } 1069 1070 unsigned SILoadStoreOptimizer::write2Opcode(unsigned EltSize) const { 1071 if (STM->ldsRequiresM0Init()) 1072 return (EltSize == 4) ? AMDGPU::DS_WRITE2_B32 : AMDGPU::DS_WRITE2_B64; 1073 return (EltSize == 4) ? AMDGPU::DS_WRITE2_B32_gfx9 1074 : AMDGPU::DS_WRITE2_B64_gfx9; 1075 } 1076 1077 unsigned SILoadStoreOptimizer::write2ST64Opcode(unsigned EltSize) const { 1078 if (STM->ldsRequiresM0Init()) 1079 return (EltSize == 4) ? AMDGPU::DS_WRITE2ST64_B32 1080 : AMDGPU::DS_WRITE2ST64_B64; 1081 1082 return (EltSize == 4) ? AMDGPU::DS_WRITE2ST64_B32_gfx9 1083 : AMDGPU::DS_WRITE2ST64_B64_gfx9; 1084 } 1085 1086 MachineBasicBlock::iterator 1087 SILoadStoreOptimizer::mergeWrite2Pair(CombineInfo &CI, CombineInfo &Paired, 1088 const SmallVectorImpl<MachineInstr *> &InstsToMove) { 1089 MachineBasicBlock *MBB = CI.I->getParent(); 1090 1091 // Be sure to use .addOperand(), and not .addReg() with these. We want to be 1092 // sure we preserve the subregister index and any register flags set on them. 1093 const MachineOperand *AddrReg = 1094 TII->getNamedOperand(*CI.I, AMDGPU::OpName::addr); 1095 const MachineOperand *Data0 = 1096 TII->getNamedOperand(*CI.I, AMDGPU::OpName::data0); 1097 const MachineOperand *Data1 = 1098 TII->getNamedOperand(*Paired.I, AMDGPU::OpName::data0); 1099 1100 unsigned NewOffset0 = CI.Offset; 1101 unsigned NewOffset1 = Paired.Offset; 1102 unsigned Opc = 1103 CI.UseST64 ? write2ST64Opcode(CI.EltSize) : write2Opcode(CI.EltSize); 1104 1105 if (NewOffset0 > NewOffset1) { 1106 // Canonicalize the merged instruction so the smaller offset comes first. 1107 std::swap(NewOffset0, NewOffset1); 1108 std::swap(Data0, Data1); 1109 } 1110 1111 assert((isUInt<8>(NewOffset0) && isUInt<8>(NewOffset1)) && 1112 (NewOffset0 != NewOffset1) && "Computed offset doesn't fit"); 1113 1114 const MCInstrDesc &Write2Desc = TII->get(Opc); 1115 DebugLoc DL = CI.I->getDebugLoc(); 1116 1117 Register BaseReg = AddrReg->getReg(); 1118 unsigned BaseSubReg = AddrReg->getSubReg(); 1119 unsigned BaseRegFlags = 0; 1120 if (CI.BaseOff) { 1121 Register ImmReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); 1122 BuildMI(*MBB, Paired.I, DL, TII->get(AMDGPU::S_MOV_B32), ImmReg) 1123 .addImm(CI.BaseOff); 1124 1125 BaseReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); 1126 BaseRegFlags = RegState::Kill; 1127 1128 TII->getAddNoCarry(*MBB, Paired.I, DL, BaseReg) 1129 .addReg(ImmReg) 1130 .addReg(AddrReg->getReg(), 0, BaseSubReg) 1131 .addImm(0); // clamp bit 1132 BaseSubReg = 0; 1133 } 1134 1135 MachineInstrBuilder Write2 = 1136 BuildMI(*MBB, Paired.I, DL, Write2Desc) 1137 .addReg(BaseReg, BaseRegFlags, BaseSubReg) // addr 1138 .add(*Data0) // data0 1139 .add(*Data1) // data1 1140 .addImm(NewOffset0) // offset0 1141 .addImm(NewOffset1) // offset1 1142 .addImm(0) // gds 1143 .cloneMergedMemRefs({&*CI.I, &*Paired.I}); 1144 1145 moveInstsAfter(Write2, InstsToMove); 1146 1147 CI.I->eraseFromParent(); 1148 Paired.I->eraseFromParent(); 1149 1150 LLVM_DEBUG(dbgs() << "Inserted write2 inst: " << *Write2 << '\n'); 1151 return Write2; 1152 } 1153 1154 MachineBasicBlock::iterator 1155 SILoadStoreOptimizer::mergeImagePair(CombineInfo &CI, CombineInfo &Paired, 1156 const SmallVectorImpl<MachineInstr *> &InstsToMove) { 1157 MachineBasicBlock *MBB = CI.I->getParent(); 1158 DebugLoc DL = CI.I->getDebugLoc(); 1159 const unsigned Opcode = getNewOpcode(CI, Paired); 1160 1161 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); 1162 1163 Register DestReg = MRI->createVirtualRegister(SuperRC); 1164 unsigned MergedDMask = CI.DMask | Paired.DMask; 1165 unsigned DMaskIdx = 1166 AMDGPU::getNamedOperandIdx(CI.I->getOpcode(), AMDGPU::OpName::dmask); 1167 1168 auto MIB = BuildMI(*MBB, Paired.I, DL, TII->get(Opcode), DestReg); 1169 for (unsigned I = 1, E = (*CI.I).getNumOperands(); I != E; ++I) { 1170 if (I == DMaskIdx) 1171 MIB.addImm(MergedDMask); 1172 else 1173 MIB.add((*CI.I).getOperand(I)); 1174 } 1175 1176 // It shouldn't be possible to get this far if the two instructions 1177 // don't have a single memoperand, because MachineInstr::mayAlias() 1178 // will return true if this is the case. 1179 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); 1180 1181 const MachineMemOperand *MMOa = *CI.I->memoperands_begin(); 1182 const MachineMemOperand *MMOb = *Paired.I->memoperands_begin(); 1183 1184 MachineInstr *New = MIB.addMemOperand(combineKnownAdjacentMMOs(*MBB->getParent(), MMOa, MMOb)); 1185 1186 unsigned SubRegIdx0, SubRegIdx1; 1187 std::tie(SubRegIdx0, SubRegIdx1) = getSubRegIdxs(CI, Paired); 1188 1189 // Copy to the old destination registers. 1190 const MCInstrDesc &CopyDesc = TII->get(TargetOpcode::COPY); 1191 const auto *Dest0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdata); 1192 const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); 1193 1194 BuildMI(*MBB, Paired.I, DL, CopyDesc) 1195 .add(*Dest0) // Copy to same destination including flags and sub reg. 1196 .addReg(DestReg, 0, SubRegIdx0); 1197 MachineInstr *Copy1 = BuildMI(*MBB, Paired.I, DL, CopyDesc) 1198 .add(*Dest1) 1199 .addReg(DestReg, RegState::Kill, SubRegIdx1); 1200 1201 moveInstsAfter(Copy1, InstsToMove); 1202 1203 CI.I->eraseFromParent(); 1204 Paired.I->eraseFromParent(); 1205 return New; 1206 } 1207 1208 MachineBasicBlock::iterator SILoadStoreOptimizer::mergeSBufferLoadImmPair( 1209 CombineInfo &CI, CombineInfo &Paired, 1210 const SmallVectorImpl<MachineInstr *> &InstsToMove) { 1211 MachineBasicBlock *MBB = CI.I->getParent(); 1212 DebugLoc DL = CI.I->getDebugLoc(); 1213 const unsigned Opcode = getNewOpcode(CI, Paired); 1214 1215 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); 1216 1217 Register DestReg = MRI->createVirtualRegister(SuperRC); 1218 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); 1219 1220 // It shouldn't be possible to get this far if the two instructions 1221 // don't have a single memoperand, because MachineInstr::mayAlias() 1222 // will return true if this is the case. 1223 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); 1224 1225 const MachineMemOperand *MMOa = *CI.I->memoperands_begin(); 1226 const MachineMemOperand *MMOb = *Paired.I->memoperands_begin(); 1227 1228 MachineInstr *New = 1229 BuildMI(*MBB, Paired.I, DL, TII->get(Opcode), DestReg) 1230 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::sbase)) 1231 .addImm(MergedOffset) // offset 1232 .addImm(CI.GLC) // glc 1233 .addImm(CI.DLC) // dlc 1234 .addMemOperand(combineKnownAdjacentMMOs(*MBB->getParent(), MMOa, MMOb)); 1235 1236 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); 1237 const unsigned SubRegIdx0 = std::get<0>(SubRegIdx); 1238 const unsigned SubRegIdx1 = std::get<1>(SubRegIdx); 1239 1240 // Copy to the old destination registers. 1241 const MCInstrDesc &CopyDesc = TII->get(TargetOpcode::COPY); 1242 const auto *Dest0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::sdst); 1243 const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::sdst); 1244 1245 BuildMI(*MBB, Paired.I, DL, CopyDesc) 1246 .add(*Dest0) // Copy to same destination including flags and sub reg. 1247 .addReg(DestReg, 0, SubRegIdx0); 1248 MachineInstr *Copy1 = BuildMI(*MBB, Paired.I, DL, CopyDesc) 1249 .add(*Dest1) 1250 .addReg(DestReg, RegState::Kill, SubRegIdx1); 1251 1252 moveInstsAfter(Copy1, InstsToMove); 1253 1254 CI.I->eraseFromParent(); 1255 Paired.I->eraseFromParent(); 1256 return New; 1257 } 1258 1259 MachineBasicBlock::iterator SILoadStoreOptimizer::mergeBufferLoadPair( 1260 CombineInfo &CI, CombineInfo &Paired, 1261 const SmallVectorImpl<MachineInstr *> &InstsToMove) { 1262 MachineBasicBlock *MBB = CI.I->getParent(); 1263 DebugLoc DL = CI.I->getDebugLoc(); 1264 1265 const unsigned Opcode = getNewOpcode(CI, Paired); 1266 1267 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); 1268 1269 // Copy to the new source register. 1270 Register DestReg = MRI->createVirtualRegister(SuperRC); 1271 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); 1272 1273 auto MIB = BuildMI(*MBB, Paired.I, DL, TII->get(Opcode), DestReg); 1274 1275 AddressRegs Regs = getRegs(Opcode, *TII); 1276 1277 if (Regs.VAddr) 1278 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr)); 1279 1280 // It shouldn't be possible to get this far if the two instructions 1281 // don't have a single memoperand, because MachineInstr::mayAlias() 1282 // will return true if this is the case. 1283 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); 1284 1285 const MachineMemOperand *MMOa = *CI.I->memoperands_begin(); 1286 const MachineMemOperand *MMOb = *Paired.I->memoperands_begin(); 1287 1288 MachineInstr *New = 1289 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc)) 1290 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset)) 1291 .addImm(MergedOffset) // offset 1292 .addImm(CI.GLC) // glc 1293 .addImm(CI.SLC) // slc 1294 .addImm(0) // tfe 1295 .addImm(CI.DLC) // dlc 1296 .addImm(0) // swz 1297 .addMemOperand(combineKnownAdjacentMMOs(*MBB->getParent(), MMOa, MMOb)); 1298 1299 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); 1300 const unsigned SubRegIdx0 = std::get<0>(SubRegIdx); 1301 const unsigned SubRegIdx1 = std::get<1>(SubRegIdx); 1302 1303 // Copy to the old destination registers. 1304 const MCInstrDesc &CopyDesc = TII->get(TargetOpcode::COPY); 1305 const auto *Dest0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdata); 1306 const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); 1307 1308 BuildMI(*MBB, Paired.I, DL, CopyDesc) 1309 .add(*Dest0) // Copy to same destination including flags and sub reg. 1310 .addReg(DestReg, 0, SubRegIdx0); 1311 MachineInstr *Copy1 = BuildMI(*MBB, Paired.I, DL, CopyDesc) 1312 .add(*Dest1) 1313 .addReg(DestReg, RegState::Kill, SubRegIdx1); 1314 1315 moveInstsAfter(Copy1, InstsToMove); 1316 1317 CI.I->eraseFromParent(); 1318 Paired.I->eraseFromParent(); 1319 return New; 1320 } 1321 1322 MachineBasicBlock::iterator SILoadStoreOptimizer::mergeTBufferLoadPair( 1323 CombineInfo &CI, CombineInfo &Paired, 1324 const SmallVectorImpl<MachineInstr *> &InstsToMove) { 1325 MachineBasicBlock *MBB = CI.I->getParent(); 1326 DebugLoc DL = CI.I->getDebugLoc(); 1327 1328 const unsigned Opcode = getNewOpcode(CI, Paired); 1329 1330 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); 1331 1332 // Copy to the new source register. 1333 Register DestReg = MRI->createVirtualRegister(SuperRC); 1334 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); 1335 1336 auto MIB = BuildMI(*MBB, Paired.I, DL, TII->get(Opcode), DestReg); 1337 1338 AddressRegs Regs = getRegs(Opcode, *TII); 1339 1340 if (Regs.VAddr) 1341 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr)); 1342 1343 unsigned JoinedFormat = 1344 getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, *STM); 1345 1346 // It shouldn't be possible to get this far if the two instructions 1347 // don't have a single memoperand, because MachineInstr::mayAlias() 1348 // will return true if this is the case. 1349 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); 1350 1351 const MachineMemOperand *MMOa = *CI.I->memoperands_begin(); 1352 const MachineMemOperand *MMOb = *Paired.I->memoperands_begin(); 1353 1354 MachineInstr *New = 1355 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc)) 1356 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset)) 1357 .addImm(MergedOffset) // offset 1358 .addImm(JoinedFormat) // format 1359 .addImm(CI.GLC) // glc 1360 .addImm(CI.SLC) // slc 1361 .addImm(0) // tfe 1362 .addImm(CI.DLC) // dlc 1363 .addImm(0) // swz 1364 .addMemOperand( 1365 combineKnownAdjacentMMOs(*MBB->getParent(), MMOa, MMOb)); 1366 1367 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); 1368 const unsigned SubRegIdx0 = std::get<0>(SubRegIdx); 1369 const unsigned SubRegIdx1 = std::get<1>(SubRegIdx); 1370 1371 // Copy to the old destination registers. 1372 const MCInstrDesc &CopyDesc = TII->get(TargetOpcode::COPY); 1373 const auto *Dest0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdata); 1374 const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); 1375 1376 BuildMI(*MBB, Paired.I, DL, CopyDesc) 1377 .add(*Dest0) // Copy to same destination including flags and sub reg. 1378 .addReg(DestReg, 0, SubRegIdx0); 1379 MachineInstr *Copy1 = BuildMI(*MBB, Paired.I, DL, CopyDesc) 1380 .add(*Dest1) 1381 .addReg(DestReg, RegState::Kill, SubRegIdx1); 1382 1383 moveInstsAfter(Copy1, InstsToMove); 1384 1385 CI.I->eraseFromParent(); 1386 Paired.I->eraseFromParent(); 1387 return New; 1388 } 1389 1390 MachineBasicBlock::iterator SILoadStoreOptimizer::mergeTBufferStorePair( 1391 CombineInfo &CI, CombineInfo &Paired, 1392 const SmallVectorImpl<MachineInstr *> &InstsToMove) { 1393 MachineBasicBlock *MBB = CI.I->getParent(); 1394 DebugLoc DL = CI.I->getDebugLoc(); 1395 1396 const unsigned Opcode = getNewOpcode(CI, Paired); 1397 1398 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); 1399 const unsigned SubRegIdx0 = std::get<0>(SubRegIdx); 1400 const unsigned SubRegIdx1 = std::get<1>(SubRegIdx); 1401 1402 // Copy to the new source register. 1403 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); 1404 Register SrcReg = MRI->createVirtualRegister(SuperRC); 1405 1406 const auto *Src0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdata); 1407 const auto *Src1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); 1408 1409 BuildMI(*MBB, Paired.I, DL, TII->get(AMDGPU::REG_SEQUENCE), SrcReg) 1410 .add(*Src0) 1411 .addImm(SubRegIdx0) 1412 .add(*Src1) 1413 .addImm(SubRegIdx1); 1414 1415 auto MIB = BuildMI(*MBB, Paired.I, DL, TII->get(Opcode)) 1416 .addReg(SrcReg, RegState::Kill); 1417 1418 AddressRegs Regs = getRegs(Opcode, *TII); 1419 1420 if (Regs.VAddr) 1421 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr)); 1422 1423 unsigned JoinedFormat = 1424 getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, *STM); 1425 1426 // It shouldn't be possible to get this far if the two instructions 1427 // don't have a single memoperand, because MachineInstr::mayAlias() 1428 // will return true if this is the case. 1429 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); 1430 1431 const MachineMemOperand *MMOa = *CI.I->memoperands_begin(); 1432 const MachineMemOperand *MMOb = *Paired.I->memoperands_begin(); 1433 1434 MachineInstr *New = 1435 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc)) 1436 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset)) 1437 .addImm(std::min(CI.Offset, Paired.Offset)) // offset 1438 .addImm(JoinedFormat) // format 1439 .addImm(CI.GLC) // glc 1440 .addImm(CI.SLC) // slc 1441 .addImm(0) // tfe 1442 .addImm(CI.DLC) // dlc 1443 .addImm(0) // swz 1444 .addMemOperand( 1445 combineKnownAdjacentMMOs(*MBB->getParent(), MMOa, MMOb)); 1446 1447 moveInstsAfter(MIB, InstsToMove); 1448 1449 CI.I->eraseFromParent(); 1450 Paired.I->eraseFromParent(); 1451 return New; 1452 } 1453 1454 unsigned SILoadStoreOptimizer::getNewOpcode(const CombineInfo &CI, 1455 const CombineInfo &Paired) { 1456 const unsigned Width = CI.Width + Paired.Width; 1457 1458 switch (CI.InstClass) { 1459 default: 1460 assert(CI.InstClass == BUFFER_LOAD || CI.InstClass == BUFFER_STORE); 1461 // FIXME: Handle d16 correctly 1462 return AMDGPU::getMUBUFOpcode(AMDGPU::getMUBUFBaseOpcode(CI.I->getOpcode()), 1463 Width); 1464 case TBUFFER_LOAD: 1465 case TBUFFER_STORE: 1466 return AMDGPU::getMTBUFOpcode(AMDGPU::getMTBUFBaseOpcode(CI.I->getOpcode()), 1467 Width); 1468 1469 case UNKNOWN: 1470 llvm_unreachable("Unknown instruction class"); 1471 case S_BUFFER_LOAD_IMM: 1472 switch (Width) { 1473 default: 1474 return 0; 1475 case 2: 1476 return AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM; 1477 case 4: 1478 return AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM; 1479 } 1480 case MIMG: 1481 assert("No overlaps" && (countPopulation(CI.DMask | Paired.DMask) == Width)); 1482 return AMDGPU::getMaskedMIMGOp(CI.I->getOpcode(), Width); 1483 } 1484 } 1485 1486 std::pair<unsigned, unsigned> 1487 SILoadStoreOptimizer::getSubRegIdxs(const CombineInfo &CI, const CombineInfo &Paired) { 1488 1489 if (CI.Width == 0 || Paired.Width == 0 || CI.Width + Paired.Width > 4) 1490 return std::make_pair(0, 0); 1491 1492 bool ReverseOrder; 1493 if (CI.InstClass == MIMG) { 1494 assert((countPopulation(CI.DMask | Paired.DMask) == CI.Width + Paired.Width) && 1495 "No overlaps"); 1496 ReverseOrder = CI.DMask > Paired.DMask; 1497 } else 1498 ReverseOrder = CI.Offset > Paired.Offset; 1499 1500 static const unsigned Idxs[4][4] = { 1501 {AMDGPU::sub0, AMDGPU::sub0_sub1, AMDGPU::sub0_sub1_sub2, AMDGPU::sub0_sub1_sub2_sub3}, 1502 {AMDGPU::sub1, AMDGPU::sub1_sub2, AMDGPU::sub1_sub2_sub3, 0}, 1503 {AMDGPU::sub2, AMDGPU::sub2_sub3, 0, 0}, 1504 {AMDGPU::sub3, 0, 0, 0}, 1505 }; 1506 unsigned Idx0; 1507 unsigned Idx1; 1508 1509 assert(CI.Width >= 1 && CI.Width <= 3); 1510 assert(Paired.Width >= 1 && Paired.Width <= 3); 1511 1512 if (ReverseOrder) { 1513 Idx1 = Idxs[0][Paired.Width - 1]; 1514 Idx0 = Idxs[Paired.Width][CI.Width - 1]; 1515 } else { 1516 Idx0 = Idxs[0][CI.Width - 1]; 1517 Idx1 = Idxs[CI.Width][Paired.Width - 1]; 1518 } 1519 1520 return std::make_pair(Idx0, Idx1); 1521 } 1522 1523 const TargetRegisterClass * 1524 SILoadStoreOptimizer::getTargetRegisterClass(const CombineInfo &CI, 1525 const CombineInfo &Paired) { 1526 if (CI.InstClass == S_BUFFER_LOAD_IMM) { 1527 switch (CI.Width + Paired.Width) { 1528 default: 1529 return nullptr; 1530 case 2: 1531 return &AMDGPU::SReg_64_XEXECRegClass; 1532 case 4: 1533 return &AMDGPU::SGPR_128RegClass; 1534 case 8: 1535 return &AMDGPU::SGPR_256RegClass; 1536 case 16: 1537 return &AMDGPU::SGPR_512RegClass; 1538 } 1539 } else { 1540 switch (CI.Width + Paired.Width) { 1541 default: 1542 return nullptr; 1543 case 2: 1544 return &AMDGPU::VReg_64RegClass; 1545 case 3: 1546 return &AMDGPU::VReg_96RegClass; 1547 case 4: 1548 return &AMDGPU::VReg_128RegClass; 1549 } 1550 } 1551 } 1552 1553 MachineBasicBlock::iterator SILoadStoreOptimizer::mergeBufferStorePair( 1554 CombineInfo &CI, CombineInfo &Paired, 1555 const SmallVectorImpl<MachineInstr *> &InstsToMove) { 1556 MachineBasicBlock *MBB = CI.I->getParent(); 1557 DebugLoc DL = CI.I->getDebugLoc(); 1558 1559 const unsigned Opcode = getNewOpcode(CI, Paired); 1560 1561 std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); 1562 const unsigned SubRegIdx0 = std::get<0>(SubRegIdx); 1563 const unsigned SubRegIdx1 = std::get<1>(SubRegIdx); 1564 1565 // Copy to the new source register. 1566 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); 1567 Register SrcReg = MRI->createVirtualRegister(SuperRC); 1568 1569 const auto *Src0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdata); 1570 const auto *Src1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); 1571 1572 BuildMI(*MBB, Paired.I, DL, TII->get(AMDGPU::REG_SEQUENCE), SrcReg) 1573 .add(*Src0) 1574 .addImm(SubRegIdx0) 1575 .add(*Src1) 1576 .addImm(SubRegIdx1); 1577 1578 auto MIB = BuildMI(*MBB, Paired.I, DL, TII->get(Opcode)) 1579 .addReg(SrcReg, RegState::Kill); 1580 1581 AddressRegs Regs = getRegs(Opcode, *TII); 1582 1583 if (Regs.VAddr) 1584 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr)); 1585 1586 1587 // It shouldn't be possible to get this far if the two instructions 1588 // don't have a single memoperand, because MachineInstr::mayAlias() 1589 // will return true if this is the case. 1590 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); 1591 1592 const MachineMemOperand *MMOa = *CI.I->memoperands_begin(); 1593 const MachineMemOperand *MMOb = *Paired.I->memoperands_begin(); 1594 1595 MachineInstr *New = 1596 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc)) 1597 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset)) 1598 .addImm(std::min(CI.Offset, Paired.Offset)) // offset 1599 .addImm(CI.GLC) // glc 1600 .addImm(CI.SLC) // slc 1601 .addImm(0) // tfe 1602 .addImm(CI.DLC) // dlc 1603 .addImm(0) // swz 1604 .addMemOperand(combineKnownAdjacentMMOs(*MBB->getParent(), MMOa, MMOb)); 1605 1606 moveInstsAfter(MIB, InstsToMove); 1607 1608 CI.I->eraseFromParent(); 1609 Paired.I->eraseFromParent(); 1610 return New; 1611 } 1612 1613 MachineOperand 1614 SILoadStoreOptimizer::createRegOrImm(int32_t Val, MachineInstr &MI) const { 1615 APInt V(32, Val, true); 1616 if (TII->isInlineConstant(V)) 1617 return MachineOperand::CreateImm(Val); 1618 1619 Register Reg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); 1620 MachineInstr *Mov = 1621 BuildMI(*MI.getParent(), MI.getIterator(), MI.getDebugLoc(), 1622 TII->get(AMDGPU::S_MOV_B32), Reg) 1623 .addImm(Val); 1624 (void)Mov; 1625 LLVM_DEBUG(dbgs() << " "; Mov->dump()); 1626 return MachineOperand::CreateReg(Reg, false); 1627 } 1628 1629 // Compute base address using Addr and return the final register. 1630 Register SILoadStoreOptimizer::computeBase(MachineInstr &MI, 1631 const MemAddress &Addr) const { 1632 MachineBasicBlock *MBB = MI.getParent(); 1633 MachineBasicBlock::iterator MBBI = MI.getIterator(); 1634 DebugLoc DL = MI.getDebugLoc(); 1635 1636 assert((TRI->getRegSizeInBits(Addr.Base.LoReg, *MRI) == 32 || 1637 Addr.Base.LoSubReg) && 1638 "Expected 32-bit Base-Register-Low!!"); 1639 1640 assert((TRI->getRegSizeInBits(Addr.Base.HiReg, *MRI) == 32 || 1641 Addr.Base.HiSubReg) && 1642 "Expected 32-bit Base-Register-Hi!!"); 1643 1644 LLVM_DEBUG(dbgs() << " Re-Computed Anchor-Base:\n"); 1645 MachineOperand OffsetLo = createRegOrImm(static_cast<int32_t>(Addr.Offset), MI); 1646 MachineOperand OffsetHi = 1647 createRegOrImm(static_cast<int32_t>(Addr.Offset >> 32), MI); 1648 1649 const auto *CarryRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 1650 Register CarryReg = MRI->createVirtualRegister(CarryRC); 1651 Register DeadCarryReg = MRI->createVirtualRegister(CarryRC); 1652 1653 Register DestSub0 = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); 1654 Register DestSub1 = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); 1655 MachineInstr *LoHalf = 1656 BuildMI(*MBB, MBBI, DL, TII->get(AMDGPU::V_ADD_CO_U32_e64), DestSub0) 1657 .addReg(CarryReg, RegState::Define) 1658 .addReg(Addr.Base.LoReg, 0, Addr.Base.LoSubReg) 1659 .add(OffsetLo) 1660 .addImm(0); // clamp bit 1661 (void)LoHalf; 1662 LLVM_DEBUG(dbgs() << " "; LoHalf->dump();); 1663 1664 MachineInstr *HiHalf = 1665 BuildMI(*MBB, MBBI, DL, TII->get(AMDGPU::V_ADDC_U32_e64), DestSub1) 1666 .addReg(DeadCarryReg, RegState::Define | RegState::Dead) 1667 .addReg(Addr.Base.HiReg, 0, Addr.Base.HiSubReg) 1668 .add(OffsetHi) 1669 .addReg(CarryReg, RegState::Kill) 1670 .addImm(0); // clamp bit 1671 (void)HiHalf; 1672 LLVM_DEBUG(dbgs() << " "; HiHalf->dump();); 1673 1674 Register FullDestReg = MRI->createVirtualRegister(&AMDGPU::VReg_64RegClass); 1675 MachineInstr *FullBase = 1676 BuildMI(*MBB, MBBI, DL, TII->get(TargetOpcode::REG_SEQUENCE), FullDestReg) 1677 .addReg(DestSub0) 1678 .addImm(AMDGPU::sub0) 1679 .addReg(DestSub1) 1680 .addImm(AMDGPU::sub1); 1681 (void)FullBase; 1682 LLVM_DEBUG(dbgs() << " "; FullBase->dump(); dbgs() << "\n";); 1683 1684 return FullDestReg; 1685 } 1686 1687 // Update base and offset with the NewBase and NewOffset in MI. 1688 void SILoadStoreOptimizer::updateBaseAndOffset(MachineInstr &MI, 1689 Register NewBase, 1690 int32_t NewOffset) const { 1691 auto Base = TII->getNamedOperand(MI, AMDGPU::OpName::vaddr); 1692 Base->setReg(NewBase); 1693 Base->setIsKill(false); 1694 TII->getNamedOperand(MI, AMDGPU::OpName::offset)->setImm(NewOffset); 1695 } 1696 1697 Optional<int32_t> 1698 SILoadStoreOptimizer::extractConstOffset(const MachineOperand &Op) const { 1699 if (Op.isImm()) 1700 return Op.getImm(); 1701 1702 if (!Op.isReg()) 1703 return None; 1704 1705 MachineInstr *Def = MRI->getUniqueVRegDef(Op.getReg()); 1706 if (!Def || Def->getOpcode() != AMDGPU::S_MOV_B32 || 1707 !Def->getOperand(1).isImm()) 1708 return None; 1709 1710 return Def->getOperand(1).getImm(); 1711 } 1712 1713 // Analyze Base and extracts: 1714 // - 32bit base registers, subregisters 1715 // - 64bit constant offset 1716 // Expecting base computation as: 1717 // %OFFSET0:sgpr_32 = S_MOV_B32 8000 1718 // %LO:vgpr_32, %c:sreg_64_xexec = 1719 // V_ADD_CO_U32_e64 %BASE_LO:vgpr_32, %103:sgpr_32, 1720 // %HI:vgpr_32, = V_ADDC_U32_e64 %BASE_HI:vgpr_32, 0, killed %c:sreg_64_xexec 1721 // %Base:vreg_64 = 1722 // REG_SEQUENCE %LO:vgpr_32, %subreg.sub0, %HI:vgpr_32, %subreg.sub1 1723 void SILoadStoreOptimizer::processBaseWithConstOffset(const MachineOperand &Base, 1724 MemAddress &Addr) const { 1725 if (!Base.isReg()) 1726 return; 1727 1728 MachineInstr *Def = MRI->getUniqueVRegDef(Base.getReg()); 1729 if (!Def || Def->getOpcode() != AMDGPU::REG_SEQUENCE 1730 || Def->getNumOperands() != 5) 1731 return; 1732 1733 MachineOperand BaseLo = Def->getOperand(1); 1734 MachineOperand BaseHi = Def->getOperand(3); 1735 if (!BaseLo.isReg() || !BaseHi.isReg()) 1736 return; 1737 1738 MachineInstr *BaseLoDef = MRI->getUniqueVRegDef(BaseLo.getReg()); 1739 MachineInstr *BaseHiDef = MRI->getUniqueVRegDef(BaseHi.getReg()); 1740 1741 if (!BaseLoDef || BaseLoDef->getOpcode() != AMDGPU::V_ADD_CO_U32_e64 || 1742 !BaseHiDef || BaseHiDef->getOpcode() != AMDGPU::V_ADDC_U32_e64) 1743 return; 1744 1745 const auto *Src0 = TII->getNamedOperand(*BaseLoDef, AMDGPU::OpName::src0); 1746 const auto *Src1 = TII->getNamedOperand(*BaseLoDef, AMDGPU::OpName::src1); 1747 1748 auto Offset0P = extractConstOffset(*Src0); 1749 if (Offset0P) 1750 BaseLo = *Src1; 1751 else { 1752 if (!(Offset0P = extractConstOffset(*Src1))) 1753 return; 1754 BaseLo = *Src0; 1755 } 1756 1757 Src0 = TII->getNamedOperand(*BaseHiDef, AMDGPU::OpName::src0); 1758 Src1 = TII->getNamedOperand(*BaseHiDef, AMDGPU::OpName::src1); 1759 1760 if (Src0->isImm()) 1761 std::swap(Src0, Src1); 1762 1763 if (!Src1->isImm()) 1764 return; 1765 1766 uint64_t Offset1 = Src1->getImm(); 1767 BaseHi = *Src0; 1768 1769 Addr.Base.LoReg = BaseLo.getReg(); 1770 Addr.Base.HiReg = BaseHi.getReg(); 1771 Addr.Base.LoSubReg = BaseLo.getSubReg(); 1772 Addr.Base.HiSubReg = BaseHi.getSubReg(); 1773 Addr.Offset = (*Offset0P & 0x00000000ffffffff) | (Offset1 << 32); 1774 } 1775 1776 bool SILoadStoreOptimizer::promoteConstantOffsetToImm( 1777 MachineInstr &MI, 1778 MemInfoMap &Visited, 1779 SmallPtrSet<MachineInstr *, 4> &AnchorList) const { 1780 1781 if (!(MI.mayLoad() ^ MI.mayStore())) 1782 return false; 1783 1784 // TODO: Support flat and scratch. 1785 if (AMDGPU::getGlobalSaddrOp(MI.getOpcode()) < 0) 1786 return false; 1787 1788 if (MI.mayLoad() && TII->getNamedOperand(MI, AMDGPU::OpName::vdata) != NULL) 1789 return false; 1790 1791 if (AnchorList.count(&MI)) 1792 return false; 1793 1794 LLVM_DEBUG(dbgs() << "\nTryToPromoteConstantOffsetToImmFor "; MI.dump()); 1795 1796 if (TII->getNamedOperand(MI, AMDGPU::OpName::offset)->getImm()) { 1797 LLVM_DEBUG(dbgs() << " Const-offset is already promoted.\n";); 1798 return false; 1799 } 1800 1801 // Step1: Find the base-registers and a 64bit constant offset. 1802 MachineOperand &Base = *TII->getNamedOperand(MI, AMDGPU::OpName::vaddr); 1803 MemAddress MAddr; 1804 if (Visited.find(&MI) == Visited.end()) { 1805 processBaseWithConstOffset(Base, MAddr); 1806 Visited[&MI] = MAddr; 1807 } else 1808 MAddr = Visited[&MI]; 1809 1810 if (MAddr.Offset == 0) { 1811 LLVM_DEBUG(dbgs() << " Failed to extract constant-offset or there are no" 1812 " constant offsets that can be promoted.\n";); 1813 return false; 1814 } 1815 1816 LLVM_DEBUG(dbgs() << " BASE: {" << MAddr.Base.HiReg << ", " 1817 << MAddr.Base.LoReg << "} Offset: " << MAddr.Offset << "\n\n";); 1818 1819 // Step2: Traverse through MI's basic block and find an anchor(that has the 1820 // same base-registers) with the highest 13bit distance from MI's offset. 1821 // E.g. (64bit loads) 1822 // bb: 1823 // addr1 = &a + 4096; load1 = load(addr1, 0) 1824 // addr2 = &a + 6144; load2 = load(addr2, 0) 1825 // addr3 = &a + 8192; load3 = load(addr3, 0) 1826 // addr4 = &a + 10240; load4 = load(addr4, 0) 1827 // addr5 = &a + 12288; load5 = load(addr5, 0) 1828 // 1829 // Starting from the first load, the optimization will try to find a new base 1830 // from which (&a + 4096) has 13 bit distance. Both &a + 6144 and &a + 8192 1831 // has 13bit distance from &a + 4096. The heuristic considers &a + 8192 1832 // as the new-base(anchor) because of the maximum distance which can 1833 // accomodate more intermediate bases presumeably. 1834 // 1835 // Step3: move (&a + 8192) above load1. Compute and promote offsets from 1836 // (&a + 8192) for load1, load2, load4. 1837 // addr = &a + 8192 1838 // load1 = load(addr, -4096) 1839 // load2 = load(addr, -2048) 1840 // load3 = load(addr, 0) 1841 // load4 = load(addr, 2048) 1842 // addr5 = &a + 12288; load5 = load(addr5, 0) 1843 // 1844 MachineInstr *AnchorInst = nullptr; 1845 MemAddress AnchorAddr; 1846 uint32_t MaxDist = std::numeric_limits<uint32_t>::min(); 1847 SmallVector<std::pair<MachineInstr *, int64_t>, 4> InstsWCommonBase; 1848 1849 MachineBasicBlock *MBB = MI.getParent(); 1850 MachineBasicBlock::iterator E = MBB->end(); 1851 MachineBasicBlock::iterator MBBI = MI.getIterator(); 1852 ++MBBI; 1853 const SITargetLowering *TLI = 1854 static_cast<const SITargetLowering *>(STM->getTargetLowering()); 1855 1856 for ( ; MBBI != E; ++MBBI) { 1857 MachineInstr &MINext = *MBBI; 1858 // TODO: Support finding an anchor(with same base) from store addresses or 1859 // any other load addresses where the opcodes are different. 1860 if (MINext.getOpcode() != MI.getOpcode() || 1861 TII->getNamedOperand(MINext, AMDGPU::OpName::offset)->getImm()) 1862 continue; 1863 1864 const MachineOperand &BaseNext = 1865 *TII->getNamedOperand(MINext, AMDGPU::OpName::vaddr); 1866 MemAddress MAddrNext; 1867 if (Visited.find(&MINext) == Visited.end()) { 1868 processBaseWithConstOffset(BaseNext, MAddrNext); 1869 Visited[&MINext] = MAddrNext; 1870 } else 1871 MAddrNext = Visited[&MINext]; 1872 1873 if (MAddrNext.Base.LoReg != MAddr.Base.LoReg || 1874 MAddrNext.Base.HiReg != MAddr.Base.HiReg || 1875 MAddrNext.Base.LoSubReg != MAddr.Base.LoSubReg || 1876 MAddrNext.Base.HiSubReg != MAddr.Base.HiSubReg) 1877 continue; 1878 1879 InstsWCommonBase.push_back(std::make_pair(&MINext, MAddrNext.Offset)); 1880 1881 int64_t Dist = MAddr.Offset - MAddrNext.Offset; 1882 TargetLoweringBase::AddrMode AM; 1883 AM.HasBaseReg = true; 1884 AM.BaseOffs = Dist; 1885 if (TLI->isLegalGlobalAddressingMode(AM) && 1886 (uint32_t)std::abs(Dist) > MaxDist) { 1887 MaxDist = std::abs(Dist); 1888 1889 AnchorAddr = MAddrNext; 1890 AnchorInst = &MINext; 1891 } 1892 } 1893 1894 if (AnchorInst) { 1895 LLVM_DEBUG(dbgs() << " Anchor-Inst(with max-distance from Offset): "; 1896 AnchorInst->dump()); 1897 LLVM_DEBUG(dbgs() << " Anchor-Offset from BASE: " 1898 << AnchorAddr.Offset << "\n\n"); 1899 1900 // Instead of moving up, just re-compute anchor-instruction's base address. 1901 Register Base = computeBase(MI, AnchorAddr); 1902 1903 updateBaseAndOffset(MI, Base, MAddr.Offset - AnchorAddr.Offset); 1904 LLVM_DEBUG(dbgs() << " After promotion: "; MI.dump();); 1905 1906 for (auto P : InstsWCommonBase) { 1907 TargetLoweringBase::AddrMode AM; 1908 AM.HasBaseReg = true; 1909 AM.BaseOffs = P.second - AnchorAddr.Offset; 1910 1911 if (TLI->isLegalGlobalAddressingMode(AM)) { 1912 LLVM_DEBUG(dbgs() << " Promote Offset(" << P.second; 1913 dbgs() << ")"; P.first->dump()); 1914 updateBaseAndOffset(*P.first, Base, P.second - AnchorAddr.Offset); 1915 LLVM_DEBUG(dbgs() << " After promotion: "; P.first->dump()); 1916 } 1917 } 1918 AnchorList.insert(AnchorInst); 1919 return true; 1920 } 1921 1922 return false; 1923 } 1924 1925 void SILoadStoreOptimizer::addInstToMergeableList(const CombineInfo &CI, 1926 std::list<std::list<CombineInfo> > &MergeableInsts) const { 1927 for (std::list<CombineInfo> &AddrList : MergeableInsts) { 1928 if (AddrList.front().InstClass == CI.InstClass && 1929 AddrList.front().hasSameBaseAddress(*CI.I)) { 1930 AddrList.emplace_back(CI); 1931 return; 1932 } 1933 } 1934 1935 // Base address not found, so add a new list. 1936 MergeableInsts.emplace_back(1, CI); 1937 } 1938 1939 std::pair<MachineBasicBlock::iterator, bool> 1940 SILoadStoreOptimizer::collectMergeableInsts( 1941 MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End, 1942 MemInfoMap &Visited, SmallPtrSet<MachineInstr *, 4> &AnchorList, 1943 std::list<std::list<CombineInfo>> &MergeableInsts) const { 1944 bool Modified = false; 1945 1946 // Sort potential mergeable instructions into lists. One list per base address. 1947 unsigned Order = 0; 1948 MachineBasicBlock::iterator BlockI = Begin; 1949 for (; BlockI != End; ++BlockI) { 1950 MachineInstr &MI = *BlockI; 1951 1952 // We run this before checking if an address is mergeable, because it can produce 1953 // better code even if the instructions aren't mergeable. 1954 if (promoteConstantOffsetToImm(MI, Visited, AnchorList)) 1955 Modified = true; 1956 1957 // Don't combine if volatile. We also won't be able to merge across this, so 1958 // break the search. We can look after this barrier for separate merges. 1959 if (MI.hasOrderedMemoryRef()) { 1960 LLVM_DEBUG(dbgs() << "Breaking search on memory fence: " << MI); 1961 1962 // Search will resume after this instruction in a separate merge list. 1963 ++BlockI; 1964 break; 1965 } 1966 1967 const InstClassEnum InstClass = getInstClass(MI.getOpcode(), *TII); 1968 if (InstClass == UNKNOWN) 1969 continue; 1970 1971 CombineInfo CI; 1972 CI.setMI(MI, *TII, *STM); 1973 CI.Order = Order++; 1974 1975 if (!CI.hasMergeableAddress(*MRI)) 1976 continue; 1977 1978 LLVM_DEBUG(dbgs() << "Mergeable: " << MI); 1979 1980 addInstToMergeableList(CI, MergeableInsts); 1981 } 1982 1983 // At this point we have lists of Mergeable instructions. 1984 // 1985 // Part 2: Sort lists by offset and then for each CombineInfo object in the 1986 // list try to find an instruction that can be merged with I. If an instruction 1987 // is found, it is stored in the Paired field. If no instructions are found, then 1988 // the CombineInfo object is deleted from the list. 1989 1990 for (std::list<std::list<CombineInfo>>::iterator I = MergeableInsts.begin(), 1991 E = MergeableInsts.end(); I != E;) { 1992 1993 std::list<CombineInfo> &MergeList = *I; 1994 if (MergeList.size() <= 1) { 1995 // This means we have found only one instruction with a given address 1996 // that can be merged, and we need at least 2 instructions to do a merge, 1997 // so this list can be discarded. 1998 I = MergeableInsts.erase(I); 1999 continue; 2000 } 2001 2002 // Sort the lists by offsets, this way mergeable instructions will be 2003 // adjacent to each other in the list, which will make it easier to find 2004 // matches. 2005 MergeList.sort( 2006 [] (const CombineInfo &A, CombineInfo &B) { 2007 return A.Offset < B.Offset; 2008 }); 2009 ++I; 2010 } 2011 2012 return std::make_pair(BlockI, Modified); 2013 } 2014 2015 // Scan through looking for adjacent LDS operations with constant offsets from 2016 // the same base register. We rely on the scheduler to do the hard work of 2017 // clustering nearby loads, and assume these are all adjacent. 2018 bool SILoadStoreOptimizer::optimizeBlock( 2019 std::list<std::list<CombineInfo> > &MergeableInsts) { 2020 bool Modified = false; 2021 2022 for (std::list<std::list<CombineInfo>>::iterator I = MergeableInsts.begin(), 2023 E = MergeableInsts.end(); I != E;) { 2024 std::list<CombineInfo> &MergeList = *I; 2025 2026 bool OptimizeListAgain = false; 2027 if (!optimizeInstsWithSameBaseAddr(MergeList, OptimizeListAgain)) { 2028 // We weren't able to make any changes, so delete the list so we don't 2029 // process the same instructions the next time we try to optimize this 2030 // block. 2031 I = MergeableInsts.erase(I); 2032 continue; 2033 } 2034 2035 Modified = true; 2036 2037 // We made changes, but also determined that there were no more optimization 2038 // opportunities, so we don't need to reprocess the list 2039 if (!OptimizeListAgain) { 2040 I = MergeableInsts.erase(I); 2041 continue; 2042 } 2043 OptimizeAgain = true; 2044 } 2045 return Modified; 2046 } 2047 2048 bool 2049 SILoadStoreOptimizer::optimizeInstsWithSameBaseAddr( 2050 std::list<CombineInfo> &MergeList, 2051 bool &OptimizeListAgain) { 2052 if (MergeList.empty()) 2053 return false; 2054 2055 bool Modified = false; 2056 2057 for (auto I = MergeList.begin(), Next = std::next(I); Next != MergeList.end(); 2058 Next = std::next(I)) { 2059 2060 auto First = I; 2061 auto Second = Next; 2062 2063 if ((*First).Order > (*Second).Order) 2064 std::swap(First, Second); 2065 CombineInfo &CI = *First; 2066 CombineInfo &Paired = *Second; 2067 2068 SmallVector<MachineInstr *, 8> InstsToMove; 2069 if (!checkAndPrepareMerge(CI, Paired, InstsToMove)) { 2070 ++I; 2071 continue; 2072 } 2073 2074 Modified = true; 2075 2076 LLVM_DEBUG(dbgs() << "Merging: " << *CI.I << " with: " << *Paired.I); 2077 2078 switch (CI.InstClass) { 2079 default: 2080 llvm_unreachable("unknown InstClass"); 2081 break; 2082 case DS_READ: { 2083 MachineBasicBlock::iterator NewMI = 2084 mergeRead2Pair(CI, Paired, InstsToMove); 2085 CI.setMI(NewMI, *TII, *STM); 2086 break; 2087 } 2088 case DS_WRITE: { 2089 MachineBasicBlock::iterator NewMI = 2090 mergeWrite2Pair(CI, Paired, InstsToMove); 2091 CI.setMI(NewMI, *TII, *STM); 2092 break; 2093 } 2094 case S_BUFFER_LOAD_IMM: { 2095 MachineBasicBlock::iterator NewMI = 2096 mergeSBufferLoadImmPair(CI, Paired, InstsToMove); 2097 CI.setMI(NewMI, *TII, *STM); 2098 OptimizeListAgain |= (CI.Width + Paired.Width) < 16; 2099 break; 2100 } 2101 case BUFFER_LOAD: { 2102 MachineBasicBlock::iterator NewMI = 2103 mergeBufferLoadPair(CI, Paired, InstsToMove); 2104 CI.setMI(NewMI, *TII, *STM); 2105 OptimizeListAgain |= (CI.Width + Paired.Width) < 4; 2106 break; 2107 } 2108 case BUFFER_STORE: { 2109 MachineBasicBlock::iterator NewMI = 2110 mergeBufferStorePair(CI, Paired, InstsToMove); 2111 CI.setMI(NewMI, *TII, *STM); 2112 OptimizeListAgain |= (CI.Width + Paired.Width) < 4; 2113 break; 2114 } 2115 case MIMG: { 2116 MachineBasicBlock::iterator NewMI = 2117 mergeImagePair(CI, Paired, InstsToMove); 2118 CI.setMI(NewMI, *TII, *STM); 2119 OptimizeListAgain |= (CI.Width + Paired.Width) < 4; 2120 break; 2121 } 2122 case TBUFFER_LOAD: { 2123 MachineBasicBlock::iterator NewMI = 2124 mergeTBufferLoadPair(CI, Paired, InstsToMove); 2125 CI.setMI(NewMI, *TII, *STM); 2126 OptimizeListAgain |= (CI.Width + Paired.Width) < 4; 2127 break; 2128 } 2129 case TBUFFER_STORE: { 2130 MachineBasicBlock::iterator NewMI = 2131 mergeTBufferStorePair(CI, Paired, InstsToMove); 2132 CI.setMI(NewMI, *TII, *STM); 2133 OptimizeListAgain |= (CI.Width + Paired.Width) < 4; 2134 break; 2135 } 2136 } 2137 CI.Order = Paired.Order; 2138 if (I == Second) 2139 I = Next; 2140 2141 MergeList.erase(Second); 2142 } 2143 2144 return Modified; 2145 } 2146 2147 bool SILoadStoreOptimizer::runOnMachineFunction(MachineFunction &MF) { 2148 if (skipFunction(MF.getFunction())) 2149 return false; 2150 2151 STM = &MF.getSubtarget<GCNSubtarget>(); 2152 if (!STM->loadStoreOptEnabled()) 2153 return false; 2154 2155 TII = STM->getInstrInfo(); 2156 TRI = &TII->getRegisterInfo(); 2157 2158 MRI = &MF.getRegInfo(); 2159 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); 2160 2161 LLVM_DEBUG(dbgs() << "Running SILoadStoreOptimizer\n"); 2162 2163 bool Modified = false; 2164 2165 // Contains the list of instructions for which constant offsets are being 2166 // promoted to the IMM. This is tracked for an entire block at time. 2167 SmallPtrSet<MachineInstr *, 4> AnchorList; 2168 MemInfoMap Visited; 2169 2170 for (MachineBasicBlock &MBB : MF) { 2171 MachineBasicBlock::iterator SectionEnd; 2172 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E; 2173 I = SectionEnd) { 2174 bool CollectModified; 2175 std::list<std::list<CombineInfo>> MergeableInsts; 2176 2177 // First pass: Collect list of all instructions we know how to merge in a 2178 // subset of the block. 2179 std::tie(SectionEnd, CollectModified) = 2180 collectMergeableInsts(I, E, Visited, AnchorList, MergeableInsts); 2181 2182 Modified |= CollectModified; 2183 2184 do { 2185 OptimizeAgain = false; 2186 Modified |= optimizeBlock(MergeableInsts); 2187 } while (OptimizeAgain); 2188 } 2189 2190 Visited.clear(); 2191 AnchorList.clear(); 2192 } 2193 2194 return Modified; 2195 } 2196