1 //===- SIInstrInfo.cpp - SI Instruction Information ----------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// SI Implementation of TargetInstrInfo. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "SIInstrInfo.h" 15 #include "AMDGPU.h" 16 #include "AMDGPUInstrInfo.h" 17 #include "GCNHazardRecognizer.h" 18 #include "GCNSubtarget.h" 19 #include "SIMachineFunctionInfo.h" 20 #include "Utils/AMDGPUBaseInfo.h" 21 #include "llvm/Analysis/ValueTracking.h" 22 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 23 #include "llvm/CodeGen/LiveIntervals.h" 24 #include "llvm/CodeGen/LiveVariables.h" 25 #include "llvm/CodeGen/MachineDominators.h" 26 #include "llvm/CodeGen/MachineFrameInfo.h" 27 #include "llvm/CodeGen/MachineScheduler.h" 28 #include "llvm/CodeGen/RegisterScavenging.h" 29 #include "llvm/CodeGen/ScheduleDAG.h" 30 #include "llvm/IR/DiagnosticInfo.h" 31 #include "llvm/IR/IntrinsicsAMDGPU.h" 32 #include "llvm/MC/MCContext.h" 33 #include "llvm/Support/CommandLine.h" 34 #include "llvm/Target/TargetMachine.h" 35 36 using namespace llvm; 37 38 #define DEBUG_TYPE "si-instr-info" 39 40 #define GET_INSTRINFO_CTOR_DTOR 41 #include "AMDGPUGenInstrInfo.inc" 42 43 namespace llvm { 44 namespace AMDGPU { 45 #define GET_D16ImageDimIntrinsics_IMPL 46 #define GET_ImageDimIntrinsicTable_IMPL 47 #define GET_RsrcIntrinsics_IMPL 48 #include "AMDGPUGenSearchableTables.inc" 49 } 50 } 51 52 53 // Must be at least 4 to be able to branch over minimum unconditional branch 54 // code. This is only for making it possible to write reasonably small tests for 55 // long branches. 56 static cl::opt<unsigned> 57 BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16), 58 cl::desc("Restrict range of branch instructions (DEBUG)")); 59 60 static cl::opt<bool> Fix16BitCopies( 61 "amdgpu-fix-16-bit-physreg-copies", 62 cl::desc("Fix copies between 32 and 16 bit registers by extending to 32 bit"), 63 cl::init(true), 64 cl::ReallyHidden); 65 66 SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST) 67 : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN), 68 RI(ST), ST(ST) { 69 SchedModel.init(&ST); 70 } 71 72 //===----------------------------------------------------------------------===// 73 // TargetInstrInfo callbacks 74 //===----------------------------------------------------------------------===// 75 76 static unsigned getNumOperandsNoGlue(SDNode *Node) { 77 unsigned N = Node->getNumOperands(); 78 while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue) 79 --N; 80 return N; 81 } 82 83 /// Returns true if both nodes have the same value for the given 84 /// operand \p Op, or if both nodes do not have this operand. 85 static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) { 86 unsigned Opc0 = N0->getMachineOpcode(); 87 unsigned Opc1 = N1->getMachineOpcode(); 88 89 int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName); 90 int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName); 91 92 if (Op0Idx == -1 && Op1Idx == -1) 93 return true; 94 95 96 if ((Op0Idx == -1 && Op1Idx != -1) || 97 (Op1Idx == -1 && Op0Idx != -1)) 98 return false; 99 100 // getNamedOperandIdx returns the index for the MachineInstr's operands, 101 // which includes the result as the first operand. We are indexing into the 102 // MachineSDNode's operands, so we need to skip the result operand to get 103 // the real index. 104 --Op0Idx; 105 --Op1Idx; 106 107 return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx); 108 } 109 110 static bool canRemat(const MachineInstr &MI) { 111 112 if (SIInstrInfo::isVOP1(MI) || SIInstrInfo::isVOP2(MI) || 113 SIInstrInfo::isVOP3(MI) || SIInstrInfo::isSDWA(MI) || 114 SIInstrInfo::isSALU(MI)) 115 return true; 116 117 if (SIInstrInfo::isSMRD(MI)) { 118 return !MI.memoperands_empty() && 119 llvm::all_of(MI.memoperands(), [](const MachineMemOperand *MMO) { 120 return MMO->isLoad() && MMO->isInvariant(); 121 }); 122 } 123 124 return false; 125 } 126 127 bool SIInstrInfo::isReallyTriviallyReMaterializable( 128 const MachineInstr &MI) const { 129 130 if (canRemat(MI)) { 131 // Normally VALU use of exec would block the rematerialization, but that 132 // is OK in this case to have an implicit exec read as all VALU do. 133 // We really want all of the generic logic for this except for this. 134 135 // Another potential implicit use is mode register. The core logic of 136 // the RA will not attempt rematerialization if mode is set anywhere 137 // in the function, otherwise it is safe since mode is not changed. 138 139 // There is difference to generic method which does not allow 140 // rematerialization if there are virtual register uses. We allow this, 141 // therefore this method includes SOP instructions as well. 142 if (!MI.hasImplicitDef() && 143 MI.getNumImplicitOperands() == MI.getDesc().implicit_uses().size() && 144 !MI.mayRaiseFPException()) 145 return true; 146 } 147 148 return TargetInstrInfo::isReallyTriviallyReMaterializable(MI); 149 } 150 151 // Returns true if the scalar result of a VALU instruction depends on exec. 152 static bool resultDependsOnExec(const MachineInstr &MI) { 153 // Ignore comparisons which are only used masked with exec. 154 // This allows some hoisting/sinking of VALU comparisons. 155 if (MI.isCompare()) { 156 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 157 Register DstReg = MI.getOperand(0).getReg(); 158 if (!DstReg.isVirtual()) 159 return true; 160 for (MachineInstr &Use : MRI.use_nodbg_instructions(DstReg)) { 161 switch (Use.getOpcode()) { 162 case AMDGPU::S_AND_SAVEEXEC_B32: 163 case AMDGPU::S_AND_SAVEEXEC_B64: 164 break; 165 case AMDGPU::S_AND_B32: 166 case AMDGPU::S_AND_B64: 167 if (!Use.readsRegister(AMDGPU::EXEC)) 168 return true; 169 break; 170 default: 171 return true; 172 } 173 } 174 return false; 175 } 176 177 switch (MI.getOpcode()) { 178 default: 179 break; 180 case AMDGPU::V_READFIRSTLANE_B32: 181 return true; 182 } 183 184 return false; 185 } 186 187 bool SIInstrInfo::isIgnorableUse(const MachineOperand &MO) const { 188 // Any implicit use of exec by VALU is not a real register read. 189 return MO.getReg() == AMDGPU::EXEC && MO.isImplicit() && 190 isVALU(*MO.getParent()) && !resultDependsOnExec(*MO.getParent()); 191 } 192 193 bool SIInstrInfo::isSafeToSink(MachineInstr &MI, 194 MachineBasicBlock *SuccToSinkTo, 195 MachineCycleInfo *CI) const { 196 // Allow sinking if MI edits lane mask (divergent i1 in sgpr). 197 if (MI.getOpcode() == AMDGPU::SI_IF_BREAK) 198 return true; 199 200 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo(); 201 // Check if sinking of MI would create temporal divergent use. 202 for (auto Op : MI.uses()) { 203 if (Op.isReg() && Op.getReg().isVirtual() && 204 RI.isSGPRClass(MRI.getRegClass(Op.getReg()))) { 205 MachineInstr *SgprDef = MRI.getVRegDef(Op.getReg()); 206 207 // SgprDef defined inside cycle 208 MachineCycle *FromCycle = CI->getCycle(SgprDef->getParent()); 209 if (FromCycle == nullptr) 210 continue; 211 212 MachineCycle *ToCycle = CI->getCycle(SuccToSinkTo); 213 // Check if there is a FromCycle that contains SgprDef's basic block but 214 // does not contain SuccToSinkTo and also has divergent exit condition. 215 while (FromCycle && !FromCycle->contains(ToCycle)) { 216 // After structurize-cfg, there should be exactly one cycle exit. 217 SmallVector<MachineBasicBlock *, 1> ExitBlocks; 218 FromCycle->getExitBlocks(ExitBlocks); 219 assert(ExitBlocks.size() == 1); 220 assert(ExitBlocks[0]->getSinglePredecessor()); 221 222 // FromCycle has divergent exit condition. 223 if (hasDivergentBranch(ExitBlocks[0]->getSinglePredecessor())) { 224 return false; 225 } 226 227 FromCycle = FromCycle->getParentCycle(); 228 } 229 } 230 } 231 232 return true; 233 } 234 235 bool SIInstrInfo::areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1, 236 int64_t &Offset0, 237 int64_t &Offset1) const { 238 if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode()) 239 return false; 240 241 unsigned Opc0 = Load0->getMachineOpcode(); 242 unsigned Opc1 = Load1->getMachineOpcode(); 243 244 // Make sure both are actually loads. 245 if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad()) 246 return false; 247 248 // A mayLoad instruction without a def is not a load. Likely a prefetch. 249 if (!get(Opc0).getNumDefs() || !get(Opc1).getNumDefs()) 250 return false; 251 252 if (isDS(Opc0) && isDS(Opc1)) { 253 254 // FIXME: Handle this case: 255 if (getNumOperandsNoGlue(Load0) != getNumOperandsNoGlue(Load1)) 256 return false; 257 258 // Check base reg. 259 if (Load0->getOperand(0) != Load1->getOperand(0)) 260 return false; 261 262 // Skip read2 / write2 variants for simplicity. 263 // TODO: We should report true if the used offsets are adjacent (excluded 264 // st64 versions). 265 int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset); 266 int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset); 267 if (Offset0Idx == -1 || Offset1Idx == -1) 268 return false; 269 270 // XXX - be careful of dataless loads 271 // getNamedOperandIdx returns the index for MachineInstrs. Since they 272 // include the output in the operand list, but SDNodes don't, we need to 273 // subtract the index by one. 274 Offset0Idx -= get(Opc0).NumDefs; 275 Offset1Idx -= get(Opc1).NumDefs; 276 Offset0 = Load0->getConstantOperandVal(Offset0Idx); 277 Offset1 = Load1->getConstantOperandVal(Offset1Idx); 278 return true; 279 } 280 281 if (isSMRD(Opc0) && isSMRD(Opc1)) { 282 // Skip time and cache invalidation instructions. 283 if (!AMDGPU::hasNamedOperand(Opc0, AMDGPU::OpName::sbase) || 284 !AMDGPU::hasNamedOperand(Opc1, AMDGPU::OpName::sbase)) 285 return false; 286 287 unsigned NumOps = getNumOperandsNoGlue(Load0); 288 if (NumOps != getNumOperandsNoGlue(Load1)) 289 return false; 290 291 // Check base reg. 292 if (Load0->getOperand(0) != Load1->getOperand(0)) 293 return false; 294 295 // Match register offsets, if both register and immediate offsets present. 296 assert(NumOps == 4 || NumOps == 5); 297 if (NumOps == 5 && Load0->getOperand(1) != Load1->getOperand(1)) 298 return false; 299 300 const ConstantSDNode *Load0Offset = 301 dyn_cast<ConstantSDNode>(Load0->getOperand(NumOps - 3)); 302 const ConstantSDNode *Load1Offset = 303 dyn_cast<ConstantSDNode>(Load1->getOperand(NumOps - 3)); 304 305 if (!Load0Offset || !Load1Offset) 306 return false; 307 308 Offset0 = Load0Offset->getZExtValue(); 309 Offset1 = Load1Offset->getZExtValue(); 310 return true; 311 } 312 313 // MUBUF and MTBUF can access the same addresses. 314 if ((isMUBUF(Opc0) || isMTBUF(Opc0)) && (isMUBUF(Opc1) || isMTBUF(Opc1))) { 315 316 // MUBUF and MTBUF have vaddr at different indices. 317 if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) || 318 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) || 319 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc)) 320 return false; 321 322 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset); 323 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset); 324 325 if (OffIdx0 == -1 || OffIdx1 == -1) 326 return false; 327 328 // getNamedOperandIdx returns the index for MachineInstrs. Since they 329 // include the output in the operand list, but SDNodes don't, we need to 330 // subtract the index by one. 331 OffIdx0 -= get(Opc0).NumDefs; 332 OffIdx1 -= get(Opc1).NumDefs; 333 334 SDValue Off0 = Load0->getOperand(OffIdx0); 335 SDValue Off1 = Load1->getOperand(OffIdx1); 336 337 // The offset might be a FrameIndexSDNode. 338 if (!isa<ConstantSDNode>(Off0) || !isa<ConstantSDNode>(Off1)) 339 return false; 340 341 Offset0 = Off0->getAsZExtVal(); 342 Offset1 = Off1->getAsZExtVal(); 343 return true; 344 } 345 346 return false; 347 } 348 349 static bool isStride64(unsigned Opc) { 350 switch (Opc) { 351 case AMDGPU::DS_READ2ST64_B32: 352 case AMDGPU::DS_READ2ST64_B64: 353 case AMDGPU::DS_WRITE2ST64_B32: 354 case AMDGPU::DS_WRITE2ST64_B64: 355 return true; 356 default: 357 return false; 358 } 359 } 360 361 bool SIInstrInfo::getMemOperandsWithOffsetWidth( 362 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps, 363 int64_t &Offset, bool &OffsetIsScalable, unsigned &Width, 364 const TargetRegisterInfo *TRI) const { 365 if (!LdSt.mayLoadOrStore()) 366 return false; 367 368 unsigned Opc = LdSt.getOpcode(); 369 OffsetIsScalable = false; 370 const MachineOperand *BaseOp, *OffsetOp; 371 int DataOpIdx; 372 373 if (isDS(LdSt)) { 374 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr); 375 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset); 376 if (OffsetOp) { 377 // Normal, single offset LDS instruction. 378 if (!BaseOp) { 379 // DS_CONSUME/DS_APPEND use M0 for the base address. 380 // TODO: find the implicit use operand for M0 and use that as BaseOp? 381 return false; 382 } 383 BaseOps.push_back(BaseOp); 384 Offset = OffsetOp->getImm(); 385 // Get appropriate operand, and compute width accordingly. 386 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 387 if (DataOpIdx == -1) 388 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); 389 Width = getOpSize(LdSt, DataOpIdx); 390 } else { 391 // The 2 offset instructions use offset0 and offset1 instead. We can treat 392 // these as a load with a single offset if the 2 offsets are consecutive. 393 // We will use this for some partially aligned loads. 394 const MachineOperand *Offset0Op = 395 getNamedOperand(LdSt, AMDGPU::OpName::offset0); 396 const MachineOperand *Offset1Op = 397 getNamedOperand(LdSt, AMDGPU::OpName::offset1); 398 399 unsigned Offset0 = Offset0Op->getImm() & 0xff; 400 unsigned Offset1 = Offset1Op->getImm() & 0xff; 401 if (Offset0 + 1 != Offset1) 402 return false; 403 404 // Each of these offsets is in element sized units, so we need to convert 405 // to bytes of the individual reads. 406 407 unsigned EltSize; 408 if (LdSt.mayLoad()) 409 EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, 0)) / 16; 410 else { 411 assert(LdSt.mayStore()); 412 int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); 413 EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, Data0Idx)) / 8; 414 } 415 416 if (isStride64(Opc)) 417 EltSize *= 64; 418 419 BaseOps.push_back(BaseOp); 420 Offset = EltSize * Offset0; 421 // Get appropriate operand(s), and compute width accordingly. 422 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 423 if (DataOpIdx == -1) { 424 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); 425 Width = getOpSize(LdSt, DataOpIdx); 426 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1); 427 Width += getOpSize(LdSt, DataOpIdx); 428 } else { 429 Width = getOpSize(LdSt, DataOpIdx); 430 } 431 } 432 return true; 433 } 434 435 if (isMUBUF(LdSt) || isMTBUF(LdSt)) { 436 const MachineOperand *RSrc = getNamedOperand(LdSt, AMDGPU::OpName::srsrc); 437 if (!RSrc) // e.g. BUFFER_WBINVL1_VOL 438 return false; 439 BaseOps.push_back(RSrc); 440 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr); 441 if (BaseOp && !BaseOp->isFI()) 442 BaseOps.push_back(BaseOp); 443 const MachineOperand *OffsetImm = 444 getNamedOperand(LdSt, AMDGPU::OpName::offset); 445 Offset = OffsetImm->getImm(); 446 const MachineOperand *SOffset = 447 getNamedOperand(LdSt, AMDGPU::OpName::soffset); 448 if (SOffset) { 449 if (SOffset->isReg()) 450 BaseOps.push_back(SOffset); 451 else 452 Offset += SOffset->getImm(); 453 } 454 // Get appropriate operand, and compute width accordingly. 455 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 456 if (DataOpIdx == -1) 457 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); 458 if (DataOpIdx == -1) // LDS DMA 459 return false; 460 Width = getOpSize(LdSt, DataOpIdx); 461 return true; 462 } 463 464 if (isMIMG(LdSt)) { 465 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc); 466 BaseOps.push_back(&LdSt.getOperand(SRsrcIdx)); 467 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0); 468 if (VAddr0Idx >= 0) { 469 // GFX10 possible NSA encoding. 470 for (int I = VAddr0Idx; I < SRsrcIdx; ++I) 471 BaseOps.push_back(&LdSt.getOperand(I)); 472 } else { 473 BaseOps.push_back(getNamedOperand(LdSt, AMDGPU::OpName::vaddr)); 474 } 475 Offset = 0; 476 // Get appropriate operand, and compute width accordingly. 477 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); 478 Width = getOpSize(LdSt, DataOpIdx); 479 return true; 480 } 481 482 if (isSMRD(LdSt)) { 483 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::sbase); 484 if (!BaseOp) // e.g. S_MEMTIME 485 return false; 486 BaseOps.push_back(BaseOp); 487 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset); 488 Offset = OffsetOp ? OffsetOp->getImm() : 0; 489 // Get appropriate operand, and compute width accordingly. 490 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::sdst); 491 if (DataOpIdx == -1) 492 return false; 493 Width = getOpSize(LdSt, DataOpIdx); 494 return true; 495 } 496 497 if (isFLAT(LdSt)) { 498 // Instructions have either vaddr or saddr or both or none. 499 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr); 500 if (BaseOp) 501 BaseOps.push_back(BaseOp); 502 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr); 503 if (BaseOp) 504 BaseOps.push_back(BaseOp); 505 Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm(); 506 // Get appropriate operand, and compute width accordingly. 507 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 508 if (DataOpIdx == -1) 509 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); 510 if (DataOpIdx == -1) // LDS DMA 511 return false; 512 Width = getOpSize(LdSt, DataOpIdx); 513 return true; 514 } 515 516 return false; 517 } 518 519 static bool memOpsHaveSameBasePtr(const MachineInstr &MI1, 520 ArrayRef<const MachineOperand *> BaseOps1, 521 const MachineInstr &MI2, 522 ArrayRef<const MachineOperand *> BaseOps2) { 523 // Only examine the first "base" operand of each instruction, on the 524 // assumption that it represents the real base address of the memory access. 525 // Other operands are typically offsets or indices from this base address. 526 if (BaseOps1.front()->isIdenticalTo(*BaseOps2.front())) 527 return true; 528 529 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand()) 530 return false; 531 532 auto MO1 = *MI1.memoperands_begin(); 533 auto MO2 = *MI2.memoperands_begin(); 534 if (MO1->getAddrSpace() != MO2->getAddrSpace()) 535 return false; 536 537 auto Base1 = MO1->getValue(); 538 auto Base2 = MO2->getValue(); 539 if (!Base1 || !Base2) 540 return false; 541 Base1 = getUnderlyingObject(Base1); 542 Base2 = getUnderlyingObject(Base2); 543 544 if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2)) 545 return false; 546 547 return Base1 == Base2; 548 } 549 550 bool SIInstrInfo::shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1, 551 int64_t Offset1, bool OffsetIsScalable1, 552 ArrayRef<const MachineOperand *> BaseOps2, 553 int64_t Offset2, bool OffsetIsScalable2, 554 unsigned ClusterSize, 555 unsigned NumBytes) const { 556 // If the mem ops (to be clustered) do not have the same base ptr, then they 557 // should not be clustered 558 if (!BaseOps1.empty() && !BaseOps2.empty()) { 559 const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent(); 560 const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent(); 561 if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOps1, SecondLdSt, BaseOps2)) 562 return false; 563 } else if (!BaseOps1.empty() || !BaseOps2.empty()) { 564 // If only one base op is empty, they do not have the same base ptr 565 return false; 566 } 567 568 // In order to avoid register pressure, on an average, the number of DWORDS 569 // loaded together by all clustered mem ops should not exceed 8. This is an 570 // empirical value based on certain observations and performance related 571 // experiments. 572 // The good thing about this heuristic is - it avoids clustering of too many 573 // sub-word loads, and also avoids clustering of wide loads. Below is the 574 // brief summary of how the heuristic behaves for various `LoadSize`. 575 // (1) 1 <= LoadSize <= 4: cluster at max 8 mem ops 576 // (2) 5 <= LoadSize <= 8: cluster at max 4 mem ops 577 // (3) 9 <= LoadSize <= 12: cluster at max 2 mem ops 578 // (4) 13 <= LoadSize <= 16: cluster at max 2 mem ops 579 // (5) LoadSize >= 17: do not cluster 580 const unsigned LoadSize = NumBytes / ClusterSize; 581 const unsigned NumDWORDs = ((LoadSize + 3) / 4) * ClusterSize; 582 return NumDWORDs <= 8; 583 } 584 585 // FIXME: This behaves strangely. If, for example, you have 32 load + stores, 586 // the first 16 loads will be interleaved with the stores, and the next 16 will 587 // be clustered as expected. It should really split into 2 16 store batches. 588 // 589 // Loads are clustered until this returns false, rather than trying to schedule 590 // groups of stores. This also means we have to deal with saying different 591 // address space loads should be clustered, and ones which might cause bank 592 // conflicts. 593 // 594 // This might be deprecated so it might not be worth that much effort to fix. 595 bool SIInstrInfo::shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1, 596 int64_t Offset0, int64_t Offset1, 597 unsigned NumLoads) const { 598 assert(Offset1 > Offset0 && 599 "Second offset should be larger than first offset!"); 600 // If we have less than 16 loads in a row, and the offsets are within 64 601 // bytes, then schedule together. 602 603 // A cacheline is 64 bytes (for global memory). 604 return (NumLoads <= 16 && (Offset1 - Offset0) < 64); 605 } 606 607 static void reportIllegalCopy(const SIInstrInfo *TII, MachineBasicBlock &MBB, 608 MachineBasicBlock::iterator MI, 609 const DebugLoc &DL, MCRegister DestReg, 610 MCRegister SrcReg, bool KillSrc, 611 const char *Msg = "illegal VGPR to SGPR copy") { 612 MachineFunction *MF = MBB.getParent(); 613 DiagnosticInfoUnsupported IllegalCopy(MF->getFunction(), Msg, DL, DS_Error); 614 LLVMContext &C = MF->getFunction().getContext(); 615 C.diagnose(IllegalCopy); 616 617 BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg) 618 .addReg(SrcReg, getKillRegState(KillSrc)); 619 } 620 621 /// Handle copying from SGPR to AGPR, or from AGPR to AGPR on GFX908. It is not 622 /// possible to have a direct copy in these cases on GFX908, so an intermediate 623 /// VGPR copy is required. 624 static void indirectCopyToAGPR(const SIInstrInfo &TII, 625 MachineBasicBlock &MBB, 626 MachineBasicBlock::iterator MI, 627 const DebugLoc &DL, MCRegister DestReg, 628 MCRegister SrcReg, bool KillSrc, 629 RegScavenger &RS, bool RegsOverlap, 630 Register ImpDefSuperReg = Register(), 631 Register ImpUseSuperReg = Register()) { 632 assert((TII.getSubtarget().hasMAIInsts() && 633 !TII.getSubtarget().hasGFX90AInsts()) && 634 "Expected GFX908 subtarget."); 635 636 assert((AMDGPU::SReg_32RegClass.contains(SrcReg) || 637 AMDGPU::AGPR_32RegClass.contains(SrcReg)) && 638 "Source register of the copy should be either an SGPR or an AGPR."); 639 640 assert(AMDGPU::AGPR_32RegClass.contains(DestReg) && 641 "Destination register of the copy should be an AGPR."); 642 643 const SIRegisterInfo &RI = TII.getRegisterInfo(); 644 645 // First try to find defining accvgpr_write to avoid temporary registers. 646 // In the case of copies of overlapping AGPRs, we conservatively do not 647 // reuse previous accvgpr_writes. Otherwise, we may incorrectly pick up 648 // an accvgpr_write used for this same copy due to implicit-defs 649 if (!RegsOverlap) { 650 for (auto Def = MI, E = MBB.begin(); Def != E; ) { 651 --Def; 652 653 if (!Def->modifiesRegister(SrcReg, &RI)) 654 continue; 655 656 if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 || 657 Def->getOperand(0).getReg() != SrcReg) 658 break; 659 660 MachineOperand &DefOp = Def->getOperand(1); 661 assert(DefOp.isReg() || DefOp.isImm()); 662 663 if (DefOp.isReg()) { 664 bool SafeToPropagate = true; 665 // Check that register source operand is not clobbered before MI. 666 // Immediate operands are always safe to propagate. 667 for (auto I = Def; I != MI && SafeToPropagate; ++I) 668 if (I->modifiesRegister(DefOp.getReg(), &RI)) 669 SafeToPropagate = false; 670 671 if (!SafeToPropagate) 672 break; 673 674 DefOp.setIsKill(false); 675 } 676 677 MachineInstrBuilder Builder = 678 BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) 679 .add(DefOp); 680 if (ImpDefSuperReg) 681 Builder.addReg(ImpDefSuperReg, RegState::Define | RegState::Implicit); 682 683 if (ImpUseSuperReg) { 684 Builder.addReg(ImpUseSuperReg, 685 getKillRegState(KillSrc) | RegState::Implicit); 686 } 687 688 return; 689 } 690 } 691 692 RS.enterBasicBlockEnd(MBB); 693 RS.backward(std::next(MI)); 694 695 // Ideally we want to have three registers for a long reg_sequence copy 696 // to hide 2 waitstates between v_mov_b32 and accvgpr_write. 697 unsigned MaxVGPRs = RI.getRegPressureLimit(&AMDGPU::VGPR_32RegClass, 698 *MBB.getParent()); 699 700 // Registers in the sequence are allocated contiguously so we can just 701 // use register number to pick one of three round-robin temps. 702 unsigned RegNo = (DestReg - AMDGPU::AGPR0) % 3; 703 Register Tmp = 704 MBB.getParent()->getInfo<SIMachineFunctionInfo>()->getVGPRForAGPRCopy(); 705 assert(MBB.getParent()->getRegInfo().isReserved(Tmp) && 706 "VGPR used for an intermediate copy should have been reserved."); 707 708 // Only loop through if there are any free registers left. We don't want to 709 // spill. 710 while (RegNo--) { 711 Register Tmp2 = RS.scavengeRegisterBackwards(AMDGPU::VGPR_32RegClass, MI, 712 /* RestoreAfter */ false, 0, 713 /* AllowSpill */ false); 714 if (!Tmp2 || RI.getHWRegIndex(Tmp2) >= MaxVGPRs) 715 break; 716 Tmp = Tmp2; 717 RS.setRegUsed(Tmp); 718 } 719 720 // Insert copy to temporary VGPR. 721 unsigned TmpCopyOp = AMDGPU::V_MOV_B32_e32; 722 if (AMDGPU::AGPR_32RegClass.contains(SrcReg)) { 723 TmpCopyOp = AMDGPU::V_ACCVGPR_READ_B32_e64; 724 } else { 725 assert(AMDGPU::SReg_32RegClass.contains(SrcReg)); 726 } 727 728 MachineInstrBuilder UseBuilder = BuildMI(MBB, MI, DL, TII.get(TmpCopyOp), Tmp) 729 .addReg(SrcReg, getKillRegState(KillSrc)); 730 if (ImpUseSuperReg) { 731 UseBuilder.addReg(ImpUseSuperReg, 732 getKillRegState(KillSrc) | RegState::Implicit); 733 } 734 735 MachineInstrBuilder DefBuilder 736 = BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) 737 .addReg(Tmp, RegState::Kill); 738 739 if (ImpDefSuperReg) 740 DefBuilder.addReg(ImpDefSuperReg, RegState::Define | RegState::Implicit); 741 } 742 743 static void expandSGPRCopy(const SIInstrInfo &TII, MachineBasicBlock &MBB, 744 MachineBasicBlock::iterator MI, const DebugLoc &DL, 745 MCRegister DestReg, MCRegister SrcReg, bool KillSrc, 746 const TargetRegisterClass *RC, bool Forward) { 747 const SIRegisterInfo &RI = TII.getRegisterInfo(); 748 ArrayRef<int16_t> BaseIndices = RI.getRegSplitParts(RC, 4); 749 MachineBasicBlock::iterator I = MI; 750 MachineInstr *FirstMI = nullptr, *LastMI = nullptr; 751 752 for (unsigned Idx = 0; Idx < BaseIndices.size(); ++Idx) { 753 int16_t SubIdx = BaseIndices[Idx]; 754 Register DestSubReg = RI.getSubReg(DestReg, SubIdx); 755 Register SrcSubReg = RI.getSubReg(SrcReg, SubIdx); 756 assert(DestSubReg && SrcSubReg && "Failed to find subregs!"); 757 unsigned Opcode = AMDGPU::S_MOV_B32; 758 759 // Is SGPR aligned? If so try to combine with next. 760 bool AlignedDest = ((DestSubReg - AMDGPU::SGPR0) % 2) == 0; 761 bool AlignedSrc = ((SrcSubReg - AMDGPU::SGPR0) % 2) == 0; 762 if (AlignedDest && AlignedSrc && (Idx + 1 < BaseIndices.size())) { 763 // Can use SGPR64 copy 764 unsigned Channel = RI.getChannelFromSubReg(SubIdx); 765 SubIdx = RI.getSubRegFromChannel(Channel, 2); 766 DestSubReg = RI.getSubReg(DestReg, SubIdx); 767 SrcSubReg = RI.getSubReg(SrcReg, SubIdx); 768 assert(DestSubReg && SrcSubReg && "Failed to find subregs!"); 769 Opcode = AMDGPU::S_MOV_B64; 770 Idx++; 771 } 772 773 LastMI = BuildMI(MBB, I, DL, TII.get(Opcode), DestSubReg) 774 .addReg(SrcSubReg) 775 .addReg(SrcReg, RegState::Implicit); 776 777 if (!FirstMI) 778 FirstMI = LastMI; 779 780 if (!Forward) 781 I--; 782 } 783 784 assert(FirstMI && LastMI); 785 if (!Forward) 786 std::swap(FirstMI, LastMI); 787 788 FirstMI->addOperand( 789 MachineOperand::CreateReg(DestReg, true /*IsDef*/, true /*IsImp*/)); 790 791 if (KillSrc) 792 LastMI->addRegisterKilled(SrcReg, &RI); 793 } 794 795 void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB, 796 MachineBasicBlock::iterator MI, 797 const DebugLoc &DL, MCRegister DestReg, 798 MCRegister SrcReg, bool KillSrc) const { 799 const TargetRegisterClass *RC = RI.getPhysRegBaseClass(DestReg); 800 unsigned Size = RI.getRegSizeInBits(*RC); 801 const TargetRegisterClass *SrcRC = RI.getPhysRegBaseClass(SrcReg); 802 unsigned SrcSize = RI.getRegSizeInBits(*SrcRC); 803 804 // The rest of copyPhysReg assumes Src and Dst size are the same size. 805 // TODO-GFX11_16BIT If all true 16 bit instruction patterns are completed can 806 // we remove Fix16BitCopies and this code block? 807 if (Fix16BitCopies) { 808 if (((Size == 16) != (SrcSize == 16))) { 809 // Non-VGPR Src and Dst will later be expanded back to 32 bits. 810 assert(ST.hasTrue16BitInsts()); 811 MCRegister &RegToFix = (Size == 32) ? DestReg : SrcReg; 812 MCRegister SubReg = RI.getSubReg(RegToFix, AMDGPU::lo16); 813 RegToFix = SubReg; 814 815 if (DestReg == SrcReg) { 816 // Identity copy. Insert empty bundle since ExpandPostRA expects an 817 // instruction here. 818 BuildMI(MBB, MI, DL, get(AMDGPU::BUNDLE)); 819 return; 820 } 821 RC = RI.getPhysRegBaseClass(DestReg); 822 Size = RI.getRegSizeInBits(*RC); 823 SrcRC = RI.getPhysRegBaseClass(SrcReg); 824 SrcSize = RI.getRegSizeInBits(*SrcRC); 825 } 826 } 827 828 if (RC == &AMDGPU::VGPR_32RegClass) { 829 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) || 830 AMDGPU::SReg_32RegClass.contains(SrcReg) || 831 AMDGPU::AGPR_32RegClass.contains(SrcReg)); 832 unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ? 833 AMDGPU::V_ACCVGPR_READ_B32_e64 : AMDGPU::V_MOV_B32_e32; 834 BuildMI(MBB, MI, DL, get(Opc), DestReg) 835 .addReg(SrcReg, getKillRegState(KillSrc)); 836 return; 837 } 838 839 if (RC == &AMDGPU::SReg_32_XM0RegClass || 840 RC == &AMDGPU::SReg_32RegClass) { 841 if (SrcReg == AMDGPU::SCC) { 842 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg) 843 .addImm(1) 844 .addImm(0); 845 return; 846 } 847 848 if (DestReg == AMDGPU::VCC_LO) { 849 if (AMDGPU::SReg_32RegClass.contains(SrcReg)) { 850 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), AMDGPU::VCC_LO) 851 .addReg(SrcReg, getKillRegState(KillSrc)); 852 } else { 853 // FIXME: Hack until VReg_1 removed. 854 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg)); 855 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32)) 856 .addImm(0) 857 .addReg(SrcReg, getKillRegState(KillSrc)); 858 } 859 860 return; 861 } 862 863 if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) { 864 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc); 865 return; 866 } 867 868 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg) 869 .addReg(SrcReg, getKillRegState(KillSrc)); 870 return; 871 } 872 873 if (RC == &AMDGPU::SReg_64RegClass) { 874 if (SrcReg == AMDGPU::SCC) { 875 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B64), DestReg) 876 .addImm(1) 877 .addImm(0); 878 return; 879 } 880 881 if (DestReg == AMDGPU::VCC) { 882 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) { 883 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC) 884 .addReg(SrcReg, getKillRegState(KillSrc)); 885 } else { 886 // FIXME: Hack until VReg_1 removed. 887 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg)); 888 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32)) 889 .addImm(0) 890 .addReg(SrcReg, getKillRegState(KillSrc)); 891 } 892 893 return; 894 } 895 896 if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) { 897 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc); 898 return; 899 } 900 901 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg) 902 .addReg(SrcReg, getKillRegState(KillSrc)); 903 return; 904 } 905 906 if (DestReg == AMDGPU::SCC) { 907 // Copying 64-bit or 32-bit sources to SCC barely makes sense, 908 // but SelectionDAG emits such copies for i1 sources. 909 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) { 910 // This copy can only be produced by patterns 911 // with explicit SCC, which are known to be enabled 912 // only for subtargets with S_CMP_LG_U64 present. 913 assert(ST.hasScalarCompareEq64()); 914 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U64)) 915 .addReg(SrcReg, getKillRegState(KillSrc)) 916 .addImm(0); 917 } else { 918 assert(AMDGPU::SReg_32RegClass.contains(SrcReg)); 919 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32)) 920 .addReg(SrcReg, getKillRegState(KillSrc)) 921 .addImm(0); 922 } 923 924 return; 925 } 926 927 if (RC == &AMDGPU::AGPR_32RegClass) { 928 if (AMDGPU::VGPR_32RegClass.contains(SrcReg) || 929 (ST.hasGFX90AInsts() && AMDGPU::SReg_32RegClass.contains(SrcReg))) { 930 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) 931 .addReg(SrcReg, getKillRegState(KillSrc)); 932 return; 933 } 934 935 if (AMDGPU::AGPR_32RegClass.contains(SrcReg) && ST.hasGFX90AInsts()) { 936 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_MOV_B32), DestReg) 937 .addReg(SrcReg, getKillRegState(KillSrc)); 938 return; 939 } 940 941 // FIXME: Pass should maintain scavenger to avoid scan through the block on 942 // every AGPR spill. 943 RegScavenger RS; 944 const bool Overlap = RI.regsOverlap(SrcReg, DestReg); 945 indirectCopyToAGPR(*this, MBB, MI, DL, DestReg, SrcReg, KillSrc, RS, Overlap); 946 return; 947 } 948 949 if (Size == 16) { 950 assert(AMDGPU::VGPR_16RegClass.contains(SrcReg) || 951 AMDGPU::SReg_LO16RegClass.contains(SrcReg) || 952 AMDGPU::AGPR_LO16RegClass.contains(SrcReg)); 953 954 bool IsSGPRDst = AMDGPU::SReg_LO16RegClass.contains(DestReg); 955 bool IsSGPRSrc = AMDGPU::SReg_LO16RegClass.contains(SrcReg); 956 bool IsAGPRDst = AMDGPU::AGPR_LO16RegClass.contains(DestReg); 957 bool IsAGPRSrc = AMDGPU::AGPR_LO16RegClass.contains(SrcReg); 958 bool DstLow = !AMDGPU::isHi(DestReg, RI); 959 bool SrcLow = !AMDGPU::isHi(SrcReg, RI); 960 MCRegister NewDestReg = RI.get32BitRegister(DestReg); 961 MCRegister NewSrcReg = RI.get32BitRegister(SrcReg); 962 963 if (IsSGPRDst) { 964 if (!IsSGPRSrc) { 965 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc); 966 return; 967 } 968 969 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), NewDestReg) 970 .addReg(NewSrcReg, getKillRegState(KillSrc)); 971 return; 972 } 973 974 if (IsAGPRDst || IsAGPRSrc) { 975 if (!DstLow || !SrcLow) { 976 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc, 977 "Cannot use hi16 subreg with an AGPR!"); 978 } 979 980 copyPhysReg(MBB, MI, DL, NewDestReg, NewSrcReg, KillSrc); 981 return; 982 } 983 984 if (ST.hasTrue16BitInsts()) { 985 if (IsSGPRSrc) { 986 assert(SrcLow); 987 SrcReg = NewSrcReg; 988 } 989 // Use the smaller instruction encoding if possible. 990 if (AMDGPU::VGPR_16_Lo128RegClass.contains(DestReg) && 991 (IsSGPRSrc || AMDGPU::VGPR_16_Lo128RegClass.contains(SrcReg))) { 992 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B16_t16_e32), DestReg) 993 .addReg(SrcReg); 994 } else { 995 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B16_t16_e64), DestReg) 996 .addImm(0) // src0_modifiers 997 .addReg(SrcReg) 998 .addImm(0); // op_sel 999 } 1000 return; 1001 } 1002 1003 if (IsSGPRSrc && !ST.hasSDWAScalar()) { 1004 if (!DstLow || !SrcLow) { 1005 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc, 1006 "Cannot use hi16 subreg on VI!"); 1007 } 1008 1009 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), NewDestReg) 1010 .addReg(NewSrcReg, getKillRegState(KillSrc)); 1011 return; 1012 } 1013 1014 auto MIB = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_sdwa), NewDestReg) 1015 .addImm(0) // src0_modifiers 1016 .addReg(NewSrcReg) 1017 .addImm(0) // clamp 1018 .addImm(DstLow ? AMDGPU::SDWA::SdwaSel::WORD_0 1019 : AMDGPU::SDWA::SdwaSel::WORD_1) 1020 .addImm(AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE) 1021 .addImm(SrcLow ? AMDGPU::SDWA::SdwaSel::WORD_0 1022 : AMDGPU::SDWA::SdwaSel::WORD_1) 1023 .addReg(NewDestReg, RegState::Implicit | RegState::Undef); 1024 // First implicit operand is $exec. 1025 MIB->tieOperands(0, MIB->getNumOperands() - 1); 1026 return; 1027 } 1028 1029 if (RC == RI.getVGPR64Class() && (SrcRC == RC || RI.isSGPRClass(SrcRC))) { 1030 if (ST.hasMovB64()) { 1031 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_e32), DestReg) 1032 .addReg(SrcReg, getKillRegState(KillSrc)); 1033 return; 1034 } 1035 if (ST.hasPkMovB32()) { 1036 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DestReg) 1037 .addImm(SISrcMods::OP_SEL_1) 1038 .addReg(SrcReg) 1039 .addImm(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1) 1040 .addReg(SrcReg) 1041 .addImm(0) // op_sel_lo 1042 .addImm(0) // op_sel_hi 1043 .addImm(0) // neg_lo 1044 .addImm(0) // neg_hi 1045 .addImm(0) // clamp 1046 .addReg(SrcReg, getKillRegState(KillSrc) | RegState::Implicit); 1047 return; 1048 } 1049 } 1050 1051 const bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg); 1052 if (RI.isSGPRClass(RC)) { 1053 if (!RI.isSGPRClass(SrcRC)) { 1054 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc); 1055 return; 1056 } 1057 const bool CanKillSuperReg = KillSrc && !RI.regsOverlap(SrcReg, DestReg); 1058 expandSGPRCopy(*this, MBB, MI, DL, DestReg, SrcReg, CanKillSuperReg, RC, 1059 Forward); 1060 return; 1061 } 1062 1063 unsigned EltSize = 4; 1064 unsigned Opcode = AMDGPU::V_MOV_B32_e32; 1065 if (RI.isAGPRClass(RC)) { 1066 if (ST.hasGFX90AInsts() && RI.isAGPRClass(SrcRC)) 1067 Opcode = AMDGPU::V_ACCVGPR_MOV_B32; 1068 else if (RI.hasVGPRs(SrcRC) || 1069 (ST.hasGFX90AInsts() && RI.isSGPRClass(SrcRC))) 1070 Opcode = AMDGPU::V_ACCVGPR_WRITE_B32_e64; 1071 else 1072 Opcode = AMDGPU::INSTRUCTION_LIST_END; 1073 } else if (RI.hasVGPRs(RC) && RI.isAGPRClass(SrcRC)) { 1074 Opcode = AMDGPU::V_ACCVGPR_READ_B32_e64; 1075 } else if ((Size % 64 == 0) && RI.hasVGPRs(RC) && 1076 (RI.isProperlyAlignedRC(*RC) && 1077 (SrcRC == RC || RI.isSGPRClass(SrcRC)))) { 1078 // TODO: In 96-bit case, could do a 64-bit mov and then a 32-bit mov. 1079 if (ST.hasMovB64()) { 1080 Opcode = AMDGPU::V_MOV_B64_e32; 1081 EltSize = 8; 1082 } else if (ST.hasPkMovB32()) { 1083 Opcode = AMDGPU::V_PK_MOV_B32; 1084 EltSize = 8; 1085 } 1086 } 1087 1088 // For the cases where we need an intermediate instruction/temporary register 1089 // (destination is an AGPR), we need a scavenger. 1090 // 1091 // FIXME: The pass should maintain this for us so we don't have to re-scan the 1092 // whole block for every handled copy. 1093 std::unique_ptr<RegScavenger> RS; 1094 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) 1095 RS.reset(new RegScavenger()); 1096 1097 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize); 1098 1099 // If there is an overlap, we can't kill the super-register on the last 1100 // instruction, since it will also kill the components made live by this def. 1101 const bool Overlap = RI.regsOverlap(SrcReg, DestReg); 1102 const bool CanKillSuperReg = KillSrc && !Overlap; 1103 1104 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) { 1105 unsigned SubIdx; 1106 if (Forward) 1107 SubIdx = SubIndices[Idx]; 1108 else 1109 SubIdx = SubIndices[SubIndices.size() - Idx - 1]; 1110 Register DestSubReg = RI.getSubReg(DestReg, SubIdx); 1111 Register SrcSubReg = RI.getSubReg(SrcReg, SubIdx); 1112 assert(DestSubReg && SrcSubReg && "Failed to find subregs!"); 1113 1114 bool IsFirstSubreg = Idx == 0; 1115 bool UseKill = CanKillSuperReg && Idx == SubIndices.size() - 1; 1116 1117 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) { 1118 Register ImpDefSuper = IsFirstSubreg ? Register(DestReg) : Register(); 1119 Register ImpUseSuper = SrcReg; 1120 indirectCopyToAGPR(*this, MBB, MI, DL, DestSubReg, SrcSubReg, UseKill, 1121 *RS, Overlap, ImpDefSuper, ImpUseSuper); 1122 } else if (Opcode == AMDGPU::V_PK_MOV_B32) { 1123 MachineInstrBuilder MIB = 1124 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DestSubReg) 1125 .addImm(SISrcMods::OP_SEL_1) 1126 .addReg(SrcSubReg) 1127 .addImm(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1) 1128 .addReg(SrcSubReg) 1129 .addImm(0) // op_sel_lo 1130 .addImm(0) // op_sel_hi 1131 .addImm(0) // neg_lo 1132 .addImm(0) // neg_hi 1133 .addImm(0) // clamp 1134 .addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit); 1135 if (IsFirstSubreg) 1136 MIB.addReg(DestReg, RegState::Define | RegState::Implicit); 1137 } else { 1138 MachineInstrBuilder Builder = 1139 BuildMI(MBB, MI, DL, get(Opcode), DestSubReg).addReg(SrcSubReg); 1140 if (IsFirstSubreg) 1141 Builder.addReg(DestReg, RegState::Define | RegState::Implicit); 1142 1143 Builder.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit); 1144 } 1145 } 1146 } 1147 1148 int SIInstrInfo::commuteOpcode(unsigned Opcode) const { 1149 int NewOpc; 1150 1151 // Try to map original to commuted opcode 1152 NewOpc = AMDGPU::getCommuteRev(Opcode); 1153 if (NewOpc != -1) 1154 // Check if the commuted (REV) opcode exists on the target. 1155 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1; 1156 1157 // Try to map commuted to original opcode 1158 NewOpc = AMDGPU::getCommuteOrig(Opcode); 1159 if (NewOpc != -1) 1160 // Check if the original (non-REV) opcode exists on the target. 1161 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1; 1162 1163 return Opcode; 1164 } 1165 1166 void SIInstrInfo::materializeImmediate(MachineBasicBlock &MBB, 1167 MachineBasicBlock::iterator MI, 1168 const DebugLoc &DL, Register DestReg, 1169 int64_t Value) const { 1170 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 1171 const TargetRegisterClass *RegClass = MRI.getRegClass(DestReg); 1172 if (RegClass == &AMDGPU::SReg_32RegClass || 1173 RegClass == &AMDGPU::SGPR_32RegClass || 1174 RegClass == &AMDGPU::SReg_32_XM0RegClass || 1175 RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) { 1176 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg) 1177 .addImm(Value); 1178 return; 1179 } 1180 1181 if (RegClass == &AMDGPU::SReg_64RegClass || 1182 RegClass == &AMDGPU::SGPR_64RegClass || 1183 RegClass == &AMDGPU::SReg_64_XEXECRegClass) { 1184 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg) 1185 .addImm(Value); 1186 return; 1187 } 1188 1189 if (RegClass == &AMDGPU::VGPR_32RegClass) { 1190 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg) 1191 .addImm(Value); 1192 return; 1193 } 1194 if (RegClass->hasSuperClassEq(&AMDGPU::VReg_64RegClass)) { 1195 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg) 1196 .addImm(Value); 1197 return; 1198 } 1199 1200 unsigned EltSize = 4; 1201 unsigned Opcode = AMDGPU::V_MOV_B32_e32; 1202 if (RI.isSGPRClass(RegClass)) { 1203 if (RI.getRegSizeInBits(*RegClass) > 32) { 1204 Opcode = AMDGPU::S_MOV_B64; 1205 EltSize = 8; 1206 } else { 1207 Opcode = AMDGPU::S_MOV_B32; 1208 EltSize = 4; 1209 } 1210 } 1211 1212 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RegClass, EltSize); 1213 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) { 1214 int64_t IdxValue = Idx == 0 ? Value : 0; 1215 1216 MachineInstrBuilder Builder = BuildMI(MBB, MI, DL, 1217 get(Opcode), RI.getSubReg(DestReg, SubIndices[Idx])); 1218 Builder.addImm(IdxValue); 1219 } 1220 } 1221 1222 const TargetRegisterClass * 1223 SIInstrInfo::getPreferredSelectRegClass(unsigned Size) const { 1224 return &AMDGPU::VGPR_32RegClass; 1225 } 1226 1227 void SIInstrInfo::insertVectorSelect(MachineBasicBlock &MBB, 1228 MachineBasicBlock::iterator I, 1229 const DebugLoc &DL, Register DstReg, 1230 ArrayRef<MachineOperand> Cond, 1231 Register TrueReg, 1232 Register FalseReg) const { 1233 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 1234 const TargetRegisterClass *BoolXExecRC = 1235 RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 1236 assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass && 1237 "Not a VGPR32 reg"); 1238 1239 if (Cond.size() == 1) { 1240 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1241 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) 1242 .add(Cond[0]); 1243 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1244 .addImm(0) 1245 .addReg(FalseReg) 1246 .addImm(0) 1247 .addReg(TrueReg) 1248 .addReg(SReg); 1249 } else if (Cond.size() == 2) { 1250 assert(Cond[0].isImm() && "Cond[0] is not an immediate"); 1251 switch (Cond[0].getImm()) { 1252 case SIInstrInfo::SCC_TRUE: { 1253 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1254 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 1255 : AMDGPU::S_CSELECT_B64), SReg) 1256 .addImm(1) 1257 .addImm(0); 1258 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1259 .addImm(0) 1260 .addReg(FalseReg) 1261 .addImm(0) 1262 .addReg(TrueReg) 1263 .addReg(SReg); 1264 break; 1265 } 1266 case SIInstrInfo::SCC_FALSE: { 1267 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1268 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 1269 : AMDGPU::S_CSELECT_B64), SReg) 1270 .addImm(0) 1271 .addImm(1); 1272 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1273 .addImm(0) 1274 .addReg(FalseReg) 1275 .addImm(0) 1276 .addReg(TrueReg) 1277 .addReg(SReg); 1278 break; 1279 } 1280 case SIInstrInfo::VCCNZ: { 1281 MachineOperand RegOp = Cond[1]; 1282 RegOp.setImplicit(false); 1283 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1284 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) 1285 .add(RegOp); 1286 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1287 .addImm(0) 1288 .addReg(FalseReg) 1289 .addImm(0) 1290 .addReg(TrueReg) 1291 .addReg(SReg); 1292 break; 1293 } 1294 case SIInstrInfo::VCCZ: { 1295 MachineOperand RegOp = Cond[1]; 1296 RegOp.setImplicit(false); 1297 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1298 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) 1299 .add(RegOp); 1300 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1301 .addImm(0) 1302 .addReg(TrueReg) 1303 .addImm(0) 1304 .addReg(FalseReg) 1305 .addReg(SReg); 1306 break; 1307 } 1308 case SIInstrInfo::EXECNZ: { 1309 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1310 Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC()); 1311 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 1312 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2) 1313 .addImm(0); 1314 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 1315 : AMDGPU::S_CSELECT_B64), SReg) 1316 .addImm(1) 1317 .addImm(0); 1318 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1319 .addImm(0) 1320 .addReg(FalseReg) 1321 .addImm(0) 1322 .addReg(TrueReg) 1323 .addReg(SReg); 1324 break; 1325 } 1326 case SIInstrInfo::EXECZ: { 1327 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1328 Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC()); 1329 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 1330 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2) 1331 .addImm(0); 1332 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 1333 : AMDGPU::S_CSELECT_B64), SReg) 1334 .addImm(0) 1335 .addImm(1); 1336 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1337 .addImm(0) 1338 .addReg(FalseReg) 1339 .addImm(0) 1340 .addReg(TrueReg) 1341 .addReg(SReg); 1342 llvm_unreachable("Unhandled branch predicate EXECZ"); 1343 break; 1344 } 1345 default: 1346 llvm_unreachable("invalid branch predicate"); 1347 } 1348 } else { 1349 llvm_unreachable("Can only handle Cond size 1 or 2"); 1350 } 1351 } 1352 1353 Register SIInstrInfo::insertEQ(MachineBasicBlock *MBB, 1354 MachineBasicBlock::iterator I, 1355 const DebugLoc &DL, 1356 Register SrcReg, int Value) const { 1357 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 1358 Register Reg = MRI.createVirtualRegister(RI.getBoolRC()); 1359 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg) 1360 .addImm(Value) 1361 .addReg(SrcReg); 1362 1363 return Reg; 1364 } 1365 1366 Register SIInstrInfo::insertNE(MachineBasicBlock *MBB, 1367 MachineBasicBlock::iterator I, 1368 const DebugLoc &DL, 1369 Register SrcReg, int Value) const { 1370 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 1371 Register Reg = MRI.createVirtualRegister(RI.getBoolRC()); 1372 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg) 1373 .addImm(Value) 1374 .addReg(SrcReg); 1375 1376 return Reg; 1377 } 1378 1379 unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const { 1380 1381 if (RI.isAGPRClass(DstRC)) 1382 return AMDGPU::COPY; 1383 if (RI.getRegSizeInBits(*DstRC) == 16) { 1384 // Assume hi bits are unneeded. Only _e64 true16 instructions are legal 1385 // before RA. 1386 return RI.isSGPRClass(DstRC) ? AMDGPU::COPY : AMDGPU::V_MOV_B16_t16_e64; 1387 } else if (RI.getRegSizeInBits(*DstRC) == 32) { 1388 return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; 1389 } else if (RI.getRegSizeInBits(*DstRC) == 64 && RI.isSGPRClass(DstRC)) { 1390 return AMDGPU::S_MOV_B64; 1391 } else if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC)) { 1392 return AMDGPU::V_MOV_B64_PSEUDO; 1393 } 1394 return AMDGPU::COPY; 1395 } 1396 1397 const MCInstrDesc & 1398 SIInstrInfo::getIndirectGPRIDXPseudo(unsigned VecSize, 1399 bool IsIndirectSrc) const { 1400 if (IsIndirectSrc) { 1401 if (VecSize <= 32) // 4 bytes 1402 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1); 1403 if (VecSize <= 64) // 8 bytes 1404 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2); 1405 if (VecSize <= 96) // 12 bytes 1406 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3); 1407 if (VecSize <= 128) // 16 bytes 1408 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4); 1409 if (VecSize <= 160) // 20 bytes 1410 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5); 1411 if (VecSize <= 256) // 32 bytes 1412 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8); 1413 if (VecSize <= 288) // 36 bytes 1414 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9); 1415 if (VecSize <= 320) // 40 bytes 1416 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10); 1417 if (VecSize <= 352) // 44 bytes 1418 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11); 1419 if (VecSize <= 384) // 48 bytes 1420 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12); 1421 if (VecSize <= 512) // 64 bytes 1422 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16); 1423 if (VecSize <= 1024) // 128 bytes 1424 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32); 1425 1426 llvm_unreachable("unsupported size for IndirectRegReadGPRIDX pseudos"); 1427 } 1428 1429 if (VecSize <= 32) // 4 bytes 1430 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1); 1431 if (VecSize <= 64) // 8 bytes 1432 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2); 1433 if (VecSize <= 96) // 12 bytes 1434 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3); 1435 if (VecSize <= 128) // 16 bytes 1436 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4); 1437 if (VecSize <= 160) // 20 bytes 1438 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5); 1439 if (VecSize <= 256) // 32 bytes 1440 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8); 1441 if (VecSize <= 288) // 36 bytes 1442 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9); 1443 if (VecSize <= 320) // 40 bytes 1444 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10); 1445 if (VecSize <= 352) // 44 bytes 1446 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11); 1447 if (VecSize <= 384) // 48 bytes 1448 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12); 1449 if (VecSize <= 512) // 64 bytes 1450 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16); 1451 if (VecSize <= 1024) // 128 bytes 1452 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32); 1453 1454 llvm_unreachable("unsupported size for IndirectRegWriteGPRIDX pseudos"); 1455 } 1456 1457 static unsigned getIndirectVGPRWriteMovRelPseudoOpc(unsigned VecSize) { 1458 if (VecSize <= 32) // 4 bytes 1459 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1; 1460 if (VecSize <= 64) // 8 bytes 1461 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2; 1462 if (VecSize <= 96) // 12 bytes 1463 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3; 1464 if (VecSize <= 128) // 16 bytes 1465 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4; 1466 if (VecSize <= 160) // 20 bytes 1467 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5; 1468 if (VecSize <= 256) // 32 bytes 1469 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8; 1470 if (VecSize <= 288) // 36 bytes 1471 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9; 1472 if (VecSize <= 320) // 40 bytes 1473 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10; 1474 if (VecSize <= 352) // 44 bytes 1475 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11; 1476 if (VecSize <= 384) // 48 bytes 1477 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12; 1478 if (VecSize <= 512) // 64 bytes 1479 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16; 1480 if (VecSize <= 1024) // 128 bytes 1481 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32; 1482 1483 llvm_unreachable("unsupported size for IndirectRegWrite pseudos"); 1484 } 1485 1486 static unsigned getIndirectSGPRWriteMovRelPseudo32(unsigned VecSize) { 1487 if (VecSize <= 32) // 4 bytes 1488 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1; 1489 if (VecSize <= 64) // 8 bytes 1490 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2; 1491 if (VecSize <= 96) // 12 bytes 1492 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3; 1493 if (VecSize <= 128) // 16 bytes 1494 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4; 1495 if (VecSize <= 160) // 20 bytes 1496 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5; 1497 if (VecSize <= 256) // 32 bytes 1498 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8; 1499 if (VecSize <= 288) // 36 bytes 1500 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9; 1501 if (VecSize <= 320) // 40 bytes 1502 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10; 1503 if (VecSize <= 352) // 44 bytes 1504 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11; 1505 if (VecSize <= 384) // 48 bytes 1506 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12; 1507 if (VecSize <= 512) // 64 bytes 1508 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16; 1509 if (VecSize <= 1024) // 128 bytes 1510 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32; 1511 1512 llvm_unreachable("unsupported size for IndirectRegWrite pseudos"); 1513 } 1514 1515 static unsigned getIndirectSGPRWriteMovRelPseudo64(unsigned VecSize) { 1516 if (VecSize <= 64) // 8 bytes 1517 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1; 1518 if (VecSize <= 128) // 16 bytes 1519 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2; 1520 if (VecSize <= 256) // 32 bytes 1521 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4; 1522 if (VecSize <= 512) // 64 bytes 1523 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8; 1524 if (VecSize <= 1024) // 128 bytes 1525 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16; 1526 1527 llvm_unreachable("unsupported size for IndirectRegWrite pseudos"); 1528 } 1529 1530 const MCInstrDesc & 1531 SIInstrInfo::getIndirectRegWriteMovRelPseudo(unsigned VecSize, unsigned EltSize, 1532 bool IsSGPR) const { 1533 if (IsSGPR) { 1534 switch (EltSize) { 1535 case 32: 1536 return get(getIndirectSGPRWriteMovRelPseudo32(VecSize)); 1537 case 64: 1538 return get(getIndirectSGPRWriteMovRelPseudo64(VecSize)); 1539 default: 1540 llvm_unreachable("invalid reg indexing elt size"); 1541 } 1542 } 1543 1544 assert(EltSize == 32 && "invalid reg indexing elt size"); 1545 return get(getIndirectVGPRWriteMovRelPseudoOpc(VecSize)); 1546 } 1547 1548 static unsigned getSGPRSpillSaveOpcode(unsigned Size) { 1549 switch (Size) { 1550 case 4: 1551 return AMDGPU::SI_SPILL_S32_SAVE; 1552 case 8: 1553 return AMDGPU::SI_SPILL_S64_SAVE; 1554 case 12: 1555 return AMDGPU::SI_SPILL_S96_SAVE; 1556 case 16: 1557 return AMDGPU::SI_SPILL_S128_SAVE; 1558 case 20: 1559 return AMDGPU::SI_SPILL_S160_SAVE; 1560 case 24: 1561 return AMDGPU::SI_SPILL_S192_SAVE; 1562 case 28: 1563 return AMDGPU::SI_SPILL_S224_SAVE; 1564 case 32: 1565 return AMDGPU::SI_SPILL_S256_SAVE; 1566 case 36: 1567 return AMDGPU::SI_SPILL_S288_SAVE; 1568 case 40: 1569 return AMDGPU::SI_SPILL_S320_SAVE; 1570 case 44: 1571 return AMDGPU::SI_SPILL_S352_SAVE; 1572 case 48: 1573 return AMDGPU::SI_SPILL_S384_SAVE; 1574 case 64: 1575 return AMDGPU::SI_SPILL_S512_SAVE; 1576 case 128: 1577 return AMDGPU::SI_SPILL_S1024_SAVE; 1578 default: 1579 llvm_unreachable("unknown register size"); 1580 } 1581 } 1582 1583 static unsigned getVGPRSpillSaveOpcode(unsigned Size) { 1584 switch (Size) { 1585 case 4: 1586 return AMDGPU::SI_SPILL_V32_SAVE; 1587 case 8: 1588 return AMDGPU::SI_SPILL_V64_SAVE; 1589 case 12: 1590 return AMDGPU::SI_SPILL_V96_SAVE; 1591 case 16: 1592 return AMDGPU::SI_SPILL_V128_SAVE; 1593 case 20: 1594 return AMDGPU::SI_SPILL_V160_SAVE; 1595 case 24: 1596 return AMDGPU::SI_SPILL_V192_SAVE; 1597 case 28: 1598 return AMDGPU::SI_SPILL_V224_SAVE; 1599 case 32: 1600 return AMDGPU::SI_SPILL_V256_SAVE; 1601 case 36: 1602 return AMDGPU::SI_SPILL_V288_SAVE; 1603 case 40: 1604 return AMDGPU::SI_SPILL_V320_SAVE; 1605 case 44: 1606 return AMDGPU::SI_SPILL_V352_SAVE; 1607 case 48: 1608 return AMDGPU::SI_SPILL_V384_SAVE; 1609 case 64: 1610 return AMDGPU::SI_SPILL_V512_SAVE; 1611 case 128: 1612 return AMDGPU::SI_SPILL_V1024_SAVE; 1613 default: 1614 llvm_unreachable("unknown register size"); 1615 } 1616 } 1617 1618 static unsigned getAGPRSpillSaveOpcode(unsigned Size) { 1619 switch (Size) { 1620 case 4: 1621 return AMDGPU::SI_SPILL_A32_SAVE; 1622 case 8: 1623 return AMDGPU::SI_SPILL_A64_SAVE; 1624 case 12: 1625 return AMDGPU::SI_SPILL_A96_SAVE; 1626 case 16: 1627 return AMDGPU::SI_SPILL_A128_SAVE; 1628 case 20: 1629 return AMDGPU::SI_SPILL_A160_SAVE; 1630 case 24: 1631 return AMDGPU::SI_SPILL_A192_SAVE; 1632 case 28: 1633 return AMDGPU::SI_SPILL_A224_SAVE; 1634 case 32: 1635 return AMDGPU::SI_SPILL_A256_SAVE; 1636 case 36: 1637 return AMDGPU::SI_SPILL_A288_SAVE; 1638 case 40: 1639 return AMDGPU::SI_SPILL_A320_SAVE; 1640 case 44: 1641 return AMDGPU::SI_SPILL_A352_SAVE; 1642 case 48: 1643 return AMDGPU::SI_SPILL_A384_SAVE; 1644 case 64: 1645 return AMDGPU::SI_SPILL_A512_SAVE; 1646 case 128: 1647 return AMDGPU::SI_SPILL_A1024_SAVE; 1648 default: 1649 llvm_unreachable("unknown register size"); 1650 } 1651 } 1652 1653 static unsigned getAVSpillSaveOpcode(unsigned Size) { 1654 switch (Size) { 1655 case 4: 1656 return AMDGPU::SI_SPILL_AV32_SAVE; 1657 case 8: 1658 return AMDGPU::SI_SPILL_AV64_SAVE; 1659 case 12: 1660 return AMDGPU::SI_SPILL_AV96_SAVE; 1661 case 16: 1662 return AMDGPU::SI_SPILL_AV128_SAVE; 1663 case 20: 1664 return AMDGPU::SI_SPILL_AV160_SAVE; 1665 case 24: 1666 return AMDGPU::SI_SPILL_AV192_SAVE; 1667 case 28: 1668 return AMDGPU::SI_SPILL_AV224_SAVE; 1669 case 32: 1670 return AMDGPU::SI_SPILL_AV256_SAVE; 1671 case 36: 1672 return AMDGPU::SI_SPILL_AV288_SAVE; 1673 case 40: 1674 return AMDGPU::SI_SPILL_AV320_SAVE; 1675 case 44: 1676 return AMDGPU::SI_SPILL_AV352_SAVE; 1677 case 48: 1678 return AMDGPU::SI_SPILL_AV384_SAVE; 1679 case 64: 1680 return AMDGPU::SI_SPILL_AV512_SAVE; 1681 case 128: 1682 return AMDGPU::SI_SPILL_AV1024_SAVE; 1683 default: 1684 llvm_unreachable("unknown register size"); 1685 } 1686 } 1687 1688 static unsigned getWWMRegSpillSaveOpcode(unsigned Size, 1689 bool IsVectorSuperClass) { 1690 // Currently, there is only 32-bit WWM register spills needed. 1691 if (Size != 4) 1692 llvm_unreachable("unknown wwm register spill size"); 1693 1694 if (IsVectorSuperClass) 1695 return AMDGPU::SI_SPILL_WWM_AV32_SAVE; 1696 1697 return AMDGPU::SI_SPILL_WWM_V32_SAVE; 1698 } 1699 1700 static unsigned getVectorRegSpillSaveOpcode(Register Reg, 1701 const TargetRegisterClass *RC, 1702 unsigned Size, 1703 const SIRegisterInfo &TRI, 1704 const SIMachineFunctionInfo &MFI) { 1705 bool IsVectorSuperClass = TRI.isVectorSuperClass(RC); 1706 1707 // Choose the right opcode if spilling a WWM register. 1708 if (MFI.checkFlag(Reg, AMDGPU::VirtRegFlag::WWM_REG)) 1709 return getWWMRegSpillSaveOpcode(Size, IsVectorSuperClass); 1710 1711 if (IsVectorSuperClass) 1712 return getAVSpillSaveOpcode(Size); 1713 1714 return TRI.isAGPRClass(RC) ? getAGPRSpillSaveOpcode(Size) 1715 : getVGPRSpillSaveOpcode(Size); 1716 } 1717 1718 void SIInstrInfo::storeRegToStackSlot( 1719 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg, 1720 bool isKill, int FrameIndex, const TargetRegisterClass *RC, 1721 const TargetRegisterInfo *TRI, Register VReg) const { 1722 MachineFunction *MF = MBB.getParent(); 1723 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 1724 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 1725 const DebugLoc &DL = MBB.findDebugLoc(MI); 1726 1727 MachinePointerInfo PtrInfo 1728 = MachinePointerInfo::getFixedStack(*MF, FrameIndex); 1729 MachineMemOperand *MMO = MF->getMachineMemOperand( 1730 PtrInfo, MachineMemOperand::MOStore, FrameInfo.getObjectSize(FrameIndex), 1731 FrameInfo.getObjectAlign(FrameIndex)); 1732 unsigned SpillSize = TRI->getSpillSize(*RC); 1733 1734 MachineRegisterInfo &MRI = MF->getRegInfo(); 1735 if (RI.isSGPRClass(RC)) { 1736 MFI->setHasSpilledSGPRs(); 1737 assert(SrcReg != AMDGPU::M0 && "m0 should not be spilled"); 1738 assert(SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI && 1739 SrcReg != AMDGPU::EXEC && "exec should not be spilled"); 1740 1741 // We are only allowed to create one new instruction when spilling 1742 // registers, so we need to use pseudo instruction for spilling SGPRs. 1743 const MCInstrDesc &OpDesc = get(getSGPRSpillSaveOpcode(SpillSize)); 1744 1745 // The SGPR spill/restore instructions only work on number sgprs, so we need 1746 // to make sure we are using the correct register class. 1747 if (SrcReg.isVirtual() && SpillSize == 4) { 1748 MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); 1749 } 1750 1751 BuildMI(MBB, MI, DL, OpDesc) 1752 .addReg(SrcReg, getKillRegState(isKill)) // data 1753 .addFrameIndex(FrameIndex) // addr 1754 .addMemOperand(MMO) 1755 .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit); 1756 1757 if (RI.spillSGPRToVGPR()) 1758 FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill); 1759 return; 1760 } 1761 1762 unsigned Opcode = getVectorRegSpillSaveOpcode(VReg ? VReg : SrcReg, RC, 1763 SpillSize, RI, *MFI); 1764 MFI->setHasSpilledVGPRs(); 1765 1766 BuildMI(MBB, MI, DL, get(Opcode)) 1767 .addReg(SrcReg, getKillRegState(isKill)) // data 1768 .addFrameIndex(FrameIndex) // addr 1769 .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset 1770 .addImm(0) // offset 1771 .addMemOperand(MMO); 1772 } 1773 1774 static unsigned getSGPRSpillRestoreOpcode(unsigned Size) { 1775 switch (Size) { 1776 case 4: 1777 return AMDGPU::SI_SPILL_S32_RESTORE; 1778 case 8: 1779 return AMDGPU::SI_SPILL_S64_RESTORE; 1780 case 12: 1781 return AMDGPU::SI_SPILL_S96_RESTORE; 1782 case 16: 1783 return AMDGPU::SI_SPILL_S128_RESTORE; 1784 case 20: 1785 return AMDGPU::SI_SPILL_S160_RESTORE; 1786 case 24: 1787 return AMDGPU::SI_SPILL_S192_RESTORE; 1788 case 28: 1789 return AMDGPU::SI_SPILL_S224_RESTORE; 1790 case 32: 1791 return AMDGPU::SI_SPILL_S256_RESTORE; 1792 case 36: 1793 return AMDGPU::SI_SPILL_S288_RESTORE; 1794 case 40: 1795 return AMDGPU::SI_SPILL_S320_RESTORE; 1796 case 44: 1797 return AMDGPU::SI_SPILL_S352_RESTORE; 1798 case 48: 1799 return AMDGPU::SI_SPILL_S384_RESTORE; 1800 case 64: 1801 return AMDGPU::SI_SPILL_S512_RESTORE; 1802 case 128: 1803 return AMDGPU::SI_SPILL_S1024_RESTORE; 1804 default: 1805 llvm_unreachable("unknown register size"); 1806 } 1807 } 1808 1809 static unsigned getVGPRSpillRestoreOpcode(unsigned Size) { 1810 switch (Size) { 1811 case 4: 1812 return AMDGPU::SI_SPILL_V32_RESTORE; 1813 case 8: 1814 return AMDGPU::SI_SPILL_V64_RESTORE; 1815 case 12: 1816 return AMDGPU::SI_SPILL_V96_RESTORE; 1817 case 16: 1818 return AMDGPU::SI_SPILL_V128_RESTORE; 1819 case 20: 1820 return AMDGPU::SI_SPILL_V160_RESTORE; 1821 case 24: 1822 return AMDGPU::SI_SPILL_V192_RESTORE; 1823 case 28: 1824 return AMDGPU::SI_SPILL_V224_RESTORE; 1825 case 32: 1826 return AMDGPU::SI_SPILL_V256_RESTORE; 1827 case 36: 1828 return AMDGPU::SI_SPILL_V288_RESTORE; 1829 case 40: 1830 return AMDGPU::SI_SPILL_V320_RESTORE; 1831 case 44: 1832 return AMDGPU::SI_SPILL_V352_RESTORE; 1833 case 48: 1834 return AMDGPU::SI_SPILL_V384_RESTORE; 1835 case 64: 1836 return AMDGPU::SI_SPILL_V512_RESTORE; 1837 case 128: 1838 return AMDGPU::SI_SPILL_V1024_RESTORE; 1839 default: 1840 llvm_unreachable("unknown register size"); 1841 } 1842 } 1843 1844 static unsigned getAGPRSpillRestoreOpcode(unsigned Size) { 1845 switch (Size) { 1846 case 4: 1847 return AMDGPU::SI_SPILL_A32_RESTORE; 1848 case 8: 1849 return AMDGPU::SI_SPILL_A64_RESTORE; 1850 case 12: 1851 return AMDGPU::SI_SPILL_A96_RESTORE; 1852 case 16: 1853 return AMDGPU::SI_SPILL_A128_RESTORE; 1854 case 20: 1855 return AMDGPU::SI_SPILL_A160_RESTORE; 1856 case 24: 1857 return AMDGPU::SI_SPILL_A192_RESTORE; 1858 case 28: 1859 return AMDGPU::SI_SPILL_A224_RESTORE; 1860 case 32: 1861 return AMDGPU::SI_SPILL_A256_RESTORE; 1862 case 36: 1863 return AMDGPU::SI_SPILL_A288_RESTORE; 1864 case 40: 1865 return AMDGPU::SI_SPILL_A320_RESTORE; 1866 case 44: 1867 return AMDGPU::SI_SPILL_A352_RESTORE; 1868 case 48: 1869 return AMDGPU::SI_SPILL_A384_RESTORE; 1870 case 64: 1871 return AMDGPU::SI_SPILL_A512_RESTORE; 1872 case 128: 1873 return AMDGPU::SI_SPILL_A1024_RESTORE; 1874 default: 1875 llvm_unreachable("unknown register size"); 1876 } 1877 } 1878 1879 static unsigned getAVSpillRestoreOpcode(unsigned Size) { 1880 switch (Size) { 1881 case 4: 1882 return AMDGPU::SI_SPILL_AV32_RESTORE; 1883 case 8: 1884 return AMDGPU::SI_SPILL_AV64_RESTORE; 1885 case 12: 1886 return AMDGPU::SI_SPILL_AV96_RESTORE; 1887 case 16: 1888 return AMDGPU::SI_SPILL_AV128_RESTORE; 1889 case 20: 1890 return AMDGPU::SI_SPILL_AV160_RESTORE; 1891 case 24: 1892 return AMDGPU::SI_SPILL_AV192_RESTORE; 1893 case 28: 1894 return AMDGPU::SI_SPILL_AV224_RESTORE; 1895 case 32: 1896 return AMDGPU::SI_SPILL_AV256_RESTORE; 1897 case 36: 1898 return AMDGPU::SI_SPILL_AV288_RESTORE; 1899 case 40: 1900 return AMDGPU::SI_SPILL_AV320_RESTORE; 1901 case 44: 1902 return AMDGPU::SI_SPILL_AV352_RESTORE; 1903 case 48: 1904 return AMDGPU::SI_SPILL_AV384_RESTORE; 1905 case 64: 1906 return AMDGPU::SI_SPILL_AV512_RESTORE; 1907 case 128: 1908 return AMDGPU::SI_SPILL_AV1024_RESTORE; 1909 default: 1910 llvm_unreachable("unknown register size"); 1911 } 1912 } 1913 1914 static unsigned getWWMRegSpillRestoreOpcode(unsigned Size, 1915 bool IsVectorSuperClass) { 1916 // Currently, there is only 32-bit WWM register spills needed. 1917 if (Size != 4) 1918 llvm_unreachable("unknown wwm register spill size"); 1919 1920 if (IsVectorSuperClass) 1921 return AMDGPU::SI_SPILL_WWM_AV32_RESTORE; 1922 1923 return AMDGPU::SI_SPILL_WWM_V32_RESTORE; 1924 } 1925 1926 static unsigned 1927 getVectorRegSpillRestoreOpcode(Register Reg, const TargetRegisterClass *RC, 1928 unsigned Size, const SIRegisterInfo &TRI, 1929 const SIMachineFunctionInfo &MFI) { 1930 bool IsVectorSuperClass = TRI.isVectorSuperClass(RC); 1931 1932 // Choose the right opcode if restoring a WWM register. 1933 if (MFI.checkFlag(Reg, AMDGPU::VirtRegFlag::WWM_REG)) 1934 return getWWMRegSpillRestoreOpcode(Size, IsVectorSuperClass); 1935 1936 if (IsVectorSuperClass) 1937 return getAVSpillRestoreOpcode(Size); 1938 1939 return TRI.isAGPRClass(RC) ? getAGPRSpillRestoreOpcode(Size) 1940 : getVGPRSpillRestoreOpcode(Size); 1941 } 1942 1943 void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, 1944 MachineBasicBlock::iterator MI, 1945 Register DestReg, int FrameIndex, 1946 const TargetRegisterClass *RC, 1947 const TargetRegisterInfo *TRI, 1948 Register VReg) const { 1949 MachineFunction *MF = MBB.getParent(); 1950 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 1951 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 1952 const DebugLoc &DL = MBB.findDebugLoc(MI); 1953 unsigned SpillSize = TRI->getSpillSize(*RC); 1954 1955 MachinePointerInfo PtrInfo 1956 = MachinePointerInfo::getFixedStack(*MF, FrameIndex); 1957 1958 MachineMemOperand *MMO = MF->getMachineMemOperand( 1959 PtrInfo, MachineMemOperand::MOLoad, FrameInfo.getObjectSize(FrameIndex), 1960 FrameInfo.getObjectAlign(FrameIndex)); 1961 1962 if (RI.isSGPRClass(RC)) { 1963 MFI->setHasSpilledSGPRs(); 1964 assert(DestReg != AMDGPU::M0 && "m0 should not be reloaded into"); 1965 assert(DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI && 1966 DestReg != AMDGPU::EXEC && "exec should not be spilled"); 1967 1968 // FIXME: Maybe this should not include a memoperand because it will be 1969 // lowered to non-memory instructions. 1970 const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(SpillSize)); 1971 if (DestReg.isVirtual() && SpillSize == 4) { 1972 MachineRegisterInfo &MRI = MF->getRegInfo(); 1973 MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); 1974 } 1975 1976 if (RI.spillSGPRToVGPR()) 1977 FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill); 1978 BuildMI(MBB, MI, DL, OpDesc, DestReg) 1979 .addFrameIndex(FrameIndex) // addr 1980 .addMemOperand(MMO) 1981 .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit); 1982 1983 return; 1984 } 1985 1986 unsigned Opcode = getVectorRegSpillRestoreOpcode(VReg ? VReg : DestReg, RC, 1987 SpillSize, RI, *MFI); 1988 BuildMI(MBB, MI, DL, get(Opcode), DestReg) 1989 .addFrameIndex(FrameIndex) // vaddr 1990 .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset 1991 .addImm(0) // offset 1992 .addMemOperand(MMO); 1993 } 1994 1995 void SIInstrInfo::insertNoop(MachineBasicBlock &MBB, 1996 MachineBasicBlock::iterator MI) const { 1997 insertNoops(MBB, MI, 1); 1998 } 1999 2000 void SIInstrInfo::insertNoops(MachineBasicBlock &MBB, 2001 MachineBasicBlock::iterator MI, 2002 unsigned Quantity) const { 2003 DebugLoc DL = MBB.findDebugLoc(MI); 2004 while (Quantity > 0) { 2005 unsigned Arg = std::min(Quantity, 8u); 2006 Quantity -= Arg; 2007 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP)).addImm(Arg - 1); 2008 } 2009 } 2010 2011 void SIInstrInfo::insertReturn(MachineBasicBlock &MBB) const { 2012 auto MF = MBB.getParent(); 2013 SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>(); 2014 2015 assert(Info->isEntryFunction()); 2016 2017 if (MBB.succ_empty()) { 2018 bool HasNoTerminator = MBB.getFirstTerminator() == MBB.end(); 2019 if (HasNoTerminator) { 2020 if (Info->returnsVoid()) { 2021 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::S_ENDPGM)).addImm(0); 2022 } else { 2023 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::SI_RETURN_TO_EPILOG)); 2024 } 2025 } 2026 } 2027 } 2028 2029 unsigned SIInstrInfo::getNumWaitStates(const MachineInstr &MI) { 2030 switch (MI.getOpcode()) { 2031 default: 2032 if (MI.isMetaInstruction()) 2033 return 0; 2034 return 1; // FIXME: Do wait states equal cycles? 2035 2036 case AMDGPU::S_NOP: 2037 return MI.getOperand(0).getImm() + 1; 2038 // SI_RETURN_TO_EPILOG is a fallthrough to code outside of the function. The 2039 // hazard, even if one exist, won't really be visible. Should we handle it? 2040 } 2041 } 2042 2043 bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const { 2044 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 2045 MachineBasicBlock &MBB = *MI.getParent(); 2046 DebugLoc DL = MBB.findDebugLoc(MI); 2047 switch (MI.getOpcode()) { 2048 default: return TargetInstrInfo::expandPostRAPseudo(MI); 2049 case AMDGPU::S_MOV_B64_term: 2050 // This is only a terminator to get the correct spill code placement during 2051 // register allocation. 2052 MI.setDesc(get(AMDGPU::S_MOV_B64)); 2053 break; 2054 2055 case AMDGPU::S_MOV_B32_term: 2056 // This is only a terminator to get the correct spill code placement during 2057 // register allocation. 2058 MI.setDesc(get(AMDGPU::S_MOV_B32)); 2059 break; 2060 2061 case AMDGPU::S_XOR_B64_term: 2062 // This is only a terminator to get the correct spill code placement during 2063 // register allocation. 2064 MI.setDesc(get(AMDGPU::S_XOR_B64)); 2065 break; 2066 2067 case AMDGPU::S_XOR_B32_term: 2068 // This is only a terminator to get the correct spill code placement during 2069 // register allocation. 2070 MI.setDesc(get(AMDGPU::S_XOR_B32)); 2071 break; 2072 case AMDGPU::S_OR_B64_term: 2073 // This is only a terminator to get the correct spill code placement during 2074 // register allocation. 2075 MI.setDesc(get(AMDGPU::S_OR_B64)); 2076 break; 2077 case AMDGPU::S_OR_B32_term: 2078 // This is only a terminator to get the correct spill code placement during 2079 // register allocation. 2080 MI.setDesc(get(AMDGPU::S_OR_B32)); 2081 break; 2082 2083 case AMDGPU::S_ANDN2_B64_term: 2084 // This is only a terminator to get the correct spill code placement during 2085 // register allocation. 2086 MI.setDesc(get(AMDGPU::S_ANDN2_B64)); 2087 break; 2088 2089 case AMDGPU::S_ANDN2_B32_term: 2090 // This is only a terminator to get the correct spill code placement during 2091 // register allocation. 2092 MI.setDesc(get(AMDGPU::S_ANDN2_B32)); 2093 break; 2094 2095 case AMDGPU::S_AND_B64_term: 2096 // This is only a terminator to get the correct spill code placement during 2097 // register allocation. 2098 MI.setDesc(get(AMDGPU::S_AND_B64)); 2099 break; 2100 2101 case AMDGPU::S_AND_B32_term: 2102 // This is only a terminator to get the correct spill code placement during 2103 // register allocation. 2104 MI.setDesc(get(AMDGPU::S_AND_B32)); 2105 break; 2106 2107 case AMDGPU::S_AND_SAVEEXEC_B64_term: 2108 // This is only a terminator to get the correct spill code placement during 2109 // register allocation. 2110 MI.setDesc(get(AMDGPU::S_AND_SAVEEXEC_B64)); 2111 break; 2112 2113 case AMDGPU::S_AND_SAVEEXEC_B32_term: 2114 // This is only a terminator to get the correct spill code placement during 2115 // register allocation. 2116 MI.setDesc(get(AMDGPU::S_AND_SAVEEXEC_B32)); 2117 break; 2118 2119 case AMDGPU::SI_SPILL_S32_TO_VGPR: 2120 MI.setDesc(get(AMDGPU::V_WRITELANE_B32)); 2121 break; 2122 2123 case AMDGPU::SI_RESTORE_S32_FROM_VGPR: 2124 MI.setDesc(get(AMDGPU::V_READLANE_B32)); 2125 break; 2126 2127 case AMDGPU::V_MOV_B64_PSEUDO: { 2128 Register Dst = MI.getOperand(0).getReg(); 2129 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0); 2130 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1); 2131 2132 const MachineOperand &SrcOp = MI.getOperand(1); 2133 // FIXME: Will this work for 64-bit floating point immediates? 2134 assert(!SrcOp.isFPImm()); 2135 if (ST.hasMovB64()) { 2136 MI.setDesc(get(AMDGPU::V_MOV_B64_e32)); 2137 if (SrcOp.isReg() || isInlineConstant(MI, 1) || 2138 isUInt<32>(SrcOp.getImm())) 2139 break; 2140 } 2141 if (SrcOp.isImm()) { 2142 APInt Imm(64, SrcOp.getImm()); 2143 APInt Lo(32, Imm.getLoBits(32).getZExtValue()); 2144 APInt Hi(32, Imm.getHiBits(32).getZExtValue()); 2145 if (ST.hasPkMovB32() && Lo == Hi && isInlineConstant(Lo)) { 2146 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst) 2147 .addImm(SISrcMods::OP_SEL_1) 2148 .addImm(Lo.getSExtValue()) 2149 .addImm(SISrcMods::OP_SEL_1) 2150 .addImm(Lo.getSExtValue()) 2151 .addImm(0) // op_sel_lo 2152 .addImm(0) // op_sel_hi 2153 .addImm(0) // neg_lo 2154 .addImm(0) // neg_hi 2155 .addImm(0); // clamp 2156 } else { 2157 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo) 2158 .addImm(Lo.getSExtValue()) 2159 .addReg(Dst, RegState::Implicit | RegState::Define); 2160 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi) 2161 .addImm(Hi.getSExtValue()) 2162 .addReg(Dst, RegState::Implicit | RegState::Define); 2163 } 2164 } else { 2165 assert(SrcOp.isReg()); 2166 if (ST.hasPkMovB32() && 2167 !RI.isAGPR(MBB.getParent()->getRegInfo(), SrcOp.getReg())) { 2168 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst) 2169 .addImm(SISrcMods::OP_SEL_1) // src0_mod 2170 .addReg(SrcOp.getReg()) 2171 .addImm(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1) // src1_mod 2172 .addReg(SrcOp.getReg()) 2173 .addImm(0) // op_sel_lo 2174 .addImm(0) // op_sel_hi 2175 .addImm(0) // neg_lo 2176 .addImm(0) // neg_hi 2177 .addImm(0); // clamp 2178 } else { 2179 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo) 2180 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0)) 2181 .addReg(Dst, RegState::Implicit | RegState::Define); 2182 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi) 2183 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1)) 2184 .addReg(Dst, RegState::Implicit | RegState::Define); 2185 } 2186 } 2187 MI.eraseFromParent(); 2188 break; 2189 } 2190 case AMDGPU::V_MOV_B64_DPP_PSEUDO: { 2191 expandMovDPP64(MI); 2192 break; 2193 } 2194 case AMDGPU::S_MOV_B64_IMM_PSEUDO: { 2195 const MachineOperand &SrcOp = MI.getOperand(1); 2196 assert(!SrcOp.isFPImm()); 2197 APInt Imm(64, SrcOp.getImm()); 2198 if (Imm.isIntN(32) || isInlineConstant(Imm)) { 2199 MI.setDesc(get(AMDGPU::S_MOV_B64)); 2200 break; 2201 } 2202 2203 Register Dst = MI.getOperand(0).getReg(); 2204 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0); 2205 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1); 2206 2207 APInt Lo(32, Imm.getLoBits(32).getZExtValue()); 2208 APInt Hi(32, Imm.getHiBits(32).getZExtValue()); 2209 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstLo) 2210 .addImm(Lo.getSExtValue()) 2211 .addReg(Dst, RegState::Implicit | RegState::Define); 2212 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstHi) 2213 .addImm(Hi.getSExtValue()) 2214 .addReg(Dst, RegState::Implicit | RegState::Define); 2215 MI.eraseFromParent(); 2216 break; 2217 } 2218 case AMDGPU::V_SET_INACTIVE_B32: { 2219 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64; 2220 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 2221 // FIXME: We may possibly optimize the COPY once we find ways to make LLVM 2222 // optimizations (mainly Register Coalescer) aware of WWM register liveness. 2223 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg()) 2224 .add(MI.getOperand(1)); 2225 auto FirstNot = BuildMI(MBB, MI, DL, get(NotOpc), Exec).addReg(Exec); 2226 FirstNot->addRegisterDead(AMDGPU::SCC, TRI); // SCC is overwritten 2227 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg()) 2228 .add(MI.getOperand(2)); 2229 BuildMI(MBB, MI, DL, get(NotOpc), Exec) 2230 .addReg(Exec); 2231 MI.eraseFromParent(); 2232 break; 2233 } 2234 case AMDGPU::V_SET_INACTIVE_B64: { 2235 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64; 2236 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 2237 MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), 2238 MI.getOperand(0).getReg()) 2239 .add(MI.getOperand(1)); 2240 expandPostRAPseudo(*Copy); 2241 auto FirstNot = BuildMI(MBB, MI, DL, get(NotOpc), Exec).addReg(Exec); 2242 FirstNot->addRegisterDead(AMDGPU::SCC, TRI); // SCC is overwritten 2243 Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), 2244 MI.getOperand(0).getReg()) 2245 .add(MI.getOperand(2)); 2246 expandPostRAPseudo(*Copy); 2247 BuildMI(MBB, MI, DL, get(NotOpc), Exec) 2248 .addReg(Exec); 2249 MI.eraseFromParent(); 2250 break; 2251 } 2252 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1: 2253 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2: 2254 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3: 2255 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4: 2256 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5: 2257 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8: 2258 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9: 2259 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10: 2260 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11: 2261 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12: 2262 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16: 2263 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32: 2264 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1: 2265 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2: 2266 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3: 2267 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4: 2268 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5: 2269 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8: 2270 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9: 2271 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10: 2272 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11: 2273 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12: 2274 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16: 2275 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32: 2276 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1: 2277 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2: 2278 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4: 2279 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8: 2280 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16: { 2281 const TargetRegisterClass *EltRC = getOpRegClass(MI, 2); 2282 2283 unsigned Opc; 2284 if (RI.hasVGPRs(EltRC)) { 2285 Opc = AMDGPU::V_MOVRELD_B32_e32; 2286 } else { 2287 Opc = RI.getRegSizeInBits(*EltRC) == 64 ? AMDGPU::S_MOVRELD_B64 2288 : AMDGPU::S_MOVRELD_B32; 2289 } 2290 2291 const MCInstrDesc &OpDesc = get(Opc); 2292 Register VecReg = MI.getOperand(0).getReg(); 2293 bool IsUndef = MI.getOperand(1).isUndef(); 2294 unsigned SubReg = MI.getOperand(3).getImm(); 2295 assert(VecReg == MI.getOperand(1).getReg()); 2296 2297 MachineInstrBuilder MIB = 2298 BuildMI(MBB, MI, DL, OpDesc) 2299 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef) 2300 .add(MI.getOperand(2)) 2301 .addReg(VecReg, RegState::ImplicitDefine) 2302 .addReg(VecReg, RegState::Implicit | (IsUndef ? RegState::Undef : 0)); 2303 2304 const int ImpDefIdx = 2305 OpDesc.getNumOperands() + OpDesc.implicit_uses().size(); 2306 const int ImpUseIdx = ImpDefIdx + 1; 2307 MIB->tieOperands(ImpDefIdx, ImpUseIdx); 2308 MI.eraseFromParent(); 2309 break; 2310 } 2311 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1: 2312 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2: 2313 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3: 2314 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4: 2315 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5: 2316 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8: 2317 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9: 2318 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10: 2319 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11: 2320 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12: 2321 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16: 2322 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32: { 2323 assert(ST.useVGPRIndexMode()); 2324 Register VecReg = MI.getOperand(0).getReg(); 2325 bool IsUndef = MI.getOperand(1).isUndef(); 2326 Register Idx = MI.getOperand(3).getReg(); 2327 Register SubReg = MI.getOperand(4).getImm(); 2328 2329 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON)) 2330 .addReg(Idx) 2331 .addImm(AMDGPU::VGPRIndexMode::DST_ENABLE); 2332 SetOn->getOperand(3).setIsUndef(); 2333 2334 const MCInstrDesc &OpDesc = get(AMDGPU::V_MOV_B32_indirect_write); 2335 MachineInstrBuilder MIB = 2336 BuildMI(MBB, MI, DL, OpDesc) 2337 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef) 2338 .add(MI.getOperand(2)) 2339 .addReg(VecReg, RegState::ImplicitDefine) 2340 .addReg(VecReg, 2341 RegState::Implicit | (IsUndef ? RegState::Undef : 0)); 2342 2343 const int ImpDefIdx = 2344 OpDesc.getNumOperands() + OpDesc.implicit_uses().size(); 2345 const int ImpUseIdx = ImpDefIdx + 1; 2346 MIB->tieOperands(ImpDefIdx, ImpUseIdx); 2347 2348 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF)); 2349 2350 finalizeBundle(MBB, SetOn->getIterator(), std::next(SetOff->getIterator())); 2351 2352 MI.eraseFromParent(); 2353 break; 2354 } 2355 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1: 2356 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2: 2357 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3: 2358 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4: 2359 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5: 2360 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8: 2361 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9: 2362 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10: 2363 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11: 2364 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12: 2365 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16: 2366 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32: { 2367 assert(ST.useVGPRIndexMode()); 2368 Register Dst = MI.getOperand(0).getReg(); 2369 Register VecReg = MI.getOperand(1).getReg(); 2370 bool IsUndef = MI.getOperand(1).isUndef(); 2371 Register Idx = MI.getOperand(2).getReg(); 2372 Register SubReg = MI.getOperand(3).getImm(); 2373 2374 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON)) 2375 .addReg(Idx) 2376 .addImm(AMDGPU::VGPRIndexMode::SRC0_ENABLE); 2377 SetOn->getOperand(3).setIsUndef(); 2378 2379 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_indirect_read)) 2380 .addDef(Dst) 2381 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef) 2382 .addReg(VecReg, RegState::Implicit | (IsUndef ? RegState::Undef : 0)); 2383 2384 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF)); 2385 2386 finalizeBundle(MBB, SetOn->getIterator(), std::next(SetOff->getIterator())); 2387 2388 MI.eraseFromParent(); 2389 break; 2390 } 2391 case AMDGPU::SI_PC_ADD_REL_OFFSET: { 2392 MachineFunction &MF = *MBB.getParent(); 2393 Register Reg = MI.getOperand(0).getReg(); 2394 Register RegLo = RI.getSubReg(Reg, AMDGPU::sub0); 2395 Register RegHi = RI.getSubReg(Reg, AMDGPU::sub1); 2396 MachineOperand OpLo = MI.getOperand(1); 2397 MachineOperand OpHi = MI.getOperand(2); 2398 2399 // Create a bundle so these instructions won't be re-ordered by the 2400 // post-RA scheduler. 2401 MIBundleBuilder Bundler(MBB, MI); 2402 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg)); 2403 2404 // What we want here is an offset from the value returned by s_getpc (which 2405 // is the address of the s_add_u32 instruction) to the global variable, but 2406 // since the encoding of $symbol starts 4 bytes after the start of the 2407 // s_add_u32 instruction, we end up with an offset that is 4 bytes too 2408 // small. This requires us to add 4 to the global variable offset in order 2409 // to compute the correct address. Similarly for the s_addc_u32 instruction, 2410 // the encoding of $symbol starts 12 bytes after the start of the s_add_u32 2411 // instruction. 2412 2413 int64_t Adjust = 0; 2414 if (ST.hasGetPCZeroExtension()) { 2415 // Fix up hardware that does not sign-extend the 48-bit PC value by 2416 // inserting: s_sext_i32_i16 reghi, reghi 2417 Bundler.append( 2418 BuildMI(MF, DL, get(AMDGPU::S_SEXT_I32_I16), RegHi).addReg(RegHi)); 2419 Adjust += 4; 2420 } 2421 2422 if (OpLo.isGlobal()) 2423 OpLo.setOffset(OpLo.getOffset() + Adjust + 4); 2424 Bundler.append( 2425 BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo).addReg(RegLo).add(OpLo)); 2426 2427 if (OpHi.isGlobal()) 2428 OpHi.setOffset(OpHi.getOffset() + Adjust + 12); 2429 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi) 2430 .addReg(RegHi) 2431 .add(OpHi)); 2432 2433 finalizeBundle(MBB, Bundler.begin()); 2434 2435 MI.eraseFromParent(); 2436 break; 2437 } 2438 case AMDGPU::ENTER_STRICT_WWM: { 2439 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when 2440 // Whole Wave Mode is entered. 2441 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 2442 : AMDGPU::S_OR_SAVEEXEC_B64)); 2443 break; 2444 } 2445 case AMDGPU::ENTER_STRICT_WQM: { 2446 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when 2447 // STRICT_WQM is entered. 2448 const unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 2449 const unsigned WQMOp = ST.isWave32() ? AMDGPU::S_WQM_B32 : AMDGPU::S_WQM_B64; 2450 const unsigned MovOp = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; 2451 BuildMI(MBB, MI, DL, get(MovOp), MI.getOperand(0).getReg()).addReg(Exec); 2452 BuildMI(MBB, MI, DL, get(WQMOp), Exec).addReg(Exec); 2453 2454 MI.eraseFromParent(); 2455 break; 2456 } 2457 case AMDGPU::EXIT_STRICT_WWM: 2458 case AMDGPU::EXIT_STRICT_WQM: { 2459 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when 2460 // WWM/STICT_WQM is exited. 2461 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64)); 2462 break; 2463 } 2464 case AMDGPU::ENTER_PSEUDO_WM: 2465 case AMDGPU::EXIT_PSEUDO_WM: { 2466 // These do nothing. 2467 MI.eraseFromParent(); 2468 break; 2469 } 2470 case AMDGPU::SI_RETURN: { 2471 const MachineFunction *MF = MBB.getParent(); 2472 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); 2473 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 2474 // Hiding the return address use with SI_RETURN may lead to extra kills in 2475 // the function and missing live-ins. We are fine in practice because callee 2476 // saved register handling ensures the register value is restored before 2477 // RET, but we need the undef flag here to appease the MachineVerifier 2478 // liveness checks. 2479 MachineInstrBuilder MIB = 2480 BuildMI(MBB, MI, DL, get(AMDGPU::S_SETPC_B64_return)) 2481 .addReg(TRI->getReturnAddressReg(*MF), RegState::Undef); 2482 2483 MIB.copyImplicitOps(MI); 2484 MI.eraseFromParent(); 2485 break; 2486 } 2487 2488 case AMDGPU::S_MUL_U64_U32_PSEUDO: 2489 case AMDGPU::S_MUL_I64_I32_PSEUDO: 2490 MI.setDesc(get(AMDGPU::S_MUL_U64)); 2491 break; 2492 2493 case AMDGPU::S_GETPC_B64_pseudo: 2494 MI.setDesc(get(AMDGPU::S_GETPC_B64)); 2495 if (ST.hasGetPCZeroExtension()) { 2496 Register Dst = MI.getOperand(0).getReg(); 2497 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1); 2498 // Fix up hardware that does not sign-extend the 48-bit PC value by 2499 // inserting: s_sext_i32_i16 dsthi, dsthi 2500 BuildMI(MBB, std::next(MI.getIterator()), DL, get(AMDGPU::S_SEXT_I32_I16), 2501 DstHi) 2502 .addReg(DstHi); 2503 } 2504 break; 2505 } 2506 return true; 2507 } 2508 2509 void SIInstrInfo::reMaterialize(MachineBasicBlock &MBB, 2510 MachineBasicBlock::iterator I, Register DestReg, 2511 unsigned SubIdx, const MachineInstr &Orig, 2512 const TargetRegisterInfo &RI) const { 2513 2514 // Try shrinking the instruction to remat only the part needed for current 2515 // context. 2516 // TODO: Handle more cases. 2517 unsigned Opcode = Orig.getOpcode(); 2518 switch (Opcode) { 2519 case AMDGPU::S_LOAD_DWORDX16_IMM: 2520 case AMDGPU::S_LOAD_DWORDX8_IMM: { 2521 if (SubIdx != 0) 2522 break; 2523 2524 if (I == MBB.end()) 2525 break; 2526 2527 if (I->isBundled()) 2528 break; 2529 2530 // Look for a single use of the register that is also a subreg. 2531 Register RegToFind = Orig.getOperand(0).getReg(); 2532 MachineOperand *UseMO = nullptr; 2533 for (auto &CandMO : I->operands()) { 2534 if (!CandMO.isReg() || CandMO.getReg() != RegToFind || CandMO.isDef()) 2535 continue; 2536 if (UseMO) { 2537 UseMO = nullptr; 2538 break; 2539 } 2540 UseMO = &CandMO; 2541 } 2542 if (!UseMO || UseMO->getSubReg() == AMDGPU::NoSubRegister) 2543 break; 2544 2545 unsigned Offset = RI.getSubRegIdxOffset(UseMO->getSubReg()); 2546 unsigned SubregSize = RI.getSubRegIdxSize(UseMO->getSubReg()); 2547 2548 MachineFunction *MF = MBB.getParent(); 2549 MachineRegisterInfo &MRI = MF->getRegInfo(); 2550 assert(MRI.use_nodbg_empty(DestReg) && "DestReg should have no users yet."); 2551 2552 unsigned NewOpcode = -1; 2553 if (SubregSize == 256) 2554 NewOpcode = AMDGPU::S_LOAD_DWORDX8_IMM; 2555 else if (SubregSize == 128) 2556 NewOpcode = AMDGPU::S_LOAD_DWORDX4_IMM; 2557 else 2558 break; 2559 2560 const MCInstrDesc &TID = get(NewOpcode); 2561 const TargetRegisterClass *NewRC = 2562 RI.getAllocatableClass(getRegClass(TID, 0, &RI, *MF)); 2563 MRI.setRegClass(DestReg, NewRC); 2564 2565 UseMO->setReg(DestReg); 2566 UseMO->setSubReg(AMDGPU::NoSubRegister); 2567 2568 // Use a smaller load with the desired size, possibly with updated offset. 2569 MachineInstr *MI = MF->CloneMachineInstr(&Orig); 2570 MI->setDesc(TID); 2571 MI->getOperand(0).setReg(DestReg); 2572 MI->getOperand(0).setSubReg(AMDGPU::NoSubRegister); 2573 if (Offset) { 2574 MachineOperand *OffsetMO = getNamedOperand(*MI, AMDGPU::OpName::offset); 2575 int64_t FinalOffset = OffsetMO->getImm() + Offset / 8; 2576 OffsetMO->setImm(FinalOffset); 2577 } 2578 SmallVector<MachineMemOperand *> NewMMOs; 2579 for (const MachineMemOperand *MemOp : Orig.memoperands()) 2580 NewMMOs.push_back(MF->getMachineMemOperand(MemOp, MemOp->getPointerInfo(), 2581 SubregSize / 8)); 2582 MI->setMemRefs(*MF, NewMMOs); 2583 2584 MBB.insert(I, MI); 2585 return; 2586 } 2587 2588 default: 2589 break; 2590 } 2591 2592 TargetInstrInfo::reMaterialize(MBB, I, DestReg, SubIdx, Orig, RI); 2593 } 2594 2595 std::pair<MachineInstr*, MachineInstr*> 2596 SIInstrInfo::expandMovDPP64(MachineInstr &MI) const { 2597 assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO); 2598 2599 if (ST.hasMovB64() && 2600 AMDGPU::isLegalDPALU_DPPControl( 2601 getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl)->getImm())) { 2602 MI.setDesc(get(AMDGPU::V_MOV_B64_dpp)); 2603 return std::pair(&MI, nullptr); 2604 } 2605 2606 MachineBasicBlock &MBB = *MI.getParent(); 2607 DebugLoc DL = MBB.findDebugLoc(MI); 2608 MachineFunction *MF = MBB.getParent(); 2609 MachineRegisterInfo &MRI = MF->getRegInfo(); 2610 Register Dst = MI.getOperand(0).getReg(); 2611 unsigned Part = 0; 2612 MachineInstr *Split[2]; 2613 2614 for (auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) { 2615 auto MovDPP = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_dpp)); 2616 if (Dst.isPhysical()) { 2617 MovDPP.addDef(RI.getSubReg(Dst, Sub)); 2618 } else { 2619 assert(MRI.isSSA()); 2620 auto Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 2621 MovDPP.addDef(Tmp); 2622 } 2623 2624 for (unsigned I = 1; I <= 2; ++I) { // old and src operands. 2625 const MachineOperand &SrcOp = MI.getOperand(I); 2626 assert(!SrcOp.isFPImm()); 2627 if (SrcOp.isImm()) { 2628 APInt Imm(64, SrcOp.getImm()); 2629 Imm.ashrInPlace(Part * 32); 2630 MovDPP.addImm(Imm.getLoBits(32).getZExtValue()); 2631 } else { 2632 assert(SrcOp.isReg()); 2633 Register Src = SrcOp.getReg(); 2634 if (Src.isPhysical()) 2635 MovDPP.addReg(RI.getSubReg(Src, Sub)); 2636 else 2637 MovDPP.addReg(Src, SrcOp.isUndef() ? RegState::Undef : 0, Sub); 2638 } 2639 } 2640 2641 for (const MachineOperand &MO : llvm::drop_begin(MI.explicit_operands(), 3)) 2642 MovDPP.addImm(MO.getImm()); 2643 2644 Split[Part] = MovDPP; 2645 ++Part; 2646 } 2647 2648 if (Dst.isVirtual()) 2649 BuildMI(MBB, MI, DL, get(AMDGPU::REG_SEQUENCE), Dst) 2650 .addReg(Split[0]->getOperand(0).getReg()) 2651 .addImm(AMDGPU::sub0) 2652 .addReg(Split[1]->getOperand(0).getReg()) 2653 .addImm(AMDGPU::sub1); 2654 2655 MI.eraseFromParent(); 2656 return std::pair(Split[0], Split[1]); 2657 } 2658 2659 std::optional<DestSourcePair> 2660 SIInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const { 2661 if (MI.getOpcode() == AMDGPU::WWM_COPY) 2662 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)}; 2663 2664 return std::nullopt; 2665 } 2666 2667 bool SIInstrInfo::swapSourceModifiers(MachineInstr &MI, 2668 MachineOperand &Src0, 2669 unsigned Src0OpName, 2670 MachineOperand &Src1, 2671 unsigned Src1OpName) const { 2672 MachineOperand *Src0Mods = getNamedOperand(MI, Src0OpName); 2673 if (!Src0Mods) 2674 return false; 2675 2676 MachineOperand *Src1Mods = getNamedOperand(MI, Src1OpName); 2677 assert(Src1Mods && 2678 "All commutable instructions have both src0 and src1 modifiers"); 2679 2680 int Src0ModsVal = Src0Mods->getImm(); 2681 int Src1ModsVal = Src1Mods->getImm(); 2682 2683 Src1Mods->setImm(Src0ModsVal); 2684 Src0Mods->setImm(Src1ModsVal); 2685 return true; 2686 } 2687 2688 static MachineInstr *swapRegAndNonRegOperand(MachineInstr &MI, 2689 MachineOperand &RegOp, 2690 MachineOperand &NonRegOp) { 2691 Register Reg = RegOp.getReg(); 2692 unsigned SubReg = RegOp.getSubReg(); 2693 bool IsKill = RegOp.isKill(); 2694 bool IsDead = RegOp.isDead(); 2695 bool IsUndef = RegOp.isUndef(); 2696 bool IsDebug = RegOp.isDebug(); 2697 2698 if (NonRegOp.isImm()) 2699 RegOp.ChangeToImmediate(NonRegOp.getImm()); 2700 else if (NonRegOp.isFI()) 2701 RegOp.ChangeToFrameIndex(NonRegOp.getIndex()); 2702 else if (NonRegOp.isGlobal()) { 2703 RegOp.ChangeToGA(NonRegOp.getGlobal(), NonRegOp.getOffset(), 2704 NonRegOp.getTargetFlags()); 2705 } else 2706 return nullptr; 2707 2708 // Make sure we don't reinterpret a subreg index in the target flags. 2709 RegOp.setTargetFlags(NonRegOp.getTargetFlags()); 2710 2711 NonRegOp.ChangeToRegister(Reg, false, false, IsKill, IsDead, IsUndef, IsDebug); 2712 NonRegOp.setSubReg(SubReg); 2713 2714 return &MI; 2715 } 2716 2717 MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI, 2718 unsigned Src0Idx, 2719 unsigned Src1Idx) const { 2720 assert(!NewMI && "this should never be used"); 2721 2722 unsigned Opc = MI.getOpcode(); 2723 int CommutedOpcode = commuteOpcode(Opc); 2724 if (CommutedOpcode == -1) 2725 return nullptr; 2726 2727 if (Src0Idx > Src1Idx) 2728 std::swap(Src0Idx, Src1Idx); 2729 2730 assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) == 2731 static_cast<int>(Src0Idx) && 2732 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) == 2733 static_cast<int>(Src1Idx) && 2734 "inconsistency with findCommutedOpIndices"); 2735 2736 MachineOperand &Src0 = MI.getOperand(Src0Idx); 2737 MachineOperand &Src1 = MI.getOperand(Src1Idx); 2738 2739 MachineInstr *CommutedMI = nullptr; 2740 if (Src0.isReg() && Src1.isReg()) { 2741 if (isOperandLegal(MI, Src1Idx, &Src0)) { 2742 // Be sure to copy the source modifiers to the right place. 2743 CommutedMI 2744 = TargetInstrInfo::commuteInstructionImpl(MI, NewMI, Src0Idx, Src1Idx); 2745 } 2746 2747 } else if (Src0.isReg() && !Src1.isReg()) { 2748 // src0 should always be able to support any operand type, so no need to 2749 // check operand legality. 2750 CommutedMI = swapRegAndNonRegOperand(MI, Src0, Src1); 2751 } else if (!Src0.isReg() && Src1.isReg()) { 2752 if (isOperandLegal(MI, Src1Idx, &Src0)) 2753 CommutedMI = swapRegAndNonRegOperand(MI, Src1, Src0); 2754 } else { 2755 // FIXME: Found two non registers to commute. This does happen. 2756 return nullptr; 2757 } 2758 2759 if (CommutedMI) { 2760 swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers, 2761 Src1, AMDGPU::OpName::src1_modifiers); 2762 2763 CommutedMI->setDesc(get(CommutedOpcode)); 2764 } 2765 2766 return CommutedMI; 2767 } 2768 2769 // This needs to be implemented because the source modifiers may be inserted 2770 // between the true commutable operands, and the base 2771 // TargetInstrInfo::commuteInstruction uses it. 2772 bool SIInstrInfo::findCommutedOpIndices(const MachineInstr &MI, 2773 unsigned &SrcOpIdx0, 2774 unsigned &SrcOpIdx1) const { 2775 return findCommutedOpIndices(MI.getDesc(), SrcOpIdx0, SrcOpIdx1); 2776 } 2777 2778 bool SIInstrInfo::findCommutedOpIndices(const MCInstrDesc &Desc, 2779 unsigned &SrcOpIdx0, 2780 unsigned &SrcOpIdx1) const { 2781 if (!Desc.isCommutable()) 2782 return false; 2783 2784 unsigned Opc = Desc.getOpcode(); 2785 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); 2786 if (Src0Idx == -1) 2787 return false; 2788 2789 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); 2790 if (Src1Idx == -1) 2791 return false; 2792 2793 return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx); 2794 } 2795 2796 bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp, 2797 int64_t BrOffset) const { 2798 // BranchRelaxation should never have to check s_setpc_b64 because its dest 2799 // block is unanalyzable. 2800 assert(BranchOp != AMDGPU::S_SETPC_B64); 2801 2802 // Convert to dwords. 2803 BrOffset /= 4; 2804 2805 // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is 2806 // from the next instruction. 2807 BrOffset -= 1; 2808 2809 return isIntN(BranchOffsetBits, BrOffset); 2810 } 2811 2812 MachineBasicBlock * 2813 SIInstrInfo::getBranchDestBlock(const MachineInstr &MI) const { 2814 return MI.getOperand(0).getMBB(); 2815 } 2816 2817 bool SIInstrInfo::hasDivergentBranch(const MachineBasicBlock *MBB) const { 2818 for (const MachineInstr &MI : MBB->terminators()) { 2819 if (MI.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO || 2820 MI.getOpcode() == AMDGPU::SI_IF || MI.getOpcode() == AMDGPU::SI_ELSE || 2821 MI.getOpcode() == AMDGPU::SI_LOOP) 2822 return true; 2823 } 2824 return false; 2825 } 2826 2827 void SIInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB, 2828 MachineBasicBlock &DestBB, 2829 MachineBasicBlock &RestoreBB, 2830 const DebugLoc &DL, int64_t BrOffset, 2831 RegScavenger *RS) const { 2832 assert(RS && "RegScavenger required for long branching"); 2833 assert(MBB.empty() && 2834 "new block should be inserted for expanding unconditional branch"); 2835 assert(MBB.pred_size() == 1); 2836 assert(RestoreBB.empty() && 2837 "restore block should be inserted for restoring clobbered registers"); 2838 2839 MachineFunction *MF = MBB.getParent(); 2840 MachineRegisterInfo &MRI = MF->getRegInfo(); 2841 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 2842 2843 // FIXME: Virtual register workaround for RegScavenger not working with empty 2844 // blocks. 2845 Register PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); 2846 2847 auto I = MBB.end(); 2848 2849 // We need to compute the offset relative to the instruction immediately after 2850 // s_getpc_b64. Insert pc arithmetic code before last terminator. 2851 MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg); 2852 2853 auto &MCCtx = MF->getContext(); 2854 MCSymbol *PostGetPCLabel = 2855 MCCtx.createTempSymbol("post_getpc", /*AlwaysAddSuffix=*/true); 2856 GetPC->setPostInstrSymbol(*MF, PostGetPCLabel); 2857 2858 MCSymbol *OffsetLo = 2859 MCCtx.createTempSymbol("offset_lo", /*AlwaysAddSuffix=*/true); 2860 MCSymbol *OffsetHi = 2861 MCCtx.createTempSymbol("offset_hi", /*AlwaysAddSuffix=*/true); 2862 BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32)) 2863 .addReg(PCReg, RegState::Define, AMDGPU::sub0) 2864 .addReg(PCReg, 0, AMDGPU::sub0) 2865 .addSym(OffsetLo, MO_FAR_BRANCH_OFFSET); 2866 BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32)) 2867 .addReg(PCReg, RegState::Define, AMDGPU::sub1) 2868 .addReg(PCReg, 0, AMDGPU::sub1) 2869 .addSym(OffsetHi, MO_FAR_BRANCH_OFFSET); 2870 2871 // Insert the indirect branch after the other terminator. 2872 BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64)) 2873 .addReg(PCReg); 2874 2875 // If a spill is needed for the pc register pair, we need to insert a spill 2876 // restore block right before the destination block, and insert a short branch 2877 // into the old destination block's fallthrough predecessor. 2878 // e.g.: 2879 // 2880 // s_cbranch_scc0 skip_long_branch: 2881 // 2882 // long_branch_bb: 2883 // spill s[8:9] 2884 // s_getpc_b64 s[8:9] 2885 // s_add_u32 s8, s8, restore_bb 2886 // s_addc_u32 s9, s9, 0 2887 // s_setpc_b64 s[8:9] 2888 // 2889 // skip_long_branch: 2890 // foo; 2891 // 2892 // ..... 2893 // 2894 // dest_bb_fallthrough_predecessor: 2895 // bar; 2896 // s_branch dest_bb 2897 // 2898 // restore_bb: 2899 // restore s[8:9] 2900 // fallthrough dest_bb 2901 /// 2902 // dest_bb: 2903 // buzz; 2904 2905 Register LongBranchReservedReg = MFI->getLongBranchReservedReg(); 2906 Register Scav; 2907 2908 // If we've previously reserved a register for long branches 2909 // avoid running the scavenger and just use those registers 2910 if (LongBranchReservedReg) { 2911 RS->enterBasicBlock(MBB); 2912 Scav = LongBranchReservedReg; 2913 } else { 2914 RS->enterBasicBlockEnd(MBB); 2915 Scav = RS->scavengeRegisterBackwards( 2916 AMDGPU::SReg_64RegClass, MachineBasicBlock::iterator(GetPC), 2917 /* RestoreAfter */ false, 0, /* AllowSpill */ false); 2918 } 2919 if (Scav) { 2920 RS->setRegUsed(Scav); 2921 MRI.replaceRegWith(PCReg, Scav); 2922 MRI.clearVirtRegs(); 2923 } else { 2924 // As SGPR needs VGPR to be spilled, we reuse the slot of temporary VGPR for 2925 // SGPR spill. 2926 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); 2927 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 2928 TRI->spillEmergencySGPR(GetPC, RestoreBB, AMDGPU::SGPR0_SGPR1, RS); 2929 MRI.replaceRegWith(PCReg, AMDGPU::SGPR0_SGPR1); 2930 MRI.clearVirtRegs(); 2931 } 2932 2933 MCSymbol *DestLabel = Scav ? DestBB.getSymbol() : RestoreBB.getSymbol(); 2934 // Now, the distance could be defined. 2935 auto *Offset = MCBinaryExpr::createSub( 2936 MCSymbolRefExpr::create(DestLabel, MCCtx), 2937 MCSymbolRefExpr::create(PostGetPCLabel, MCCtx), MCCtx); 2938 // Add offset assignments. 2939 auto *Mask = MCConstantExpr::create(0xFFFFFFFFULL, MCCtx); 2940 OffsetLo->setVariableValue(MCBinaryExpr::createAnd(Offset, Mask, MCCtx)); 2941 auto *ShAmt = MCConstantExpr::create(32, MCCtx); 2942 OffsetHi->setVariableValue(MCBinaryExpr::createAShr(Offset, ShAmt, MCCtx)); 2943 } 2944 2945 unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) { 2946 switch (Cond) { 2947 case SIInstrInfo::SCC_TRUE: 2948 return AMDGPU::S_CBRANCH_SCC1; 2949 case SIInstrInfo::SCC_FALSE: 2950 return AMDGPU::S_CBRANCH_SCC0; 2951 case SIInstrInfo::VCCNZ: 2952 return AMDGPU::S_CBRANCH_VCCNZ; 2953 case SIInstrInfo::VCCZ: 2954 return AMDGPU::S_CBRANCH_VCCZ; 2955 case SIInstrInfo::EXECNZ: 2956 return AMDGPU::S_CBRANCH_EXECNZ; 2957 case SIInstrInfo::EXECZ: 2958 return AMDGPU::S_CBRANCH_EXECZ; 2959 default: 2960 llvm_unreachable("invalid branch predicate"); 2961 } 2962 } 2963 2964 SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) { 2965 switch (Opcode) { 2966 case AMDGPU::S_CBRANCH_SCC0: 2967 return SCC_FALSE; 2968 case AMDGPU::S_CBRANCH_SCC1: 2969 return SCC_TRUE; 2970 case AMDGPU::S_CBRANCH_VCCNZ: 2971 return VCCNZ; 2972 case AMDGPU::S_CBRANCH_VCCZ: 2973 return VCCZ; 2974 case AMDGPU::S_CBRANCH_EXECNZ: 2975 return EXECNZ; 2976 case AMDGPU::S_CBRANCH_EXECZ: 2977 return EXECZ; 2978 default: 2979 return INVALID_BR; 2980 } 2981 } 2982 2983 bool SIInstrInfo::analyzeBranchImpl(MachineBasicBlock &MBB, 2984 MachineBasicBlock::iterator I, 2985 MachineBasicBlock *&TBB, 2986 MachineBasicBlock *&FBB, 2987 SmallVectorImpl<MachineOperand> &Cond, 2988 bool AllowModify) const { 2989 if (I->getOpcode() == AMDGPU::S_BRANCH) { 2990 // Unconditional Branch 2991 TBB = I->getOperand(0).getMBB(); 2992 return false; 2993 } 2994 2995 MachineBasicBlock *CondBB = nullptr; 2996 2997 if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { 2998 CondBB = I->getOperand(1).getMBB(); 2999 Cond.push_back(I->getOperand(0)); 3000 } else { 3001 BranchPredicate Pred = getBranchPredicate(I->getOpcode()); 3002 if (Pred == INVALID_BR) 3003 return true; 3004 3005 CondBB = I->getOperand(0).getMBB(); 3006 Cond.push_back(MachineOperand::CreateImm(Pred)); 3007 Cond.push_back(I->getOperand(1)); // Save the branch register. 3008 } 3009 ++I; 3010 3011 if (I == MBB.end()) { 3012 // Conditional branch followed by fall-through. 3013 TBB = CondBB; 3014 return false; 3015 } 3016 3017 if (I->getOpcode() == AMDGPU::S_BRANCH) { 3018 TBB = CondBB; 3019 FBB = I->getOperand(0).getMBB(); 3020 return false; 3021 } 3022 3023 return true; 3024 } 3025 3026 bool SIInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, 3027 MachineBasicBlock *&FBB, 3028 SmallVectorImpl<MachineOperand> &Cond, 3029 bool AllowModify) const { 3030 MachineBasicBlock::iterator I = MBB.getFirstTerminator(); 3031 auto E = MBB.end(); 3032 if (I == E) 3033 return false; 3034 3035 // Skip over the instructions that are artificially terminators for special 3036 // exec management. 3037 while (I != E && !I->isBranch() && !I->isReturn()) { 3038 switch (I->getOpcode()) { 3039 case AMDGPU::S_MOV_B64_term: 3040 case AMDGPU::S_XOR_B64_term: 3041 case AMDGPU::S_OR_B64_term: 3042 case AMDGPU::S_ANDN2_B64_term: 3043 case AMDGPU::S_AND_B64_term: 3044 case AMDGPU::S_AND_SAVEEXEC_B64_term: 3045 case AMDGPU::S_MOV_B32_term: 3046 case AMDGPU::S_XOR_B32_term: 3047 case AMDGPU::S_OR_B32_term: 3048 case AMDGPU::S_ANDN2_B32_term: 3049 case AMDGPU::S_AND_B32_term: 3050 case AMDGPU::S_AND_SAVEEXEC_B32_term: 3051 break; 3052 case AMDGPU::SI_IF: 3053 case AMDGPU::SI_ELSE: 3054 case AMDGPU::SI_KILL_I1_TERMINATOR: 3055 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR: 3056 // FIXME: It's messy that these need to be considered here at all. 3057 return true; 3058 default: 3059 llvm_unreachable("unexpected non-branch terminator inst"); 3060 } 3061 3062 ++I; 3063 } 3064 3065 if (I == E) 3066 return false; 3067 3068 return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify); 3069 } 3070 3071 unsigned SIInstrInfo::removeBranch(MachineBasicBlock &MBB, 3072 int *BytesRemoved) const { 3073 unsigned Count = 0; 3074 unsigned RemovedSize = 0; 3075 for (MachineInstr &MI : llvm::make_early_inc_range(MBB.terminators())) { 3076 // Skip over artificial terminators when removing instructions. 3077 if (MI.isBranch() || MI.isReturn()) { 3078 RemovedSize += getInstSizeInBytes(MI); 3079 MI.eraseFromParent(); 3080 ++Count; 3081 } 3082 } 3083 3084 if (BytesRemoved) 3085 *BytesRemoved = RemovedSize; 3086 3087 return Count; 3088 } 3089 3090 // Copy the flags onto the implicit condition register operand. 3091 static void preserveCondRegFlags(MachineOperand &CondReg, 3092 const MachineOperand &OrigCond) { 3093 CondReg.setIsUndef(OrigCond.isUndef()); 3094 CondReg.setIsKill(OrigCond.isKill()); 3095 } 3096 3097 unsigned SIInstrInfo::insertBranch(MachineBasicBlock &MBB, 3098 MachineBasicBlock *TBB, 3099 MachineBasicBlock *FBB, 3100 ArrayRef<MachineOperand> Cond, 3101 const DebugLoc &DL, 3102 int *BytesAdded) const { 3103 if (!FBB && Cond.empty()) { 3104 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH)) 3105 .addMBB(TBB); 3106 if (BytesAdded) 3107 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4; 3108 return 1; 3109 } 3110 3111 if(Cond.size() == 1 && Cond[0].isReg()) { 3112 BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO)) 3113 .add(Cond[0]) 3114 .addMBB(TBB); 3115 return 1; 3116 } 3117 3118 assert(TBB && Cond[0].isImm()); 3119 3120 unsigned Opcode 3121 = getBranchOpcode(static_cast<BranchPredicate>(Cond[0].getImm())); 3122 3123 if (!FBB) { 3124 MachineInstr *CondBr = 3125 BuildMI(&MBB, DL, get(Opcode)) 3126 .addMBB(TBB); 3127 3128 // Copy the flags onto the implicit condition register operand. 3129 preserveCondRegFlags(CondBr->getOperand(1), Cond[1]); 3130 fixImplicitOperands(*CondBr); 3131 3132 if (BytesAdded) 3133 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4; 3134 return 1; 3135 } 3136 3137 assert(TBB && FBB); 3138 3139 MachineInstr *CondBr = 3140 BuildMI(&MBB, DL, get(Opcode)) 3141 .addMBB(TBB); 3142 fixImplicitOperands(*CondBr); 3143 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH)) 3144 .addMBB(FBB); 3145 3146 MachineOperand &CondReg = CondBr->getOperand(1); 3147 CondReg.setIsUndef(Cond[1].isUndef()); 3148 CondReg.setIsKill(Cond[1].isKill()); 3149 3150 if (BytesAdded) 3151 *BytesAdded = ST.hasOffset3fBug() ? 16 : 8; 3152 3153 return 2; 3154 } 3155 3156 bool SIInstrInfo::reverseBranchCondition( 3157 SmallVectorImpl<MachineOperand> &Cond) const { 3158 if (Cond.size() != 2) { 3159 return true; 3160 } 3161 3162 if (Cond[0].isImm()) { 3163 Cond[0].setImm(-Cond[0].getImm()); 3164 return false; 3165 } 3166 3167 return true; 3168 } 3169 3170 bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB, 3171 ArrayRef<MachineOperand> Cond, 3172 Register DstReg, Register TrueReg, 3173 Register FalseReg, int &CondCycles, 3174 int &TrueCycles, int &FalseCycles) const { 3175 switch (Cond[0].getImm()) { 3176 case VCCNZ: 3177 case VCCZ: { 3178 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 3179 const TargetRegisterClass *RC = MRI.getRegClass(TrueReg); 3180 if (MRI.getRegClass(FalseReg) != RC) 3181 return false; 3182 3183 int NumInsts = AMDGPU::getRegBitWidth(*RC) / 32; 3184 CondCycles = TrueCycles = FalseCycles = NumInsts; // ??? 3185 3186 // Limit to equal cost for branch vs. N v_cndmask_b32s. 3187 return RI.hasVGPRs(RC) && NumInsts <= 6; 3188 } 3189 case SCC_TRUE: 3190 case SCC_FALSE: { 3191 // FIXME: We could insert for VGPRs if we could replace the original compare 3192 // with a vector one. 3193 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 3194 const TargetRegisterClass *RC = MRI.getRegClass(TrueReg); 3195 if (MRI.getRegClass(FalseReg) != RC) 3196 return false; 3197 3198 int NumInsts = AMDGPU::getRegBitWidth(*RC) / 32; 3199 3200 // Multiples of 8 can do s_cselect_b64 3201 if (NumInsts % 2 == 0) 3202 NumInsts /= 2; 3203 3204 CondCycles = TrueCycles = FalseCycles = NumInsts; // ??? 3205 return RI.isSGPRClass(RC); 3206 } 3207 default: 3208 return false; 3209 } 3210 } 3211 3212 void SIInstrInfo::insertSelect(MachineBasicBlock &MBB, 3213 MachineBasicBlock::iterator I, const DebugLoc &DL, 3214 Register DstReg, ArrayRef<MachineOperand> Cond, 3215 Register TrueReg, Register FalseReg) const { 3216 BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm()); 3217 if (Pred == VCCZ || Pred == SCC_FALSE) { 3218 Pred = static_cast<BranchPredicate>(-Pred); 3219 std::swap(TrueReg, FalseReg); 3220 } 3221 3222 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 3223 const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg); 3224 unsigned DstSize = RI.getRegSizeInBits(*DstRC); 3225 3226 if (DstSize == 32) { 3227 MachineInstr *Select; 3228 if (Pred == SCC_TRUE) { 3229 Select = BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B32), DstReg) 3230 .addReg(TrueReg) 3231 .addReg(FalseReg); 3232 } else { 3233 // Instruction's operands are backwards from what is expected. 3234 Select = BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e32), DstReg) 3235 .addReg(FalseReg) 3236 .addReg(TrueReg); 3237 } 3238 3239 preserveCondRegFlags(Select->getOperand(3), Cond[1]); 3240 return; 3241 } 3242 3243 if (DstSize == 64 && Pred == SCC_TRUE) { 3244 MachineInstr *Select = 3245 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg) 3246 .addReg(TrueReg) 3247 .addReg(FalseReg); 3248 3249 preserveCondRegFlags(Select->getOperand(3), Cond[1]); 3250 return; 3251 } 3252 3253 static const int16_t Sub0_15[] = { 3254 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, 3255 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7, 3256 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11, 3257 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15, 3258 }; 3259 3260 static const int16_t Sub0_15_64[] = { 3261 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3, 3262 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7, 3263 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11, 3264 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15, 3265 }; 3266 3267 unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32; 3268 const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass; 3269 const int16_t *SubIndices = Sub0_15; 3270 int NElts = DstSize / 32; 3271 3272 // 64-bit select is only available for SALU. 3273 // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit. 3274 if (Pred == SCC_TRUE) { 3275 if (NElts % 2) { 3276 SelOp = AMDGPU::S_CSELECT_B32; 3277 EltRC = &AMDGPU::SGPR_32RegClass; 3278 } else { 3279 SelOp = AMDGPU::S_CSELECT_B64; 3280 EltRC = &AMDGPU::SGPR_64RegClass; 3281 SubIndices = Sub0_15_64; 3282 NElts /= 2; 3283 } 3284 } 3285 3286 MachineInstrBuilder MIB = BuildMI( 3287 MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg); 3288 3289 I = MIB->getIterator(); 3290 3291 SmallVector<Register, 8> Regs; 3292 for (int Idx = 0; Idx != NElts; ++Idx) { 3293 Register DstElt = MRI.createVirtualRegister(EltRC); 3294 Regs.push_back(DstElt); 3295 3296 unsigned SubIdx = SubIndices[Idx]; 3297 3298 MachineInstr *Select; 3299 if (SelOp == AMDGPU::V_CNDMASK_B32_e32) { 3300 Select = 3301 BuildMI(MBB, I, DL, get(SelOp), DstElt) 3302 .addReg(FalseReg, 0, SubIdx) 3303 .addReg(TrueReg, 0, SubIdx); 3304 } else { 3305 Select = 3306 BuildMI(MBB, I, DL, get(SelOp), DstElt) 3307 .addReg(TrueReg, 0, SubIdx) 3308 .addReg(FalseReg, 0, SubIdx); 3309 } 3310 3311 preserveCondRegFlags(Select->getOperand(3), Cond[1]); 3312 fixImplicitOperands(*Select); 3313 3314 MIB.addReg(DstElt) 3315 .addImm(SubIdx); 3316 } 3317 } 3318 3319 bool SIInstrInfo::isFoldableCopy(const MachineInstr &MI) { 3320 switch (MI.getOpcode()) { 3321 case AMDGPU::V_MOV_B32_e32: 3322 case AMDGPU::V_MOV_B32_e64: 3323 case AMDGPU::V_MOV_B64_PSEUDO: 3324 case AMDGPU::V_MOV_B64_e32: 3325 case AMDGPU::V_MOV_B64_e64: 3326 case AMDGPU::S_MOV_B32: 3327 case AMDGPU::S_MOV_B64: 3328 case AMDGPU::S_MOV_B64_IMM_PSEUDO: 3329 case AMDGPU::COPY: 3330 case AMDGPU::WWM_COPY: 3331 case AMDGPU::V_ACCVGPR_WRITE_B32_e64: 3332 case AMDGPU::V_ACCVGPR_READ_B32_e64: 3333 case AMDGPU::V_ACCVGPR_MOV_B32: 3334 return true; 3335 default: 3336 return false; 3337 } 3338 } 3339 3340 static constexpr unsigned ModifierOpNames[] = { 3341 AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src1_modifiers, 3342 AMDGPU::OpName::src2_modifiers, AMDGPU::OpName::clamp, 3343 AMDGPU::OpName::omod, AMDGPU::OpName::op_sel}; 3344 3345 void SIInstrInfo::removeModOperands(MachineInstr &MI) const { 3346 unsigned Opc = MI.getOpcode(); 3347 for (unsigned Name : reverse(ModifierOpNames)) { 3348 int Idx = AMDGPU::getNamedOperandIdx(Opc, Name); 3349 if (Idx >= 0) 3350 MI.removeOperand(Idx); 3351 } 3352 } 3353 3354 bool SIInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, 3355 Register Reg, MachineRegisterInfo *MRI) const { 3356 if (!MRI->hasOneNonDBGUse(Reg)) 3357 return false; 3358 3359 switch (DefMI.getOpcode()) { 3360 default: 3361 return false; 3362 case AMDGPU::V_MOV_B64_e32: 3363 case AMDGPU::S_MOV_B64: 3364 case AMDGPU::V_MOV_B64_PSEUDO: 3365 case AMDGPU::S_MOV_B64_IMM_PSEUDO: 3366 case AMDGPU::V_MOV_B32_e32: 3367 case AMDGPU::S_MOV_B32: 3368 case AMDGPU::V_ACCVGPR_WRITE_B32_e64: 3369 break; 3370 } 3371 3372 const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0); 3373 assert(ImmOp); 3374 // FIXME: We could handle FrameIndex values here. 3375 if (!ImmOp->isImm()) 3376 return false; 3377 3378 auto getImmFor = [ImmOp](const MachineOperand &UseOp) -> int64_t { 3379 int64_t Imm = ImmOp->getImm(); 3380 switch (UseOp.getSubReg()) { 3381 default: 3382 return Imm; 3383 case AMDGPU::sub0: 3384 return Lo_32(Imm); 3385 case AMDGPU::sub1: 3386 return Hi_32(Imm); 3387 case AMDGPU::lo16: 3388 return APInt(16, Imm).getSExtValue(); 3389 case AMDGPU::hi16: 3390 return APInt(32, Imm).ashr(16).getSExtValue(); 3391 case AMDGPU::sub1_lo16: 3392 return APInt(16, Hi_32(Imm)).getSExtValue(); 3393 case AMDGPU::sub1_hi16: 3394 return APInt(32, Hi_32(Imm)).ashr(16).getSExtValue(); 3395 } 3396 }; 3397 3398 assert(!DefMI.getOperand(0).getSubReg() && "Expected SSA form"); 3399 3400 unsigned Opc = UseMI.getOpcode(); 3401 if (Opc == AMDGPU::COPY) { 3402 assert(!UseMI.getOperand(0).getSubReg() && "Expected SSA form"); 3403 3404 Register DstReg = UseMI.getOperand(0).getReg(); 3405 unsigned OpSize = getOpSize(UseMI, 0); 3406 bool Is16Bit = OpSize == 2; 3407 bool Is64Bit = OpSize == 8; 3408 bool isVGPRCopy = RI.isVGPR(*MRI, DstReg); 3409 unsigned NewOpc = isVGPRCopy ? Is64Bit ? AMDGPU::V_MOV_B64_PSEUDO 3410 : AMDGPU::V_MOV_B32_e32 3411 : Is64Bit ? AMDGPU::S_MOV_B64_IMM_PSEUDO 3412 : AMDGPU::S_MOV_B32; 3413 APInt Imm(Is64Bit ? 64 : 32, getImmFor(UseMI.getOperand(1))); 3414 3415 if (RI.isAGPR(*MRI, DstReg)) { 3416 if (Is64Bit || !isInlineConstant(Imm)) 3417 return false; 3418 NewOpc = AMDGPU::V_ACCVGPR_WRITE_B32_e64; 3419 } 3420 3421 if (Is16Bit) { 3422 if (isVGPRCopy) 3423 return false; // Do not clobber vgpr_hi16 3424 3425 if (DstReg.isVirtual() && UseMI.getOperand(0).getSubReg() != AMDGPU::lo16) 3426 return false; 3427 3428 UseMI.getOperand(0).setSubReg(0); 3429 if (DstReg.isPhysical()) { 3430 DstReg = RI.get32BitRegister(DstReg); 3431 UseMI.getOperand(0).setReg(DstReg); 3432 } 3433 assert(UseMI.getOperand(1).getReg().isVirtual()); 3434 } 3435 3436 const MCInstrDesc &NewMCID = get(NewOpc); 3437 if (DstReg.isPhysical() && 3438 !RI.getRegClass(NewMCID.operands()[0].RegClass)->contains(DstReg)) 3439 return false; 3440 3441 UseMI.setDesc(NewMCID); 3442 UseMI.getOperand(1).ChangeToImmediate(Imm.getSExtValue()); 3443 UseMI.addImplicitDefUseOperands(*UseMI.getParent()->getParent()); 3444 return true; 3445 } 3446 3447 if (Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 || 3448 Opc == AMDGPU::V_MAD_F16_e64 || Opc == AMDGPU::V_MAC_F16_e64 || 3449 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 || 3450 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64 || 3451 Opc == AMDGPU::V_FMAC_F16_t16_e64) { 3452 // Don't fold if we are using source or output modifiers. The new VOP2 3453 // instructions don't have them. 3454 if (hasAnyModifiersSet(UseMI)) 3455 return false; 3456 3457 // If this is a free constant, there's no reason to do this. 3458 // TODO: We could fold this here instead of letting SIFoldOperands do it 3459 // later. 3460 MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0); 3461 3462 // Any src operand can be used for the legality check. 3463 if (isInlineConstant(UseMI, *Src0, *ImmOp)) 3464 return false; 3465 3466 bool IsF32 = Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 || 3467 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64; 3468 bool IsFMA = 3469 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 || 3470 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64 || 3471 Opc == AMDGPU::V_FMAC_F16_t16_e64; 3472 MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1); 3473 MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2); 3474 3475 // Multiplied part is the constant: Use v_madmk_{f16, f32}. 3476 if ((Src0->isReg() && Src0->getReg() == Reg) || 3477 (Src1->isReg() && Src1->getReg() == Reg)) { 3478 MachineOperand *RegSrc = 3479 Src1->isReg() && Src1->getReg() == Reg ? Src0 : Src1; 3480 if (!RegSrc->isReg()) 3481 return false; 3482 if (RI.isSGPRClass(MRI->getRegClass(RegSrc->getReg())) && 3483 ST.getConstantBusLimit(Opc) < 2) 3484 return false; 3485 3486 if (!Src2->isReg() || RI.isSGPRClass(MRI->getRegClass(Src2->getReg()))) 3487 return false; 3488 3489 // If src2 is also a literal constant then we have to choose which one to 3490 // fold. In general it is better to choose madak so that the other literal 3491 // can be materialized in an sgpr instead of a vgpr: 3492 // s_mov_b32 s0, literal 3493 // v_madak_f32 v0, s0, v0, literal 3494 // Instead of: 3495 // v_mov_b32 v1, literal 3496 // v_madmk_f32 v0, v0, literal, v1 3497 MachineInstr *Def = MRI->getUniqueVRegDef(Src2->getReg()); 3498 if (Def && Def->isMoveImmediate() && 3499 !isInlineConstant(Def->getOperand(1))) 3500 return false; 3501 3502 unsigned NewOpc = 3503 IsFMA ? (IsF32 ? AMDGPU::V_FMAMK_F32 3504 : ST.hasTrue16BitInsts() ? AMDGPU::V_FMAMK_F16_t16 3505 : AMDGPU::V_FMAMK_F16) 3506 : (IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16); 3507 if (pseudoToMCOpcode(NewOpc) == -1) 3508 return false; 3509 3510 // V_FMAMK_F16_t16 takes VGPR_32_Lo128 operands, so the rewrite 3511 // would also require restricting their register classes. For now 3512 // just bail out. 3513 if (NewOpc == AMDGPU::V_FMAMK_F16_t16) 3514 return false; 3515 3516 const int64_t Imm = getImmFor(RegSrc == Src1 ? *Src0 : *Src1); 3517 3518 // FIXME: This would be a lot easier if we could return a new instruction 3519 // instead of having to modify in place. 3520 3521 Register SrcReg = RegSrc->getReg(); 3522 unsigned SrcSubReg = RegSrc->getSubReg(); 3523 Src0->setReg(SrcReg); 3524 Src0->setSubReg(SrcSubReg); 3525 Src0->setIsKill(RegSrc->isKill()); 3526 3527 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 || 3528 Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 || 3529 Opc == AMDGPU::V_FMAC_F16_e64) 3530 UseMI.untieRegOperand( 3531 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)); 3532 3533 Src1->ChangeToImmediate(Imm); 3534 3535 removeModOperands(UseMI); 3536 UseMI.setDesc(get(NewOpc)); 3537 3538 bool DeleteDef = MRI->use_nodbg_empty(Reg); 3539 if (DeleteDef) 3540 DefMI.eraseFromParent(); 3541 3542 return true; 3543 } 3544 3545 // Added part is the constant: Use v_madak_{f16, f32}. 3546 if (Src2->isReg() && Src2->getReg() == Reg) { 3547 if (ST.getConstantBusLimit(Opc) < 2) { 3548 // Not allowed to use constant bus for another operand. 3549 // We can however allow an inline immediate as src0. 3550 bool Src0Inlined = false; 3551 if (Src0->isReg()) { 3552 // Try to inline constant if possible. 3553 // If the Def moves immediate and the use is single 3554 // We are saving VGPR here. 3555 MachineInstr *Def = MRI->getUniqueVRegDef(Src0->getReg()); 3556 if (Def && Def->isMoveImmediate() && 3557 isInlineConstant(Def->getOperand(1)) && 3558 MRI->hasOneUse(Src0->getReg())) { 3559 Src0->ChangeToImmediate(Def->getOperand(1).getImm()); 3560 Src0Inlined = true; 3561 } else if (ST.getConstantBusLimit(Opc) <= 1 && 3562 RI.isSGPRReg(*MRI, Src0->getReg())) { 3563 return false; 3564 } 3565 // VGPR is okay as Src0 - fallthrough 3566 } 3567 3568 if (Src1->isReg() && !Src0Inlined) { 3569 // We have one slot for inlinable constant so far - try to fill it 3570 MachineInstr *Def = MRI->getUniqueVRegDef(Src1->getReg()); 3571 if (Def && Def->isMoveImmediate() && 3572 isInlineConstant(Def->getOperand(1)) && 3573 MRI->hasOneUse(Src1->getReg()) && commuteInstruction(UseMI)) 3574 Src0->ChangeToImmediate(Def->getOperand(1).getImm()); 3575 else if (RI.isSGPRReg(*MRI, Src1->getReg())) 3576 return false; 3577 // VGPR is okay as Src1 - fallthrough 3578 } 3579 } 3580 3581 unsigned NewOpc = 3582 IsFMA ? (IsF32 ? AMDGPU::V_FMAAK_F32 3583 : ST.hasTrue16BitInsts() ? AMDGPU::V_FMAAK_F16_t16 3584 : AMDGPU::V_FMAAK_F16) 3585 : (IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16); 3586 if (pseudoToMCOpcode(NewOpc) == -1) 3587 return false; 3588 3589 // V_FMAAK_F16_t16 takes VGPR_32_Lo128 operands, so the rewrite 3590 // would also require restricting their register classes. For now 3591 // just bail out. 3592 if (NewOpc == AMDGPU::V_FMAAK_F16_t16) 3593 return false; 3594 3595 // FIXME: This would be a lot easier if we could return a new instruction 3596 // instead of having to modify in place. 3597 3598 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 || 3599 Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 || 3600 Opc == AMDGPU::V_FMAC_F16_e64) 3601 UseMI.untieRegOperand( 3602 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)); 3603 3604 // ChangingToImmediate adds Src2 back to the instruction. 3605 Src2->ChangeToImmediate(getImmFor(*Src2)); 3606 3607 // These come before src2. 3608 removeModOperands(UseMI); 3609 UseMI.setDesc(get(NewOpc)); 3610 // It might happen that UseMI was commuted 3611 // and we now have SGPR as SRC1. If so 2 inlined 3612 // constant and SGPR are illegal. 3613 legalizeOperands(UseMI); 3614 3615 bool DeleteDef = MRI->use_nodbg_empty(Reg); 3616 if (DeleteDef) 3617 DefMI.eraseFromParent(); 3618 3619 return true; 3620 } 3621 } 3622 3623 return false; 3624 } 3625 3626 static bool 3627 memOpsHaveSameBaseOperands(ArrayRef<const MachineOperand *> BaseOps1, 3628 ArrayRef<const MachineOperand *> BaseOps2) { 3629 if (BaseOps1.size() != BaseOps2.size()) 3630 return false; 3631 for (size_t I = 0, E = BaseOps1.size(); I < E; ++I) { 3632 if (!BaseOps1[I]->isIdenticalTo(*BaseOps2[I])) 3633 return false; 3634 } 3635 return true; 3636 } 3637 3638 static bool offsetsDoNotOverlap(int WidthA, int OffsetA, 3639 int WidthB, int OffsetB) { 3640 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB; 3641 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA; 3642 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB; 3643 return LowOffset + LowWidth <= HighOffset; 3644 } 3645 3646 bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa, 3647 const MachineInstr &MIb) const { 3648 SmallVector<const MachineOperand *, 4> BaseOps0, BaseOps1; 3649 int64_t Offset0, Offset1; 3650 unsigned Dummy0, Dummy1; 3651 bool Offset0IsScalable, Offset1IsScalable; 3652 if (!getMemOperandsWithOffsetWidth(MIa, BaseOps0, Offset0, Offset0IsScalable, 3653 Dummy0, &RI) || 3654 !getMemOperandsWithOffsetWidth(MIb, BaseOps1, Offset1, Offset1IsScalable, 3655 Dummy1, &RI)) 3656 return false; 3657 3658 if (!memOpsHaveSameBaseOperands(BaseOps0, BaseOps1)) 3659 return false; 3660 3661 if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) { 3662 // FIXME: Handle ds_read2 / ds_write2. 3663 return false; 3664 } 3665 unsigned Width0 = MIa.memoperands().front()->getSize(); 3666 unsigned Width1 = MIb.memoperands().front()->getSize(); 3667 return offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1); 3668 } 3669 3670 bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, 3671 const MachineInstr &MIb) const { 3672 assert(MIa.mayLoadOrStore() && 3673 "MIa must load from or modify a memory location"); 3674 assert(MIb.mayLoadOrStore() && 3675 "MIb must load from or modify a memory location"); 3676 3677 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects()) 3678 return false; 3679 3680 // XXX - Can we relax this between address spaces? 3681 if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef()) 3682 return false; 3683 3684 if (isLDSDMA(MIa) || isLDSDMA(MIb)) 3685 return false; 3686 3687 // TODO: Should we check the address space from the MachineMemOperand? That 3688 // would allow us to distinguish objects we know don't alias based on the 3689 // underlying address space, even if it was lowered to a different one, 3690 // e.g. private accesses lowered to use MUBUF instructions on a scratch 3691 // buffer. 3692 if (isDS(MIa)) { 3693 if (isDS(MIb)) 3694 return checkInstOffsetsDoNotOverlap(MIa, MIb); 3695 3696 return !isFLAT(MIb) || isSegmentSpecificFLAT(MIb); 3697 } 3698 3699 if (isMUBUF(MIa) || isMTBUF(MIa)) { 3700 if (isMUBUF(MIb) || isMTBUF(MIb)) 3701 return checkInstOffsetsDoNotOverlap(MIa, MIb); 3702 3703 if (isFLAT(MIb)) 3704 return isFLATScratch(MIb); 3705 3706 return !isSMRD(MIb); 3707 } 3708 3709 if (isSMRD(MIa)) { 3710 if (isSMRD(MIb)) 3711 return checkInstOffsetsDoNotOverlap(MIa, MIb); 3712 3713 if (isFLAT(MIb)) 3714 return isFLATScratch(MIb); 3715 3716 return !isMUBUF(MIb) && !isMTBUF(MIb); 3717 } 3718 3719 if (isFLAT(MIa)) { 3720 if (isFLAT(MIb)) { 3721 if ((isFLATScratch(MIa) && isFLATGlobal(MIb)) || 3722 (isFLATGlobal(MIa) && isFLATScratch(MIb))) 3723 return true; 3724 3725 return checkInstOffsetsDoNotOverlap(MIa, MIb); 3726 } 3727 3728 return false; 3729 } 3730 3731 return false; 3732 } 3733 3734 static bool getFoldableImm(Register Reg, const MachineRegisterInfo &MRI, 3735 int64_t &Imm, MachineInstr **DefMI = nullptr) { 3736 if (Reg.isPhysical()) 3737 return false; 3738 auto *Def = MRI.getUniqueVRegDef(Reg); 3739 if (Def && SIInstrInfo::isFoldableCopy(*Def) && Def->getOperand(1).isImm()) { 3740 Imm = Def->getOperand(1).getImm(); 3741 if (DefMI) 3742 *DefMI = Def; 3743 return true; 3744 } 3745 return false; 3746 } 3747 3748 static bool getFoldableImm(const MachineOperand *MO, int64_t &Imm, 3749 MachineInstr **DefMI = nullptr) { 3750 if (!MO->isReg()) 3751 return false; 3752 const MachineFunction *MF = MO->getParent()->getParent()->getParent(); 3753 const MachineRegisterInfo &MRI = MF->getRegInfo(); 3754 return getFoldableImm(MO->getReg(), MRI, Imm, DefMI); 3755 } 3756 3757 static void updateLiveVariables(LiveVariables *LV, MachineInstr &MI, 3758 MachineInstr &NewMI) { 3759 if (LV) { 3760 unsigned NumOps = MI.getNumOperands(); 3761 for (unsigned I = 1; I < NumOps; ++I) { 3762 MachineOperand &Op = MI.getOperand(I); 3763 if (Op.isReg() && Op.isKill()) 3764 LV->replaceKillInstruction(Op.getReg(), MI, NewMI); 3765 } 3766 } 3767 } 3768 3769 MachineInstr *SIInstrInfo::convertToThreeAddress(MachineInstr &MI, 3770 LiveVariables *LV, 3771 LiveIntervals *LIS) const { 3772 MachineBasicBlock &MBB = *MI.getParent(); 3773 unsigned Opc = MI.getOpcode(); 3774 3775 // Handle MFMA. 3776 int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(Opc); 3777 if (NewMFMAOpc != -1) { 3778 MachineInstrBuilder MIB = 3779 BuildMI(MBB, MI, MI.getDebugLoc(), get(NewMFMAOpc)); 3780 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) 3781 MIB.add(MI.getOperand(I)); 3782 updateLiveVariables(LV, MI, *MIB); 3783 if (LIS) 3784 LIS->ReplaceMachineInstrInMaps(MI, *MIB); 3785 return MIB; 3786 } 3787 3788 if (SIInstrInfo::isWMMA(MI)) { 3789 unsigned NewOpc = AMDGPU::mapWMMA2AddrTo3AddrOpcode(MI.getOpcode()); 3790 MachineInstrBuilder MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3791 .setMIFlags(MI.getFlags()); 3792 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) 3793 MIB->addOperand(MI.getOperand(I)); 3794 3795 updateLiveVariables(LV, MI, *MIB); 3796 if (LIS) 3797 LIS->ReplaceMachineInstrInMaps(MI, *MIB); 3798 3799 return MIB; 3800 } 3801 3802 assert(Opc != AMDGPU::V_FMAC_F16_t16_e32 && 3803 "V_FMAC_F16_t16_e32 is not supported and not expected to be present " 3804 "pre-RA"); 3805 3806 // Handle MAC/FMAC. 3807 bool IsF16 = Opc == AMDGPU::V_MAC_F16_e32 || Opc == AMDGPU::V_MAC_F16_e64 || 3808 Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64 || 3809 Opc == AMDGPU::V_FMAC_F16_t16_e64; 3810 bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64 || 3811 Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 || 3812 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64 || 3813 Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64 || 3814 Opc == AMDGPU::V_FMAC_F16_t16_e64 || 3815 Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64; 3816 bool IsF64 = Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64; 3817 bool IsLegacy = Opc == AMDGPU::V_MAC_LEGACY_F32_e32 || 3818 Opc == AMDGPU::V_MAC_LEGACY_F32_e64 || 3819 Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 || 3820 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64; 3821 bool Src0Literal = false; 3822 3823 switch (Opc) { 3824 default: 3825 return nullptr; 3826 case AMDGPU::V_MAC_F16_e64: 3827 case AMDGPU::V_FMAC_F16_e64: 3828 case AMDGPU::V_FMAC_F16_t16_e64: 3829 case AMDGPU::V_MAC_F32_e64: 3830 case AMDGPU::V_MAC_LEGACY_F32_e64: 3831 case AMDGPU::V_FMAC_F32_e64: 3832 case AMDGPU::V_FMAC_LEGACY_F32_e64: 3833 case AMDGPU::V_FMAC_F64_e64: 3834 break; 3835 case AMDGPU::V_MAC_F16_e32: 3836 case AMDGPU::V_FMAC_F16_e32: 3837 case AMDGPU::V_MAC_F32_e32: 3838 case AMDGPU::V_MAC_LEGACY_F32_e32: 3839 case AMDGPU::V_FMAC_F32_e32: 3840 case AMDGPU::V_FMAC_LEGACY_F32_e32: 3841 case AMDGPU::V_FMAC_F64_e32: { 3842 int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), 3843 AMDGPU::OpName::src0); 3844 const MachineOperand *Src0 = &MI.getOperand(Src0Idx); 3845 if (!Src0->isReg() && !Src0->isImm()) 3846 return nullptr; 3847 3848 if (Src0->isImm() && !isInlineConstant(MI, Src0Idx, *Src0)) 3849 Src0Literal = true; 3850 3851 break; 3852 } 3853 } 3854 3855 MachineInstrBuilder MIB; 3856 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); 3857 const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0); 3858 const MachineOperand *Src0Mods = 3859 getNamedOperand(MI, AMDGPU::OpName::src0_modifiers); 3860 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); 3861 const MachineOperand *Src1Mods = 3862 getNamedOperand(MI, AMDGPU::OpName::src1_modifiers); 3863 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); 3864 const MachineOperand *Src2Mods = 3865 getNamedOperand(MI, AMDGPU::OpName::src2_modifiers); 3866 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp); 3867 const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod); 3868 const MachineOperand *OpSel = getNamedOperand(MI, AMDGPU::OpName::op_sel); 3869 3870 if (!Src0Mods && !Src1Mods && !Src2Mods && !Clamp && !Omod && !IsF64 && 3871 !IsLegacy && 3872 // If we have an SGPR input, we will violate the constant bus restriction. 3873 (ST.getConstantBusLimit(Opc) > 1 || !Src0->isReg() || 3874 !RI.isSGPRReg(MBB.getParent()->getRegInfo(), Src0->getReg()))) { 3875 MachineInstr *DefMI; 3876 const auto killDef = [&]() -> void { 3877 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 3878 // The only user is the instruction which will be killed. 3879 Register DefReg = DefMI->getOperand(0).getReg(); 3880 if (!MRI.hasOneNonDBGUse(DefReg)) 3881 return; 3882 // We cannot just remove the DefMI here, calling pass will crash. 3883 DefMI->setDesc(get(AMDGPU::IMPLICIT_DEF)); 3884 for (unsigned I = DefMI->getNumOperands() - 1; I != 0; --I) 3885 DefMI->removeOperand(I); 3886 if (LV) 3887 LV->getVarInfo(DefReg).AliveBlocks.clear(); 3888 }; 3889 3890 int64_t Imm; 3891 if (!Src0Literal && getFoldableImm(Src2, Imm, &DefMI)) { 3892 unsigned NewOpc = 3893 IsFMA ? (IsF16 ? (ST.hasTrue16BitInsts() ? AMDGPU::V_FMAAK_F16_t16 3894 : AMDGPU::V_FMAAK_F16) 3895 : AMDGPU::V_FMAAK_F32) 3896 : (IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32); 3897 if (pseudoToMCOpcode(NewOpc) != -1) { 3898 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3899 .add(*Dst) 3900 .add(*Src0) 3901 .add(*Src1) 3902 .addImm(Imm); 3903 updateLiveVariables(LV, MI, *MIB); 3904 if (LIS) 3905 LIS->ReplaceMachineInstrInMaps(MI, *MIB); 3906 killDef(); 3907 return MIB; 3908 } 3909 } 3910 unsigned NewOpc = 3911 IsFMA ? (IsF16 ? (ST.hasTrue16BitInsts() ? AMDGPU::V_FMAMK_F16_t16 3912 : AMDGPU::V_FMAMK_F16) 3913 : AMDGPU::V_FMAMK_F32) 3914 : (IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32); 3915 if (!Src0Literal && getFoldableImm(Src1, Imm, &DefMI)) { 3916 if (pseudoToMCOpcode(NewOpc) != -1) { 3917 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3918 .add(*Dst) 3919 .add(*Src0) 3920 .addImm(Imm) 3921 .add(*Src2); 3922 updateLiveVariables(LV, MI, *MIB); 3923 if (LIS) 3924 LIS->ReplaceMachineInstrInMaps(MI, *MIB); 3925 killDef(); 3926 return MIB; 3927 } 3928 } 3929 if (Src0Literal || getFoldableImm(Src0, Imm, &DefMI)) { 3930 if (Src0Literal) { 3931 Imm = Src0->getImm(); 3932 DefMI = nullptr; 3933 } 3934 if (pseudoToMCOpcode(NewOpc) != -1 && 3935 isOperandLegal( 3936 MI, AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::src0), 3937 Src1)) { 3938 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3939 .add(*Dst) 3940 .add(*Src1) 3941 .addImm(Imm) 3942 .add(*Src2); 3943 updateLiveVariables(LV, MI, *MIB); 3944 if (LIS) 3945 LIS->ReplaceMachineInstrInMaps(MI, *MIB); 3946 if (DefMI) 3947 killDef(); 3948 return MIB; 3949 } 3950 } 3951 } 3952 3953 // VOP2 mac/fmac with a literal operand cannot be converted to VOP3 mad/fma 3954 // if VOP3 does not allow a literal operand. 3955 if (Src0Literal && !ST.hasVOP3Literal()) 3956 return nullptr; 3957 3958 unsigned NewOpc = IsFMA ? IsF16 ? AMDGPU::V_FMA_F16_gfx9_e64 3959 : IsF64 ? AMDGPU::V_FMA_F64_e64 3960 : IsLegacy 3961 ? AMDGPU::V_FMA_LEGACY_F32_e64 3962 : AMDGPU::V_FMA_F32_e64 3963 : IsF16 ? AMDGPU::V_MAD_F16_e64 3964 : IsLegacy ? AMDGPU::V_MAD_LEGACY_F32_e64 3965 : AMDGPU::V_MAD_F32_e64; 3966 if (pseudoToMCOpcode(NewOpc) == -1) 3967 return nullptr; 3968 3969 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3970 .add(*Dst) 3971 .addImm(Src0Mods ? Src0Mods->getImm() : 0) 3972 .add(*Src0) 3973 .addImm(Src1Mods ? Src1Mods->getImm() : 0) 3974 .add(*Src1) 3975 .addImm(Src2Mods ? Src2Mods->getImm() : 0) 3976 .add(*Src2) 3977 .addImm(Clamp ? Clamp->getImm() : 0) 3978 .addImm(Omod ? Omod->getImm() : 0); 3979 if (AMDGPU::hasNamedOperand(NewOpc, AMDGPU::OpName::op_sel)) 3980 MIB.addImm(OpSel ? OpSel->getImm() : 0); 3981 updateLiveVariables(LV, MI, *MIB); 3982 if (LIS) 3983 LIS->ReplaceMachineInstrInMaps(MI, *MIB); 3984 return MIB; 3985 } 3986 3987 // It's not generally safe to move VALU instructions across these since it will 3988 // start using the register as a base index rather than directly. 3989 // XXX - Why isn't hasSideEffects sufficient for these? 3990 static bool changesVGPRIndexingMode(const MachineInstr &MI) { 3991 switch (MI.getOpcode()) { 3992 case AMDGPU::S_SET_GPR_IDX_ON: 3993 case AMDGPU::S_SET_GPR_IDX_MODE: 3994 case AMDGPU::S_SET_GPR_IDX_OFF: 3995 return true; 3996 default: 3997 return false; 3998 } 3999 } 4000 4001 bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI, 4002 const MachineBasicBlock *MBB, 4003 const MachineFunction &MF) const { 4004 // Skipping the check for SP writes in the base implementation. The reason it 4005 // was added was apparently due to compile time concerns. 4006 // 4007 // TODO: Do we really want this barrier? It triggers unnecessary hazard nops 4008 // but is probably avoidable. 4009 4010 // Copied from base implementation. 4011 // Terminators and labels can't be scheduled around. 4012 if (MI.isTerminator() || MI.isPosition()) 4013 return true; 4014 4015 // INLINEASM_BR can jump to another block 4016 if (MI.getOpcode() == TargetOpcode::INLINEASM_BR) 4017 return true; 4018 4019 if (MI.getOpcode() == AMDGPU::SCHED_BARRIER && MI.getOperand(0).getImm() == 0) 4020 return true; 4021 4022 // Target-independent instructions do not have an implicit-use of EXEC, even 4023 // when they operate on VGPRs. Treating EXEC modifications as scheduling 4024 // boundaries prevents incorrect movements of such instructions. 4025 return MI.modifiesRegister(AMDGPU::EXEC, &RI) || 4026 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 || 4027 MI.getOpcode() == AMDGPU::S_SETREG_B32 || 4028 MI.getOpcode() == AMDGPU::S_SETPRIO || 4029 changesVGPRIndexingMode(MI); 4030 } 4031 4032 bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const { 4033 return Opcode == AMDGPU::DS_ORDERED_COUNT || isGWS(Opcode); 4034 } 4035 4036 bool SIInstrInfo::modifiesModeRegister(const MachineInstr &MI) { 4037 // Skip the full operand and register alias search modifiesRegister 4038 // does. There's only a handful of instructions that touch this, it's only an 4039 // implicit def, and doesn't alias any other registers. 4040 return is_contained(MI.getDesc().implicit_defs(), AMDGPU::MODE); 4041 } 4042 4043 bool SIInstrInfo::hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const { 4044 unsigned Opcode = MI.getOpcode(); 4045 4046 if (MI.mayStore() && isSMRD(MI)) 4047 return true; // scalar store or atomic 4048 4049 // This will terminate the function when other lanes may need to continue. 4050 if (MI.isReturn()) 4051 return true; 4052 4053 // These instructions cause shader I/O that may cause hardware lockups 4054 // when executed with an empty EXEC mask. 4055 // 4056 // Note: exp with VM = DONE = 0 is automatically skipped by hardware when 4057 // EXEC = 0, but checking for that case here seems not worth it 4058 // given the typical code patterns. 4059 if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT || 4060 isEXP(Opcode) || 4061 Opcode == AMDGPU::DS_ORDERED_COUNT || Opcode == AMDGPU::S_TRAP || 4062 Opcode == AMDGPU::DS_GWS_INIT || Opcode == AMDGPU::DS_GWS_BARRIER) 4063 return true; 4064 4065 if (MI.isCall() || MI.isInlineAsm()) 4066 return true; // conservative assumption 4067 4068 // A mode change is a scalar operation that influences vector instructions. 4069 if (modifiesModeRegister(MI)) 4070 return true; 4071 4072 // These are like SALU instructions in terms of effects, so it's questionable 4073 // whether we should return true for those. 4074 // 4075 // However, executing them with EXEC = 0 causes them to operate on undefined 4076 // data, which we avoid by returning true here. 4077 if (Opcode == AMDGPU::V_READFIRSTLANE_B32 || 4078 Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32 || 4079 Opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR || 4080 Opcode == AMDGPU::SI_SPILL_S32_TO_VGPR) 4081 return true; 4082 4083 return false; 4084 } 4085 4086 bool SIInstrInfo::mayReadEXEC(const MachineRegisterInfo &MRI, 4087 const MachineInstr &MI) const { 4088 if (MI.isMetaInstruction()) 4089 return false; 4090 4091 // This won't read exec if this is an SGPR->SGPR copy. 4092 if (MI.isCopyLike()) { 4093 if (!RI.isSGPRReg(MRI, MI.getOperand(0).getReg())) 4094 return true; 4095 4096 // Make sure this isn't copying exec as a normal operand 4097 return MI.readsRegister(AMDGPU::EXEC, &RI); 4098 } 4099 4100 // Make a conservative assumption about the callee. 4101 if (MI.isCall()) 4102 return true; 4103 4104 // Be conservative with any unhandled generic opcodes. 4105 if (!isTargetSpecificOpcode(MI.getOpcode())) 4106 return true; 4107 4108 return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI); 4109 } 4110 4111 bool SIInstrInfo::isInlineConstant(const APInt &Imm) const { 4112 switch (Imm.getBitWidth()) { 4113 case 1: // This likely will be a condition code mask. 4114 return true; 4115 4116 case 32: 4117 return AMDGPU::isInlinableLiteral32(Imm.getSExtValue(), 4118 ST.hasInv2PiInlineImm()); 4119 case 64: 4120 return AMDGPU::isInlinableLiteral64(Imm.getSExtValue(), 4121 ST.hasInv2PiInlineImm()); 4122 case 16: 4123 return ST.has16BitInsts() && 4124 AMDGPU::isInlinableLiteral16(Imm.getSExtValue(), 4125 ST.hasInv2PiInlineImm()); 4126 default: 4127 llvm_unreachable("invalid bitwidth"); 4128 } 4129 } 4130 4131 bool SIInstrInfo::isInlineConstant(const MachineOperand &MO, 4132 uint8_t OperandType) const { 4133 assert(!MO.isReg() && "isInlineConstant called on register operand!"); 4134 if (!MO.isImm()) 4135 return false; 4136 4137 // MachineOperand provides no way to tell the true operand size, since it only 4138 // records a 64-bit value. We need to know the size to determine if a 32-bit 4139 // floating point immediate bit pattern is legal for an integer immediate. It 4140 // would be for any 32-bit integer operand, but would not be for a 64-bit one. 4141 4142 int64_t Imm = MO.getImm(); 4143 switch (OperandType) { 4144 case AMDGPU::OPERAND_REG_IMM_INT32: 4145 case AMDGPU::OPERAND_REG_IMM_FP32: 4146 case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED: 4147 case AMDGPU::OPERAND_REG_INLINE_C_INT32: 4148 case AMDGPU::OPERAND_REG_INLINE_C_FP32: 4149 case AMDGPU::OPERAND_REG_IMM_V2FP32: 4150 case AMDGPU::OPERAND_REG_INLINE_C_V2FP32: 4151 case AMDGPU::OPERAND_REG_IMM_V2INT32: 4152 case AMDGPU::OPERAND_REG_INLINE_C_V2INT32: 4153 case AMDGPU::OPERAND_REG_INLINE_AC_INT32: 4154 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: 4155 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32: { 4156 int32_t Trunc = static_cast<int32_t>(Imm); 4157 return AMDGPU::isInlinableLiteral32(Trunc, ST.hasInv2PiInlineImm()); 4158 } 4159 case AMDGPU::OPERAND_REG_IMM_INT64: 4160 case AMDGPU::OPERAND_REG_IMM_FP64: 4161 case AMDGPU::OPERAND_REG_INLINE_C_INT64: 4162 case AMDGPU::OPERAND_REG_INLINE_C_FP64: 4163 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: 4164 return AMDGPU::isInlinableLiteral64(MO.getImm(), 4165 ST.hasInv2PiInlineImm()); 4166 case AMDGPU::OPERAND_REG_IMM_INT16: 4167 case AMDGPU::OPERAND_REG_INLINE_C_INT16: 4168 case AMDGPU::OPERAND_REG_INLINE_AC_INT16: 4169 // We would expect inline immediates to not be concerned with an integer/fp 4170 // distinction. However, in the case of 16-bit integer operations, the 4171 // "floating point" values appear to not work. It seems read the low 16-bits 4172 // of 32-bit immediates, which happens to always work for the integer 4173 // values. 4174 // 4175 // See llvm bugzilla 46302. 4176 // 4177 // TODO: Theoretically we could use op-sel to use the high bits of the 4178 // 32-bit FP values. 4179 return AMDGPU::isInlinableIntLiteral(Imm); 4180 case AMDGPU::OPERAND_REG_IMM_V2INT16: 4181 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: 4182 case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16: 4183 return AMDGPU::isInlinableLiteralV2I16(Imm); 4184 case AMDGPU::OPERAND_REG_IMM_V2FP16: 4185 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: 4186 case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: 4187 return AMDGPU::isInlinableLiteralV2F16(Imm); 4188 case AMDGPU::OPERAND_REG_IMM_FP16: 4189 case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED: 4190 case AMDGPU::OPERAND_REG_INLINE_C_FP16: 4191 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: { 4192 if (isInt<16>(Imm) || isUInt<16>(Imm)) { 4193 // A few special case instructions have 16-bit operands on subtargets 4194 // where 16-bit instructions are not legal. 4195 // TODO: Do the 32-bit immediates work? We shouldn't really need to handle 4196 // constants in these cases 4197 int16_t Trunc = static_cast<int16_t>(Imm); 4198 return ST.has16BitInsts() && 4199 AMDGPU::isInlinableLiteral16(Trunc, ST.hasInv2PiInlineImm()); 4200 } 4201 4202 return false; 4203 } 4204 case AMDGPU::OPERAND_KIMM32: 4205 case AMDGPU::OPERAND_KIMM16: 4206 return false; 4207 case AMDGPU::OPERAND_INPUT_MODS: 4208 case MCOI::OPERAND_IMMEDIATE: 4209 // Always embedded in the instruction for free. 4210 return true; 4211 case MCOI::OPERAND_UNKNOWN: 4212 case MCOI::OPERAND_REGISTER: 4213 case MCOI::OPERAND_PCREL: 4214 case MCOI::OPERAND_GENERIC_0: 4215 case MCOI::OPERAND_GENERIC_1: 4216 case MCOI::OPERAND_GENERIC_2: 4217 case MCOI::OPERAND_GENERIC_3: 4218 case MCOI::OPERAND_GENERIC_4: 4219 case MCOI::OPERAND_GENERIC_5: 4220 // Just ignore anything else. 4221 return true; 4222 default: 4223 llvm_unreachable("invalid operand type"); 4224 } 4225 } 4226 4227 static bool compareMachineOp(const MachineOperand &Op0, 4228 const MachineOperand &Op1) { 4229 if (Op0.getType() != Op1.getType()) 4230 return false; 4231 4232 switch (Op0.getType()) { 4233 case MachineOperand::MO_Register: 4234 return Op0.getReg() == Op1.getReg(); 4235 case MachineOperand::MO_Immediate: 4236 return Op0.getImm() == Op1.getImm(); 4237 default: 4238 llvm_unreachable("Didn't expect to be comparing these operand types"); 4239 } 4240 } 4241 4242 bool SIInstrInfo::isImmOperandLegal(const MachineInstr &MI, unsigned OpNo, 4243 const MachineOperand &MO) const { 4244 const MCInstrDesc &InstDesc = MI.getDesc(); 4245 const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo]; 4246 4247 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal()); 4248 4249 if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE) 4250 return true; 4251 4252 if (OpInfo.RegClass < 0) 4253 return false; 4254 4255 if (MO.isImm() && isInlineConstant(MO, OpInfo)) { 4256 if (isMAI(MI) && ST.hasMFMAInlineLiteralBug() && 4257 OpNo ==(unsigned)AMDGPU::getNamedOperandIdx(MI.getOpcode(), 4258 AMDGPU::OpName::src2)) 4259 return false; 4260 return RI.opCanUseInlineConstant(OpInfo.OperandType); 4261 } 4262 4263 if (!RI.opCanUseLiteralConstant(OpInfo.OperandType)) 4264 return false; 4265 4266 if (!isVOP3(MI) || !AMDGPU::isSISrcOperand(InstDesc, OpNo)) 4267 return true; 4268 4269 return ST.hasVOP3Literal(); 4270 } 4271 4272 bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const { 4273 // GFX90A does not have V_MUL_LEGACY_F32_e32. 4274 if (Opcode == AMDGPU::V_MUL_LEGACY_F32_e64 && ST.hasGFX90AInsts()) 4275 return false; 4276 4277 int Op32 = AMDGPU::getVOPe32(Opcode); 4278 if (Op32 == -1) 4279 return false; 4280 4281 return pseudoToMCOpcode(Op32) != -1; 4282 } 4283 4284 bool SIInstrInfo::hasModifiers(unsigned Opcode) const { 4285 // The src0_modifier operand is present on all instructions 4286 // that have modifiers. 4287 4288 return AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src0_modifiers); 4289 } 4290 4291 bool SIInstrInfo::hasModifiersSet(const MachineInstr &MI, 4292 unsigned OpName) const { 4293 const MachineOperand *Mods = getNamedOperand(MI, OpName); 4294 return Mods && Mods->getImm(); 4295 } 4296 4297 bool SIInstrInfo::hasAnyModifiersSet(const MachineInstr &MI) const { 4298 return any_of(ModifierOpNames, 4299 [&](unsigned Name) { return hasModifiersSet(MI, Name); }); 4300 } 4301 4302 bool SIInstrInfo::canShrink(const MachineInstr &MI, 4303 const MachineRegisterInfo &MRI) const { 4304 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); 4305 // Can't shrink instruction with three operands. 4306 if (Src2) { 4307 switch (MI.getOpcode()) { 4308 default: return false; 4309 4310 case AMDGPU::V_ADDC_U32_e64: 4311 case AMDGPU::V_SUBB_U32_e64: 4312 case AMDGPU::V_SUBBREV_U32_e64: { 4313 const MachineOperand *Src1 4314 = getNamedOperand(MI, AMDGPU::OpName::src1); 4315 if (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg())) 4316 return false; 4317 // Additional verification is needed for sdst/src2. 4318 return true; 4319 } 4320 case AMDGPU::V_MAC_F16_e64: 4321 case AMDGPU::V_MAC_F32_e64: 4322 case AMDGPU::V_MAC_LEGACY_F32_e64: 4323 case AMDGPU::V_FMAC_F16_e64: 4324 case AMDGPU::V_FMAC_F16_t16_e64: 4325 case AMDGPU::V_FMAC_F32_e64: 4326 case AMDGPU::V_FMAC_F64_e64: 4327 case AMDGPU::V_FMAC_LEGACY_F32_e64: 4328 if (!Src2->isReg() || !RI.isVGPR(MRI, Src2->getReg()) || 4329 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers)) 4330 return false; 4331 break; 4332 4333 case AMDGPU::V_CNDMASK_B32_e64: 4334 break; 4335 } 4336 } 4337 4338 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); 4339 if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()) || 4340 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers))) 4341 return false; 4342 4343 // We don't need to check src0, all input types are legal, so just make sure 4344 // src0 isn't using any modifiers. 4345 if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers)) 4346 return false; 4347 4348 // Can it be shrunk to a valid 32 bit opcode? 4349 if (!hasVALU32BitEncoding(MI.getOpcode())) 4350 return false; 4351 4352 // Check output modifiers 4353 return !hasModifiersSet(MI, AMDGPU::OpName::omod) && 4354 !hasModifiersSet(MI, AMDGPU::OpName::clamp); 4355 } 4356 4357 // Set VCC operand with all flags from \p Orig, except for setting it as 4358 // implicit. 4359 static void copyFlagsToImplicitVCC(MachineInstr &MI, 4360 const MachineOperand &Orig) { 4361 4362 for (MachineOperand &Use : MI.implicit_operands()) { 4363 if (Use.isUse() && 4364 (Use.getReg() == AMDGPU::VCC || Use.getReg() == AMDGPU::VCC_LO)) { 4365 Use.setIsUndef(Orig.isUndef()); 4366 Use.setIsKill(Orig.isKill()); 4367 return; 4368 } 4369 } 4370 } 4371 4372 MachineInstr *SIInstrInfo::buildShrunkInst(MachineInstr &MI, 4373 unsigned Op32) const { 4374 MachineBasicBlock *MBB = MI.getParent(); 4375 MachineInstrBuilder Inst32 = 4376 BuildMI(*MBB, MI, MI.getDebugLoc(), get(Op32)) 4377 .setMIFlags(MI.getFlags()); 4378 4379 // Add the dst operand if the 32-bit encoding also has an explicit $vdst. 4380 // For VOPC instructions, this is replaced by an implicit def of vcc. 4381 if (AMDGPU::hasNamedOperand(Op32, AMDGPU::OpName::vdst)) { 4382 // dst 4383 Inst32.add(MI.getOperand(0)); 4384 } else if (AMDGPU::hasNamedOperand(Op32, AMDGPU::OpName::sdst)) { 4385 // VOPCX instructions won't be writing to an explicit dst, so this should 4386 // not fail for these instructions. 4387 assert(((MI.getOperand(0).getReg() == AMDGPU::VCC) || 4388 (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) && 4389 "Unexpected case"); 4390 } 4391 4392 Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0)); 4393 4394 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); 4395 if (Src1) 4396 Inst32.add(*Src1); 4397 4398 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); 4399 4400 if (Src2) { 4401 int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2); 4402 if (Op32Src2Idx != -1) { 4403 Inst32.add(*Src2); 4404 } else { 4405 // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is 4406 // replaced with an implicit read of vcc or vcc_lo. The implicit read 4407 // of vcc was already added during the initial BuildMI, but we 4408 // 1) may need to change vcc to vcc_lo to preserve the original register 4409 // 2) have to preserve the original flags. 4410 fixImplicitOperands(*Inst32); 4411 copyFlagsToImplicitVCC(*Inst32, *Src2); 4412 } 4413 } 4414 4415 return Inst32; 4416 } 4417 4418 bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo &MRI, 4419 const MachineOperand &MO, 4420 const MCOperandInfo &OpInfo) const { 4421 // Literal constants use the constant bus. 4422 if (!MO.isReg()) 4423 return !isInlineConstant(MO, OpInfo); 4424 4425 if (!MO.isUse()) 4426 return false; 4427 4428 if (MO.getReg().isVirtual()) 4429 return RI.isSGPRClass(MRI.getRegClass(MO.getReg())); 4430 4431 // Null is free 4432 if (MO.getReg() == AMDGPU::SGPR_NULL || MO.getReg() == AMDGPU::SGPR_NULL64) 4433 return false; 4434 4435 // SGPRs use the constant bus 4436 if (MO.isImplicit()) { 4437 return MO.getReg() == AMDGPU::M0 || 4438 MO.getReg() == AMDGPU::VCC || 4439 MO.getReg() == AMDGPU::VCC_LO; 4440 } else { 4441 return AMDGPU::SReg_32RegClass.contains(MO.getReg()) || 4442 AMDGPU::SReg_64RegClass.contains(MO.getReg()); 4443 } 4444 } 4445 4446 static Register findImplicitSGPRRead(const MachineInstr &MI) { 4447 for (const MachineOperand &MO : MI.implicit_operands()) { 4448 // We only care about reads. 4449 if (MO.isDef()) 4450 continue; 4451 4452 switch (MO.getReg()) { 4453 case AMDGPU::VCC: 4454 case AMDGPU::VCC_LO: 4455 case AMDGPU::VCC_HI: 4456 case AMDGPU::M0: 4457 case AMDGPU::FLAT_SCR: 4458 return MO.getReg(); 4459 4460 default: 4461 break; 4462 } 4463 } 4464 4465 return Register(); 4466 } 4467 4468 static bool shouldReadExec(const MachineInstr &MI) { 4469 if (SIInstrInfo::isVALU(MI)) { 4470 switch (MI.getOpcode()) { 4471 case AMDGPU::V_READLANE_B32: 4472 case AMDGPU::SI_RESTORE_S32_FROM_VGPR: 4473 case AMDGPU::V_WRITELANE_B32: 4474 case AMDGPU::SI_SPILL_S32_TO_VGPR: 4475 return false; 4476 } 4477 4478 return true; 4479 } 4480 4481 if (MI.isPreISelOpcode() || 4482 SIInstrInfo::isGenericOpcode(MI.getOpcode()) || 4483 SIInstrInfo::isSALU(MI) || 4484 SIInstrInfo::isSMRD(MI)) 4485 return false; 4486 4487 return true; 4488 } 4489 4490 static bool isSubRegOf(const SIRegisterInfo &TRI, 4491 const MachineOperand &SuperVec, 4492 const MachineOperand &SubReg) { 4493 if (SubReg.getReg().isPhysical()) 4494 return TRI.isSubRegister(SuperVec.getReg(), SubReg.getReg()); 4495 4496 return SubReg.getSubReg() != AMDGPU::NoSubRegister && 4497 SubReg.getReg() == SuperVec.getReg(); 4498 } 4499 4500 bool SIInstrInfo::verifyInstruction(const MachineInstr &MI, 4501 StringRef &ErrInfo) const { 4502 uint16_t Opcode = MI.getOpcode(); 4503 if (SIInstrInfo::isGenericOpcode(MI.getOpcode())) 4504 return true; 4505 4506 const MachineFunction *MF = MI.getParent()->getParent(); 4507 const MachineRegisterInfo &MRI = MF->getRegInfo(); 4508 4509 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0); 4510 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1); 4511 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2); 4512 int Src3Idx = -1; 4513 if (Src0Idx == -1) { 4514 // VOPD V_DUAL_* instructions use different operand names. 4515 Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0X); 4516 Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vsrc1X); 4517 Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0Y); 4518 Src3Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vsrc1Y); 4519 } 4520 4521 // Make sure the number of operands is correct. 4522 const MCInstrDesc &Desc = get(Opcode); 4523 if (!Desc.isVariadic() && 4524 Desc.getNumOperands() != MI.getNumExplicitOperands()) { 4525 ErrInfo = "Instruction has wrong number of operands."; 4526 return false; 4527 } 4528 4529 if (MI.isInlineAsm()) { 4530 // Verify register classes for inlineasm constraints. 4531 for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands(); 4532 I != E; ++I) { 4533 const TargetRegisterClass *RC = MI.getRegClassConstraint(I, this, &RI); 4534 if (!RC) 4535 continue; 4536 4537 const MachineOperand &Op = MI.getOperand(I); 4538 if (!Op.isReg()) 4539 continue; 4540 4541 Register Reg = Op.getReg(); 4542 if (!Reg.isVirtual() && !RC->contains(Reg)) { 4543 ErrInfo = "inlineasm operand has incorrect register class."; 4544 return false; 4545 } 4546 } 4547 4548 return true; 4549 } 4550 4551 if (isImage(MI) && MI.memoperands_empty() && MI.mayLoadOrStore()) { 4552 ErrInfo = "missing memory operand from image instruction."; 4553 return false; 4554 } 4555 4556 // Make sure the register classes are correct. 4557 for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) { 4558 const MachineOperand &MO = MI.getOperand(i); 4559 if (MO.isFPImm()) { 4560 ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast " 4561 "all fp values to integers."; 4562 return false; 4563 } 4564 4565 int RegClass = Desc.operands()[i].RegClass; 4566 4567 switch (Desc.operands()[i].OperandType) { 4568 case MCOI::OPERAND_REGISTER: 4569 if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) { 4570 ErrInfo = "Illegal immediate value for operand."; 4571 return false; 4572 } 4573 break; 4574 case AMDGPU::OPERAND_REG_IMM_INT32: 4575 case AMDGPU::OPERAND_REG_IMM_FP32: 4576 case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED: 4577 case AMDGPU::OPERAND_REG_IMM_V2FP32: 4578 break; 4579 case AMDGPU::OPERAND_REG_INLINE_C_INT32: 4580 case AMDGPU::OPERAND_REG_INLINE_C_FP32: 4581 case AMDGPU::OPERAND_REG_INLINE_C_INT64: 4582 case AMDGPU::OPERAND_REG_INLINE_C_FP64: 4583 case AMDGPU::OPERAND_REG_INLINE_C_INT16: 4584 case AMDGPU::OPERAND_REG_INLINE_C_FP16: 4585 case AMDGPU::OPERAND_REG_INLINE_AC_INT32: 4586 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: 4587 case AMDGPU::OPERAND_REG_INLINE_AC_INT16: 4588 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: 4589 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: { 4590 if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, i))) { 4591 ErrInfo = "Illegal immediate value for operand."; 4592 return false; 4593 } 4594 break; 4595 } 4596 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32: 4597 if (!MI.getOperand(i).isImm() || !isInlineConstant(MI, i)) { 4598 ErrInfo = "Expected inline constant for operand."; 4599 return false; 4600 } 4601 break; 4602 case MCOI::OPERAND_IMMEDIATE: 4603 case AMDGPU::OPERAND_KIMM32: 4604 // Check if this operand is an immediate. 4605 // FrameIndex operands will be replaced by immediates, so they are 4606 // allowed. 4607 if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) { 4608 ErrInfo = "Expected immediate, but got non-immediate"; 4609 return false; 4610 } 4611 [[fallthrough]]; 4612 default: 4613 continue; 4614 } 4615 4616 if (!MO.isReg()) 4617 continue; 4618 Register Reg = MO.getReg(); 4619 if (!Reg) 4620 continue; 4621 4622 // FIXME: Ideally we would have separate instruction definitions with the 4623 // aligned register constraint. 4624 // FIXME: We do not verify inline asm operands, but custom inline asm 4625 // verification is broken anyway 4626 if (ST.needsAlignedVGPRs()) { 4627 const TargetRegisterClass *RC = RI.getRegClassForReg(MRI, Reg); 4628 if (RI.hasVectorRegisters(RC) && MO.getSubReg()) { 4629 const TargetRegisterClass *SubRC = 4630 RI.getSubRegisterClass(RC, MO.getSubReg()); 4631 RC = RI.getCompatibleSubRegClass(RC, SubRC, MO.getSubReg()); 4632 if (RC) 4633 RC = SubRC; 4634 } 4635 4636 // Check that this is the aligned version of the class. 4637 if (!RC || !RI.isProperlyAlignedRC(*RC)) { 4638 ErrInfo = "Subtarget requires even aligned vector registers"; 4639 return false; 4640 } 4641 } 4642 4643 if (RegClass != -1) { 4644 if (Reg.isVirtual()) 4645 continue; 4646 4647 const TargetRegisterClass *RC = RI.getRegClass(RegClass); 4648 if (!RC->contains(Reg)) { 4649 ErrInfo = "Operand has incorrect register class."; 4650 return false; 4651 } 4652 } 4653 } 4654 4655 // Verify SDWA 4656 if (isSDWA(MI)) { 4657 if (!ST.hasSDWA()) { 4658 ErrInfo = "SDWA is not supported on this target"; 4659 return false; 4660 } 4661 4662 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst); 4663 4664 for (int OpIdx : {DstIdx, Src0Idx, Src1Idx, Src2Idx}) { 4665 if (OpIdx == -1) 4666 continue; 4667 const MachineOperand &MO = MI.getOperand(OpIdx); 4668 4669 if (!ST.hasSDWAScalar()) { 4670 // Only VGPRS on VI 4671 if (!MO.isReg() || !RI.hasVGPRs(RI.getRegClassForReg(MRI, MO.getReg()))) { 4672 ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI"; 4673 return false; 4674 } 4675 } else { 4676 // No immediates on GFX9 4677 if (!MO.isReg()) { 4678 ErrInfo = 4679 "Only reg allowed as operands in SDWA instructions on GFX9+"; 4680 return false; 4681 } 4682 } 4683 } 4684 4685 if (!ST.hasSDWAOmod()) { 4686 // No omod allowed on VI 4687 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod); 4688 if (OMod != nullptr && 4689 (!OMod->isImm() || OMod->getImm() != 0)) { 4690 ErrInfo = "OMod not allowed in SDWA instructions on VI"; 4691 return false; 4692 } 4693 } 4694 4695 uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode); 4696 if (isVOPC(BasicOpcode)) { 4697 if (!ST.hasSDWASdst() && DstIdx != -1) { 4698 // Only vcc allowed as dst on VI for VOPC 4699 const MachineOperand &Dst = MI.getOperand(DstIdx); 4700 if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) { 4701 ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI"; 4702 return false; 4703 } 4704 } else if (!ST.hasSDWAOutModsVOPC()) { 4705 // No clamp allowed on GFX9 for VOPC 4706 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp); 4707 if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) { 4708 ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI"; 4709 return false; 4710 } 4711 4712 // No omod allowed on GFX9 for VOPC 4713 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod); 4714 if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) { 4715 ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI"; 4716 return false; 4717 } 4718 } 4719 } 4720 4721 const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused); 4722 if (DstUnused && DstUnused->isImm() && 4723 DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) { 4724 const MachineOperand &Dst = MI.getOperand(DstIdx); 4725 if (!Dst.isReg() || !Dst.isTied()) { 4726 ErrInfo = "Dst register should have tied register"; 4727 return false; 4728 } 4729 4730 const MachineOperand &TiedMO = 4731 MI.getOperand(MI.findTiedOperandIdx(DstIdx)); 4732 if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) { 4733 ErrInfo = 4734 "Dst register should be tied to implicit use of preserved register"; 4735 return false; 4736 } else if (TiedMO.getReg().isPhysical() && 4737 Dst.getReg() != TiedMO.getReg()) { 4738 ErrInfo = "Dst register should use same physical register as preserved"; 4739 return false; 4740 } 4741 } 4742 } 4743 4744 // Verify MIMG / VIMAGE / VSAMPLE 4745 if (isImage(MI.getOpcode()) && !MI.mayStore()) { 4746 // Ensure that the return type used is large enough for all the options 4747 // being used TFE/LWE require an extra result register. 4748 const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask); 4749 if (DMask) { 4750 uint64_t DMaskImm = DMask->getImm(); 4751 uint32_t RegCount = 4752 isGather4(MI.getOpcode()) ? 4 : llvm::popcount(DMaskImm); 4753 const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe); 4754 const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe); 4755 const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16); 4756 4757 // Adjust for packed 16 bit values 4758 if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem()) 4759 RegCount = divideCeil(RegCount, 2); 4760 4761 // Adjust if using LWE or TFE 4762 if ((LWE && LWE->getImm()) || (TFE && TFE->getImm())) 4763 RegCount += 1; 4764 4765 const uint32_t DstIdx = 4766 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata); 4767 const MachineOperand &Dst = MI.getOperand(DstIdx); 4768 if (Dst.isReg()) { 4769 const TargetRegisterClass *DstRC = getOpRegClass(MI, DstIdx); 4770 uint32_t DstSize = RI.getRegSizeInBits(*DstRC) / 32; 4771 if (RegCount > DstSize) { 4772 ErrInfo = "Image instruction returns too many registers for dst " 4773 "register class"; 4774 return false; 4775 } 4776 } 4777 } 4778 } 4779 4780 // Verify VOP*. Ignore multiple sgpr operands on writelane. 4781 if (isVALU(MI) && Desc.getOpcode() != AMDGPU::V_WRITELANE_B32) { 4782 unsigned ConstantBusCount = 0; 4783 bool UsesLiteral = false; 4784 const MachineOperand *LiteralVal = nullptr; 4785 4786 int ImmIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm); 4787 if (ImmIdx != -1) { 4788 ++ConstantBusCount; 4789 UsesLiteral = true; 4790 LiteralVal = &MI.getOperand(ImmIdx); 4791 } 4792 4793 SmallVector<Register, 2> SGPRsUsed; 4794 Register SGPRUsed; 4795 4796 // Only look at the true operands. Only a real operand can use the constant 4797 // bus, and we don't want to check pseudo-operands like the source modifier 4798 // flags. 4799 for (int OpIdx : {Src0Idx, Src1Idx, Src2Idx, Src3Idx}) { 4800 if (OpIdx == -1) 4801 continue; 4802 const MachineOperand &MO = MI.getOperand(OpIdx); 4803 if (usesConstantBus(MRI, MO, MI.getDesc().operands()[OpIdx])) { 4804 if (MO.isReg()) { 4805 SGPRUsed = MO.getReg(); 4806 if (!llvm::is_contained(SGPRsUsed, SGPRUsed)) { 4807 ++ConstantBusCount; 4808 SGPRsUsed.push_back(SGPRUsed); 4809 } 4810 } else { 4811 if (!UsesLiteral) { 4812 ++ConstantBusCount; 4813 UsesLiteral = true; 4814 LiteralVal = &MO; 4815 } else if (!MO.isIdenticalTo(*LiteralVal)) { 4816 assert(isVOP2(MI) || isVOP3(MI)); 4817 ErrInfo = "VOP2/VOP3 instruction uses more than one literal"; 4818 return false; 4819 } 4820 } 4821 } 4822 } 4823 4824 SGPRUsed = findImplicitSGPRRead(MI); 4825 if (SGPRUsed) { 4826 // Implicit uses may safely overlap true operands 4827 if (llvm::all_of(SGPRsUsed, [this, SGPRUsed](unsigned SGPR) { 4828 return !RI.regsOverlap(SGPRUsed, SGPR); 4829 })) { 4830 ++ConstantBusCount; 4831 SGPRsUsed.push_back(SGPRUsed); 4832 } 4833 } 4834 4835 // v_writelane_b32 is an exception from constant bus restriction: 4836 // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const 4837 if (ConstantBusCount > ST.getConstantBusLimit(Opcode) && 4838 Opcode != AMDGPU::V_WRITELANE_B32) { 4839 ErrInfo = "VOP* instruction violates constant bus restriction"; 4840 return false; 4841 } 4842 4843 if (isVOP3(MI) && UsesLiteral && !ST.hasVOP3Literal()) { 4844 ErrInfo = "VOP3 instruction uses literal"; 4845 return false; 4846 } 4847 } 4848 4849 // Special case for writelane - this can break the multiple constant bus rule, 4850 // but still can't use more than one SGPR register 4851 if (Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) { 4852 unsigned SGPRCount = 0; 4853 Register SGPRUsed; 4854 4855 for (int OpIdx : {Src0Idx, Src1Idx}) { 4856 if (OpIdx == -1) 4857 break; 4858 4859 const MachineOperand &MO = MI.getOperand(OpIdx); 4860 4861 if (usesConstantBus(MRI, MO, MI.getDesc().operands()[OpIdx])) { 4862 if (MO.isReg() && MO.getReg() != AMDGPU::M0) { 4863 if (MO.getReg() != SGPRUsed) 4864 ++SGPRCount; 4865 SGPRUsed = MO.getReg(); 4866 } 4867 } 4868 if (SGPRCount > ST.getConstantBusLimit(Opcode)) { 4869 ErrInfo = "WRITELANE instruction violates constant bus restriction"; 4870 return false; 4871 } 4872 } 4873 } 4874 4875 // Verify misc. restrictions on specific instructions. 4876 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32_e64 || 4877 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64_e64) { 4878 const MachineOperand &Src0 = MI.getOperand(Src0Idx); 4879 const MachineOperand &Src1 = MI.getOperand(Src1Idx); 4880 const MachineOperand &Src2 = MI.getOperand(Src2Idx); 4881 if (Src0.isReg() && Src1.isReg() && Src2.isReg()) { 4882 if (!compareMachineOp(Src0, Src1) && 4883 !compareMachineOp(Src0, Src2)) { 4884 ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2"; 4885 return false; 4886 } 4887 } 4888 if ((getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm() & 4889 SISrcMods::ABS) || 4890 (getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm() & 4891 SISrcMods::ABS) || 4892 (getNamedOperand(MI, AMDGPU::OpName::src2_modifiers)->getImm() & 4893 SISrcMods::ABS)) { 4894 ErrInfo = "ABS not allowed in VOP3B instructions"; 4895 return false; 4896 } 4897 } 4898 4899 if (isSOP2(MI) || isSOPC(MI)) { 4900 const MachineOperand &Src0 = MI.getOperand(Src0Idx); 4901 const MachineOperand &Src1 = MI.getOperand(Src1Idx); 4902 4903 if (!Src0.isReg() && !Src1.isReg() && 4904 !isInlineConstant(Src0, Desc.operands()[Src0Idx]) && 4905 !isInlineConstant(Src1, Desc.operands()[Src1Idx]) && 4906 !Src0.isIdenticalTo(Src1)) { 4907 ErrInfo = "SOP2/SOPC instruction requires too many immediate constants"; 4908 return false; 4909 } 4910 } 4911 4912 if (isSOPK(MI)) { 4913 auto Op = getNamedOperand(MI, AMDGPU::OpName::simm16); 4914 if (Desc.isBranch()) { 4915 if (!Op->isMBB()) { 4916 ErrInfo = "invalid branch target for SOPK instruction"; 4917 return false; 4918 } 4919 } else { 4920 uint64_t Imm = Op->getImm(); 4921 if (sopkIsZext(MI)) { 4922 if (!isUInt<16>(Imm)) { 4923 ErrInfo = "invalid immediate for SOPK instruction"; 4924 return false; 4925 } 4926 } else { 4927 if (!isInt<16>(Imm)) { 4928 ErrInfo = "invalid immediate for SOPK instruction"; 4929 return false; 4930 } 4931 } 4932 } 4933 } 4934 4935 if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 || 4936 Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 || 4937 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 || 4938 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) { 4939 const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 || 4940 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64; 4941 4942 const unsigned StaticNumOps = 4943 Desc.getNumOperands() + Desc.implicit_uses().size(); 4944 const unsigned NumImplicitOps = IsDst ? 2 : 1; 4945 4946 // Allow additional implicit operands. This allows a fixup done by the post 4947 // RA scheduler where the main implicit operand is killed and implicit-defs 4948 // are added for sub-registers that remain live after this instruction. 4949 if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) { 4950 ErrInfo = "missing implicit register operands"; 4951 return false; 4952 } 4953 4954 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); 4955 if (IsDst) { 4956 if (!Dst->isUse()) { 4957 ErrInfo = "v_movreld_b32 vdst should be a use operand"; 4958 return false; 4959 } 4960 4961 unsigned UseOpIdx; 4962 if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) || 4963 UseOpIdx != StaticNumOps + 1) { 4964 ErrInfo = "movrel implicit operands should be tied"; 4965 return false; 4966 } 4967 } 4968 4969 const MachineOperand &Src0 = MI.getOperand(Src0Idx); 4970 const MachineOperand &ImpUse 4971 = MI.getOperand(StaticNumOps + NumImplicitOps - 1); 4972 if (!ImpUse.isReg() || !ImpUse.isUse() || 4973 !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) { 4974 ErrInfo = "src0 should be subreg of implicit vector use"; 4975 return false; 4976 } 4977 } 4978 4979 // Make sure we aren't losing exec uses in the td files. This mostly requires 4980 // being careful when using let Uses to try to add other use registers. 4981 if (shouldReadExec(MI)) { 4982 if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) { 4983 ErrInfo = "VALU instruction does not implicitly read exec mask"; 4984 return false; 4985 } 4986 } 4987 4988 if (isSMRD(MI)) { 4989 if (MI.mayStore() && 4990 ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS) { 4991 // The register offset form of scalar stores may only use m0 as the 4992 // soffset register. 4993 const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soffset); 4994 if (Soff && Soff->getReg() != AMDGPU::M0) { 4995 ErrInfo = "scalar stores must use m0 as offset register"; 4996 return false; 4997 } 4998 } 4999 } 5000 5001 if (isFLAT(MI) && !ST.hasFlatInstOffsets()) { 5002 const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset); 5003 if (Offset->getImm() != 0) { 5004 ErrInfo = "subtarget does not support offsets in flat instructions"; 5005 return false; 5006 } 5007 } 5008 5009 if (isDS(MI) && !ST.hasGDS()) { 5010 const MachineOperand *GDSOp = getNamedOperand(MI, AMDGPU::OpName::gds); 5011 if (GDSOp && GDSOp->getImm() != 0) { 5012 ErrInfo = "GDS is not supported on this subtarget"; 5013 return false; 5014 } 5015 } 5016 5017 if (isImage(MI)) { 5018 const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim); 5019 if (DimOp) { 5020 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode, 5021 AMDGPU::OpName::vaddr0); 5022 int RSrcOpName = 5023 isMIMG(MI) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc; 5024 int RsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, RSrcOpName); 5025 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opcode); 5026 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = 5027 AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode); 5028 const AMDGPU::MIMGDimInfo *Dim = 5029 AMDGPU::getMIMGDimInfoByEncoding(DimOp->getImm()); 5030 5031 if (!Dim) { 5032 ErrInfo = "dim is out of range"; 5033 return false; 5034 } 5035 5036 bool IsA16 = false; 5037 if (ST.hasR128A16()) { 5038 const MachineOperand *R128A16 = getNamedOperand(MI, AMDGPU::OpName::r128); 5039 IsA16 = R128A16->getImm() != 0; 5040 } else if (ST.hasA16()) { 5041 const MachineOperand *A16 = getNamedOperand(MI, AMDGPU::OpName::a16); 5042 IsA16 = A16->getImm() != 0; 5043 } 5044 5045 bool IsNSA = RsrcIdx - VAddr0Idx > 1; 5046 5047 unsigned AddrWords = 5048 AMDGPU::getAddrSizeMIMGOp(BaseOpcode, Dim, IsA16, ST.hasG16()); 5049 5050 unsigned VAddrWords; 5051 if (IsNSA) { 5052 VAddrWords = RsrcIdx - VAddr0Idx; 5053 if (ST.hasPartialNSAEncoding() && 5054 AddrWords > ST.getNSAMaxSize(isVSAMPLE(MI))) { 5055 unsigned LastVAddrIdx = RsrcIdx - 1; 5056 VAddrWords += getOpSize(MI, LastVAddrIdx) / 4 - 1; 5057 } 5058 } else { 5059 VAddrWords = getOpSize(MI, VAddr0Idx) / 4; 5060 if (AddrWords > 12) 5061 AddrWords = 16; 5062 } 5063 5064 if (VAddrWords != AddrWords) { 5065 LLVM_DEBUG(dbgs() << "bad vaddr size, expected " << AddrWords 5066 << " but got " << VAddrWords << "\n"); 5067 ErrInfo = "bad vaddr size"; 5068 return false; 5069 } 5070 } 5071 } 5072 5073 const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl); 5074 if (DppCt) { 5075 using namespace AMDGPU::DPP; 5076 5077 unsigned DC = DppCt->getImm(); 5078 if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 || 5079 DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST || 5080 (DC >= DppCtrl::DPP_UNUSED4_FIRST && DC <= DppCtrl::DPP_UNUSED4_LAST) || 5081 (DC >= DppCtrl::DPP_UNUSED5_FIRST && DC <= DppCtrl::DPP_UNUSED5_LAST) || 5082 (DC >= DppCtrl::DPP_UNUSED6_FIRST && DC <= DppCtrl::DPP_UNUSED6_LAST) || 5083 (DC >= DppCtrl::DPP_UNUSED7_FIRST && DC <= DppCtrl::DPP_UNUSED7_LAST) || 5084 (DC >= DppCtrl::DPP_UNUSED8_FIRST && DC <= DppCtrl::DPP_UNUSED8_LAST)) { 5085 ErrInfo = "Invalid dpp_ctrl value"; 5086 return false; 5087 } 5088 if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 && 5089 ST.getGeneration() >= AMDGPUSubtarget::GFX10) { 5090 ErrInfo = "Invalid dpp_ctrl value: " 5091 "wavefront shifts are not supported on GFX10+"; 5092 return false; 5093 } 5094 if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 && 5095 ST.getGeneration() >= AMDGPUSubtarget::GFX10) { 5096 ErrInfo = "Invalid dpp_ctrl value: " 5097 "broadcasts are not supported on GFX10+"; 5098 return false; 5099 } 5100 if (DC >= DppCtrl::ROW_SHARE_FIRST && DC <= DppCtrl::ROW_XMASK_LAST && 5101 ST.getGeneration() < AMDGPUSubtarget::GFX10) { 5102 if (DC >= DppCtrl::ROW_NEWBCAST_FIRST && 5103 DC <= DppCtrl::ROW_NEWBCAST_LAST && 5104 !ST.hasGFX90AInsts()) { 5105 ErrInfo = "Invalid dpp_ctrl value: " 5106 "row_newbroadcast/row_share is not supported before " 5107 "GFX90A/GFX10"; 5108 return false; 5109 } else if (DC > DppCtrl::ROW_NEWBCAST_LAST || !ST.hasGFX90AInsts()) { 5110 ErrInfo = "Invalid dpp_ctrl value: " 5111 "row_share and row_xmask are not supported before GFX10"; 5112 return false; 5113 } 5114 } 5115 5116 if (Opcode != AMDGPU::V_MOV_B64_DPP_PSEUDO && 5117 !AMDGPU::isLegalDPALU_DPPControl(DC) && AMDGPU::isDPALU_DPP(Desc)) { 5118 ErrInfo = "Invalid dpp_ctrl value: " 5119 "DP ALU dpp only support row_newbcast"; 5120 return false; 5121 } 5122 } 5123 5124 if ((MI.mayStore() || MI.mayLoad()) && !isVGPRSpill(MI)) { 5125 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); 5126 uint16_t DataNameIdx = isDS(Opcode) ? AMDGPU::OpName::data0 5127 : AMDGPU::OpName::vdata; 5128 const MachineOperand *Data = getNamedOperand(MI, DataNameIdx); 5129 const MachineOperand *Data2 = getNamedOperand(MI, AMDGPU::OpName::data1); 5130 if (Data && !Data->isReg()) 5131 Data = nullptr; 5132 5133 if (ST.hasGFX90AInsts()) { 5134 if (Dst && Data && 5135 (RI.isAGPR(MRI, Dst->getReg()) != RI.isAGPR(MRI, Data->getReg()))) { 5136 ErrInfo = "Invalid register class: " 5137 "vdata and vdst should be both VGPR or AGPR"; 5138 return false; 5139 } 5140 if (Data && Data2 && 5141 (RI.isAGPR(MRI, Data->getReg()) != RI.isAGPR(MRI, Data2->getReg()))) { 5142 ErrInfo = "Invalid register class: " 5143 "both data operands should be VGPR or AGPR"; 5144 return false; 5145 } 5146 } else { 5147 if ((Dst && RI.isAGPR(MRI, Dst->getReg())) || 5148 (Data && RI.isAGPR(MRI, Data->getReg())) || 5149 (Data2 && RI.isAGPR(MRI, Data2->getReg()))) { 5150 ErrInfo = "Invalid register class: " 5151 "agpr loads and stores not supported on this GPU"; 5152 return false; 5153 } 5154 } 5155 } 5156 5157 if (ST.needsAlignedVGPRs()) { 5158 const auto isAlignedReg = [&MI, &MRI, this](unsigned OpName) -> bool { 5159 const MachineOperand *Op = getNamedOperand(MI, OpName); 5160 if (!Op) 5161 return true; 5162 Register Reg = Op->getReg(); 5163 if (Reg.isPhysical()) 5164 return !(RI.getHWRegIndex(Reg) & 1); 5165 const TargetRegisterClass &RC = *MRI.getRegClass(Reg); 5166 return RI.getRegSizeInBits(RC) > 32 && RI.isProperlyAlignedRC(RC) && 5167 !(RI.getChannelFromSubReg(Op->getSubReg()) & 1); 5168 }; 5169 5170 if (MI.getOpcode() == AMDGPU::DS_GWS_INIT || 5171 MI.getOpcode() == AMDGPU::DS_GWS_SEMA_BR || 5172 MI.getOpcode() == AMDGPU::DS_GWS_BARRIER) { 5173 5174 if (!isAlignedReg(AMDGPU::OpName::data0)) { 5175 ErrInfo = "Subtarget requires even aligned vector registers " 5176 "for DS_GWS instructions"; 5177 return false; 5178 } 5179 } 5180 5181 if (isMIMG(MI)) { 5182 if (!isAlignedReg(AMDGPU::OpName::vaddr)) { 5183 ErrInfo = "Subtarget requires even aligned vector registers " 5184 "for vaddr operand of image instructions"; 5185 return false; 5186 } 5187 } 5188 } 5189 5190 if (MI.getOpcode() == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && 5191 !ST.hasGFX90AInsts()) { 5192 const MachineOperand *Src = getNamedOperand(MI, AMDGPU::OpName::src0); 5193 if (Src->isReg() && RI.isSGPRReg(MRI, Src->getReg())) { 5194 ErrInfo = "Invalid register class: " 5195 "v_accvgpr_write with an SGPR is not supported on this GPU"; 5196 return false; 5197 } 5198 } 5199 5200 if (Desc.getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS) { 5201 const MachineOperand &SrcOp = MI.getOperand(1); 5202 if (!SrcOp.isReg() || SrcOp.getReg().isVirtual()) { 5203 ErrInfo = "pseudo expects only physical SGPRs"; 5204 return false; 5205 } 5206 } 5207 5208 return true; 5209 } 5210 5211 // It is more readable to list mapped opcodes on the same line. 5212 // clang-format off 5213 5214 unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const { 5215 switch (MI.getOpcode()) { 5216 default: return AMDGPU::INSTRUCTION_LIST_END; 5217 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE; 5218 case AMDGPU::COPY: return AMDGPU::COPY; 5219 case AMDGPU::PHI: return AMDGPU::PHI; 5220 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG; 5221 case AMDGPU::WQM: return AMDGPU::WQM; 5222 case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM; 5223 case AMDGPU::STRICT_WWM: return AMDGPU::STRICT_WWM; 5224 case AMDGPU::STRICT_WQM: return AMDGPU::STRICT_WQM; 5225 case AMDGPU::S_MOV_B32: { 5226 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 5227 return MI.getOperand(1).isReg() || 5228 RI.isAGPR(MRI, MI.getOperand(0).getReg()) ? 5229 AMDGPU::COPY : AMDGPU::V_MOV_B32_e32; 5230 } 5231 case AMDGPU::S_ADD_I32: 5232 return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32; 5233 case AMDGPU::S_ADDC_U32: 5234 return AMDGPU::V_ADDC_U32_e32; 5235 case AMDGPU::S_SUB_I32: 5236 return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_CO_U32_e32; 5237 // FIXME: These are not consistently handled, and selected when the carry is 5238 // used. 5239 case AMDGPU::S_ADD_U32: 5240 return AMDGPU::V_ADD_CO_U32_e32; 5241 case AMDGPU::S_SUB_U32: 5242 return AMDGPU::V_SUB_CO_U32_e32; 5243 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32; 5244 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32_e64; 5245 case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32_e64; 5246 case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32_e64; 5247 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64; 5248 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64; 5249 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64; 5250 case AMDGPU::S_XNOR_B32: 5251 return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END; 5252 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64; 5253 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64; 5254 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64; 5255 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64; 5256 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32; 5257 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64_e64; 5258 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32; 5259 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64_e64; 5260 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32; 5261 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64_e64; 5262 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32_e64; 5263 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32_e64; 5264 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32_e64; 5265 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32_e64; 5266 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64; 5267 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32; 5268 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32; 5269 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32; 5270 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e64; 5271 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e64; 5272 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e64; 5273 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e64; 5274 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e64; 5275 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e64; 5276 case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e64; 5277 case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e64; 5278 case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e64; 5279 case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e64; 5280 case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e64; 5281 case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e64; 5282 case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e64; 5283 case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e64; 5284 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64; 5285 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32; 5286 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32; 5287 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64; 5288 case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ; 5289 case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ; 5290 case AMDGPU::S_CVT_F32_I32: return AMDGPU::V_CVT_F32_I32_e64; 5291 case AMDGPU::S_CVT_F32_U32: return AMDGPU::V_CVT_F32_U32_e64; 5292 case AMDGPU::S_CVT_I32_F32: return AMDGPU::V_CVT_I32_F32_e64; 5293 case AMDGPU::S_CVT_U32_F32: return AMDGPU::V_CVT_U32_F32_e64; 5294 case AMDGPU::S_CVT_F32_F16: return AMDGPU::V_CVT_F32_F16_t16_e64; 5295 case AMDGPU::S_CVT_HI_F32_F16: return AMDGPU::V_CVT_F32_F16_t16_e64; 5296 case AMDGPU::S_CVT_F16_F32: return AMDGPU::V_CVT_F16_F32_t16_e64; 5297 case AMDGPU::S_CEIL_F32: return AMDGPU::V_CEIL_F32_e64; 5298 case AMDGPU::S_FLOOR_F32: return AMDGPU::V_FLOOR_F32_e64; 5299 case AMDGPU::S_TRUNC_F32: return AMDGPU::V_TRUNC_F32_e64; 5300 case AMDGPU::S_RNDNE_F32: return AMDGPU::V_RNDNE_F32_e64; 5301 case AMDGPU::S_CEIL_F16: 5302 return ST.useRealTrue16Insts() ? AMDGPU::V_CEIL_F16_t16_e64 5303 : AMDGPU::V_CEIL_F16_fake16_e64; 5304 case AMDGPU::S_FLOOR_F16: 5305 return ST.useRealTrue16Insts() ? AMDGPU::V_FLOOR_F16_t16_e64 5306 : AMDGPU::V_FLOOR_F16_fake16_e64; 5307 case AMDGPU::S_TRUNC_F16: 5308 return AMDGPU::V_TRUNC_F16_fake16_e64; 5309 case AMDGPU::S_RNDNE_F16: 5310 return AMDGPU::V_RNDNE_F16_fake16_e64; 5311 case AMDGPU::S_ADD_F32: return AMDGPU::V_ADD_F32_e64; 5312 case AMDGPU::S_SUB_F32: return AMDGPU::V_SUB_F32_e64; 5313 case AMDGPU::S_MIN_F32: return AMDGPU::V_MIN_F32_e64; 5314 case AMDGPU::S_MAX_F32: return AMDGPU::V_MAX_F32_e64; 5315 case AMDGPU::S_MINIMUM_F32: return AMDGPU::V_MINIMUM_F32_e64; 5316 case AMDGPU::S_MAXIMUM_F32: return AMDGPU::V_MAXIMUM_F32_e64; 5317 case AMDGPU::S_MUL_F32: return AMDGPU::V_MUL_F32_e64; 5318 case AMDGPU::S_ADD_F16: return AMDGPU::V_ADD_F16_fake16_e64; 5319 case AMDGPU::S_SUB_F16: return AMDGPU::V_SUB_F16_fake16_e64; 5320 case AMDGPU::S_MIN_F16: return AMDGPU::V_MIN_F16_fake16_e64; 5321 case AMDGPU::S_MAX_F16: return AMDGPU::V_MAX_F16_fake16_e64; 5322 case AMDGPU::S_MINIMUM_F16: return AMDGPU::V_MINIMUM_F16_e64; 5323 case AMDGPU::S_MAXIMUM_F16: return AMDGPU::V_MAXIMUM_F16_e64; 5324 case AMDGPU::S_MUL_F16: return AMDGPU::V_MUL_F16_fake16_e64; 5325 case AMDGPU::S_CVT_PK_RTZ_F16_F32: return AMDGPU::V_CVT_PKRTZ_F16_F32_e64; 5326 case AMDGPU::S_FMAC_F32: return AMDGPU::V_FMAC_F32_e64; 5327 case AMDGPU::S_FMAC_F16: return AMDGPU::V_FMAC_F16_t16_e64; 5328 case AMDGPU::S_FMAMK_F32: return AMDGPU::V_FMAMK_F32; 5329 case AMDGPU::S_FMAAK_F32: return AMDGPU::V_FMAAK_F32; 5330 case AMDGPU::S_CMP_LT_F32: return AMDGPU::V_CMP_LT_F32_e64; 5331 case AMDGPU::S_CMP_EQ_F32: return AMDGPU::V_CMP_EQ_F32_e64; 5332 case AMDGPU::S_CMP_LE_F32: return AMDGPU::V_CMP_LE_F32_e64; 5333 case AMDGPU::S_CMP_GT_F32: return AMDGPU::V_CMP_GT_F32_e64; 5334 case AMDGPU::S_CMP_LG_F32: return AMDGPU::V_CMP_LG_F32_e64; 5335 case AMDGPU::S_CMP_GE_F32: return AMDGPU::V_CMP_GE_F32_e64; 5336 case AMDGPU::S_CMP_O_F32: return AMDGPU::V_CMP_O_F32_e64; 5337 case AMDGPU::S_CMP_U_F32: return AMDGPU::V_CMP_U_F32_e64; 5338 case AMDGPU::S_CMP_NGE_F32: return AMDGPU::V_CMP_NGE_F32_e64; 5339 case AMDGPU::S_CMP_NLG_F32: return AMDGPU::V_CMP_NLG_F32_e64; 5340 case AMDGPU::S_CMP_NGT_F32: return AMDGPU::V_CMP_NGT_F32_e64; 5341 case AMDGPU::S_CMP_NLE_F32: return AMDGPU::V_CMP_NLE_F32_e64; 5342 case AMDGPU::S_CMP_NEQ_F32: return AMDGPU::V_CMP_NEQ_F32_e64; 5343 case AMDGPU::S_CMP_NLT_F32: return AMDGPU::V_CMP_NLT_F32_e64; 5344 case AMDGPU::S_CMP_LT_F16: return AMDGPU::V_CMP_LT_F16_t16_e64; 5345 case AMDGPU::S_CMP_EQ_F16: return AMDGPU::V_CMP_EQ_F16_t16_e64; 5346 case AMDGPU::S_CMP_LE_F16: return AMDGPU::V_CMP_LE_F16_t16_e64; 5347 case AMDGPU::S_CMP_GT_F16: return AMDGPU::V_CMP_GT_F16_t16_e64; 5348 case AMDGPU::S_CMP_LG_F16: return AMDGPU::V_CMP_LG_F16_t16_e64; 5349 case AMDGPU::S_CMP_GE_F16: return AMDGPU::V_CMP_GE_F16_t16_e64; 5350 case AMDGPU::S_CMP_O_F16: return AMDGPU::V_CMP_O_F16_t16_e64; 5351 case AMDGPU::S_CMP_U_F16: return AMDGPU::V_CMP_U_F16_t16_e64; 5352 case AMDGPU::S_CMP_NGE_F16: return AMDGPU::V_CMP_NGE_F16_t16_e64; 5353 case AMDGPU::S_CMP_NLG_F16: return AMDGPU::V_CMP_NLG_F16_t16_e64; 5354 case AMDGPU::S_CMP_NGT_F16: return AMDGPU::V_CMP_NGT_F16_t16_e64; 5355 case AMDGPU::S_CMP_NLE_F16: return AMDGPU::V_CMP_NLE_F16_t16_e64; 5356 case AMDGPU::S_CMP_NEQ_F16: return AMDGPU::V_CMP_NEQ_F16_t16_e64; 5357 case AMDGPU::S_CMP_NLT_F16: return AMDGPU::V_CMP_NLT_F16_t16_e64; 5358 case AMDGPU::V_S_EXP_F32_e64: return AMDGPU::V_EXP_F32_e64; 5359 case AMDGPU::V_S_EXP_F16_e64: return AMDGPU::V_EXP_F16_fake16_e64; 5360 case AMDGPU::V_S_LOG_F32_e64: return AMDGPU::V_LOG_F32_e64; 5361 case AMDGPU::V_S_LOG_F16_e64: return AMDGPU::V_LOG_F16_fake16_e64; 5362 case AMDGPU::V_S_RCP_F32_e64: return AMDGPU::V_RCP_F32_e64; 5363 case AMDGPU::V_S_RCP_F16_e64: return AMDGPU::V_RCP_F16_fake16_e64; 5364 case AMDGPU::V_S_RSQ_F32_e64: return AMDGPU::V_RSQ_F32_e64; 5365 case AMDGPU::V_S_RSQ_F16_e64: return AMDGPU::V_RSQ_F16_fake16_e64; 5366 case AMDGPU::V_S_SQRT_F32_e64: return AMDGPU::V_SQRT_F32_e64; 5367 case AMDGPU::V_S_SQRT_F16_e64: return AMDGPU::V_SQRT_F16_fake16_e64; 5368 } 5369 llvm_unreachable( 5370 "Unexpected scalar opcode without corresponding vector one!"); 5371 } 5372 5373 // clang-format on 5374 5375 void SIInstrInfo::insertScratchExecCopy(MachineFunction &MF, 5376 MachineBasicBlock &MBB, 5377 MachineBasicBlock::iterator MBBI, 5378 const DebugLoc &DL, Register Reg, 5379 bool IsSCCLive, 5380 SlotIndexes *Indexes) const { 5381 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 5382 const SIInstrInfo *TII = ST.getInstrInfo(); 5383 bool IsWave32 = ST.isWave32(); 5384 if (IsSCCLive) { 5385 // Insert two move instructions, one to save the original value of EXEC and 5386 // the other to turn on all bits in EXEC. This is required as we can't use 5387 // the single instruction S_OR_SAVEEXEC that clobbers SCC. 5388 unsigned MovOpc = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; 5389 MCRegister Exec = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 5390 auto StoreExecMI = BuildMI(MBB, MBBI, DL, TII->get(MovOpc), Reg) 5391 .addReg(Exec, RegState::Kill); 5392 auto FlipExecMI = BuildMI(MBB, MBBI, DL, TII->get(MovOpc), Exec).addImm(-1); 5393 if (Indexes) { 5394 Indexes->insertMachineInstrInMaps(*StoreExecMI); 5395 Indexes->insertMachineInstrInMaps(*FlipExecMI); 5396 } 5397 } else { 5398 const unsigned OrSaveExec = 5399 IsWave32 ? AMDGPU::S_OR_SAVEEXEC_B32 : AMDGPU::S_OR_SAVEEXEC_B64; 5400 auto SaveExec = 5401 BuildMI(MBB, MBBI, DL, TII->get(OrSaveExec), Reg).addImm(-1); 5402 SaveExec->getOperand(3).setIsDead(); // Mark SCC as dead. 5403 if (Indexes) 5404 Indexes->insertMachineInstrInMaps(*SaveExec); 5405 } 5406 } 5407 5408 void SIInstrInfo::restoreExec(MachineFunction &MF, MachineBasicBlock &MBB, 5409 MachineBasicBlock::iterator MBBI, 5410 const DebugLoc &DL, Register Reg, 5411 SlotIndexes *Indexes) const { 5412 unsigned ExecMov = isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; 5413 MCRegister Exec = isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 5414 auto ExecRestoreMI = 5415 BuildMI(MBB, MBBI, DL, get(ExecMov), Exec).addReg(Reg, RegState::Kill); 5416 if (Indexes) 5417 Indexes->insertMachineInstrInMaps(*ExecRestoreMI); 5418 } 5419 5420 static const TargetRegisterClass * 5421 adjustAllocatableRegClass(const GCNSubtarget &ST, const SIRegisterInfo &RI, 5422 const MachineRegisterInfo &MRI, 5423 const MCInstrDesc &TID, unsigned RCID, 5424 bool IsAllocatable) { 5425 if ((IsAllocatable || !ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) && 5426 (((TID.mayLoad() || TID.mayStore()) && 5427 !(TID.TSFlags & SIInstrFlags::VGPRSpill)) || 5428 (TID.TSFlags & (SIInstrFlags::DS | SIInstrFlags::MIMG)))) { 5429 switch (RCID) { 5430 case AMDGPU::AV_32RegClassID: 5431 RCID = AMDGPU::VGPR_32RegClassID; 5432 break; 5433 case AMDGPU::AV_64RegClassID: 5434 RCID = AMDGPU::VReg_64RegClassID; 5435 break; 5436 case AMDGPU::AV_96RegClassID: 5437 RCID = AMDGPU::VReg_96RegClassID; 5438 break; 5439 case AMDGPU::AV_128RegClassID: 5440 RCID = AMDGPU::VReg_128RegClassID; 5441 break; 5442 case AMDGPU::AV_160RegClassID: 5443 RCID = AMDGPU::VReg_160RegClassID; 5444 break; 5445 case AMDGPU::AV_512RegClassID: 5446 RCID = AMDGPU::VReg_512RegClassID; 5447 break; 5448 default: 5449 break; 5450 } 5451 } 5452 5453 return RI.getProperlyAlignedRC(RI.getRegClass(RCID)); 5454 } 5455 5456 const TargetRegisterClass *SIInstrInfo::getRegClass(const MCInstrDesc &TID, 5457 unsigned OpNum, const TargetRegisterInfo *TRI, 5458 const MachineFunction &MF) 5459 const { 5460 if (OpNum >= TID.getNumOperands()) 5461 return nullptr; 5462 auto RegClass = TID.operands()[OpNum].RegClass; 5463 bool IsAllocatable = false; 5464 if (TID.TSFlags & (SIInstrFlags::DS | SIInstrFlags::FLAT)) { 5465 // vdst and vdata should be both VGPR or AGPR, same for the DS instructions 5466 // with two data operands. Request register class constrained to VGPR only 5467 // of both operands present as Machine Copy Propagation can not check this 5468 // constraint and possibly other passes too. 5469 // 5470 // The check is limited to FLAT and DS because atomics in non-flat encoding 5471 // have their vdst and vdata tied to be the same register. 5472 const int VDstIdx = AMDGPU::getNamedOperandIdx(TID.Opcode, 5473 AMDGPU::OpName::vdst); 5474 const int DataIdx = AMDGPU::getNamedOperandIdx(TID.Opcode, 5475 (TID.TSFlags & SIInstrFlags::DS) ? AMDGPU::OpName::data0 5476 : AMDGPU::OpName::vdata); 5477 if (DataIdx != -1) { 5478 IsAllocatable = VDstIdx != -1 || AMDGPU::hasNamedOperand( 5479 TID.Opcode, AMDGPU::OpName::data1); 5480 } 5481 } 5482 return adjustAllocatableRegClass(ST, RI, MF.getRegInfo(), TID, RegClass, 5483 IsAllocatable); 5484 } 5485 5486 const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI, 5487 unsigned OpNo) const { 5488 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 5489 const MCInstrDesc &Desc = get(MI.getOpcode()); 5490 if (MI.isVariadic() || OpNo >= Desc.getNumOperands() || 5491 Desc.operands()[OpNo].RegClass == -1) { 5492 Register Reg = MI.getOperand(OpNo).getReg(); 5493 5494 if (Reg.isVirtual()) 5495 return MRI.getRegClass(Reg); 5496 return RI.getPhysRegBaseClass(Reg); 5497 } 5498 5499 unsigned RCID = Desc.operands()[OpNo].RegClass; 5500 return adjustAllocatableRegClass(ST, RI, MRI, Desc, RCID, true); 5501 } 5502 5503 void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const { 5504 MachineBasicBlock::iterator I = MI; 5505 MachineBasicBlock *MBB = MI.getParent(); 5506 MachineOperand &MO = MI.getOperand(OpIdx); 5507 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 5508 unsigned RCID = get(MI.getOpcode()).operands()[OpIdx].RegClass; 5509 const TargetRegisterClass *RC = RI.getRegClass(RCID); 5510 unsigned Size = RI.getRegSizeInBits(*RC); 5511 unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO : AMDGPU::V_MOV_B32_e32; 5512 if (MO.isReg()) 5513 Opcode = AMDGPU::COPY; 5514 else if (RI.isSGPRClass(RC)) 5515 Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32; 5516 5517 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(RC); 5518 Register Reg = MRI.createVirtualRegister(VRC); 5519 DebugLoc DL = MBB->findDebugLoc(I); 5520 BuildMI(*MI.getParent(), I, DL, get(Opcode), Reg).add(MO); 5521 MO.ChangeToRegister(Reg, false); 5522 } 5523 5524 unsigned SIInstrInfo::buildExtractSubReg( 5525 MachineBasicBlock::iterator MI, MachineRegisterInfo &MRI, 5526 const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC, 5527 unsigned SubIdx, const TargetRegisterClass *SubRC) const { 5528 MachineBasicBlock *MBB = MI->getParent(); 5529 DebugLoc DL = MI->getDebugLoc(); 5530 Register SubReg = MRI.createVirtualRegister(SubRC); 5531 5532 if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) { 5533 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg) 5534 .addReg(SuperReg.getReg(), 0, SubIdx); 5535 return SubReg; 5536 } 5537 5538 // Just in case the super register is itself a sub-register, copy it to a new 5539 // value so we don't need to worry about merging its subreg index with the 5540 // SubIdx passed to this function. The register coalescer should be able to 5541 // eliminate this extra copy. 5542 Register NewSuperReg = MRI.createVirtualRegister(SuperRC); 5543 5544 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), NewSuperReg) 5545 .addReg(SuperReg.getReg(), 0, SuperReg.getSubReg()); 5546 5547 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg) 5548 .addReg(NewSuperReg, 0, SubIdx); 5549 5550 return SubReg; 5551 } 5552 5553 MachineOperand SIInstrInfo::buildExtractSubRegOrImm( 5554 MachineBasicBlock::iterator MII, MachineRegisterInfo &MRI, 5555 const MachineOperand &Op, const TargetRegisterClass *SuperRC, 5556 unsigned SubIdx, const TargetRegisterClass *SubRC) const { 5557 if (Op.isImm()) { 5558 if (SubIdx == AMDGPU::sub0) 5559 return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm())); 5560 if (SubIdx == AMDGPU::sub1) 5561 return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm() >> 32)); 5562 5563 llvm_unreachable("Unhandled register index for immediate"); 5564 } 5565 5566 unsigned SubReg = buildExtractSubReg(MII, MRI, Op, SuperRC, 5567 SubIdx, SubRC); 5568 return MachineOperand::CreateReg(SubReg, false); 5569 } 5570 5571 // Change the order of operands from (0, 1, 2) to (0, 2, 1) 5572 void SIInstrInfo::swapOperands(MachineInstr &Inst) const { 5573 assert(Inst.getNumExplicitOperands() == 3); 5574 MachineOperand Op1 = Inst.getOperand(1); 5575 Inst.removeOperand(1); 5576 Inst.addOperand(Op1); 5577 } 5578 5579 bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo &MRI, 5580 const MCOperandInfo &OpInfo, 5581 const MachineOperand &MO) const { 5582 if (!MO.isReg()) 5583 return false; 5584 5585 Register Reg = MO.getReg(); 5586 5587 const TargetRegisterClass *DRC = RI.getRegClass(OpInfo.RegClass); 5588 if (Reg.isPhysical()) 5589 return DRC->contains(Reg); 5590 5591 const TargetRegisterClass *RC = MRI.getRegClass(Reg); 5592 5593 if (MO.getSubReg()) { 5594 const MachineFunction *MF = MO.getParent()->getParent()->getParent(); 5595 const TargetRegisterClass *SuperRC = RI.getLargestLegalSuperClass(RC, *MF); 5596 if (!SuperRC) 5597 return false; 5598 5599 DRC = RI.getMatchingSuperRegClass(SuperRC, DRC, MO.getSubReg()); 5600 if (!DRC) 5601 return false; 5602 } 5603 return RC->hasSuperClassEq(DRC); 5604 } 5605 5606 bool SIInstrInfo::isLegalVSrcOperand(const MachineRegisterInfo &MRI, 5607 const MCOperandInfo &OpInfo, 5608 const MachineOperand &MO) const { 5609 if (MO.isReg()) 5610 return isLegalRegOperand(MRI, OpInfo, MO); 5611 5612 // Handle non-register types that are treated like immediates. 5613 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal()); 5614 return true; 5615 } 5616 5617 bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx, 5618 const MachineOperand *MO) const { 5619 const MachineFunction &MF = *MI.getParent()->getParent(); 5620 const MachineRegisterInfo &MRI = MF.getRegInfo(); 5621 const MCInstrDesc &InstDesc = MI.getDesc(); 5622 const MCOperandInfo &OpInfo = InstDesc.operands()[OpIdx]; 5623 const TargetRegisterClass *DefinedRC = 5624 OpInfo.RegClass != -1 ? RI.getRegClass(OpInfo.RegClass) : nullptr; 5625 if (!MO) 5626 MO = &MI.getOperand(OpIdx); 5627 5628 int ConstantBusLimit = ST.getConstantBusLimit(MI.getOpcode()); 5629 int LiteralLimit = !isVOP3(MI) || ST.hasVOP3Literal() ? 1 : 0; 5630 if (isVALU(MI) && usesConstantBus(MRI, *MO, OpInfo)) { 5631 if (!MO->isReg() && !isInlineConstant(*MO, OpInfo) && !LiteralLimit--) 5632 return false; 5633 5634 SmallDenseSet<RegSubRegPair> SGPRsUsed; 5635 if (MO->isReg()) 5636 SGPRsUsed.insert(RegSubRegPair(MO->getReg(), MO->getSubReg())); 5637 5638 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 5639 if (i == OpIdx) 5640 continue; 5641 const MachineOperand &Op = MI.getOperand(i); 5642 if (Op.isReg()) { 5643 RegSubRegPair SGPR(Op.getReg(), Op.getSubReg()); 5644 if (!SGPRsUsed.count(SGPR) && 5645 // FIXME: This can access off the end of the operands() array. 5646 usesConstantBus(MRI, Op, InstDesc.operands().begin()[i])) { 5647 if (--ConstantBusLimit <= 0) 5648 return false; 5649 SGPRsUsed.insert(SGPR); 5650 } 5651 } else if (AMDGPU::isSISrcOperand(InstDesc, i) && 5652 !isInlineConstant(Op, InstDesc.operands()[i])) { 5653 if (!LiteralLimit--) 5654 return false; 5655 if (--ConstantBusLimit <= 0) 5656 return false; 5657 } 5658 } 5659 } 5660 5661 if (MO->isReg()) { 5662 if (!DefinedRC) 5663 return OpInfo.OperandType == MCOI::OPERAND_UNKNOWN; 5664 if (!isLegalRegOperand(MRI, OpInfo, *MO)) 5665 return false; 5666 bool IsAGPR = RI.isAGPR(MRI, MO->getReg()); 5667 if (IsAGPR && !ST.hasMAIInsts()) 5668 return false; 5669 unsigned Opc = MI.getOpcode(); 5670 if (IsAGPR && 5671 (!ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) && 5672 (MI.mayLoad() || MI.mayStore() || isDS(Opc) || isMIMG(Opc))) 5673 return false; 5674 // Atomics should have both vdst and vdata either vgpr or agpr. 5675 const int VDstIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 5676 const int DataIdx = AMDGPU::getNamedOperandIdx(Opc, 5677 isDS(Opc) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata); 5678 if ((int)OpIdx == VDstIdx && DataIdx != -1 && 5679 MI.getOperand(DataIdx).isReg() && 5680 RI.isAGPR(MRI, MI.getOperand(DataIdx).getReg()) != IsAGPR) 5681 return false; 5682 if ((int)OpIdx == DataIdx) { 5683 if (VDstIdx != -1 && 5684 RI.isAGPR(MRI, MI.getOperand(VDstIdx).getReg()) != IsAGPR) 5685 return false; 5686 // DS instructions with 2 src operands also must have tied RC. 5687 const int Data1Idx = AMDGPU::getNamedOperandIdx(Opc, 5688 AMDGPU::OpName::data1); 5689 if (Data1Idx != -1 && MI.getOperand(Data1Idx).isReg() && 5690 RI.isAGPR(MRI, MI.getOperand(Data1Idx).getReg()) != IsAGPR) 5691 return false; 5692 } 5693 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && !ST.hasGFX90AInsts() && 5694 (int)OpIdx == AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) && 5695 RI.isSGPRReg(MRI, MO->getReg())) 5696 return false; 5697 return true; 5698 } 5699 5700 if (MO->isImm()) { 5701 uint64_t Imm = MO->getImm(); 5702 bool Is64BitFPOp = OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_FP64; 5703 bool Is64BitOp = Is64BitFPOp || 5704 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_INT64 || 5705 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2INT32 || 5706 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2FP32; 5707 if (Is64BitOp && 5708 !AMDGPU::isInlinableLiteral64(Imm, ST.hasInv2PiInlineImm())) { 5709 if (!AMDGPU::isValid32BitLiteral(Imm, Is64BitFPOp)) 5710 return false; 5711 5712 // FIXME: We can use sign extended 64-bit literals, but only for signed 5713 // operands. At the moment we do not know if an operand is signed. 5714 // Such operand will be encoded as its low 32 bits and then either 5715 // correctly sign extended or incorrectly zero extended by HW. 5716 if (!Is64BitFPOp && (int32_t)Imm < 0) 5717 return false; 5718 } 5719 } 5720 5721 // Handle non-register types that are treated like immediates. 5722 assert(MO->isImm() || MO->isTargetIndex() || MO->isFI() || MO->isGlobal()); 5723 5724 if (!DefinedRC) { 5725 // This operand expects an immediate. 5726 return true; 5727 } 5728 5729 return isImmOperandLegal(MI, OpIdx, *MO); 5730 } 5731 5732 void SIInstrInfo::legalizeOperandsVOP2(MachineRegisterInfo &MRI, 5733 MachineInstr &MI) const { 5734 unsigned Opc = MI.getOpcode(); 5735 const MCInstrDesc &InstrDesc = get(Opc); 5736 5737 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); 5738 MachineOperand &Src0 = MI.getOperand(Src0Idx); 5739 5740 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); 5741 MachineOperand &Src1 = MI.getOperand(Src1Idx); 5742 5743 // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32 5744 // we need to only have one constant bus use before GFX10. 5745 bool HasImplicitSGPR = findImplicitSGPRRead(MI); 5746 if (HasImplicitSGPR && ST.getConstantBusLimit(Opc) <= 1 && Src0.isReg() && 5747 RI.isSGPRReg(MRI, Src0.getReg())) 5748 legalizeOpWithMove(MI, Src0Idx); 5749 5750 // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for 5751 // both the value to write (src0) and lane select (src1). Fix up non-SGPR 5752 // src0/src1 with V_READFIRSTLANE. 5753 if (Opc == AMDGPU::V_WRITELANE_B32) { 5754 const DebugLoc &DL = MI.getDebugLoc(); 5755 if (Src0.isReg() && RI.isVGPR(MRI, Src0.getReg())) { 5756 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 5757 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 5758 .add(Src0); 5759 Src0.ChangeToRegister(Reg, false); 5760 } 5761 if (Src1.isReg() && RI.isVGPR(MRI, Src1.getReg())) { 5762 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 5763 const DebugLoc &DL = MI.getDebugLoc(); 5764 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 5765 .add(Src1); 5766 Src1.ChangeToRegister(Reg, false); 5767 } 5768 return; 5769 } 5770 5771 // No VOP2 instructions support AGPRs. 5772 if (Src0.isReg() && RI.isAGPR(MRI, Src0.getReg())) 5773 legalizeOpWithMove(MI, Src0Idx); 5774 5775 if (Src1.isReg() && RI.isAGPR(MRI, Src1.getReg())) 5776 legalizeOpWithMove(MI, Src1Idx); 5777 5778 // Special case: V_FMAC_F32 and V_FMAC_F16 have src2. 5779 if (Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F16_e32) { 5780 int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); 5781 if (!RI.isVGPR(MRI, MI.getOperand(Src2Idx).getReg())) 5782 legalizeOpWithMove(MI, Src2Idx); 5783 } 5784 5785 // VOP2 src0 instructions support all operand types, so we don't need to check 5786 // their legality. If src1 is already legal, we don't need to do anything. 5787 if (isLegalRegOperand(MRI, InstrDesc.operands()[Src1Idx], Src1)) 5788 return; 5789 5790 // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for 5791 // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane 5792 // select is uniform. 5793 if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() && 5794 RI.isVGPR(MRI, Src1.getReg())) { 5795 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 5796 const DebugLoc &DL = MI.getDebugLoc(); 5797 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 5798 .add(Src1); 5799 Src1.ChangeToRegister(Reg, false); 5800 return; 5801 } 5802 5803 // We do not use commuteInstruction here because it is too aggressive and will 5804 // commute if it is possible. We only want to commute here if it improves 5805 // legality. This can be called a fairly large number of times so don't waste 5806 // compile time pointlessly swapping and checking legality again. 5807 if (HasImplicitSGPR || !MI.isCommutable()) { 5808 legalizeOpWithMove(MI, Src1Idx); 5809 return; 5810 } 5811 5812 // If src0 can be used as src1, commuting will make the operands legal. 5813 // Otherwise we have to give up and insert a move. 5814 // 5815 // TODO: Other immediate-like operand kinds could be commuted if there was a 5816 // MachineOperand::ChangeTo* for them. 5817 if ((!Src1.isImm() && !Src1.isReg()) || 5818 !isLegalRegOperand(MRI, InstrDesc.operands()[Src1Idx], Src0)) { 5819 legalizeOpWithMove(MI, Src1Idx); 5820 return; 5821 } 5822 5823 int CommutedOpc = commuteOpcode(MI); 5824 if (CommutedOpc == -1) { 5825 legalizeOpWithMove(MI, Src1Idx); 5826 return; 5827 } 5828 5829 MI.setDesc(get(CommutedOpc)); 5830 5831 Register Src0Reg = Src0.getReg(); 5832 unsigned Src0SubReg = Src0.getSubReg(); 5833 bool Src0Kill = Src0.isKill(); 5834 5835 if (Src1.isImm()) 5836 Src0.ChangeToImmediate(Src1.getImm()); 5837 else if (Src1.isReg()) { 5838 Src0.ChangeToRegister(Src1.getReg(), false, false, Src1.isKill()); 5839 Src0.setSubReg(Src1.getSubReg()); 5840 } else 5841 llvm_unreachable("Should only have register or immediate operands"); 5842 5843 Src1.ChangeToRegister(Src0Reg, false, false, Src0Kill); 5844 Src1.setSubReg(Src0SubReg); 5845 fixImplicitOperands(MI); 5846 } 5847 5848 // Legalize VOP3 operands. All operand types are supported for any operand 5849 // but only one literal constant and only starting from GFX10. 5850 void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI, 5851 MachineInstr &MI) const { 5852 unsigned Opc = MI.getOpcode(); 5853 5854 int VOP3Idx[3] = { 5855 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0), 5856 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1), 5857 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2) 5858 }; 5859 5860 if (Opc == AMDGPU::V_PERMLANE16_B32_e64 || 5861 Opc == AMDGPU::V_PERMLANEX16_B32_e64) { 5862 // src1 and src2 must be scalar 5863 MachineOperand &Src1 = MI.getOperand(VOP3Idx[1]); 5864 MachineOperand &Src2 = MI.getOperand(VOP3Idx[2]); 5865 const DebugLoc &DL = MI.getDebugLoc(); 5866 if (Src1.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src1.getReg()))) { 5867 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 5868 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 5869 .add(Src1); 5870 Src1.ChangeToRegister(Reg, false); 5871 } 5872 if (Src2.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src2.getReg()))) { 5873 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 5874 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 5875 .add(Src2); 5876 Src2.ChangeToRegister(Reg, false); 5877 } 5878 } 5879 5880 // Find the one SGPR operand we are allowed to use. 5881 int ConstantBusLimit = ST.getConstantBusLimit(Opc); 5882 int LiteralLimit = ST.hasVOP3Literal() ? 1 : 0; 5883 SmallDenseSet<unsigned> SGPRsUsed; 5884 Register SGPRReg = findUsedSGPR(MI, VOP3Idx); 5885 if (SGPRReg) { 5886 SGPRsUsed.insert(SGPRReg); 5887 --ConstantBusLimit; 5888 } 5889 5890 for (int Idx : VOP3Idx) { 5891 if (Idx == -1) 5892 break; 5893 MachineOperand &MO = MI.getOperand(Idx); 5894 5895 if (!MO.isReg()) { 5896 if (isInlineConstant(MO, get(Opc).operands()[Idx])) 5897 continue; 5898 5899 if (LiteralLimit > 0 && ConstantBusLimit > 0) { 5900 --LiteralLimit; 5901 --ConstantBusLimit; 5902 continue; 5903 } 5904 5905 --LiteralLimit; 5906 --ConstantBusLimit; 5907 legalizeOpWithMove(MI, Idx); 5908 continue; 5909 } 5910 5911 if (RI.hasAGPRs(RI.getRegClassForReg(MRI, MO.getReg())) && 5912 !isOperandLegal(MI, Idx, &MO)) { 5913 legalizeOpWithMove(MI, Idx); 5914 continue; 5915 } 5916 5917 if (!RI.isSGPRClass(RI.getRegClassForReg(MRI, MO.getReg()))) 5918 continue; // VGPRs are legal 5919 5920 // We can use one SGPR in each VOP3 instruction prior to GFX10 5921 // and two starting from GFX10. 5922 if (SGPRsUsed.count(MO.getReg())) 5923 continue; 5924 if (ConstantBusLimit > 0) { 5925 SGPRsUsed.insert(MO.getReg()); 5926 --ConstantBusLimit; 5927 continue; 5928 } 5929 5930 // If we make it this far, then the operand is not legal and we must 5931 // legalize it. 5932 legalizeOpWithMove(MI, Idx); 5933 } 5934 5935 // Special case: V_FMAC_F32 and V_FMAC_F16 have src2 tied to vdst. 5936 if ((Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_e64) && 5937 !RI.isVGPR(MRI, MI.getOperand(VOP3Idx[2]).getReg())) 5938 legalizeOpWithMove(MI, VOP3Idx[2]); 5939 } 5940 5941 Register SIInstrInfo::readlaneVGPRToSGPR(Register SrcReg, MachineInstr &UseMI, 5942 MachineRegisterInfo &MRI) const { 5943 const TargetRegisterClass *VRC = MRI.getRegClass(SrcReg); 5944 const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC); 5945 Register DstReg = MRI.createVirtualRegister(SRC); 5946 unsigned SubRegs = RI.getRegSizeInBits(*VRC) / 32; 5947 5948 if (RI.hasAGPRs(VRC)) { 5949 VRC = RI.getEquivalentVGPRClass(VRC); 5950 Register NewSrcReg = MRI.createVirtualRegister(VRC); 5951 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), 5952 get(TargetOpcode::COPY), NewSrcReg) 5953 .addReg(SrcReg); 5954 SrcReg = NewSrcReg; 5955 } 5956 5957 if (SubRegs == 1) { 5958 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), 5959 get(AMDGPU::V_READFIRSTLANE_B32), DstReg) 5960 .addReg(SrcReg); 5961 return DstReg; 5962 } 5963 5964 SmallVector<Register, 8> SRegs; 5965 for (unsigned i = 0; i < SubRegs; ++i) { 5966 Register SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 5967 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), 5968 get(AMDGPU::V_READFIRSTLANE_B32), SGPR) 5969 .addReg(SrcReg, 0, RI.getSubRegFromChannel(i)); 5970 SRegs.push_back(SGPR); 5971 } 5972 5973 MachineInstrBuilder MIB = 5974 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), 5975 get(AMDGPU::REG_SEQUENCE), DstReg); 5976 for (unsigned i = 0; i < SubRegs; ++i) { 5977 MIB.addReg(SRegs[i]); 5978 MIB.addImm(RI.getSubRegFromChannel(i)); 5979 } 5980 return DstReg; 5981 } 5982 5983 void SIInstrInfo::legalizeOperandsSMRD(MachineRegisterInfo &MRI, 5984 MachineInstr &MI) const { 5985 5986 // If the pointer is store in VGPRs, then we need to move them to 5987 // SGPRs using v_readfirstlane. This is safe because we only select 5988 // loads with uniform pointers to SMRD instruction so we know the 5989 // pointer value is uniform. 5990 MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase); 5991 if (SBase && !RI.isSGPRClass(MRI.getRegClass(SBase->getReg()))) { 5992 Register SGPR = readlaneVGPRToSGPR(SBase->getReg(), MI, MRI); 5993 SBase->setReg(SGPR); 5994 } 5995 MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soffset); 5996 if (SOff && !RI.isSGPRClass(MRI.getRegClass(SOff->getReg()))) { 5997 Register SGPR = readlaneVGPRToSGPR(SOff->getReg(), MI, MRI); 5998 SOff->setReg(SGPR); 5999 } 6000 } 6001 6002 bool SIInstrInfo::moveFlatAddrToVGPR(MachineInstr &Inst) const { 6003 unsigned Opc = Inst.getOpcode(); 6004 int OldSAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr); 6005 if (OldSAddrIdx < 0) 6006 return false; 6007 6008 assert(isSegmentSpecificFLAT(Inst)); 6009 6010 int NewOpc = AMDGPU::getGlobalVaddrOp(Opc); 6011 if (NewOpc < 0) 6012 NewOpc = AMDGPU::getFlatScratchInstSVfromSS(Opc); 6013 if (NewOpc < 0) 6014 return false; 6015 6016 MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo(); 6017 MachineOperand &SAddr = Inst.getOperand(OldSAddrIdx); 6018 if (RI.isSGPRReg(MRI, SAddr.getReg())) 6019 return false; 6020 6021 int NewVAddrIdx = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr); 6022 if (NewVAddrIdx < 0) 6023 return false; 6024 6025 int OldVAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr); 6026 6027 // Check vaddr, it shall be zero or absent. 6028 MachineInstr *VAddrDef = nullptr; 6029 if (OldVAddrIdx >= 0) { 6030 MachineOperand &VAddr = Inst.getOperand(OldVAddrIdx); 6031 VAddrDef = MRI.getUniqueVRegDef(VAddr.getReg()); 6032 if (!VAddrDef || VAddrDef->getOpcode() != AMDGPU::V_MOV_B32_e32 || 6033 !VAddrDef->getOperand(1).isImm() || 6034 VAddrDef->getOperand(1).getImm() != 0) 6035 return false; 6036 } 6037 6038 const MCInstrDesc &NewDesc = get(NewOpc); 6039 Inst.setDesc(NewDesc); 6040 6041 // Callers expect iterator to be valid after this call, so modify the 6042 // instruction in place. 6043 if (OldVAddrIdx == NewVAddrIdx) { 6044 MachineOperand &NewVAddr = Inst.getOperand(NewVAddrIdx); 6045 // Clear use list from the old vaddr holding a zero register. 6046 MRI.removeRegOperandFromUseList(&NewVAddr); 6047 MRI.moveOperands(&NewVAddr, &SAddr, 1); 6048 Inst.removeOperand(OldSAddrIdx); 6049 // Update the use list with the pointer we have just moved from vaddr to 6050 // saddr position. Otherwise new vaddr will be missing from the use list. 6051 MRI.removeRegOperandFromUseList(&NewVAddr); 6052 MRI.addRegOperandToUseList(&NewVAddr); 6053 } else { 6054 assert(OldSAddrIdx == NewVAddrIdx); 6055 6056 if (OldVAddrIdx >= 0) { 6057 int NewVDstIn = AMDGPU::getNamedOperandIdx(NewOpc, 6058 AMDGPU::OpName::vdst_in); 6059 6060 // removeOperand doesn't try to fixup tied operand indexes at it goes, so 6061 // it asserts. Untie the operands for now and retie them afterwards. 6062 if (NewVDstIn != -1) { 6063 int OldVDstIn = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst_in); 6064 Inst.untieRegOperand(OldVDstIn); 6065 } 6066 6067 Inst.removeOperand(OldVAddrIdx); 6068 6069 if (NewVDstIn != -1) { 6070 int NewVDst = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vdst); 6071 Inst.tieOperands(NewVDst, NewVDstIn); 6072 } 6073 } 6074 } 6075 6076 if (VAddrDef && MRI.use_nodbg_empty(VAddrDef->getOperand(0).getReg())) 6077 VAddrDef->eraseFromParent(); 6078 6079 return true; 6080 } 6081 6082 // FIXME: Remove this when SelectionDAG is obsoleted. 6083 void SIInstrInfo::legalizeOperandsFLAT(MachineRegisterInfo &MRI, 6084 MachineInstr &MI) const { 6085 if (!isSegmentSpecificFLAT(MI)) 6086 return; 6087 6088 // Fixup SGPR operands in VGPRs. We only select these when the DAG divergence 6089 // thinks they are uniform, so a readfirstlane should be valid. 6090 MachineOperand *SAddr = getNamedOperand(MI, AMDGPU::OpName::saddr); 6091 if (!SAddr || RI.isSGPRClass(MRI.getRegClass(SAddr->getReg()))) 6092 return; 6093 6094 if (moveFlatAddrToVGPR(MI)) 6095 return; 6096 6097 Register ToSGPR = readlaneVGPRToSGPR(SAddr->getReg(), MI, MRI); 6098 SAddr->setReg(ToSGPR); 6099 } 6100 6101 void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB, 6102 MachineBasicBlock::iterator I, 6103 const TargetRegisterClass *DstRC, 6104 MachineOperand &Op, 6105 MachineRegisterInfo &MRI, 6106 const DebugLoc &DL) const { 6107 Register OpReg = Op.getReg(); 6108 unsigned OpSubReg = Op.getSubReg(); 6109 6110 const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg( 6111 RI.getRegClassForReg(MRI, OpReg), OpSubReg); 6112 6113 // Check if operand is already the correct register class. 6114 if (DstRC == OpRC) 6115 return; 6116 6117 Register DstReg = MRI.createVirtualRegister(DstRC); 6118 auto Copy = BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).add(Op); 6119 6120 Op.setReg(DstReg); 6121 Op.setSubReg(0); 6122 6123 MachineInstr *Def = MRI.getVRegDef(OpReg); 6124 if (!Def) 6125 return; 6126 6127 // Try to eliminate the copy if it is copying an immediate value. 6128 if (Def->isMoveImmediate() && DstRC != &AMDGPU::VReg_1RegClass) 6129 FoldImmediate(*Copy, *Def, OpReg, &MRI); 6130 6131 bool ImpDef = Def->isImplicitDef(); 6132 while (!ImpDef && Def && Def->isCopy()) { 6133 if (Def->getOperand(1).getReg().isPhysical()) 6134 break; 6135 Def = MRI.getUniqueVRegDef(Def->getOperand(1).getReg()); 6136 ImpDef = Def && Def->isImplicitDef(); 6137 } 6138 if (!RI.isSGPRClass(DstRC) && !Copy->readsRegister(AMDGPU::EXEC, &RI) && 6139 !ImpDef) 6140 Copy.addReg(AMDGPU::EXEC, RegState::Implicit); 6141 } 6142 6143 // Emit the actual waterfall loop, executing the wrapped instruction for each 6144 // unique value of \p ScalarOps across all lanes. In the best case we execute 1 6145 // iteration, in the worst case we execute 64 (once per lane). 6146 static void emitLoadScalarOpsFromVGPRLoop( 6147 const SIInstrInfo &TII, MachineRegisterInfo &MRI, MachineBasicBlock &OrigBB, 6148 MachineBasicBlock &LoopBB, MachineBasicBlock &BodyBB, const DebugLoc &DL, 6149 ArrayRef<MachineOperand *> ScalarOps) { 6150 MachineFunction &MF = *OrigBB.getParent(); 6151 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 6152 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 6153 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 6154 unsigned SaveExecOpc = 6155 ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64; 6156 unsigned XorTermOpc = 6157 ST.isWave32() ? AMDGPU::S_XOR_B32_term : AMDGPU::S_XOR_B64_term; 6158 unsigned AndOpc = 6159 ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64; 6160 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 6161 6162 MachineBasicBlock::iterator I = LoopBB.begin(); 6163 6164 SmallVector<Register, 8> ReadlanePieces; 6165 Register CondReg; 6166 6167 for (MachineOperand *ScalarOp : ScalarOps) { 6168 unsigned RegSize = TRI->getRegSizeInBits(ScalarOp->getReg(), MRI); 6169 unsigned NumSubRegs = RegSize / 32; 6170 Register VScalarOp = ScalarOp->getReg(); 6171 6172 if (NumSubRegs == 1) { 6173 Register CurReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 6174 6175 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurReg) 6176 .addReg(VScalarOp); 6177 6178 Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC); 6179 6180 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U32_e64), NewCondReg) 6181 .addReg(CurReg) 6182 .addReg(VScalarOp); 6183 6184 // Combine the comparison results with AND. 6185 if (!CondReg) // First. 6186 CondReg = NewCondReg; 6187 else { // If not the first, we create an AND. 6188 Register AndReg = MRI.createVirtualRegister(BoolXExecRC); 6189 BuildMI(LoopBB, I, DL, TII.get(AndOpc), AndReg) 6190 .addReg(CondReg) 6191 .addReg(NewCondReg); 6192 CondReg = AndReg; 6193 } 6194 6195 // Update ScalarOp operand to use the SGPR ScalarOp. 6196 ScalarOp->setReg(CurReg); 6197 ScalarOp->setIsKill(); 6198 } else { 6199 unsigned VScalarOpUndef = getUndefRegState(ScalarOp->isUndef()); 6200 assert(NumSubRegs % 2 == 0 && NumSubRegs <= 32 && 6201 "Unhandled register size"); 6202 6203 for (unsigned Idx = 0; Idx < NumSubRegs; Idx += 2) { 6204 Register CurRegLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 6205 Register CurRegHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 6206 6207 // Read the next variant <- also loop target. 6208 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegLo) 6209 .addReg(VScalarOp, VScalarOpUndef, TRI->getSubRegFromChannel(Idx)); 6210 6211 // Read the next variant <- also loop target. 6212 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegHi) 6213 .addReg(VScalarOp, VScalarOpUndef, 6214 TRI->getSubRegFromChannel(Idx + 1)); 6215 6216 ReadlanePieces.push_back(CurRegLo); 6217 ReadlanePieces.push_back(CurRegHi); 6218 6219 // Comparison is to be done as 64-bit. 6220 Register CurReg = MRI.createVirtualRegister(&AMDGPU::SGPR_64RegClass); 6221 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), CurReg) 6222 .addReg(CurRegLo) 6223 .addImm(AMDGPU::sub0) 6224 .addReg(CurRegHi) 6225 .addImm(AMDGPU::sub1); 6226 6227 Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC); 6228 auto Cmp = BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), 6229 NewCondReg) 6230 .addReg(CurReg); 6231 if (NumSubRegs <= 2) 6232 Cmp.addReg(VScalarOp); 6233 else 6234 Cmp.addReg(VScalarOp, VScalarOpUndef, 6235 TRI->getSubRegFromChannel(Idx, 2)); 6236 6237 // Combine the comparison results with AND. 6238 if (!CondReg) // First. 6239 CondReg = NewCondReg; 6240 else { // If not the first, we create an AND. 6241 Register AndReg = MRI.createVirtualRegister(BoolXExecRC); 6242 BuildMI(LoopBB, I, DL, TII.get(AndOpc), AndReg) 6243 .addReg(CondReg) 6244 .addReg(NewCondReg); 6245 CondReg = AndReg; 6246 } 6247 } // End for loop. 6248 6249 auto SScalarOpRC = 6250 TRI->getEquivalentSGPRClass(MRI.getRegClass(VScalarOp)); 6251 Register SScalarOp = MRI.createVirtualRegister(SScalarOpRC); 6252 6253 // Build scalar ScalarOp. 6254 auto Merge = 6255 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SScalarOp); 6256 unsigned Channel = 0; 6257 for (Register Piece : ReadlanePieces) { 6258 Merge.addReg(Piece).addImm(TRI->getSubRegFromChannel(Channel++)); 6259 } 6260 6261 // Update ScalarOp operand to use the SGPR ScalarOp. 6262 ScalarOp->setReg(SScalarOp); 6263 ScalarOp->setIsKill(); 6264 } 6265 } 6266 6267 Register SaveExec = MRI.createVirtualRegister(BoolXExecRC); 6268 MRI.setSimpleHint(SaveExec, CondReg); 6269 6270 // Update EXEC to matching lanes, saving original to SaveExec. 6271 BuildMI(LoopBB, I, DL, TII.get(SaveExecOpc), SaveExec) 6272 .addReg(CondReg, RegState::Kill); 6273 6274 // The original instruction is here; we insert the terminators after it. 6275 I = BodyBB.end(); 6276 6277 // Update EXEC, switch all done bits to 0 and all todo bits to 1. 6278 BuildMI(BodyBB, I, DL, TII.get(XorTermOpc), Exec) 6279 .addReg(Exec) 6280 .addReg(SaveExec); 6281 6282 BuildMI(BodyBB, I, DL, TII.get(AMDGPU::SI_WATERFALL_LOOP)).addMBB(&LoopBB); 6283 } 6284 6285 // Build a waterfall loop around \p MI, replacing the VGPR \p ScalarOp register 6286 // with SGPRs by iterating over all unique values across all lanes. 6287 // Returns the loop basic block that now contains \p MI. 6288 static MachineBasicBlock * 6289 loadMBUFScalarOperandsFromVGPR(const SIInstrInfo &TII, MachineInstr &MI, 6290 ArrayRef<MachineOperand *> ScalarOps, 6291 MachineDominatorTree *MDT, 6292 MachineBasicBlock::iterator Begin = nullptr, 6293 MachineBasicBlock::iterator End = nullptr) { 6294 MachineBasicBlock &MBB = *MI.getParent(); 6295 MachineFunction &MF = *MBB.getParent(); 6296 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 6297 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 6298 MachineRegisterInfo &MRI = MF.getRegInfo(); 6299 if (!Begin.isValid()) 6300 Begin = &MI; 6301 if (!End.isValid()) { 6302 End = &MI; 6303 ++End; 6304 } 6305 const DebugLoc &DL = MI.getDebugLoc(); 6306 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 6307 unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; 6308 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 6309 6310 // Save SCC. Waterfall Loop may overwrite SCC. 6311 Register SaveSCCReg; 6312 bool SCCNotDead = (MBB.computeRegisterLiveness(TRI, AMDGPU::SCC, MI, 30) != 6313 MachineBasicBlock::LQR_Dead); 6314 if (SCCNotDead) { 6315 SaveSCCReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 6316 BuildMI(MBB, Begin, DL, TII.get(AMDGPU::S_CSELECT_B32), SaveSCCReg) 6317 .addImm(1) 6318 .addImm(0); 6319 } 6320 6321 Register SaveExec = MRI.createVirtualRegister(BoolXExecRC); 6322 6323 // Save the EXEC mask 6324 BuildMI(MBB, Begin, DL, TII.get(MovExecOpc), SaveExec).addReg(Exec); 6325 6326 // Killed uses in the instruction we are waterfalling around will be 6327 // incorrect due to the added control-flow. 6328 MachineBasicBlock::iterator AfterMI = MI; 6329 ++AfterMI; 6330 for (auto I = Begin; I != AfterMI; I++) { 6331 for (auto &MO : I->all_uses()) 6332 MRI.clearKillFlags(MO.getReg()); 6333 } 6334 6335 // To insert the loop we need to split the block. Move everything after this 6336 // point to a new block, and insert a new empty block between the two. 6337 MachineBasicBlock *LoopBB = MF.CreateMachineBasicBlock(); 6338 MachineBasicBlock *BodyBB = MF.CreateMachineBasicBlock(); 6339 MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock(); 6340 MachineFunction::iterator MBBI(MBB); 6341 ++MBBI; 6342 6343 MF.insert(MBBI, LoopBB); 6344 MF.insert(MBBI, BodyBB); 6345 MF.insert(MBBI, RemainderBB); 6346 6347 LoopBB->addSuccessor(BodyBB); 6348 BodyBB->addSuccessor(LoopBB); 6349 BodyBB->addSuccessor(RemainderBB); 6350 6351 // Move Begin to MI to the BodyBB, and the remainder of the block to 6352 // RemainderBB. 6353 RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB); 6354 RemainderBB->splice(RemainderBB->begin(), &MBB, End, MBB.end()); 6355 BodyBB->splice(BodyBB->begin(), &MBB, Begin, MBB.end()); 6356 6357 MBB.addSuccessor(LoopBB); 6358 6359 // Update dominators. We know that MBB immediately dominates LoopBB, that 6360 // LoopBB immediately dominates BodyBB, and BodyBB immediately dominates 6361 // RemainderBB. RemainderBB immediately dominates all of the successors 6362 // transferred to it from MBB that MBB used to properly dominate. 6363 if (MDT) { 6364 MDT->addNewBlock(LoopBB, &MBB); 6365 MDT->addNewBlock(BodyBB, LoopBB); 6366 MDT->addNewBlock(RemainderBB, BodyBB); 6367 for (auto &Succ : RemainderBB->successors()) { 6368 if (MDT->properlyDominates(&MBB, Succ)) { 6369 MDT->changeImmediateDominator(Succ, RemainderBB); 6370 } 6371 } 6372 } 6373 6374 emitLoadScalarOpsFromVGPRLoop(TII, MRI, MBB, *LoopBB, *BodyBB, DL, ScalarOps); 6375 6376 MachineBasicBlock::iterator First = RemainderBB->begin(); 6377 // Restore SCC 6378 if (SCCNotDead) { 6379 BuildMI(*RemainderBB, First, DL, TII.get(AMDGPU::S_CMP_LG_U32)) 6380 .addReg(SaveSCCReg, RegState::Kill) 6381 .addImm(0); 6382 } 6383 6384 // Restore the EXEC mask 6385 BuildMI(*RemainderBB, First, DL, TII.get(MovExecOpc), Exec).addReg(SaveExec); 6386 return BodyBB; 6387 } 6388 6389 // Extract pointer from Rsrc and return a zero-value Rsrc replacement. 6390 static std::tuple<unsigned, unsigned> 6391 extractRsrcPtr(const SIInstrInfo &TII, MachineInstr &MI, MachineOperand &Rsrc) { 6392 MachineBasicBlock &MBB = *MI.getParent(); 6393 MachineFunction &MF = *MBB.getParent(); 6394 MachineRegisterInfo &MRI = MF.getRegInfo(); 6395 6396 // Extract the ptr from the resource descriptor. 6397 unsigned RsrcPtr = 6398 TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass, 6399 AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass); 6400 6401 // Create an empty resource descriptor 6402 Register Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); 6403 Register SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 6404 Register SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 6405 Register NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass); 6406 uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat(); 6407 6408 // Zero64 = 0 6409 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64) 6410 .addImm(0); 6411 6412 // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0} 6413 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo) 6414 .addImm(RsrcDataFormat & 0xFFFFFFFF); 6415 6416 // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32} 6417 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi) 6418 .addImm(RsrcDataFormat >> 32); 6419 6420 // NewSRsrc = {Zero64, SRsrcFormat} 6421 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc) 6422 .addReg(Zero64) 6423 .addImm(AMDGPU::sub0_sub1) 6424 .addReg(SRsrcFormatLo) 6425 .addImm(AMDGPU::sub2) 6426 .addReg(SRsrcFormatHi) 6427 .addImm(AMDGPU::sub3); 6428 6429 return std::tuple(RsrcPtr, NewSRsrc); 6430 } 6431 6432 MachineBasicBlock * 6433 SIInstrInfo::legalizeOperands(MachineInstr &MI, 6434 MachineDominatorTree *MDT) const { 6435 MachineFunction &MF = *MI.getParent()->getParent(); 6436 MachineRegisterInfo &MRI = MF.getRegInfo(); 6437 MachineBasicBlock *CreatedBB = nullptr; 6438 6439 // Legalize VOP2 6440 if (isVOP2(MI) || isVOPC(MI)) { 6441 legalizeOperandsVOP2(MRI, MI); 6442 return CreatedBB; 6443 } 6444 6445 // Legalize VOP3 6446 if (isVOP3(MI)) { 6447 legalizeOperandsVOP3(MRI, MI); 6448 return CreatedBB; 6449 } 6450 6451 // Legalize SMRD 6452 if (isSMRD(MI)) { 6453 legalizeOperandsSMRD(MRI, MI); 6454 return CreatedBB; 6455 } 6456 6457 // Legalize FLAT 6458 if (isFLAT(MI)) { 6459 legalizeOperandsFLAT(MRI, MI); 6460 return CreatedBB; 6461 } 6462 6463 // Legalize REG_SEQUENCE and PHI 6464 // The register class of the operands much be the same type as the register 6465 // class of the output. 6466 if (MI.getOpcode() == AMDGPU::PHI) { 6467 const TargetRegisterClass *RC = nullptr, *SRC = nullptr, *VRC = nullptr; 6468 for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) { 6469 if (!MI.getOperand(i).isReg() || !MI.getOperand(i).getReg().isVirtual()) 6470 continue; 6471 const TargetRegisterClass *OpRC = 6472 MRI.getRegClass(MI.getOperand(i).getReg()); 6473 if (RI.hasVectorRegisters(OpRC)) { 6474 VRC = OpRC; 6475 } else { 6476 SRC = OpRC; 6477 } 6478 } 6479 6480 // If any of the operands are VGPR registers, then they all most be 6481 // otherwise we will create illegal VGPR->SGPR copies when legalizing 6482 // them. 6483 if (VRC || !RI.isSGPRClass(getOpRegClass(MI, 0))) { 6484 if (!VRC) { 6485 assert(SRC); 6486 if (getOpRegClass(MI, 0) == &AMDGPU::VReg_1RegClass) { 6487 VRC = &AMDGPU::VReg_1RegClass; 6488 } else 6489 VRC = RI.isAGPRClass(getOpRegClass(MI, 0)) 6490 ? RI.getEquivalentAGPRClass(SRC) 6491 : RI.getEquivalentVGPRClass(SRC); 6492 } else { 6493 VRC = RI.isAGPRClass(getOpRegClass(MI, 0)) 6494 ? RI.getEquivalentAGPRClass(VRC) 6495 : RI.getEquivalentVGPRClass(VRC); 6496 } 6497 RC = VRC; 6498 } else { 6499 RC = SRC; 6500 } 6501 6502 // Update all the operands so they have the same type. 6503 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { 6504 MachineOperand &Op = MI.getOperand(I); 6505 if (!Op.isReg() || !Op.getReg().isVirtual()) 6506 continue; 6507 6508 // MI is a PHI instruction. 6509 MachineBasicBlock *InsertBB = MI.getOperand(I + 1).getMBB(); 6510 MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator(); 6511 6512 // Avoid creating no-op copies with the same src and dst reg class. These 6513 // confuse some of the machine passes. 6514 legalizeGenericOperand(*InsertBB, Insert, RC, Op, MRI, MI.getDebugLoc()); 6515 } 6516 } 6517 6518 // REG_SEQUENCE doesn't really require operand legalization, but if one has a 6519 // VGPR dest type and SGPR sources, insert copies so all operands are 6520 // VGPRs. This seems to help operand folding / the register coalescer. 6521 if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) { 6522 MachineBasicBlock *MBB = MI.getParent(); 6523 const TargetRegisterClass *DstRC = getOpRegClass(MI, 0); 6524 if (RI.hasVGPRs(DstRC)) { 6525 // Update all the operands so they are VGPR register classes. These may 6526 // not be the same register class because REG_SEQUENCE supports mixing 6527 // subregister index types e.g. sub0_sub1 + sub2 + sub3 6528 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { 6529 MachineOperand &Op = MI.getOperand(I); 6530 if (!Op.isReg() || !Op.getReg().isVirtual()) 6531 continue; 6532 6533 const TargetRegisterClass *OpRC = MRI.getRegClass(Op.getReg()); 6534 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(OpRC); 6535 if (VRC == OpRC) 6536 continue; 6537 6538 legalizeGenericOperand(*MBB, MI, VRC, Op, MRI, MI.getDebugLoc()); 6539 Op.setIsKill(); 6540 } 6541 } 6542 6543 return CreatedBB; 6544 } 6545 6546 // Legalize INSERT_SUBREG 6547 // src0 must have the same register class as dst 6548 if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) { 6549 Register Dst = MI.getOperand(0).getReg(); 6550 Register Src0 = MI.getOperand(1).getReg(); 6551 const TargetRegisterClass *DstRC = MRI.getRegClass(Dst); 6552 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0); 6553 if (DstRC != Src0RC) { 6554 MachineBasicBlock *MBB = MI.getParent(); 6555 MachineOperand &Op = MI.getOperand(1); 6556 legalizeGenericOperand(*MBB, MI, DstRC, Op, MRI, MI.getDebugLoc()); 6557 } 6558 return CreatedBB; 6559 } 6560 6561 // Legalize SI_INIT_M0 6562 if (MI.getOpcode() == AMDGPU::SI_INIT_M0) { 6563 MachineOperand &Src = MI.getOperand(0); 6564 if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg()))) 6565 Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI)); 6566 return CreatedBB; 6567 } 6568 6569 // Legalize S_BITREPLICATE, S_QUADMASK and S_WQM 6570 if (MI.getOpcode() == AMDGPU::S_BITREPLICATE_B64_B32 || 6571 MI.getOpcode() == AMDGPU::S_QUADMASK_B32 || 6572 MI.getOpcode() == AMDGPU::S_QUADMASK_B64 || 6573 MI.getOpcode() == AMDGPU::S_WQM_B32 || 6574 MI.getOpcode() == AMDGPU::S_WQM_B64) { 6575 MachineOperand &Src = MI.getOperand(1); 6576 if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg()))) 6577 Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI)); 6578 return CreatedBB; 6579 } 6580 6581 // Legalize MIMG/VIMAGE/VSAMPLE and MUBUF/MTBUF for shaders. 6582 // 6583 // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via 6584 // scratch memory access. In both cases, the legalization never involves 6585 // conversion to the addr64 form. 6586 if (isImage(MI) || (AMDGPU::isGraphics(MF.getFunction().getCallingConv()) && 6587 (isMUBUF(MI) || isMTBUF(MI)))) { 6588 int RSrcOpName = (isVIMAGE(MI) || isVSAMPLE(MI)) ? AMDGPU::OpName::rsrc 6589 : AMDGPU::OpName::srsrc; 6590 MachineOperand *SRsrc = getNamedOperand(MI, RSrcOpName); 6591 if (SRsrc && !RI.isSGPRClass(MRI.getRegClass(SRsrc->getReg()))) 6592 CreatedBB = loadMBUFScalarOperandsFromVGPR(*this, MI, {SRsrc}, MDT); 6593 6594 int SampOpName = isMIMG(MI) ? AMDGPU::OpName::ssamp : AMDGPU::OpName::samp; 6595 MachineOperand *SSamp = getNamedOperand(MI, SampOpName); 6596 if (SSamp && !RI.isSGPRClass(MRI.getRegClass(SSamp->getReg()))) 6597 CreatedBB = loadMBUFScalarOperandsFromVGPR(*this, MI, {SSamp}, MDT); 6598 6599 return CreatedBB; 6600 } 6601 6602 // Legalize SI_CALL 6603 if (MI.getOpcode() == AMDGPU::SI_CALL_ISEL) { 6604 MachineOperand *Dest = &MI.getOperand(0); 6605 if (!RI.isSGPRClass(MRI.getRegClass(Dest->getReg()))) { 6606 // Move everything between ADJCALLSTACKUP and ADJCALLSTACKDOWN and 6607 // following copies, we also need to move copies from and to physical 6608 // registers into the loop block. 6609 unsigned FrameSetupOpcode = getCallFrameSetupOpcode(); 6610 unsigned FrameDestroyOpcode = getCallFrameDestroyOpcode(); 6611 6612 // Also move the copies to physical registers into the loop block 6613 MachineBasicBlock &MBB = *MI.getParent(); 6614 MachineBasicBlock::iterator Start(&MI); 6615 while (Start->getOpcode() != FrameSetupOpcode) 6616 --Start; 6617 MachineBasicBlock::iterator End(&MI); 6618 while (End->getOpcode() != FrameDestroyOpcode) 6619 ++End; 6620 // Also include following copies of the return value 6621 ++End; 6622 while (End != MBB.end() && End->isCopy() && End->getOperand(1).isReg() && 6623 MI.definesRegister(End->getOperand(1).getReg())) 6624 ++End; 6625 CreatedBB = 6626 loadMBUFScalarOperandsFromVGPR(*this, MI, {Dest}, MDT, Start, End); 6627 } 6628 } 6629 6630 // Legalize s_sleep_var. 6631 if (MI.getOpcode() == AMDGPU::S_SLEEP_VAR) { 6632 const DebugLoc &DL = MI.getDebugLoc(); 6633 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 6634 int Src0Idx = 6635 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0); 6636 MachineOperand &Src0 = MI.getOperand(Src0Idx); 6637 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 6638 .add(Src0); 6639 Src0.ChangeToRegister(Reg, false); 6640 return nullptr; 6641 } 6642 6643 // Legalize MUBUF instructions. 6644 bool isSoffsetLegal = true; 6645 int SoffsetIdx = 6646 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::soffset); 6647 if (SoffsetIdx != -1) { 6648 MachineOperand *Soffset = &MI.getOperand(SoffsetIdx); 6649 if (Soffset->isReg() && Soffset->getReg().isVirtual() && 6650 !RI.isSGPRClass(MRI.getRegClass(Soffset->getReg()))) { 6651 isSoffsetLegal = false; 6652 } 6653 } 6654 6655 bool isRsrcLegal = true; 6656 int RsrcIdx = 6657 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc); 6658 if (RsrcIdx != -1) { 6659 MachineOperand *Rsrc = &MI.getOperand(RsrcIdx); 6660 if (Rsrc->isReg() && !RI.isSGPRClass(MRI.getRegClass(Rsrc->getReg()))) { 6661 isRsrcLegal = false; 6662 } 6663 } 6664 6665 // The operands are legal. 6666 if (isRsrcLegal && isSoffsetLegal) 6667 return CreatedBB; 6668 6669 if (!isRsrcLegal) { 6670 // Legalize a VGPR Rsrc 6671 // 6672 // If the instruction is _ADDR64, we can avoid a waterfall by extracting 6673 // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using 6674 // a zero-value SRsrc. 6675 // 6676 // If the instruction is _OFFSET (both idxen and offen disabled), and we 6677 // support ADDR64 instructions, we can convert to ADDR64 and do the same as 6678 // above. 6679 // 6680 // Otherwise we are on non-ADDR64 hardware, and/or we have 6681 // idxen/offen/bothen and we fall back to a waterfall loop. 6682 6683 MachineOperand *Rsrc = &MI.getOperand(RsrcIdx); 6684 MachineBasicBlock &MBB = *MI.getParent(); 6685 6686 MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr); 6687 if (VAddr && AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -1) { 6688 // This is already an ADDR64 instruction so we need to add the pointer 6689 // extracted from the resource descriptor to the current value of VAddr. 6690 Register NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6691 Register NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6692 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 6693 6694 const auto *BoolXExecRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 6695 Register CondReg0 = MRI.createVirtualRegister(BoolXExecRC); 6696 Register CondReg1 = MRI.createVirtualRegister(BoolXExecRC); 6697 6698 unsigned RsrcPtr, NewSRsrc; 6699 std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc); 6700 6701 // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0 6702 const DebugLoc &DL = MI.getDebugLoc(); 6703 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_CO_U32_e64), NewVAddrLo) 6704 .addDef(CondReg0) 6705 .addReg(RsrcPtr, 0, AMDGPU::sub0) 6706 .addReg(VAddr->getReg(), 0, AMDGPU::sub0) 6707 .addImm(0); 6708 6709 // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1 6710 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e64), NewVAddrHi) 6711 .addDef(CondReg1, RegState::Dead) 6712 .addReg(RsrcPtr, 0, AMDGPU::sub1) 6713 .addReg(VAddr->getReg(), 0, AMDGPU::sub1) 6714 .addReg(CondReg0, RegState::Kill) 6715 .addImm(0); 6716 6717 // NewVaddr = {NewVaddrHi, NewVaddrLo} 6718 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr) 6719 .addReg(NewVAddrLo) 6720 .addImm(AMDGPU::sub0) 6721 .addReg(NewVAddrHi) 6722 .addImm(AMDGPU::sub1); 6723 6724 VAddr->setReg(NewVAddr); 6725 Rsrc->setReg(NewSRsrc); 6726 } else if (!VAddr && ST.hasAddr64()) { 6727 // This instructions is the _OFFSET variant, so we need to convert it to 6728 // ADDR64. 6729 assert(ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS && 6730 "FIXME: Need to emit flat atomics here"); 6731 6732 unsigned RsrcPtr, NewSRsrc; 6733 std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc); 6734 6735 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 6736 MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata); 6737 MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset); 6738 MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset); 6739 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode()); 6740 6741 // Atomics with return have an additional tied operand and are 6742 // missing some of the special bits. 6743 MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in); 6744 MachineInstr *Addr64; 6745 6746 if (!VDataIn) { 6747 // Regular buffer load / store. 6748 MachineInstrBuilder MIB = 6749 BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode)) 6750 .add(*VData) 6751 .addReg(NewVAddr) 6752 .addReg(NewSRsrc) 6753 .add(*SOffset) 6754 .add(*Offset); 6755 6756 if (const MachineOperand *CPol = 6757 getNamedOperand(MI, AMDGPU::OpName::cpol)) { 6758 MIB.addImm(CPol->getImm()); 6759 } 6760 6761 if (const MachineOperand *TFE = 6762 getNamedOperand(MI, AMDGPU::OpName::tfe)) { 6763 MIB.addImm(TFE->getImm()); 6764 } 6765 6766 MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::swz)); 6767 6768 MIB.cloneMemRefs(MI); 6769 Addr64 = MIB; 6770 } else { 6771 // Atomics with return. 6772 Addr64 = BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode)) 6773 .add(*VData) 6774 .add(*VDataIn) 6775 .addReg(NewVAddr) 6776 .addReg(NewSRsrc) 6777 .add(*SOffset) 6778 .add(*Offset) 6779 .addImm(getNamedImmOperand(MI, AMDGPU::OpName::cpol)) 6780 .cloneMemRefs(MI); 6781 } 6782 6783 MI.removeFromParent(); 6784 6785 // NewVaddr = {NewVaddrHi, NewVaddrLo} 6786 BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE), 6787 NewVAddr) 6788 .addReg(RsrcPtr, 0, AMDGPU::sub0) 6789 .addImm(AMDGPU::sub0) 6790 .addReg(RsrcPtr, 0, AMDGPU::sub1) 6791 .addImm(AMDGPU::sub1); 6792 } else { 6793 // Legalize a VGPR Rsrc and soffset together. 6794 if (!isSoffsetLegal) { 6795 MachineOperand *Soffset = getNamedOperand(MI, AMDGPU::OpName::soffset); 6796 CreatedBB = 6797 loadMBUFScalarOperandsFromVGPR(*this, MI, {Rsrc, Soffset}, MDT); 6798 return CreatedBB; 6799 } 6800 CreatedBB = loadMBUFScalarOperandsFromVGPR(*this, MI, {Rsrc}, MDT); 6801 return CreatedBB; 6802 } 6803 } 6804 6805 // Legalize a VGPR soffset. 6806 if (!isSoffsetLegal) { 6807 MachineOperand *Soffset = getNamedOperand(MI, AMDGPU::OpName::soffset); 6808 CreatedBB = loadMBUFScalarOperandsFromVGPR(*this, MI, {Soffset}, MDT); 6809 return CreatedBB; 6810 } 6811 return CreatedBB; 6812 } 6813 6814 void SIInstrWorklist::insert(MachineInstr *MI) { 6815 InstrList.insert(MI); 6816 // Add MBUF instructiosn to deferred list. 6817 int RsrcIdx = 6818 AMDGPU::getNamedOperandIdx(MI->getOpcode(), AMDGPU::OpName::srsrc); 6819 if (RsrcIdx != -1) { 6820 DeferredList.insert(MI); 6821 } 6822 } 6823 6824 bool SIInstrWorklist::isDeferred(MachineInstr *MI) { 6825 return DeferredList.contains(MI); 6826 } 6827 6828 void SIInstrInfo::moveToVALU(SIInstrWorklist &Worklist, 6829 MachineDominatorTree *MDT) const { 6830 6831 while (!Worklist.empty()) { 6832 MachineInstr &Inst = *Worklist.top(); 6833 Worklist.erase_top(); 6834 // Skip MachineInstr in the deferred list. 6835 if (Worklist.isDeferred(&Inst)) 6836 continue; 6837 moveToVALUImpl(Worklist, MDT, Inst); 6838 } 6839 6840 // Deferred list of instructions will be processed once 6841 // all the MachineInstr in the worklist are done. 6842 for (MachineInstr *Inst : Worklist.getDeferredList()) { 6843 moveToVALUImpl(Worklist, MDT, *Inst); 6844 assert(Worklist.empty() && 6845 "Deferred MachineInstr are not supposed to re-populate worklist"); 6846 } 6847 } 6848 6849 void SIInstrInfo::moveToVALUImpl(SIInstrWorklist &Worklist, 6850 MachineDominatorTree *MDT, 6851 MachineInstr &Inst) const { 6852 6853 MachineBasicBlock *MBB = Inst.getParent(); 6854 if (!MBB) 6855 return; 6856 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 6857 unsigned Opcode = Inst.getOpcode(); 6858 unsigned NewOpcode = getVALUOp(Inst); 6859 // Handle some special cases 6860 switch (Opcode) { 6861 default: 6862 break; 6863 case AMDGPU::S_ADD_U64_PSEUDO: 6864 NewOpcode = AMDGPU::V_ADD_U64_PSEUDO; 6865 break; 6866 case AMDGPU::S_SUB_U64_PSEUDO: 6867 NewOpcode = AMDGPU::V_SUB_U64_PSEUDO; 6868 break; 6869 case AMDGPU::S_ADD_I32: 6870 case AMDGPU::S_SUB_I32: { 6871 // FIXME: The u32 versions currently selected use the carry. 6872 bool Changed; 6873 MachineBasicBlock *CreatedBBTmp = nullptr; 6874 std::tie(Changed, CreatedBBTmp) = moveScalarAddSub(Worklist, Inst, MDT); 6875 if (Changed) 6876 return; 6877 6878 // Default handling 6879 break; 6880 } 6881 6882 case AMDGPU::S_MUL_U64: 6883 // Split s_mul_u64 in 32-bit vector multiplications. 6884 splitScalarSMulU64(Worklist, Inst, MDT); 6885 Inst.eraseFromParent(); 6886 return; 6887 6888 case AMDGPU::S_MUL_U64_U32_PSEUDO: 6889 case AMDGPU::S_MUL_I64_I32_PSEUDO: 6890 // This is a special case of s_mul_u64 where all the operands are either 6891 // zero extended or sign extended. 6892 splitScalarSMulPseudo(Worklist, Inst, MDT); 6893 Inst.eraseFromParent(); 6894 return; 6895 6896 case AMDGPU::S_AND_B64: 6897 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT); 6898 Inst.eraseFromParent(); 6899 return; 6900 6901 case AMDGPU::S_OR_B64: 6902 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT); 6903 Inst.eraseFromParent(); 6904 return; 6905 6906 case AMDGPU::S_XOR_B64: 6907 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT); 6908 Inst.eraseFromParent(); 6909 return; 6910 6911 case AMDGPU::S_NAND_B64: 6912 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT); 6913 Inst.eraseFromParent(); 6914 return; 6915 6916 case AMDGPU::S_NOR_B64: 6917 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT); 6918 Inst.eraseFromParent(); 6919 return; 6920 6921 case AMDGPU::S_XNOR_B64: 6922 if (ST.hasDLInsts()) 6923 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT); 6924 else 6925 splitScalar64BitXnor(Worklist, Inst, MDT); 6926 Inst.eraseFromParent(); 6927 return; 6928 6929 case AMDGPU::S_ANDN2_B64: 6930 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT); 6931 Inst.eraseFromParent(); 6932 return; 6933 6934 case AMDGPU::S_ORN2_B64: 6935 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT); 6936 Inst.eraseFromParent(); 6937 return; 6938 6939 case AMDGPU::S_BREV_B64: 6940 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_BREV_B32, true); 6941 Inst.eraseFromParent(); 6942 return; 6943 6944 case AMDGPU::S_NOT_B64: 6945 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32); 6946 Inst.eraseFromParent(); 6947 return; 6948 6949 case AMDGPU::S_BCNT1_I32_B64: 6950 splitScalar64BitBCNT(Worklist, Inst); 6951 Inst.eraseFromParent(); 6952 return; 6953 6954 case AMDGPU::S_BFE_I64: 6955 splitScalar64BitBFE(Worklist, Inst); 6956 Inst.eraseFromParent(); 6957 return; 6958 6959 case AMDGPU::S_FLBIT_I32_B64: 6960 splitScalar64BitCountOp(Worklist, Inst, AMDGPU::V_FFBH_U32_e32); 6961 Inst.eraseFromParent(); 6962 return; 6963 case AMDGPU::S_FF1_I32_B64: 6964 splitScalar64BitCountOp(Worklist, Inst, AMDGPU::V_FFBL_B32_e32); 6965 Inst.eraseFromParent(); 6966 return; 6967 6968 case AMDGPU::S_LSHL_B32: 6969 if (ST.hasOnlyRevVALUShifts()) { 6970 NewOpcode = AMDGPU::V_LSHLREV_B32_e64; 6971 swapOperands(Inst); 6972 } 6973 break; 6974 case AMDGPU::S_ASHR_I32: 6975 if (ST.hasOnlyRevVALUShifts()) { 6976 NewOpcode = AMDGPU::V_ASHRREV_I32_e64; 6977 swapOperands(Inst); 6978 } 6979 break; 6980 case AMDGPU::S_LSHR_B32: 6981 if (ST.hasOnlyRevVALUShifts()) { 6982 NewOpcode = AMDGPU::V_LSHRREV_B32_e64; 6983 swapOperands(Inst); 6984 } 6985 break; 6986 case AMDGPU::S_LSHL_B64: 6987 if (ST.hasOnlyRevVALUShifts()) { 6988 NewOpcode = ST.getGeneration() >= AMDGPUSubtarget::GFX12 6989 ? AMDGPU::V_LSHLREV_B64_pseudo_e64 6990 : AMDGPU::V_LSHLREV_B64_e64; 6991 swapOperands(Inst); 6992 } 6993 break; 6994 case AMDGPU::S_ASHR_I64: 6995 if (ST.hasOnlyRevVALUShifts()) { 6996 NewOpcode = AMDGPU::V_ASHRREV_I64_e64; 6997 swapOperands(Inst); 6998 } 6999 break; 7000 case AMDGPU::S_LSHR_B64: 7001 if (ST.hasOnlyRevVALUShifts()) { 7002 NewOpcode = AMDGPU::V_LSHRREV_B64_e64; 7003 swapOperands(Inst); 7004 } 7005 break; 7006 7007 case AMDGPU::S_ABS_I32: 7008 lowerScalarAbs(Worklist, Inst); 7009 Inst.eraseFromParent(); 7010 return; 7011 7012 case AMDGPU::S_CBRANCH_SCC0: 7013 case AMDGPU::S_CBRANCH_SCC1: { 7014 // Clear unused bits of vcc 7015 Register CondReg = Inst.getOperand(1).getReg(); 7016 bool IsSCC = CondReg == AMDGPU::SCC; 7017 Register VCC = RI.getVCC(); 7018 Register EXEC = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 7019 unsigned Opc = ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64; 7020 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(Opc), VCC) 7021 .addReg(EXEC) 7022 .addReg(IsSCC ? VCC : CondReg); 7023 Inst.removeOperand(1); 7024 } break; 7025 7026 case AMDGPU::S_BFE_U64: 7027 case AMDGPU::S_BFM_B64: 7028 llvm_unreachable("Moving this op to VALU not implemented"); 7029 7030 case AMDGPU::S_PACK_LL_B32_B16: 7031 case AMDGPU::S_PACK_LH_B32_B16: 7032 case AMDGPU::S_PACK_HL_B32_B16: 7033 case AMDGPU::S_PACK_HH_B32_B16: 7034 movePackToVALU(Worklist, MRI, Inst); 7035 Inst.eraseFromParent(); 7036 return; 7037 7038 case AMDGPU::S_XNOR_B32: 7039 lowerScalarXnor(Worklist, Inst); 7040 Inst.eraseFromParent(); 7041 return; 7042 7043 case AMDGPU::S_NAND_B32: 7044 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32); 7045 Inst.eraseFromParent(); 7046 return; 7047 7048 case AMDGPU::S_NOR_B32: 7049 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32); 7050 Inst.eraseFromParent(); 7051 return; 7052 7053 case AMDGPU::S_ANDN2_B32: 7054 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32); 7055 Inst.eraseFromParent(); 7056 return; 7057 7058 case AMDGPU::S_ORN2_B32: 7059 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32); 7060 Inst.eraseFromParent(); 7061 return; 7062 7063 // TODO: remove as soon as everything is ready 7064 // to replace VGPR to SGPR copy with V_READFIRSTLANEs. 7065 // S_ADD/SUB_CO_PSEUDO as well as S_UADDO/USUBO_PSEUDO 7066 // can only be selected from the uniform SDNode. 7067 case AMDGPU::S_ADD_CO_PSEUDO: 7068 case AMDGPU::S_SUB_CO_PSEUDO: { 7069 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO) 7070 ? AMDGPU::V_ADDC_U32_e64 7071 : AMDGPU::V_SUBB_U32_e64; 7072 const auto *CarryRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 7073 7074 Register CarryInReg = Inst.getOperand(4).getReg(); 7075 if (!MRI.constrainRegClass(CarryInReg, CarryRC)) { 7076 Register NewCarryReg = MRI.createVirtualRegister(CarryRC); 7077 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::COPY), NewCarryReg) 7078 .addReg(CarryInReg); 7079 } 7080 7081 Register CarryOutReg = Inst.getOperand(1).getReg(); 7082 7083 Register DestReg = MRI.createVirtualRegister(RI.getEquivalentVGPRClass( 7084 MRI.getRegClass(Inst.getOperand(0).getReg()))); 7085 MachineInstr *CarryOp = 7086 BuildMI(*MBB, &Inst, Inst.getDebugLoc(), get(Opc), DestReg) 7087 .addReg(CarryOutReg, RegState::Define) 7088 .add(Inst.getOperand(2)) 7089 .add(Inst.getOperand(3)) 7090 .addReg(CarryInReg) 7091 .addImm(0); 7092 legalizeOperands(*CarryOp); 7093 MRI.replaceRegWith(Inst.getOperand(0).getReg(), DestReg); 7094 addUsersToMoveToVALUWorklist(DestReg, MRI, Worklist); 7095 Inst.eraseFromParent(); 7096 } 7097 return; 7098 case AMDGPU::S_UADDO_PSEUDO: 7099 case AMDGPU::S_USUBO_PSEUDO: { 7100 const DebugLoc &DL = Inst.getDebugLoc(); 7101 MachineOperand &Dest0 = Inst.getOperand(0); 7102 MachineOperand &Dest1 = Inst.getOperand(1); 7103 MachineOperand &Src0 = Inst.getOperand(2); 7104 MachineOperand &Src1 = Inst.getOperand(3); 7105 7106 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_UADDO_PSEUDO) 7107 ? AMDGPU::V_ADD_CO_U32_e64 7108 : AMDGPU::V_SUB_CO_U32_e64; 7109 const TargetRegisterClass *NewRC = 7110 RI.getEquivalentVGPRClass(MRI.getRegClass(Dest0.getReg())); 7111 Register DestReg = MRI.createVirtualRegister(NewRC); 7112 MachineInstr *NewInstr = BuildMI(*MBB, &Inst, DL, get(Opc), DestReg) 7113 .addReg(Dest1.getReg(), RegState::Define) 7114 .add(Src0) 7115 .add(Src1) 7116 .addImm(0); // clamp bit 7117 7118 legalizeOperands(*NewInstr, MDT); 7119 MRI.replaceRegWith(Dest0.getReg(), DestReg); 7120 addUsersToMoveToVALUWorklist(NewInstr->getOperand(0).getReg(), MRI, 7121 Worklist); 7122 Inst.eraseFromParent(); 7123 } 7124 return; 7125 7126 case AMDGPU::S_CSELECT_B32: 7127 case AMDGPU::S_CSELECT_B64: 7128 lowerSelect(Worklist, Inst, MDT); 7129 Inst.eraseFromParent(); 7130 return; 7131 case AMDGPU::S_CMP_EQ_I32: 7132 case AMDGPU::S_CMP_LG_I32: 7133 case AMDGPU::S_CMP_GT_I32: 7134 case AMDGPU::S_CMP_GE_I32: 7135 case AMDGPU::S_CMP_LT_I32: 7136 case AMDGPU::S_CMP_LE_I32: 7137 case AMDGPU::S_CMP_EQ_U32: 7138 case AMDGPU::S_CMP_LG_U32: 7139 case AMDGPU::S_CMP_GT_U32: 7140 case AMDGPU::S_CMP_GE_U32: 7141 case AMDGPU::S_CMP_LT_U32: 7142 case AMDGPU::S_CMP_LE_U32: 7143 case AMDGPU::S_CMP_EQ_U64: 7144 case AMDGPU::S_CMP_LG_U64: 7145 case AMDGPU::S_CMP_LT_F32: 7146 case AMDGPU::S_CMP_EQ_F32: 7147 case AMDGPU::S_CMP_LE_F32: 7148 case AMDGPU::S_CMP_GT_F32: 7149 case AMDGPU::S_CMP_LG_F32: 7150 case AMDGPU::S_CMP_GE_F32: 7151 case AMDGPU::S_CMP_O_F32: 7152 case AMDGPU::S_CMP_U_F32: 7153 case AMDGPU::S_CMP_NGE_F32: 7154 case AMDGPU::S_CMP_NLG_F32: 7155 case AMDGPU::S_CMP_NGT_F32: 7156 case AMDGPU::S_CMP_NLE_F32: 7157 case AMDGPU::S_CMP_NEQ_F32: 7158 case AMDGPU::S_CMP_NLT_F32: 7159 case AMDGPU::S_CMP_LT_F16: 7160 case AMDGPU::S_CMP_EQ_F16: 7161 case AMDGPU::S_CMP_LE_F16: 7162 case AMDGPU::S_CMP_GT_F16: 7163 case AMDGPU::S_CMP_LG_F16: 7164 case AMDGPU::S_CMP_GE_F16: 7165 case AMDGPU::S_CMP_O_F16: 7166 case AMDGPU::S_CMP_U_F16: 7167 case AMDGPU::S_CMP_NGE_F16: 7168 case AMDGPU::S_CMP_NLG_F16: 7169 case AMDGPU::S_CMP_NGT_F16: 7170 case AMDGPU::S_CMP_NLE_F16: 7171 case AMDGPU::S_CMP_NEQ_F16: 7172 case AMDGPU::S_CMP_NLT_F16: { 7173 Register CondReg = MRI.createVirtualRegister(RI.getWaveMaskRegClass()); 7174 auto NewInstr = 7175 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(NewOpcode), CondReg) 7176 .setMIFlags(Inst.getFlags()); 7177 if (AMDGPU::getNamedOperandIdx(NewOpcode, 7178 AMDGPU::OpName::src0_modifiers) >= 0) { 7179 NewInstr 7180 .addImm(0) // src0_modifiers 7181 .add(Inst.getOperand(0)) // src0 7182 .addImm(0) // src1_modifiers 7183 .add(Inst.getOperand(1)) // src1 7184 .addImm(0); // clamp 7185 } else { 7186 NewInstr 7187 .add(Inst.getOperand(0)) 7188 .add(Inst.getOperand(1)); 7189 } 7190 legalizeOperands(*NewInstr, MDT); 7191 int SCCIdx = Inst.findRegisterDefOperandIdx(AMDGPU::SCC); 7192 MachineOperand SCCOp = Inst.getOperand(SCCIdx); 7193 addSCCDefUsersToVALUWorklist(SCCOp, Inst, Worklist, CondReg); 7194 Inst.eraseFromParent(); 7195 return; 7196 } 7197 case AMDGPU::S_CVT_HI_F32_F16: { 7198 const DebugLoc &DL = Inst.getDebugLoc(); 7199 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7200 Register NewDst = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7201 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg) 7202 .addImm(16) 7203 .add(Inst.getOperand(1)); 7204 BuildMI(*MBB, Inst, DL, get(NewOpcode), NewDst) 7205 .addImm(0) // src0_modifiers 7206 .addReg(TmpReg) 7207 .addImm(0) // clamp 7208 .addImm(0); // omod 7209 7210 MRI.replaceRegWith(Inst.getOperand(0).getReg(), NewDst); 7211 addUsersToMoveToVALUWorklist(NewDst, MRI, Worklist); 7212 Inst.eraseFromParent(); 7213 return; 7214 } 7215 case AMDGPU::S_MINIMUM_F32: 7216 case AMDGPU::S_MAXIMUM_F32: 7217 case AMDGPU::S_MINIMUM_F16: 7218 case AMDGPU::S_MAXIMUM_F16: { 7219 const DebugLoc &DL = Inst.getDebugLoc(); 7220 Register NewDst = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7221 MachineInstr *NewInstr = BuildMI(*MBB, Inst, DL, get(NewOpcode), NewDst) 7222 .addImm(0) // src0_modifiers 7223 .add(Inst.getOperand(1)) 7224 .addImm(0) // src1_modifiers 7225 .add(Inst.getOperand(2)) 7226 .addImm(0) // clamp 7227 .addImm(0); // omod 7228 MRI.replaceRegWith(Inst.getOperand(0).getReg(), NewDst); 7229 7230 legalizeOperands(*NewInstr, MDT); 7231 addUsersToMoveToVALUWorklist(NewDst, MRI, Worklist); 7232 Inst.eraseFromParent(); 7233 return; 7234 } 7235 } 7236 7237 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) { 7238 // We cannot move this instruction to the VALU, so we should try to 7239 // legalize its operands instead. 7240 legalizeOperands(Inst, MDT); 7241 return; 7242 } 7243 // Handle converting generic instructions like COPY-to-SGPR into 7244 // COPY-to-VGPR. 7245 if (NewOpcode == Opcode) { 7246 Register DstReg = Inst.getOperand(0).getReg(); 7247 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst); 7248 7249 // If it's a copy of a VGPR to a physical SGPR, insert a V_READFIRSTLANE and 7250 // hope for the best. 7251 if (Inst.isCopy() && DstReg.isPhysical() && 7252 RI.isVGPR(MRI, Inst.getOperand(1).getReg())) { 7253 // TODO: Only works for 32 bit registers. 7254 BuildMI(*Inst.getParent(), &Inst, Inst.getDebugLoc(), 7255 get(AMDGPU::V_READFIRSTLANE_B32), Inst.getOperand(0).getReg()) 7256 .add(Inst.getOperand(1)); 7257 Inst.eraseFromParent(); 7258 return; 7259 } 7260 7261 if (Inst.isCopy() && Inst.getOperand(1).getReg().isVirtual() && 7262 NewDstRC == RI.getRegClassForReg(MRI, Inst.getOperand(1).getReg())) { 7263 // Instead of creating a copy where src and dst are the same register 7264 // class, we just replace all uses of dst with src. These kinds of 7265 // copies interfere with the heuristics MachineSink uses to decide 7266 // whether or not to split a critical edge. Since the pass assumes 7267 // that copies will end up as machine instructions and not be 7268 // eliminated. 7269 addUsersToMoveToVALUWorklist(DstReg, MRI, Worklist); 7270 MRI.replaceRegWith(DstReg, Inst.getOperand(1).getReg()); 7271 MRI.clearKillFlags(Inst.getOperand(1).getReg()); 7272 Inst.getOperand(0).setReg(DstReg); 7273 // Make sure we don't leave around a dead VGPR->SGPR copy. Normally 7274 // these are deleted later, but at -O0 it would leave a suspicious 7275 // looking illegal copy of an undef register. 7276 for (unsigned I = Inst.getNumOperands() - 1; I != 0; --I) 7277 Inst.removeOperand(I); 7278 Inst.setDesc(get(AMDGPU::IMPLICIT_DEF)); 7279 return; 7280 } 7281 Register NewDstReg = MRI.createVirtualRegister(NewDstRC); 7282 MRI.replaceRegWith(DstReg, NewDstReg); 7283 legalizeOperands(Inst, MDT); 7284 addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist); 7285 return; 7286 } 7287 7288 // Use the new VALU Opcode. 7289 auto NewInstr = BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(NewOpcode)) 7290 .setMIFlags(Inst.getFlags()); 7291 if (isVOP3(NewOpcode) && !isVOP3(Opcode)) { 7292 // Intersperse VOP3 modifiers among the SALU operands. 7293 NewInstr->addOperand(Inst.getOperand(0)); 7294 if (AMDGPU::getNamedOperandIdx(NewOpcode, 7295 AMDGPU::OpName::src0_modifiers) >= 0) 7296 NewInstr.addImm(0); 7297 if (AMDGPU::hasNamedOperand(NewOpcode, AMDGPU::OpName::src0)) { 7298 MachineOperand Src = Inst.getOperand(1); 7299 if (AMDGPU::isTrue16Inst(NewOpcode) && ST.useRealTrue16Insts() && 7300 Src.isReg() && RI.isVGPR(MRI, Src.getReg())) 7301 NewInstr.addReg(Src.getReg(), 0, AMDGPU::lo16); 7302 else 7303 NewInstr->addOperand(Src); 7304 } 7305 7306 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) { 7307 // We are converting these to a BFE, so we need to add the missing 7308 // operands for the size and offset. 7309 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16; 7310 NewInstr.addImm(0); 7311 NewInstr.addImm(Size); 7312 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) { 7313 // The VALU version adds the second operand to the result, so insert an 7314 // extra 0 operand. 7315 NewInstr.addImm(0); 7316 } else if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) { 7317 const MachineOperand &OffsetWidthOp = Inst.getOperand(2); 7318 // If we need to move this to VGPRs, we need to unpack the second 7319 // operand back into the 2 separate ones for bit offset and width. 7320 assert(OffsetWidthOp.isImm() && 7321 "Scalar BFE is only implemented for constant width and offset"); 7322 uint32_t Imm = OffsetWidthOp.getImm(); 7323 7324 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0]. 7325 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16]. 7326 NewInstr.addImm(Offset); 7327 NewInstr.addImm(BitWidth); 7328 } else { 7329 if (AMDGPU::getNamedOperandIdx(NewOpcode, 7330 AMDGPU::OpName::src1_modifiers) >= 0) 7331 NewInstr.addImm(0); 7332 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::src1) >= 0) 7333 NewInstr->addOperand(Inst.getOperand(2)); 7334 if (AMDGPU::getNamedOperandIdx(NewOpcode, 7335 AMDGPU::OpName::src2_modifiers) >= 0) 7336 NewInstr.addImm(0); 7337 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::src2) >= 0) 7338 NewInstr->addOperand(Inst.getOperand(3)); 7339 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::clamp) >= 0) 7340 NewInstr.addImm(0); 7341 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::omod) >= 0) 7342 NewInstr.addImm(0); 7343 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::op_sel) >= 0) 7344 NewInstr.addImm(0); 7345 } 7346 } else { 7347 // Just copy the SALU operands. 7348 for (const MachineOperand &Op : Inst.explicit_operands()) 7349 NewInstr->addOperand(Op); 7350 } 7351 7352 // Remove any references to SCC. Vector instructions can't read from it, and 7353 // We're just about to add the implicit use / defs of VCC, and we don't want 7354 // both. 7355 for (MachineOperand &Op : Inst.implicit_operands()) { 7356 if (Op.getReg() == AMDGPU::SCC) { 7357 // Only propagate through live-def of SCC. 7358 if (Op.isDef() && !Op.isDead()) 7359 addSCCDefUsersToVALUWorklist(Op, Inst, Worklist); 7360 if (Op.isUse()) 7361 addSCCDefsToVALUWorklist(NewInstr, Worklist); 7362 } 7363 } 7364 Inst.eraseFromParent(); 7365 Register NewDstReg; 7366 if (NewInstr->getOperand(0).isReg() && NewInstr->getOperand(0).isDef()) { 7367 Register DstReg = NewInstr->getOperand(0).getReg(); 7368 assert(DstReg.isVirtual()); 7369 // Update the destination register class. 7370 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(*NewInstr); 7371 assert(NewDstRC); 7372 NewDstReg = MRI.createVirtualRegister(NewDstRC); 7373 MRI.replaceRegWith(DstReg, NewDstReg); 7374 } 7375 fixImplicitOperands(*NewInstr); 7376 // Legalize the operands 7377 legalizeOperands(*NewInstr, MDT); 7378 if (NewDstReg) 7379 addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist); 7380 } 7381 7382 // Add/sub require special handling to deal with carry outs. 7383 std::pair<bool, MachineBasicBlock *> 7384 SIInstrInfo::moveScalarAddSub(SIInstrWorklist &Worklist, MachineInstr &Inst, 7385 MachineDominatorTree *MDT) const { 7386 if (ST.hasAddNoCarry()) { 7387 // Assume there is no user of scc since we don't select this in that case. 7388 // Since scc isn't used, it doesn't really matter if the i32 or u32 variant 7389 // is used. 7390 7391 MachineBasicBlock &MBB = *Inst.getParent(); 7392 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7393 7394 Register OldDstReg = Inst.getOperand(0).getReg(); 7395 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7396 7397 unsigned Opc = Inst.getOpcode(); 7398 assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32); 7399 7400 unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ? 7401 AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64; 7402 7403 assert(Inst.getOperand(3).getReg() == AMDGPU::SCC); 7404 Inst.removeOperand(3); 7405 7406 Inst.setDesc(get(NewOpc)); 7407 Inst.addOperand(MachineOperand::CreateImm(0)); // clamp bit 7408 Inst.addImplicitDefUseOperands(*MBB.getParent()); 7409 MRI.replaceRegWith(OldDstReg, ResultReg); 7410 MachineBasicBlock *NewBB = legalizeOperands(Inst, MDT); 7411 7412 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 7413 return std::pair(true, NewBB); 7414 } 7415 7416 return std::pair(false, nullptr); 7417 } 7418 7419 void SIInstrInfo::lowerSelect(SIInstrWorklist &Worklist, MachineInstr &Inst, 7420 MachineDominatorTree *MDT) const { 7421 7422 MachineBasicBlock &MBB = *Inst.getParent(); 7423 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7424 MachineBasicBlock::iterator MII = Inst; 7425 DebugLoc DL = Inst.getDebugLoc(); 7426 7427 MachineOperand &Dest = Inst.getOperand(0); 7428 MachineOperand &Src0 = Inst.getOperand(1); 7429 MachineOperand &Src1 = Inst.getOperand(2); 7430 MachineOperand &Cond = Inst.getOperand(3); 7431 7432 Register CondReg = Cond.getReg(); 7433 bool IsSCC = (CondReg == AMDGPU::SCC); 7434 7435 // If this is a trivial select where the condition is effectively not SCC 7436 // (CondReg is a source of copy to SCC), then the select is semantically 7437 // equivalent to copying CondReg. Hence, there is no need to create 7438 // V_CNDMASK, we can just use that and bail out. 7439 if (!IsSCC && Src0.isImm() && (Src0.getImm() == -1) && Src1.isImm() && 7440 (Src1.getImm() == 0)) { 7441 MRI.replaceRegWith(Dest.getReg(), CondReg); 7442 return; 7443 } 7444 7445 Register NewCondReg = CondReg; 7446 if (IsSCC) { 7447 const TargetRegisterClass *TC = 7448 RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 7449 NewCondReg = MRI.createVirtualRegister(TC); 7450 7451 // Now look for the closest SCC def if it is a copy 7452 // replacing the CondReg with the COPY source register 7453 bool CopyFound = false; 7454 for (MachineInstr &CandI : 7455 make_range(std::next(MachineBasicBlock::reverse_iterator(Inst)), 7456 Inst.getParent()->rend())) { 7457 if (CandI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != 7458 -1) { 7459 if (CandI.isCopy() && CandI.getOperand(0).getReg() == AMDGPU::SCC) { 7460 BuildMI(MBB, MII, DL, get(AMDGPU::COPY), NewCondReg) 7461 .addReg(CandI.getOperand(1).getReg()); 7462 CopyFound = true; 7463 } 7464 break; 7465 } 7466 } 7467 if (!CopyFound) { 7468 // SCC def is not a copy 7469 // Insert a trivial select instead of creating a copy, because a copy from 7470 // SCC would semantically mean just copying a single bit, but we may need 7471 // the result to be a vector condition mask that needs preserving. 7472 unsigned Opcode = (ST.getWavefrontSize() == 64) ? AMDGPU::S_CSELECT_B64 7473 : AMDGPU::S_CSELECT_B32; 7474 auto NewSelect = 7475 BuildMI(MBB, MII, DL, get(Opcode), NewCondReg).addImm(-1).addImm(0); 7476 NewSelect->getOperand(3).setIsUndef(Cond.isUndef()); 7477 } 7478 } 7479 7480 Register NewDestReg = MRI.createVirtualRegister( 7481 RI.getEquivalentVGPRClass(MRI.getRegClass(Dest.getReg()))); 7482 MachineInstr *NewInst; 7483 if (Inst.getOpcode() == AMDGPU::S_CSELECT_B32) { 7484 NewInst = BuildMI(MBB, MII, DL, get(AMDGPU::V_CNDMASK_B32_e64), NewDestReg) 7485 .addImm(0) 7486 .add(Src1) // False 7487 .addImm(0) 7488 .add(Src0) // True 7489 .addReg(NewCondReg); 7490 } else { 7491 NewInst = 7492 BuildMI(MBB, MII, DL, get(AMDGPU::V_CNDMASK_B64_PSEUDO), NewDestReg) 7493 .add(Src1) // False 7494 .add(Src0) // True 7495 .addReg(NewCondReg); 7496 } 7497 MRI.replaceRegWith(Dest.getReg(), NewDestReg); 7498 legalizeOperands(*NewInst, MDT); 7499 addUsersToMoveToVALUWorklist(NewDestReg, MRI, Worklist); 7500 } 7501 7502 void SIInstrInfo::lowerScalarAbs(SIInstrWorklist &Worklist, 7503 MachineInstr &Inst) const { 7504 MachineBasicBlock &MBB = *Inst.getParent(); 7505 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7506 MachineBasicBlock::iterator MII = Inst; 7507 DebugLoc DL = Inst.getDebugLoc(); 7508 7509 MachineOperand &Dest = Inst.getOperand(0); 7510 MachineOperand &Src = Inst.getOperand(1); 7511 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7512 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7513 7514 unsigned SubOp = ST.hasAddNoCarry() ? 7515 AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_CO_U32_e32; 7516 7517 BuildMI(MBB, MII, DL, get(SubOp), TmpReg) 7518 .addImm(0) 7519 .addReg(Src.getReg()); 7520 7521 BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg) 7522 .addReg(Src.getReg()) 7523 .addReg(TmpReg); 7524 7525 MRI.replaceRegWith(Dest.getReg(), ResultReg); 7526 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 7527 } 7528 7529 void SIInstrInfo::lowerScalarXnor(SIInstrWorklist &Worklist, 7530 MachineInstr &Inst) const { 7531 MachineBasicBlock &MBB = *Inst.getParent(); 7532 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7533 MachineBasicBlock::iterator MII = Inst; 7534 const DebugLoc &DL = Inst.getDebugLoc(); 7535 7536 MachineOperand &Dest = Inst.getOperand(0); 7537 MachineOperand &Src0 = Inst.getOperand(1); 7538 MachineOperand &Src1 = Inst.getOperand(2); 7539 7540 if (ST.hasDLInsts()) { 7541 Register NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7542 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL); 7543 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL); 7544 7545 BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest) 7546 .add(Src0) 7547 .add(Src1); 7548 7549 MRI.replaceRegWith(Dest.getReg(), NewDest); 7550 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist); 7551 } else { 7552 // Using the identity !(x ^ y) == (!x ^ y) == (x ^ !y), we can 7553 // invert either source and then perform the XOR. If either source is a 7554 // scalar register, then we can leave the inversion on the scalar unit to 7555 // achieve a better distribution of scalar and vector instructions. 7556 bool Src0IsSGPR = Src0.isReg() && 7557 RI.isSGPRClass(MRI.getRegClass(Src0.getReg())); 7558 bool Src1IsSGPR = Src1.isReg() && 7559 RI.isSGPRClass(MRI.getRegClass(Src1.getReg())); 7560 MachineInstr *Xor; 7561 Register Temp = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 7562 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 7563 7564 // Build a pair of scalar instructions and add them to the work list. 7565 // The next iteration over the work list will lower these to the vector 7566 // unit as necessary. 7567 if (Src0IsSGPR) { 7568 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src0); 7569 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest) 7570 .addReg(Temp) 7571 .add(Src1); 7572 } else if (Src1IsSGPR) { 7573 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src1); 7574 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest) 7575 .add(Src0) 7576 .addReg(Temp); 7577 } else { 7578 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), Temp) 7579 .add(Src0) 7580 .add(Src1); 7581 MachineInstr *Not = 7582 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest).addReg(Temp); 7583 Worklist.insert(Not); 7584 } 7585 7586 MRI.replaceRegWith(Dest.getReg(), NewDest); 7587 7588 Worklist.insert(Xor); 7589 7590 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist); 7591 } 7592 } 7593 7594 void SIInstrInfo::splitScalarNotBinop(SIInstrWorklist &Worklist, 7595 MachineInstr &Inst, 7596 unsigned Opcode) const { 7597 MachineBasicBlock &MBB = *Inst.getParent(); 7598 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7599 MachineBasicBlock::iterator MII = Inst; 7600 const DebugLoc &DL = Inst.getDebugLoc(); 7601 7602 MachineOperand &Dest = Inst.getOperand(0); 7603 MachineOperand &Src0 = Inst.getOperand(1); 7604 MachineOperand &Src1 = Inst.getOperand(2); 7605 7606 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 7607 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 7608 7609 MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), Interm) 7610 .add(Src0) 7611 .add(Src1); 7612 7613 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest) 7614 .addReg(Interm); 7615 7616 Worklist.insert(&Op); 7617 Worklist.insert(&Not); 7618 7619 MRI.replaceRegWith(Dest.getReg(), NewDest); 7620 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist); 7621 } 7622 7623 void SIInstrInfo::splitScalarBinOpN2(SIInstrWorklist &Worklist, 7624 MachineInstr &Inst, 7625 unsigned Opcode) const { 7626 MachineBasicBlock &MBB = *Inst.getParent(); 7627 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7628 MachineBasicBlock::iterator MII = Inst; 7629 const DebugLoc &DL = Inst.getDebugLoc(); 7630 7631 MachineOperand &Dest = Inst.getOperand(0); 7632 MachineOperand &Src0 = Inst.getOperand(1); 7633 MachineOperand &Src1 = Inst.getOperand(2); 7634 7635 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 7636 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 7637 7638 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Interm) 7639 .add(Src1); 7640 7641 MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), NewDest) 7642 .add(Src0) 7643 .addReg(Interm); 7644 7645 Worklist.insert(&Not); 7646 Worklist.insert(&Op); 7647 7648 MRI.replaceRegWith(Dest.getReg(), NewDest); 7649 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist); 7650 } 7651 7652 void SIInstrInfo::splitScalar64BitUnaryOp(SIInstrWorklist &Worklist, 7653 MachineInstr &Inst, unsigned Opcode, 7654 bool Swap) const { 7655 MachineBasicBlock &MBB = *Inst.getParent(); 7656 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7657 7658 MachineOperand &Dest = Inst.getOperand(0); 7659 MachineOperand &Src0 = Inst.getOperand(1); 7660 DebugLoc DL = Inst.getDebugLoc(); 7661 7662 MachineBasicBlock::iterator MII = Inst; 7663 7664 const MCInstrDesc &InstDesc = get(Opcode); 7665 const TargetRegisterClass *Src0RC = Src0.isReg() ? 7666 MRI.getRegClass(Src0.getReg()) : 7667 &AMDGPU::SGPR_32RegClass; 7668 7669 const TargetRegisterClass *Src0SubRC = 7670 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0); 7671 7672 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 7673 AMDGPU::sub0, Src0SubRC); 7674 7675 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg()); 7676 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC); 7677 const TargetRegisterClass *NewDestSubRC = 7678 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0); 7679 7680 Register DestSub0 = MRI.createVirtualRegister(NewDestSubRC); 7681 MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0).add(SrcReg0Sub0); 7682 7683 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 7684 AMDGPU::sub1, Src0SubRC); 7685 7686 Register DestSub1 = MRI.createVirtualRegister(NewDestSubRC); 7687 MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1).add(SrcReg0Sub1); 7688 7689 if (Swap) 7690 std::swap(DestSub0, DestSub1); 7691 7692 Register FullDestReg = MRI.createVirtualRegister(NewDestRC); 7693 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg) 7694 .addReg(DestSub0) 7695 .addImm(AMDGPU::sub0) 7696 .addReg(DestSub1) 7697 .addImm(AMDGPU::sub1); 7698 7699 MRI.replaceRegWith(Dest.getReg(), FullDestReg); 7700 7701 Worklist.insert(&LoHalf); 7702 Worklist.insert(&HiHalf); 7703 7704 // We don't need to legalizeOperands here because for a single operand, src0 7705 // will support any kind of input. 7706 7707 // Move all users of this moved value. 7708 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist); 7709 } 7710 7711 // There is not a vector equivalent of s_mul_u64. For this reason, we need to 7712 // split the s_mul_u64 in 32-bit vector multiplications. 7713 void SIInstrInfo::splitScalarSMulU64(SIInstrWorklist &Worklist, 7714 MachineInstr &Inst, 7715 MachineDominatorTree *MDT) const { 7716 MachineBasicBlock &MBB = *Inst.getParent(); 7717 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7718 7719 Register FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 7720 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7721 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7722 7723 MachineOperand &Dest = Inst.getOperand(0); 7724 MachineOperand &Src0 = Inst.getOperand(1); 7725 MachineOperand &Src1 = Inst.getOperand(2); 7726 const DebugLoc &DL = Inst.getDebugLoc(); 7727 MachineBasicBlock::iterator MII = Inst; 7728 7729 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0.getReg()); 7730 const TargetRegisterClass *Src1RC = MRI.getRegClass(Src1.getReg()); 7731 const TargetRegisterClass *Src0SubRC = 7732 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0); 7733 if (RI.isSGPRClass(Src0SubRC)) 7734 Src0SubRC = RI.getEquivalentVGPRClass(Src0SubRC); 7735 const TargetRegisterClass *Src1SubRC = 7736 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0); 7737 if (RI.isSGPRClass(Src1SubRC)) 7738 Src1SubRC = RI.getEquivalentVGPRClass(Src1SubRC); 7739 7740 // First, we extract the low 32-bit and high 32-bit values from each of the 7741 // operands. 7742 MachineOperand Op0L = 7743 buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC); 7744 MachineOperand Op1L = 7745 buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC); 7746 MachineOperand Op0H = 7747 buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, AMDGPU::sub1, Src0SubRC); 7748 MachineOperand Op1H = 7749 buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, AMDGPU::sub1, Src1SubRC); 7750 7751 // The multilication is done as follows: 7752 // 7753 // Op1H Op1L 7754 // * Op0H Op0L 7755 // -------------------- 7756 // Op1H*Op0L Op1L*Op0L 7757 // + Op1H*Op0H Op1L*Op0H 7758 // ----------------------------------------- 7759 // (Op1H*Op0L + Op1L*Op0H + carry) Op1L*Op0L 7760 // 7761 // We drop Op1H*Op0H because the result of the multiplication is a 64-bit 7762 // value and that would overflow. 7763 // The low 32-bit value is Op1L*Op0L. 7764 // The high 32-bit value is Op1H*Op0L + Op1L*Op0H + carry (from Op1L*Op0L). 7765 7766 Register Op1L_Op0H_Reg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7767 MachineInstr *Op1L_Op0H = 7768 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), Op1L_Op0H_Reg) 7769 .add(Op1L) 7770 .add(Op0H); 7771 7772 Register Op1H_Op0L_Reg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7773 MachineInstr *Op1H_Op0L = 7774 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), Op1H_Op0L_Reg) 7775 .add(Op1H) 7776 .add(Op0L); 7777 7778 Register CarryReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7779 MachineInstr *Carry = 7780 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_HI_U32_e64), CarryReg) 7781 .add(Op1L) 7782 .add(Op0L); 7783 7784 MachineInstr *LoHalf = 7785 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), DestSub0) 7786 .add(Op1L) 7787 .add(Op0L); 7788 7789 Register AddReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7790 MachineInstr *Add = BuildMI(MBB, MII, DL, get(AMDGPU::V_ADD_U32_e32), AddReg) 7791 .addReg(Op1L_Op0H_Reg) 7792 .addReg(Op1H_Op0L_Reg); 7793 7794 MachineInstr *HiHalf = 7795 BuildMI(MBB, MII, DL, get(AMDGPU::V_ADD_U32_e32), DestSub1) 7796 .addReg(AddReg) 7797 .addReg(CarryReg); 7798 7799 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg) 7800 .addReg(DestSub0) 7801 .addImm(AMDGPU::sub0) 7802 .addReg(DestSub1) 7803 .addImm(AMDGPU::sub1); 7804 7805 MRI.replaceRegWith(Dest.getReg(), FullDestReg); 7806 7807 // Try to legalize the operands in case we need to swap the order to keep it 7808 // valid. 7809 legalizeOperands(*Op1L_Op0H, MDT); 7810 legalizeOperands(*Op1H_Op0L, MDT); 7811 legalizeOperands(*Carry, MDT); 7812 legalizeOperands(*LoHalf, MDT); 7813 legalizeOperands(*Add, MDT); 7814 legalizeOperands(*HiHalf, MDT); 7815 7816 // Move all users of this moved value. 7817 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist); 7818 } 7819 7820 // Lower S_MUL_U64_U32_PSEUDO/S_MUL_I64_I32_PSEUDO in two 32-bit vector 7821 // multiplications. 7822 void SIInstrInfo::splitScalarSMulPseudo(SIInstrWorklist &Worklist, 7823 MachineInstr &Inst, 7824 MachineDominatorTree *MDT) const { 7825 MachineBasicBlock &MBB = *Inst.getParent(); 7826 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7827 7828 Register FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 7829 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7830 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7831 7832 MachineOperand &Dest = Inst.getOperand(0); 7833 MachineOperand &Src0 = Inst.getOperand(1); 7834 MachineOperand &Src1 = Inst.getOperand(2); 7835 const DebugLoc &DL = Inst.getDebugLoc(); 7836 MachineBasicBlock::iterator MII = Inst; 7837 7838 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0.getReg()); 7839 const TargetRegisterClass *Src1RC = MRI.getRegClass(Src1.getReg()); 7840 const TargetRegisterClass *Src0SubRC = 7841 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0); 7842 if (RI.isSGPRClass(Src0SubRC)) 7843 Src0SubRC = RI.getEquivalentVGPRClass(Src0SubRC); 7844 const TargetRegisterClass *Src1SubRC = 7845 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0); 7846 if (RI.isSGPRClass(Src1SubRC)) 7847 Src1SubRC = RI.getEquivalentVGPRClass(Src1SubRC); 7848 7849 // First, we extract the low 32-bit and high 32-bit values from each of the 7850 // operands. 7851 MachineOperand Op0L = 7852 buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC); 7853 MachineOperand Op1L = 7854 buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC); 7855 7856 unsigned Opc = Inst.getOpcode(); 7857 unsigned NewOpc = Opc == AMDGPU::S_MUL_U64_U32_PSEUDO 7858 ? AMDGPU::V_MUL_HI_U32_e64 7859 : AMDGPU::V_MUL_HI_I32_e64; 7860 MachineInstr *HiHalf = 7861 BuildMI(MBB, MII, DL, get(NewOpc), DestSub1).add(Op1L).add(Op0L); 7862 7863 MachineInstr *LoHalf = 7864 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), DestSub0) 7865 .add(Op1L) 7866 .add(Op0L); 7867 7868 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg) 7869 .addReg(DestSub0) 7870 .addImm(AMDGPU::sub0) 7871 .addReg(DestSub1) 7872 .addImm(AMDGPU::sub1); 7873 7874 MRI.replaceRegWith(Dest.getReg(), FullDestReg); 7875 7876 // Try to legalize the operands in case we need to swap the order to keep it 7877 // valid. 7878 legalizeOperands(*HiHalf, MDT); 7879 legalizeOperands(*LoHalf, MDT); 7880 7881 // Move all users of this moved value. 7882 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist); 7883 } 7884 7885 void SIInstrInfo::splitScalar64BitBinaryOp(SIInstrWorklist &Worklist, 7886 MachineInstr &Inst, unsigned Opcode, 7887 MachineDominatorTree *MDT) const { 7888 MachineBasicBlock &MBB = *Inst.getParent(); 7889 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7890 7891 MachineOperand &Dest = Inst.getOperand(0); 7892 MachineOperand &Src0 = Inst.getOperand(1); 7893 MachineOperand &Src1 = Inst.getOperand(2); 7894 DebugLoc DL = Inst.getDebugLoc(); 7895 7896 MachineBasicBlock::iterator MII = Inst; 7897 7898 const MCInstrDesc &InstDesc = get(Opcode); 7899 const TargetRegisterClass *Src0RC = Src0.isReg() ? 7900 MRI.getRegClass(Src0.getReg()) : 7901 &AMDGPU::SGPR_32RegClass; 7902 7903 const TargetRegisterClass *Src0SubRC = 7904 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0); 7905 const TargetRegisterClass *Src1RC = Src1.isReg() ? 7906 MRI.getRegClass(Src1.getReg()) : 7907 &AMDGPU::SGPR_32RegClass; 7908 7909 const TargetRegisterClass *Src1SubRC = 7910 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0); 7911 7912 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 7913 AMDGPU::sub0, Src0SubRC); 7914 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, 7915 AMDGPU::sub0, Src1SubRC); 7916 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 7917 AMDGPU::sub1, Src0SubRC); 7918 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, 7919 AMDGPU::sub1, Src1SubRC); 7920 7921 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg()); 7922 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC); 7923 const TargetRegisterClass *NewDestSubRC = 7924 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0); 7925 7926 Register DestSub0 = MRI.createVirtualRegister(NewDestSubRC); 7927 MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0) 7928 .add(SrcReg0Sub0) 7929 .add(SrcReg1Sub0); 7930 7931 Register DestSub1 = MRI.createVirtualRegister(NewDestSubRC); 7932 MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1) 7933 .add(SrcReg0Sub1) 7934 .add(SrcReg1Sub1); 7935 7936 Register FullDestReg = MRI.createVirtualRegister(NewDestRC); 7937 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg) 7938 .addReg(DestSub0) 7939 .addImm(AMDGPU::sub0) 7940 .addReg(DestSub1) 7941 .addImm(AMDGPU::sub1); 7942 7943 MRI.replaceRegWith(Dest.getReg(), FullDestReg); 7944 7945 Worklist.insert(&LoHalf); 7946 Worklist.insert(&HiHalf); 7947 7948 // Move all users of this moved value. 7949 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist); 7950 } 7951 7952 void SIInstrInfo::splitScalar64BitXnor(SIInstrWorklist &Worklist, 7953 MachineInstr &Inst, 7954 MachineDominatorTree *MDT) const { 7955 MachineBasicBlock &MBB = *Inst.getParent(); 7956 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7957 7958 MachineOperand &Dest = Inst.getOperand(0); 7959 MachineOperand &Src0 = Inst.getOperand(1); 7960 MachineOperand &Src1 = Inst.getOperand(2); 7961 const DebugLoc &DL = Inst.getDebugLoc(); 7962 7963 MachineBasicBlock::iterator MII = Inst; 7964 7965 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg()); 7966 7967 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); 7968 7969 MachineOperand* Op0; 7970 MachineOperand* Op1; 7971 7972 if (Src0.isReg() && RI.isSGPRReg(MRI, Src0.getReg())) { 7973 Op0 = &Src0; 7974 Op1 = &Src1; 7975 } else { 7976 Op0 = &Src1; 7977 Op1 = &Src0; 7978 } 7979 7980 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B64), Interm) 7981 .add(*Op0); 7982 7983 Register NewDest = MRI.createVirtualRegister(DestRC); 7984 7985 MachineInstr &Xor = *BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B64), NewDest) 7986 .addReg(Interm) 7987 .add(*Op1); 7988 7989 MRI.replaceRegWith(Dest.getReg(), NewDest); 7990 7991 Worklist.insert(&Xor); 7992 } 7993 7994 void SIInstrInfo::splitScalar64BitBCNT(SIInstrWorklist &Worklist, 7995 MachineInstr &Inst) const { 7996 MachineBasicBlock &MBB = *Inst.getParent(); 7997 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7998 7999 MachineBasicBlock::iterator MII = Inst; 8000 const DebugLoc &DL = Inst.getDebugLoc(); 8001 8002 MachineOperand &Dest = Inst.getOperand(0); 8003 MachineOperand &Src = Inst.getOperand(1); 8004 8005 const MCInstrDesc &InstDesc = get(AMDGPU::V_BCNT_U32_B32_e64); 8006 const TargetRegisterClass *SrcRC = Src.isReg() ? 8007 MRI.getRegClass(Src.getReg()) : 8008 &AMDGPU::SGPR_32RegClass; 8009 8010 Register MidReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8011 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8012 8013 const TargetRegisterClass *SrcSubRC = 8014 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0); 8015 8016 MachineOperand SrcRegSub0 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, 8017 AMDGPU::sub0, SrcSubRC); 8018 MachineOperand SrcRegSub1 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, 8019 AMDGPU::sub1, SrcSubRC); 8020 8021 BuildMI(MBB, MII, DL, InstDesc, MidReg).add(SrcRegSub0).addImm(0); 8022 8023 BuildMI(MBB, MII, DL, InstDesc, ResultReg).add(SrcRegSub1).addReg(MidReg); 8024 8025 MRI.replaceRegWith(Dest.getReg(), ResultReg); 8026 8027 // We don't need to legalize operands here. src0 for either instruction can be 8028 // an SGPR, and the second input is unused or determined here. 8029 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 8030 } 8031 8032 void SIInstrInfo::splitScalar64BitBFE(SIInstrWorklist &Worklist, 8033 MachineInstr &Inst) const { 8034 MachineBasicBlock &MBB = *Inst.getParent(); 8035 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 8036 MachineBasicBlock::iterator MII = Inst; 8037 const DebugLoc &DL = Inst.getDebugLoc(); 8038 8039 MachineOperand &Dest = Inst.getOperand(0); 8040 uint32_t Imm = Inst.getOperand(2).getImm(); 8041 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0]. 8042 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16]. 8043 8044 (void) Offset; 8045 8046 // Only sext_inreg cases handled. 8047 assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 && 8048 Offset == 0 && "Not implemented"); 8049 8050 if (BitWidth < 32) { 8051 Register MidRegLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8052 Register MidRegHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8053 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 8054 8055 BuildMI(MBB, MII, DL, get(AMDGPU::V_BFE_I32_e64), MidRegLo) 8056 .addReg(Inst.getOperand(1).getReg(), 0, AMDGPU::sub0) 8057 .addImm(0) 8058 .addImm(BitWidth); 8059 8060 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e32), MidRegHi) 8061 .addImm(31) 8062 .addReg(MidRegLo); 8063 8064 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg) 8065 .addReg(MidRegLo) 8066 .addImm(AMDGPU::sub0) 8067 .addReg(MidRegHi) 8068 .addImm(AMDGPU::sub1); 8069 8070 MRI.replaceRegWith(Dest.getReg(), ResultReg); 8071 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 8072 return; 8073 } 8074 8075 MachineOperand &Src = Inst.getOperand(1); 8076 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8077 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 8078 8079 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e64), TmpReg) 8080 .addImm(31) 8081 .addReg(Src.getReg(), 0, AMDGPU::sub0); 8082 8083 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg) 8084 .addReg(Src.getReg(), 0, AMDGPU::sub0) 8085 .addImm(AMDGPU::sub0) 8086 .addReg(TmpReg) 8087 .addImm(AMDGPU::sub1); 8088 8089 MRI.replaceRegWith(Dest.getReg(), ResultReg); 8090 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 8091 } 8092 8093 void SIInstrInfo::splitScalar64BitCountOp(SIInstrWorklist &Worklist, 8094 MachineInstr &Inst, unsigned Opcode, 8095 MachineDominatorTree *MDT) const { 8096 // (S_FLBIT_I32_B64 hi:lo) -> 8097 // -> (umin (V_FFBH_U32_e32 hi), (uaddsat (V_FFBH_U32_e32 lo), 32)) 8098 // (S_FF1_I32_B64 hi:lo) -> 8099 // ->(umin (uaddsat (V_FFBL_B32_e32 hi), 32) (V_FFBL_B32_e32 lo)) 8100 8101 MachineBasicBlock &MBB = *Inst.getParent(); 8102 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 8103 MachineBasicBlock::iterator MII = Inst; 8104 const DebugLoc &DL = Inst.getDebugLoc(); 8105 8106 MachineOperand &Dest = Inst.getOperand(0); 8107 MachineOperand &Src = Inst.getOperand(1); 8108 8109 const MCInstrDesc &InstDesc = get(Opcode); 8110 8111 bool IsCtlz = Opcode == AMDGPU::V_FFBH_U32_e32; 8112 unsigned OpcodeAdd = 8113 ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32; 8114 8115 const TargetRegisterClass *SrcRC = 8116 Src.isReg() ? MRI.getRegClass(Src.getReg()) : &AMDGPU::SGPR_32RegClass; 8117 const TargetRegisterClass *SrcSubRC = 8118 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0); 8119 8120 MachineOperand SrcRegSub0 = 8121 buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, AMDGPU::sub0, SrcSubRC); 8122 MachineOperand SrcRegSub1 = 8123 buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, AMDGPU::sub1, SrcSubRC); 8124 8125 Register MidReg1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8126 Register MidReg2 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8127 Register MidReg3 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8128 Register MidReg4 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8129 8130 BuildMI(MBB, MII, DL, InstDesc, MidReg1).add(SrcRegSub0); 8131 8132 BuildMI(MBB, MII, DL, InstDesc, MidReg2).add(SrcRegSub1); 8133 8134 BuildMI(MBB, MII, DL, get(OpcodeAdd), MidReg3) 8135 .addReg(IsCtlz ? MidReg1 : MidReg2) 8136 .addImm(32) 8137 .addImm(1); // enable clamp 8138 8139 BuildMI(MBB, MII, DL, get(AMDGPU::V_MIN_U32_e64), MidReg4) 8140 .addReg(MidReg3) 8141 .addReg(IsCtlz ? MidReg2 : MidReg1); 8142 8143 MRI.replaceRegWith(Dest.getReg(), MidReg4); 8144 8145 addUsersToMoveToVALUWorklist(MidReg4, MRI, Worklist); 8146 } 8147 8148 void SIInstrInfo::addUsersToMoveToVALUWorklist( 8149 Register DstReg, MachineRegisterInfo &MRI, 8150 SIInstrWorklist &Worklist) const { 8151 for (MachineRegisterInfo::use_iterator I = MRI.use_begin(DstReg), 8152 E = MRI.use_end(); I != E;) { 8153 MachineInstr &UseMI = *I->getParent(); 8154 8155 unsigned OpNo = 0; 8156 8157 switch (UseMI.getOpcode()) { 8158 case AMDGPU::COPY: 8159 case AMDGPU::WQM: 8160 case AMDGPU::SOFT_WQM: 8161 case AMDGPU::STRICT_WWM: 8162 case AMDGPU::STRICT_WQM: 8163 case AMDGPU::REG_SEQUENCE: 8164 case AMDGPU::PHI: 8165 case AMDGPU::INSERT_SUBREG: 8166 break; 8167 default: 8168 OpNo = I.getOperandNo(); 8169 break; 8170 } 8171 8172 if (!RI.hasVectorRegisters(getOpRegClass(UseMI, OpNo))) { 8173 Worklist.insert(&UseMI); 8174 8175 do { 8176 ++I; 8177 } while (I != E && I->getParent() == &UseMI); 8178 } else { 8179 ++I; 8180 } 8181 } 8182 } 8183 8184 void SIInstrInfo::movePackToVALU(SIInstrWorklist &Worklist, 8185 MachineRegisterInfo &MRI, 8186 MachineInstr &Inst) const { 8187 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8188 MachineBasicBlock *MBB = Inst.getParent(); 8189 MachineOperand &Src0 = Inst.getOperand(1); 8190 MachineOperand &Src1 = Inst.getOperand(2); 8191 const DebugLoc &DL = Inst.getDebugLoc(); 8192 8193 switch (Inst.getOpcode()) { 8194 case AMDGPU::S_PACK_LL_B32_B16: { 8195 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8196 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8197 8198 // FIXME: Can do a lot better if we know the high bits of src0 or src1 are 8199 // 0. 8200 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) 8201 .addImm(0xffff); 8202 8203 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_B32_e64), TmpReg) 8204 .addReg(ImmReg, RegState::Kill) 8205 .add(Src0); 8206 8207 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32_e64), ResultReg) 8208 .add(Src1) 8209 .addImm(16) 8210 .addReg(TmpReg, RegState::Kill); 8211 break; 8212 } 8213 case AMDGPU::S_PACK_LH_B32_B16: { 8214 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8215 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) 8216 .addImm(0xffff); 8217 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_BFI_B32_e64), ResultReg) 8218 .addReg(ImmReg, RegState::Kill) 8219 .add(Src0) 8220 .add(Src1); 8221 break; 8222 } 8223 case AMDGPU::S_PACK_HL_B32_B16: { 8224 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8225 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg) 8226 .addImm(16) 8227 .add(Src0); 8228 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32_e64), ResultReg) 8229 .add(Src1) 8230 .addImm(16) 8231 .addReg(TmpReg, RegState::Kill); 8232 break; 8233 } 8234 case AMDGPU::S_PACK_HH_B32_B16: { 8235 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8236 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8237 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg) 8238 .addImm(16) 8239 .add(Src0); 8240 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) 8241 .addImm(0xffff0000); 8242 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_OR_B32_e64), ResultReg) 8243 .add(Src1) 8244 .addReg(ImmReg, RegState::Kill) 8245 .addReg(TmpReg, RegState::Kill); 8246 break; 8247 } 8248 default: 8249 llvm_unreachable("unhandled s_pack_* instruction"); 8250 } 8251 8252 MachineOperand &Dest = Inst.getOperand(0); 8253 MRI.replaceRegWith(Dest.getReg(), ResultReg); 8254 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 8255 } 8256 8257 void SIInstrInfo::addSCCDefUsersToVALUWorklist(MachineOperand &Op, 8258 MachineInstr &SCCDefInst, 8259 SIInstrWorklist &Worklist, 8260 Register NewCond) const { 8261 8262 // Ensure that def inst defines SCC, which is still live. 8263 assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isDef() && 8264 !Op.isDead() && Op.getParent() == &SCCDefInst); 8265 SmallVector<MachineInstr *, 4> CopyToDelete; 8266 // This assumes that all the users of SCC are in the same block 8267 // as the SCC def. 8268 for (MachineInstr &MI : // Skip the def inst itself. 8269 make_range(std::next(MachineBasicBlock::iterator(SCCDefInst)), 8270 SCCDefInst.getParent()->end())) { 8271 // Check if SCC is used first. 8272 int SCCIdx = MI.findRegisterUseOperandIdx(AMDGPU::SCC, false, &RI); 8273 if (SCCIdx != -1) { 8274 if (MI.isCopy()) { 8275 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 8276 Register DestReg = MI.getOperand(0).getReg(); 8277 8278 MRI.replaceRegWith(DestReg, NewCond); 8279 CopyToDelete.push_back(&MI); 8280 } else { 8281 8282 if (NewCond.isValid()) 8283 MI.getOperand(SCCIdx).setReg(NewCond); 8284 8285 Worklist.insert(&MI); 8286 } 8287 } 8288 // Exit if we find another SCC def. 8289 if (MI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != -1) 8290 break; 8291 } 8292 for (auto &Copy : CopyToDelete) 8293 Copy->eraseFromParent(); 8294 } 8295 8296 // Instructions that use SCC may be converted to VALU instructions. When that 8297 // happens, the SCC register is changed to VCC_LO. The instruction that defines 8298 // SCC must be changed to an instruction that defines VCC. This function makes 8299 // sure that the instruction that defines SCC is added to the moveToVALU 8300 // worklist. 8301 void SIInstrInfo::addSCCDefsToVALUWorklist(MachineInstr *SCCUseInst, 8302 SIInstrWorklist &Worklist) const { 8303 // Look for a preceding instruction that either defines VCC or SCC. If VCC 8304 // then there is nothing to do because the defining instruction has been 8305 // converted to a VALU already. If SCC then that instruction needs to be 8306 // converted to a VALU. 8307 for (MachineInstr &MI : 8308 make_range(std::next(MachineBasicBlock::reverse_iterator(SCCUseInst)), 8309 SCCUseInst->getParent()->rend())) { 8310 if (MI.modifiesRegister(AMDGPU::VCC, &RI)) 8311 break; 8312 if (MI.definesRegister(AMDGPU::SCC, &RI)) { 8313 Worklist.insert(&MI); 8314 break; 8315 } 8316 } 8317 } 8318 8319 const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass( 8320 const MachineInstr &Inst) const { 8321 const TargetRegisterClass *NewDstRC = getOpRegClass(Inst, 0); 8322 8323 switch (Inst.getOpcode()) { 8324 // For target instructions, getOpRegClass just returns the virtual register 8325 // class associated with the operand, so we need to find an equivalent VGPR 8326 // register class in order to move the instruction to the VALU. 8327 case AMDGPU::COPY: 8328 case AMDGPU::PHI: 8329 case AMDGPU::REG_SEQUENCE: 8330 case AMDGPU::INSERT_SUBREG: 8331 case AMDGPU::WQM: 8332 case AMDGPU::SOFT_WQM: 8333 case AMDGPU::STRICT_WWM: 8334 case AMDGPU::STRICT_WQM: { 8335 const TargetRegisterClass *SrcRC = getOpRegClass(Inst, 1); 8336 if (RI.isAGPRClass(SrcRC)) { 8337 if (RI.isAGPRClass(NewDstRC)) 8338 return nullptr; 8339 8340 switch (Inst.getOpcode()) { 8341 case AMDGPU::PHI: 8342 case AMDGPU::REG_SEQUENCE: 8343 case AMDGPU::INSERT_SUBREG: 8344 NewDstRC = RI.getEquivalentAGPRClass(NewDstRC); 8345 break; 8346 default: 8347 NewDstRC = RI.getEquivalentVGPRClass(NewDstRC); 8348 } 8349 8350 if (!NewDstRC) 8351 return nullptr; 8352 } else { 8353 if (RI.isVGPRClass(NewDstRC) || NewDstRC == &AMDGPU::VReg_1RegClass) 8354 return nullptr; 8355 8356 NewDstRC = RI.getEquivalentVGPRClass(NewDstRC); 8357 if (!NewDstRC) 8358 return nullptr; 8359 } 8360 8361 return NewDstRC; 8362 } 8363 default: 8364 return NewDstRC; 8365 } 8366 } 8367 8368 // Find the one SGPR operand we are allowed to use. 8369 Register SIInstrInfo::findUsedSGPR(const MachineInstr &MI, 8370 int OpIndices[3]) const { 8371 const MCInstrDesc &Desc = MI.getDesc(); 8372 8373 // Find the one SGPR operand we are allowed to use. 8374 // 8375 // First we need to consider the instruction's operand requirements before 8376 // legalizing. Some operands are required to be SGPRs, such as implicit uses 8377 // of VCC, but we are still bound by the constant bus requirement to only use 8378 // one. 8379 // 8380 // If the operand's class is an SGPR, we can never move it. 8381 8382 Register SGPRReg = findImplicitSGPRRead(MI); 8383 if (SGPRReg) 8384 return SGPRReg; 8385 8386 Register UsedSGPRs[3] = {Register()}; 8387 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 8388 8389 for (unsigned i = 0; i < 3; ++i) { 8390 int Idx = OpIndices[i]; 8391 if (Idx == -1) 8392 break; 8393 8394 const MachineOperand &MO = MI.getOperand(Idx); 8395 if (!MO.isReg()) 8396 continue; 8397 8398 // Is this operand statically required to be an SGPR based on the operand 8399 // constraints? 8400 const TargetRegisterClass *OpRC = 8401 RI.getRegClass(Desc.operands()[Idx].RegClass); 8402 bool IsRequiredSGPR = RI.isSGPRClass(OpRC); 8403 if (IsRequiredSGPR) 8404 return MO.getReg(); 8405 8406 // If this could be a VGPR or an SGPR, Check the dynamic register class. 8407 Register Reg = MO.getReg(); 8408 const TargetRegisterClass *RegRC = MRI.getRegClass(Reg); 8409 if (RI.isSGPRClass(RegRC)) 8410 UsedSGPRs[i] = Reg; 8411 } 8412 8413 // We don't have a required SGPR operand, so we have a bit more freedom in 8414 // selecting operands to move. 8415 8416 // Try to select the most used SGPR. If an SGPR is equal to one of the 8417 // others, we choose that. 8418 // 8419 // e.g. 8420 // V_FMA_F32 v0, s0, s0, s0 -> No moves 8421 // V_FMA_F32 v0, s0, s1, s0 -> Move s1 8422 8423 // TODO: If some of the operands are 64-bit SGPRs and some 32, we should 8424 // prefer those. 8425 8426 if (UsedSGPRs[0]) { 8427 if (UsedSGPRs[0] == UsedSGPRs[1] || UsedSGPRs[0] == UsedSGPRs[2]) 8428 SGPRReg = UsedSGPRs[0]; 8429 } 8430 8431 if (!SGPRReg && UsedSGPRs[1]) { 8432 if (UsedSGPRs[1] == UsedSGPRs[2]) 8433 SGPRReg = UsedSGPRs[1]; 8434 } 8435 8436 return SGPRReg; 8437 } 8438 8439 MachineOperand *SIInstrInfo::getNamedOperand(MachineInstr &MI, 8440 unsigned OperandName) const { 8441 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OperandName); 8442 if (Idx == -1) 8443 return nullptr; 8444 8445 return &MI.getOperand(Idx); 8446 } 8447 8448 uint64_t SIInstrInfo::getDefaultRsrcDataFormat() const { 8449 if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) { 8450 int64_t Format = ST.getGeneration() >= AMDGPUSubtarget::GFX11 8451 ? (int64_t)AMDGPU::UfmtGFX11::UFMT_32_FLOAT 8452 : (int64_t)AMDGPU::UfmtGFX10::UFMT_32_FLOAT; 8453 return (Format << 44) | 8454 (1ULL << 56) | // RESOURCE_LEVEL = 1 8455 (3ULL << 60); // OOB_SELECT = 3 8456 } 8457 8458 uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT; 8459 if (ST.isAmdHsaOS()) { 8460 // Set ATC = 1. GFX9 doesn't have this bit. 8461 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) 8462 RsrcDataFormat |= (1ULL << 56); 8463 8464 // Set MTYPE = 2 (MTYPE_UC = uncached). GFX9 doesn't have this. 8465 // BTW, it disables TC L2 and therefore decreases performance. 8466 if (ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS) 8467 RsrcDataFormat |= (2ULL << 59); 8468 } 8469 8470 return RsrcDataFormat; 8471 } 8472 8473 uint64_t SIInstrInfo::getScratchRsrcWords23() const { 8474 uint64_t Rsrc23 = getDefaultRsrcDataFormat() | 8475 AMDGPU::RSRC_TID_ENABLE | 8476 0xffffffff; // Size; 8477 8478 // GFX9 doesn't have ELEMENT_SIZE. 8479 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) { 8480 uint64_t EltSizeValue = Log2_32(ST.getMaxPrivateElementSize(true)) - 1; 8481 Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT; 8482 } 8483 8484 // IndexStride = 64 / 32. 8485 uint64_t IndexStride = ST.getWavefrontSize() == 64 ? 3 : 2; 8486 Rsrc23 |= IndexStride << AMDGPU::RSRC_INDEX_STRIDE_SHIFT; 8487 8488 // If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17]. 8489 // Clear them unless we want a huge stride. 8490 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS && 8491 ST.getGeneration() <= AMDGPUSubtarget::GFX9) 8492 Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT; 8493 8494 return Rsrc23; 8495 } 8496 8497 bool SIInstrInfo::isLowLatencyInstruction(const MachineInstr &MI) const { 8498 unsigned Opc = MI.getOpcode(); 8499 8500 return isSMRD(Opc); 8501 } 8502 8503 bool SIInstrInfo::isHighLatencyDef(int Opc) const { 8504 return get(Opc).mayLoad() && 8505 (isMUBUF(Opc) || isMTBUF(Opc) || isMIMG(Opc) || isFLAT(Opc)); 8506 } 8507 8508 unsigned SIInstrInfo::isStackAccess(const MachineInstr &MI, 8509 int &FrameIndex) const { 8510 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::vaddr); 8511 if (!Addr || !Addr->isFI()) 8512 return Register(); 8513 8514 assert(!MI.memoperands_empty() && 8515 (*MI.memoperands_begin())->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS); 8516 8517 FrameIndex = Addr->getIndex(); 8518 return getNamedOperand(MI, AMDGPU::OpName::vdata)->getReg(); 8519 } 8520 8521 unsigned SIInstrInfo::isSGPRStackAccess(const MachineInstr &MI, 8522 int &FrameIndex) const { 8523 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::addr); 8524 assert(Addr && Addr->isFI()); 8525 FrameIndex = Addr->getIndex(); 8526 return getNamedOperand(MI, AMDGPU::OpName::data)->getReg(); 8527 } 8528 8529 unsigned SIInstrInfo::isLoadFromStackSlot(const MachineInstr &MI, 8530 int &FrameIndex) const { 8531 if (!MI.mayLoad()) 8532 return Register(); 8533 8534 if (isMUBUF(MI) || isVGPRSpill(MI)) 8535 return isStackAccess(MI, FrameIndex); 8536 8537 if (isSGPRSpill(MI)) 8538 return isSGPRStackAccess(MI, FrameIndex); 8539 8540 return Register(); 8541 } 8542 8543 unsigned SIInstrInfo::isStoreToStackSlot(const MachineInstr &MI, 8544 int &FrameIndex) const { 8545 if (!MI.mayStore()) 8546 return Register(); 8547 8548 if (isMUBUF(MI) || isVGPRSpill(MI)) 8549 return isStackAccess(MI, FrameIndex); 8550 8551 if (isSGPRSpill(MI)) 8552 return isSGPRStackAccess(MI, FrameIndex); 8553 8554 return Register(); 8555 } 8556 8557 unsigned SIInstrInfo::getInstBundleSize(const MachineInstr &MI) const { 8558 unsigned Size = 0; 8559 MachineBasicBlock::const_instr_iterator I = MI.getIterator(); 8560 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end(); 8561 while (++I != E && I->isInsideBundle()) { 8562 assert(!I->isBundle() && "No nested bundle!"); 8563 Size += getInstSizeInBytes(*I); 8564 } 8565 8566 return Size; 8567 } 8568 8569 unsigned SIInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { 8570 unsigned Opc = MI.getOpcode(); 8571 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opc); 8572 unsigned DescSize = Desc.getSize(); 8573 8574 // If we have a definitive size, we can use it. Otherwise we need to inspect 8575 // the operands to know the size. 8576 if (isFixedSize(MI)) { 8577 unsigned Size = DescSize; 8578 8579 // If we hit the buggy offset, an extra nop will be inserted in MC so 8580 // estimate the worst case. 8581 if (MI.isBranch() && ST.hasOffset3fBug()) 8582 Size += 4; 8583 8584 return Size; 8585 } 8586 8587 // Instructions may have a 32-bit literal encoded after them. Check 8588 // operands that could ever be literals. 8589 if (isVALU(MI) || isSALU(MI)) { 8590 if (isDPP(MI)) 8591 return DescSize; 8592 bool HasLiteral = false; 8593 for (int I = 0, E = MI.getNumExplicitOperands(); I != E; ++I) { 8594 const MachineOperand &Op = MI.getOperand(I); 8595 const MCOperandInfo &OpInfo = Desc.operands()[I]; 8596 if (!Op.isReg() && !isInlineConstant(Op, OpInfo)) { 8597 HasLiteral = true; 8598 break; 8599 } 8600 } 8601 return HasLiteral ? DescSize + 4 : DescSize; 8602 } 8603 8604 // Check whether we have extra NSA words. 8605 if (isMIMG(MI)) { 8606 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0); 8607 if (VAddr0Idx < 0) 8608 return 8; 8609 8610 int RSrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc); 8611 return 8 + 4 * ((RSrcIdx - VAddr0Idx + 2) / 4); 8612 } 8613 8614 switch (Opc) { 8615 case TargetOpcode::BUNDLE: 8616 return getInstBundleSize(MI); 8617 case TargetOpcode::INLINEASM: 8618 case TargetOpcode::INLINEASM_BR: { 8619 const MachineFunction *MF = MI.getParent()->getParent(); 8620 const char *AsmStr = MI.getOperand(0).getSymbolName(); 8621 return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo(), &ST); 8622 } 8623 default: 8624 if (MI.isMetaInstruction()) 8625 return 0; 8626 return DescSize; 8627 } 8628 } 8629 8630 bool SIInstrInfo::mayAccessFlatAddressSpace(const MachineInstr &MI) const { 8631 if (!isFLAT(MI)) 8632 return false; 8633 8634 if (MI.memoperands_empty()) 8635 return true; 8636 8637 for (const MachineMemOperand *MMO : MI.memoperands()) { 8638 if (MMO->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS) 8639 return true; 8640 } 8641 return false; 8642 } 8643 8644 bool SIInstrInfo::isNonUniformBranchInstr(MachineInstr &Branch) const { 8645 return Branch.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO; 8646 } 8647 8648 void SIInstrInfo::convertNonUniformIfRegion(MachineBasicBlock *IfEntry, 8649 MachineBasicBlock *IfEnd) const { 8650 MachineBasicBlock::iterator TI = IfEntry->getFirstTerminator(); 8651 assert(TI != IfEntry->end()); 8652 8653 MachineInstr *Branch = &(*TI); 8654 MachineFunction *MF = IfEntry->getParent(); 8655 MachineRegisterInfo &MRI = IfEntry->getParent()->getRegInfo(); 8656 8657 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { 8658 Register DstReg = MRI.createVirtualRegister(RI.getBoolRC()); 8659 MachineInstr *SIIF = 8660 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_IF), DstReg) 8661 .add(Branch->getOperand(0)) 8662 .add(Branch->getOperand(1)); 8663 MachineInstr *SIEND = 8664 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_END_CF)) 8665 .addReg(DstReg); 8666 8667 IfEntry->erase(TI); 8668 IfEntry->insert(IfEntry->end(), SIIF); 8669 IfEnd->insert(IfEnd->getFirstNonPHI(), SIEND); 8670 } 8671 } 8672 8673 void SIInstrInfo::convertNonUniformLoopRegion( 8674 MachineBasicBlock *LoopEntry, MachineBasicBlock *LoopEnd) const { 8675 MachineBasicBlock::iterator TI = LoopEnd->getFirstTerminator(); 8676 // We expect 2 terminators, one conditional and one unconditional. 8677 assert(TI != LoopEnd->end()); 8678 8679 MachineInstr *Branch = &(*TI); 8680 MachineFunction *MF = LoopEnd->getParent(); 8681 MachineRegisterInfo &MRI = LoopEnd->getParent()->getRegInfo(); 8682 8683 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { 8684 8685 Register DstReg = MRI.createVirtualRegister(RI.getBoolRC()); 8686 Register BackEdgeReg = MRI.createVirtualRegister(RI.getBoolRC()); 8687 MachineInstrBuilder HeaderPHIBuilder = 8688 BuildMI(*(MF), Branch->getDebugLoc(), get(TargetOpcode::PHI), DstReg); 8689 for (MachineBasicBlock *PMBB : LoopEntry->predecessors()) { 8690 if (PMBB == LoopEnd) { 8691 HeaderPHIBuilder.addReg(BackEdgeReg); 8692 } else { 8693 Register ZeroReg = MRI.createVirtualRegister(RI.getBoolRC()); 8694 materializeImmediate(*PMBB, PMBB->getFirstTerminator(), DebugLoc(), 8695 ZeroReg, 0); 8696 HeaderPHIBuilder.addReg(ZeroReg); 8697 } 8698 HeaderPHIBuilder.addMBB(PMBB); 8699 } 8700 MachineInstr *HeaderPhi = HeaderPHIBuilder; 8701 MachineInstr *SIIFBREAK = BuildMI(*(MF), Branch->getDebugLoc(), 8702 get(AMDGPU::SI_IF_BREAK), BackEdgeReg) 8703 .addReg(DstReg) 8704 .add(Branch->getOperand(0)); 8705 MachineInstr *SILOOP = 8706 BuildMI(*(MF), Branch->getDebugLoc(), get(AMDGPU::SI_LOOP)) 8707 .addReg(BackEdgeReg) 8708 .addMBB(LoopEntry); 8709 8710 LoopEntry->insert(LoopEntry->begin(), HeaderPhi); 8711 LoopEnd->erase(TI); 8712 LoopEnd->insert(LoopEnd->end(), SIIFBREAK); 8713 LoopEnd->insert(LoopEnd->end(), SILOOP); 8714 } 8715 } 8716 8717 ArrayRef<std::pair<int, const char *>> 8718 SIInstrInfo::getSerializableTargetIndices() const { 8719 static const std::pair<int, const char *> TargetIndices[] = { 8720 {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"}, 8721 {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"}, 8722 {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"}, 8723 {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"}, 8724 {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}}; 8725 return ArrayRef(TargetIndices); 8726 } 8727 8728 /// This is used by the post-RA scheduler (SchedulePostRAList.cpp). The 8729 /// post-RA version of misched uses CreateTargetMIHazardRecognizer. 8730 ScheduleHazardRecognizer * 8731 SIInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II, 8732 const ScheduleDAG *DAG) const { 8733 return new GCNHazardRecognizer(DAG->MF); 8734 } 8735 8736 /// This is the hazard recognizer used at -O0 by the PostRAHazardRecognizer 8737 /// pass. 8738 ScheduleHazardRecognizer * 8739 SIInstrInfo::CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const { 8740 return new GCNHazardRecognizer(MF); 8741 } 8742 8743 // Called during: 8744 // - pre-RA scheduling and post-RA scheduling 8745 ScheduleHazardRecognizer * 8746 SIInstrInfo::CreateTargetMIHazardRecognizer(const InstrItineraryData *II, 8747 const ScheduleDAGMI *DAG) const { 8748 // Borrowed from Arm Target 8749 // We would like to restrict this hazard recognizer to only 8750 // post-RA scheduling; we can tell that we're post-RA because we don't 8751 // track VRegLiveness. 8752 if (!DAG->hasVRegLiveness()) 8753 return new GCNHazardRecognizer(DAG->MF); 8754 return TargetInstrInfo::CreateTargetMIHazardRecognizer(II, DAG); 8755 } 8756 8757 std::pair<unsigned, unsigned> 8758 SIInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const { 8759 return std::pair(TF & MO_MASK, TF & ~MO_MASK); 8760 } 8761 8762 ArrayRef<std::pair<unsigned, const char *>> 8763 SIInstrInfo::getSerializableDirectMachineOperandTargetFlags() const { 8764 static const std::pair<unsigned, const char *> TargetFlags[] = { 8765 { MO_GOTPCREL, "amdgpu-gotprel" }, 8766 { MO_GOTPCREL32_LO, "amdgpu-gotprel32-lo" }, 8767 { MO_GOTPCREL32_HI, "amdgpu-gotprel32-hi" }, 8768 { MO_REL32_LO, "amdgpu-rel32-lo" }, 8769 { MO_REL32_HI, "amdgpu-rel32-hi" }, 8770 { MO_ABS32_LO, "amdgpu-abs32-lo" }, 8771 { MO_ABS32_HI, "amdgpu-abs32-hi" }, 8772 }; 8773 8774 return ArrayRef(TargetFlags); 8775 } 8776 8777 ArrayRef<std::pair<MachineMemOperand::Flags, const char *>> 8778 SIInstrInfo::getSerializableMachineMemOperandTargetFlags() const { 8779 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] = 8780 { 8781 {MONoClobber, "amdgpu-noclobber"}, 8782 {MOLastUse, "amdgpu-last-use"}, 8783 }; 8784 8785 return ArrayRef(TargetFlags); 8786 } 8787 8788 unsigned SIInstrInfo::getLiveRangeSplitOpcode(Register SrcReg, 8789 const MachineFunction &MF) const { 8790 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 8791 assert(SrcReg.isVirtual()); 8792 if (MFI->checkFlag(SrcReg, AMDGPU::VirtRegFlag::WWM_REG)) 8793 return AMDGPU::WWM_COPY; 8794 8795 return AMDGPU::COPY; 8796 } 8797 8798 bool SIInstrInfo::isBasicBlockPrologue(const MachineInstr &MI, 8799 Register Reg) const { 8800 // We need to handle instructions which may be inserted during register 8801 // allocation to handle the prolog. The initial prolog instruction may have 8802 // been separated from the start of the block by spills and copies inserted 8803 // needed by the prolog. However, the insertions for scalar registers can 8804 // always be placed at the BB top as they are independent of the exec mask 8805 // value. 8806 bool IsNullOrVectorRegister = true; 8807 if (Reg) { 8808 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 8809 IsNullOrVectorRegister = !RI.isSGPRClass(RI.getRegClassForReg(MRI, Reg)); 8810 } 8811 8812 uint16_t Opc = MI.getOpcode(); 8813 // FIXME: Copies inserted in the block prolog for live-range split should also 8814 // be included. 8815 return IsNullOrVectorRegister && 8816 (isSpillOpcode(Opc) || (!MI.isTerminator() && Opc != AMDGPU::COPY && 8817 MI.modifiesRegister(AMDGPU::EXEC, &RI))); 8818 } 8819 8820 MachineInstrBuilder 8821 SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB, 8822 MachineBasicBlock::iterator I, 8823 const DebugLoc &DL, 8824 Register DestReg) const { 8825 if (ST.hasAddNoCarry()) 8826 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e64), DestReg); 8827 8828 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 8829 Register UnusedCarry = MRI.createVirtualRegister(RI.getBoolRC()); 8830 MRI.setRegAllocationHint(UnusedCarry, 0, RI.getVCC()); 8831 8832 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg) 8833 .addReg(UnusedCarry, RegState::Define | RegState::Dead); 8834 } 8835 8836 MachineInstrBuilder SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB, 8837 MachineBasicBlock::iterator I, 8838 const DebugLoc &DL, 8839 Register DestReg, 8840 RegScavenger &RS) const { 8841 if (ST.hasAddNoCarry()) 8842 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e32), DestReg); 8843 8844 // If available, prefer to use vcc. 8845 Register UnusedCarry = !RS.isRegUsed(AMDGPU::VCC) 8846 ? Register(RI.getVCC()) 8847 : RS.scavengeRegisterBackwards( 8848 *RI.getBoolRC(), I, /* RestoreAfter */ false, 8849 0, /* AllowSpill */ false); 8850 8851 // TODO: Users need to deal with this. 8852 if (!UnusedCarry.isValid()) 8853 return MachineInstrBuilder(); 8854 8855 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg) 8856 .addReg(UnusedCarry, RegState::Define | RegState::Dead); 8857 } 8858 8859 bool SIInstrInfo::isKillTerminator(unsigned Opcode) { 8860 switch (Opcode) { 8861 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR: 8862 case AMDGPU::SI_KILL_I1_TERMINATOR: 8863 return true; 8864 default: 8865 return false; 8866 } 8867 } 8868 8869 const MCInstrDesc &SIInstrInfo::getKillTerminatorFromPseudo(unsigned Opcode) const { 8870 switch (Opcode) { 8871 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO: 8872 return get(AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR); 8873 case AMDGPU::SI_KILL_I1_PSEUDO: 8874 return get(AMDGPU::SI_KILL_I1_TERMINATOR); 8875 default: 8876 llvm_unreachable("invalid opcode, expected SI_KILL_*_PSEUDO"); 8877 } 8878 } 8879 8880 bool SIInstrInfo::isLegalMUBUFImmOffset(unsigned Imm) const { 8881 return Imm <= getMaxMUBUFImmOffset(ST); 8882 } 8883 8884 unsigned SIInstrInfo::getMaxMUBUFImmOffset(const GCNSubtarget &ST) { 8885 // GFX12 field is non-negative 24-bit signed byte offset. 8886 const unsigned OffsetBits = 8887 ST.getGeneration() >= AMDGPUSubtarget::GFX12 ? 23 : 12; 8888 return (1 << OffsetBits) - 1; 8889 } 8890 8891 void SIInstrInfo::fixImplicitOperands(MachineInstr &MI) const { 8892 if (!ST.isWave32()) 8893 return; 8894 8895 if (MI.isInlineAsm()) 8896 return; 8897 8898 for (auto &Op : MI.implicit_operands()) { 8899 if (Op.isReg() && Op.getReg() == AMDGPU::VCC) 8900 Op.setReg(AMDGPU::VCC_LO); 8901 } 8902 } 8903 8904 bool SIInstrInfo::isBufferSMRD(const MachineInstr &MI) const { 8905 if (!isSMRD(MI)) 8906 return false; 8907 8908 // Check that it is using a buffer resource. 8909 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::sbase); 8910 if (Idx == -1) // e.g. s_memtime 8911 return false; 8912 8913 const auto RCID = MI.getDesc().operands()[Idx].RegClass; 8914 return RI.getRegClass(RCID)->hasSubClassEq(&AMDGPU::SGPR_128RegClass); 8915 } 8916 8917 // Given Imm, split it into the values to put into the SOffset and ImmOffset 8918 // fields in an MUBUF instruction. Return false if it is not possible (due to a 8919 // hardware bug needing a workaround). 8920 // 8921 // The required alignment ensures that individual address components remain 8922 // aligned if they are aligned to begin with. It also ensures that additional 8923 // offsets within the given alignment can be added to the resulting ImmOffset. 8924 bool SIInstrInfo::splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, 8925 uint32_t &ImmOffset, Align Alignment) const { 8926 const uint32_t MaxOffset = SIInstrInfo::getMaxMUBUFImmOffset(ST); 8927 const uint32_t MaxImm = alignDown(MaxOffset, Alignment.value()); 8928 uint32_t Overflow = 0; 8929 8930 if (Imm > MaxImm) { 8931 if (Imm <= MaxImm + 64) { 8932 // Use an SOffset inline constant for 4..64 8933 Overflow = Imm - MaxImm; 8934 Imm = MaxImm; 8935 } else { 8936 // Try to keep the same value in SOffset for adjacent loads, so that 8937 // the corresponding register contents can be re-used. 8938 // 8939 // Load values with all low-bits (except for alignment bits) set into 8940 // SOffset, so that a larger range of values can be covered using 8941 // s_movk_i32. 8942 // 8943 // Atomic operations fail to work correctly when individual address 8944 // components are unaligned, even if their sum is aligned. 8945 uint32_t High = (Imm + Alignment.value()) & ~MaxOffset; 8946 uint32_t Low = (Imm + Alignment.value()) & MaxOffset; 8947 Imm = Low; 8948 Overflow = High - Alignment.value(); 8949 } 8950 } 8951 8952 if (Overflow > 0) { 8953 // There is a hardware bug in SI and CI which prevents address clamping in 8954 // MUBUF instructions from working correctly with SOffsets. The immediate 8955 // offset is unaffected. 8956 if (ST.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS) 8957 return false; 8958 8959 // It is not possible to set immediate in SOffset field on some targets. 8960 if (ST.hasRestrictedSOffset()) 8961 return false; 8962 } 8963 8964 ImmOffset = Imm; 8965 SOffset = Overflow; 8966 return true; 8967 } 8968 8969 // Depending on the used address space and instructions, some immediate offsets 8970 // are allowed and some are not. 8971 // Pre-GFX12, flat instruction offsets can only be non-negative, global and 8972 // scratch instruction offsets can also be negative. On GFX12, offsets can be 8973 // negative for all variants. 8974 // 8975 // There are several bugs related to these offsets: 8976 // On gfx10.1, flat instructions that go into the global address space cannot 8977 // use an offset. 8978 // 8979 // For scratch instructions, the address can be either an SGPR or a VGPR. 8980 // The following offsets can be used, depending on the architecture (x means 8981 // cannot be used): 8982 // +----------------------------+------+------+ 8983 // | Address-Mode | SGPR | VGPR | 8984 // +----------------------------+------+------+ 8985 // | gfx9 | | | 8986 // | negative, 4-aligned offset | x | ok | 8987 // | negative, unaligned offset | x | ok | 8988 // +----------------------------+------+------+ 8989 // | gfx10 | | | 8990 // | negative, 4-aligned offset | ok | ok | 8991 // | negative, unaligned offset | ok | x | 8992 // +----------------------------+------+------+ 8993 // | gfx10.3 | | | 8994 // | negative, 4-aligned offset | ok | ok | 8995 // | negative, unaligned offset | ok | ok | 8996 // +----------------------------+------+------+ 8997 // 8998 // This function ignores the addressing mode, so if an offset cannot be used in 8999 // one addressing mode, it is considered illegal. 9000 bool SIInstrInfo::isLegalFLATOffset(int64_t Offset, unsigned AddrSpace, 9001 uint64_t FlatVariant) const { 9002 // TODO: Should 0 be special cased? 9003 if (!ST.hasFlatInstOffsets()) 9004 return false; 9005 9006 if (ST.hasFlatSegmentOffsetBug() && FlatVariant == SIInstrFlags::FLAT && 9007 (AddrSpace == AMDGPUAS::FLAT_ADDRESS || 9008 AddrSpace == AMDGPUAS::GLOBAL_ADDRESS)) 9009 return false; 9010 9011 if (ST.hasNegativeUnalignedScratchOffsetBug() && 9012 FlatVariant == SIInstrFlags::FlatScratch && Offset < 0 && 9013 (Offset % 4) != 0) { 9014 return false; 9015 } 9016 9017 bool AllowNegative = allowNegativeFlatOffset(FlatVariant); 9018 unsigned N = AMDGPU::getNumFlatOffsetBits(ST); 9019 return isIntN(N, Offset) && (AllowNegative || Offset >= 0); 9020 } 9021 9022 // See comment on SIInstrInfo::isLegalFLATOffset for what is legal and what not. 9023 std::pair<int64_t, int64_t> 9024 SIInstrInfo::splitFlatOffset(int64_t COffsetVal, unsigned AddrSpace, 9025 uint64_t FlatVariant) const { 9026 int64_t RemainderOffset = COffsetVal; 9027 int64_t ImmField = 0; 9028 9029 bool AllowNegative = allowNegativeFlatOffset(FlatVariant); 9030 const unsigned NumBits = AMDGPU::getNumFlatOffsetBits(ST) - 1; 9031 9032 if (AllowNegative) { 9033 // Use signed division by a power of two to truncate towards 0. 9034 int64_t D = 1LL << NumBits; 9035 RemainderOffset = (COffsetVal / D) * D; 9036 ImmField = COffsetVal - RemainderOffset; 9037 9038 if (ST.hasNegativeUnalignedScratchOffsetBug() && 9039 FlatVariant == SIInstrFlags::FlatScratch && ImmField < 0 && 9040 (ImmField % 4) != 0) { 9041 // Make ImmField a multiple of 4 9042 RemainderOffset += ImmField % 4; 9043 ImmField -= ImmField % 4; 9044 } 9045 } else if (COffsetVal >= 0) { 9046 ImmField = COffsetVal & maskTrailingOnes<uint64_t>(NumBits); 9047 RemainderOffset = COffsetVal - ImmField; 9048 } 9049 9050 assert(isLegalFLATOffset(ImmField, AddrSpace, FlatVariant)); 9051 assert(RemainderOffset + ImmField == COffsetVal); 9052 return {ImmField, RemainderOffset}; 9053 } 9054 9055 bool SIInstrInfo::allowNegativeFlatOffset(uint64_t FlatVariant) const { 9056 if (ST.hasNegativeScratchOffsetBug() && 9057 FlatVariant == SIInstrFlags::FlatScratch) 9058 return false; 9059 9060 return FlatVariant != SIInstrFlags::FLAT || AMDGPU::isGFX12Plus(ST); 9061 } 9062 9063 static unsigned subtargetEncodingFamily(const GCNSubtarget &ST) { 9064 switch (ST.getGeneration()) { 9065 default: 9066 break; 9067 case AMDGPUSubtarget::SOUTHERN_ISLANDS: 9068 case AMDGPUSubtarget::SEA_ISLANDS: 9069 return SIEncodingFamily::SI; 9070 case AMDGPUSubtarget::VOLCANIC_ISLANDS: 9071 case AMDGPUSubtarget::GFX9: 9072 return SIEncodingFamily::VI; 9073 case AMDGPUSubtarget::GFX10: 9074 return SIEncodingFamily::GFX10; 9075 case AMDGPUSubtarget::GFX11: 9076 return SIEncodingFamily::GFX11; 9077 case AMDGPUSubtarget::GFX12: 9078 return SIEncodingFamily::GFX12; 9079 } 9080 llvm_unreachable("Unknown subtarget generation!"); 9081 } 9082 9083 bool SIInstrInfo::isAsmOnlyOpcode(int MCOp) const { 9084 switch(MCOp) { 9085 // These opcodes use indirect register addressing so 9086 // they need special handling by codegen (currently missing). 9087 // Therefore it is too risky to allow these opcodes 9088 // to be selected by dpp combiner or sdwa peepholer. 9089 case AMDGPU::V_MOVRELS_B32_dpp_gfx10: 9090 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10: 9091 case AMDGPU::V_MOVRELD_B32_dpp_gfx10: 9092 case AMDGPU::V_MOVRELD_B32_sdwa_gfx10: 9093 case AMDGPU::V_MOVRELSD_B32_dpp_gfx10: 9094 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10: 9095 case AMDGPU::V_MOVRELSD_2_B32_dpp_gfx10: 9096 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10: 9097 return true; 9098 default: 9099 return false; 9100 } 9101 } 9102 9103 int SIInstrInfo::pseudoToMCOpcode(int Opcode) const { 9104 Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode); 9105 9106 unsigned Gen = subtargetEncodingFamily(ST); 9107 9108 if ((get(Opcode).TSFlags & SIInstrFlags::renamedInGFX9) != 0 && 9109 ST.getGeneration() == AMDGPUSubtarget::GFX9) 9110 Gen = SIEncodingFamily::GFX9; 9111 9112 // Adjust the encoding family to GFX80 for D16 buffer instructions when the 9113 // subtarget has UnpackedD16VMem feature. 9114 // TODO: remove this when we discard GFX80 encoding. 9115 if (ST.hasUnpackedD16VMem() && (get(Opcode).TSFlags & SIInstrFlags::D16Buf)) 9116 Gen = SIEncodingFamily::GFX80; 9117 9118 if (get(Opcode).TSFlags & SIInstrFlags::SDWA) { 9119 switch (ST.getGeneration()) { 9120 default: 9121 Gen = SIEncodingFamily::SDWA; 9122 break; 9123 case AMDGPUSubtarget::GFX9: 9124 Gen = SIEncodingFamily::SDWA9; 9125 break; 9126 case AMDGPUSubtarget::GFX10: 9127 Gen = SIEncodingFamily::SDWA10; 9128 break; 9129 } 9130 } 9131 9132 if (isMAI(Opcode)) { 9133 int MFMAOp = AMDGPU::getMFMAEarlyClobberOp(Opcode); 9134 if (MFMAOp != -1) 9135 Opcode = MFMAOp; 9136 } 9137 9138 int MCOp = AMDGPU::getMCOpcode(Opcode, Gen); 9139 9140 // -1 means that Opcode is already a native instruction. 9141 if (MCOp == -1) 9142 return Opcode; 9143 9144 if (ST.hasGFX90AInsts()) { 9145 uint16_t NMCOp = (uint16_t)-1; 9146 if (ST.hasGFX940Insts()) 9147 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX940); 9148 if (NMCOp == (uint16_t)-1) 9149 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX90A); 9150 if (NMCOp == (uint16_t)-1) 9151 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX9); 9152 if (NMCOp != (uint16_t)-1) 9153 MCOp = NMCOp; 9154 } 9155 9156 // (uint16_t)-1 means that Opcode is a pseudo instruction that has 9157 // no encoding in the given subtarget generation. 9158 if (MCOp == (uint16_t)-1) 9159 return -1; 9160 9161 if (isAsmOnlyOpcode(MCOp)) 9162 return -1; 9163 9164 return MCOp; 9165 } 9166 9167 static 9168 TargetInstrInfo::RegSubRegPair getRegOrUndef(const MachineOperand &RegOpnd) { 9169 assert(RegOpnd.isReg()); 9170 return RegOpnd.isUndef() ? TargetInstrInfo::RegSubRegPair() : 9171 getRegSubRegPair(RegOpnd); 9172 } 9173 9174 TargetInstrInfo::RegSubRegPair 9175 llvm::getRegSequenceSubReg(MachineInstr &MI, unsigned SubReg) { 9176 assert(MI.isRegSequence()); 9177 for (unsigned I = 0, E = (MI.getNumOperands() - 1)/ 2; I < E; ++I) 9178 if (MI.getOperand(1 + 2 * I + 1).getImm() == SubReg) { 9179 auto &RegOp = MI.getOperand(1 + 2 * I); 9180 return getRegOrUndef(RegOp); 9181 } 9182 return TargetInstrInfo::RegSubRegPair(); 9183 } 9184 9185 // Try to find the definition of reg:subreg in subreg-manipulation pseudos 9186 // Following a subreg of reg:subreg isn't supported 9187 static bool followSubRegDef(MachineInstr &MI, 9188 TargetInstrInfo::RegSubRegPair &RSR) { 9189 if (!RSR.SubReg) 9190 return false; 9191 switch (MI.getOpcode()) { 9192 default: break; 9193 case AMDGPU::REG_SEQUENCE: 9194 RSR = getRegSequenceSubReg(MI, RSR.SubReg); 9195 return true; 9196 // EXTRACT_SUBREG ins't supported as this would follow a subreg of subreg 9197 case AMDGPU::INSERT_SUBREG: 9198 if (RSR.SubReg == (unsigned)MI.getOperand(3).getImm()) 9199 // inserted the subreg we're looking for 9200 RSR = getRegOrUndef(MI.getOperand(2)); 9201 else { // the subreg in the rest of the reg 9202 auto R1 = getRegOrUndef(MI.getOperand(1)); 9203 if (R1.SubReg) // subreg of subreg isn't supported 9204 return false; 9205 RSR.Reg = R1.Reg; 9206 } 9207 return true; 9208 } 9209 return false; 9210 } 9211 9212 MachineInstr *llvm::getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P, 9213 MachineRegisterInfo &MRI) { 9214 assert(MRI.isSSA()); 9215 if (!P.Reg.isVirtual()) 9216 return nullptr; 9217 9218 auto RSR = P; 9219 auto *DefInst = MRI.getVRegDef(RSR.Reg); 9220 while (auto *MI = DefInst) { 9221 DefInst = nullptr; 9222 switch (MI->getOpcode()) { 9223 case AMDGPU::COPY: 9224 case AMDGPU::V_MOV_B32_e32: { 9225 auto &Op1 = MI->getOperand(1); 9226 if (Op1.isReg() && Op1.getReg().isVirtual()) { 9227 if (Op1.isUndef()) 9228 return nullptr; 9229 RSR = getRegSubRegPair(Op1); 9230 DefInst = MRI.getVRegDef(RSR.Reg); 9231 } 9232 break; 9233 } 9234 default: 9235 if (followSubRegDef(*MI, RSR)) { 9236 if (!RSR.Reg) 9237 return nullptr; 9238 DefInst = MRI.getVRegDef(RSR.Reg); 9239 } 9240 } 9241 if (!DefInst) 9242 return MI; 9243 } 9244 return nullptr; 9245 } 9246 9247 bool llvm::execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI, 9248 Register VReg, 9249 const MachineInstr &DefMI, 9250 const MachineInstr &UseMI) { 9251 assert(MRI.isSSA() && "Must be run on SSA"); 9252 9253 auto *TRI = MRI.getTargetRegisterInfo(); 9254 auto *DefBB = DefMI.getParent(); 9255 9256 // Don't bother searching between blocks, although it is possible this block 9257 // doesn't modify exec. 9258 if (UseMI.getParent() != DefBB) 9259 return true; 9260 9261 const int MaxInstScan = 20; 9262 int NumInst = 0; 9263 9264 // Stop scan at the use. 9265 auto E = UseMI.getIterator(); 9266 for (auto I = std::next(DefMI.getIterator()); I != E; ++I) { 9267 if (I->isDebugInstr()) 9268 continue; 9269 9270 if (++NumInst > MaxInstScan) 9271 return true; 9272 9273 if (I->modifiesRegister(AMDGPU::EXEC, TRI)) 9274 return true; 9275 } 9276 9277 return false; 9278 } 9279 9280 bool llvm::execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI, 9281 Register VReg, 9282 const MachineInstr &DefMI) { 9283 assert(MRI.isSSA() && "Must be run on SSA"); 9284 9285 auto *TRI = MRI.getTargetRegisterInfo(); 9286 auto *DefBB = DefMI.getParent(); 9287 9288 const int MaxUseScan = 10; 9289 int NumUse = 0; 9290 9291 for (auto &Use : MRI.use_nodbg_operands(VReg)) { 9292 auto &UseInst = *Use.getParent(); 9293 // Don't bother searching between blocks, although it is possible this block 9294 // doesn't modify exec. 9295 if (UseInst.getParent() != DefBB || UseInst.isPHI()) 9296 return true; 9297 9298 if (++NumUse > MaxUseScan) 9299 return true; 9300 } 9301 9302 if (NumUse == 0) 9303 return false; 9304 9305 const int MaxInstScan = 20; 9306 int NumInst = 0; 9307 9308 // Stop scan when we have seen all the uses. 9309 for (auto I = std::next(DefMI.getIterator()); ; ++I) { 9310 assert(I != DefBB->end()); 9311 9312 if (I->isDebugInstr()) 9313 continue; 9314 9315 if (++NumInst > MaxInstScan) 9316 return true; 9317 9318 for (const MachineOperand &Op : I->operands()) { 9319 // We don't check reg masks here as they're used only on calls: 9320 // 1. EXEC is only considered const within one BB 9321 // 2. Call should be a terminator instruction if present in a BB 9322 9323 if (!Op.isReg()) 9324 continue; 9325 9326 Register Reg = Op.getReg(); 9327 if (Op.isUse()) { 9328 if (Reg == VReg && --NumUse == 0) 9329 return false; 9330 } else if (TRI->regsOverlap(Reg, AMDGPU::EXEC)) 9331 return true; 9332 } 9333 } 9334 } 9335 9336 MachineInstr *SIInstrInfo::createPHIDestinationCopy( 9337 MachineBasicBlock &MBB, MachineBasicBlock::iterator LastPHIIt, 9338 const DebugLoc &DL, Register Src, Register Dst) const { 9339 auto Cur = MBB.begin(); 9340 if (Cur != MBB.end()) 9341 do { 9342 if (!Cur->isPHI() && Cur->readsRegister(Dst)) 9343 return BuildMI(MBB, Cur, DL, get(TargetOpcode::COPY), Dst).addReg(Src); 9344 ++Cur; 9345 } while (Cur != MBB.end() && Cur != LastPHIIt); 9346 9347 return TargetInstrInfo::createPHIDestinationCopy(MBB, LastPHIIt, DL, Src, 9348 Dst); 9349 } 9350 9351 MachineInstr *SIInstrInfo::createPHISourceCopy( 9352 MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt, 9353 const DebugLoc &DL, Register Src, unsigned SrcSubReg, Register Dst) const { 9354 if (InsPt != MBB.end() && 9355 (InsPt->getOpcode() == AMDGPU::SI_IF || 9356 InsPt->getOpcode() == AMDGPU::SI_ELSE || 9357 InsPt->getOpcode() == AMDGPU::SI_IF_BREAK) && 9358 InsPt->definesRegister(Src)) { 9359 InsPt++; 9360 return BuildMI(MBB, InsPt, DL, 9361 get(ST.isWave32() ? AMDGPU::S_MOV_B32_term 9362 : AMDGPU::S_MOV_B64_term), 9363 Dst) 9364 .addReg(Src, 0, SrcSubReg) 9365 .addReg(AMDGPU::EXEC, RegState::Implicit); 9366 } 9367 return TargetInstrInfo::createPHISourceCopy(MBB, InsPt, DL, Src, SrcSubReg, 9368 Dst); 9369 } 9370 9371 bool llvm::SIInstrInfo::isWave32() const { return ST.isWave32(); } 9372 9373 MachineInstr *SIInstrInfo::foldMemoryOperandImpl( 9374 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, 9375 MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS, 9376 VirtRegMap *VRM) const { 9377 // This is a bit of a hack (copied from AArch64). Consider this instruction: 9378 // 9379 // %0:sreg_32 = COPY $m0 9380 // 9381 // We explicitly chose SReg_32 for the virtual register so such a copy might 9382 // be eliminated by RegisterCoalescer. However, that may not be possible, and 9383 // %0 may even spill. We can't spill $m0 normally (it would require copying to 9384 // a numbered SGPR anyway), and since it is in the SReg_32 register class, 9385 // TargetInstrInfo::foldMemoryOperand() is going to try. 9386 // A similar issue also exists with spilling and reloading $exec registers. 9387 // 9388 // To prevent that, constrain the %0 register class here. 9389 if (isFullCopyInstr(MI)) { 9390 Register DstReg = MI.getOperand(0).getReg(); 9391 Register SrcReg = MI.getOperand(1).getReg(); 9392 if ((DstReg.isVirtual() || SrcReg.isVirtual()) && 9393 (DstReg.isVirtual() != SrcReg.isVirtual())) { 9394 MachineRegisterInfo &MRI = MF.getRegInfo(); 9395 Register VirtReg = DstReg.isVirtual() ? DstReg : SrcReg; 9396 const TargetRegisterClass *RC = MRI.getRegClass(VirtReg); 9397 if (RC->hasSuperClassEq(&AMDGPU::SReg_32RegClass)) { 9398 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); 9399 return nullptr; 9400 } else if (RC->hasSuperClassEq(&AMDGPU::SReg_64RegClass)) { 9401 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_64_XEXECRegClass); 9402 return nullptr; 9403 } 9404 } 9405 } 9406 9407 return nullptr; 9408 } 9409 9410 unsigned SIInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, 9411 const MachineInstr &MI, 9412 unsigned *PredCost) const { 9413 if (MI.isBundle()) { 9414 MachineBasicBlock::const_instr_iterator I(MI.getIterator()); 9415 MachineBasicBlock::const_instr_iterator E(MI.getParent()->instr_end()); 9416 unsigned Lat = 0, Count = 0; 9417 for (++I; I != E && I->isBundledWithPred(); ++I) { 9418 ++Count; 9419 Lat = std::max(Lat, SchedModel.computeInstrLatency(&*I)); 9420 } 9421 return Lat + Count - 1; 9422 } 9423 9424 return SchedModel.computeInstrLatency(&MI); 9425 } 9426 9427 InstructionUniformity 9428 SIInstrInfo::getGenericInstructionUniformity(const MachineInstr &MI) const { 9429 unsigned opcode = MI.getOpcode(); 9430 if (auto *GI = dyn_cast<GIntrinsic>(&MI)) { 9431 auto IID = GI->getIntrinsicID(); 9432 if (AMDGPU::isIntrinsicSourceOfDivergence(IID)) 9433 return InstructionUniformity::NeverUniform; 9434 if (AMDGPU::isIntrinsicAlwaysUniform(IID)) 9435 return InstructionUniformity::AlwaysUniform; 9436 9437 switch (IID) { 9438 case Intrinsic::amdgcn_if: 9439 case Intrinsic::amdgcn_else: 9440 // FIXME: Uniform if second result 9441 break; 9442 } 9443 9444 return InstructionUniformity::Default; 9445 } 9446 9447 // Loads from the private and flat address spaces are divergent, because 9448 // threads can execute the load instruction with the same inputs and get 9449 // different results. 9450 // 9451 // All other loads are not divergent, because if threads issue loads with the 9452 // same arguments, they will always get the same result. 9453 if (opcode == AMDGPU::G_LOAD) { 9454 if (MI.memoperands_empty()) 9455 return InstructionUniformity::NeverUniform; // conservative assumption 9456 9457 if (llvm::any_of(MI.memoperands(), [](const MachineMemOperand *mmo) { 9458 return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS || 9459 mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS; 9460 })) { 9461 // At least one MMO in a non-global address space. 9462 return InstructionUniformity::NeverUniform; 9463 } 9464 return InstructionUniformity::Default; 9465 } 9466 9467 if (SIInstrInfo::isGenericAtomicRMWOpcode(opcode) || 9468 opcode == AMDGPU::G_ATOMIC_CMPXCHG || 9469 opcode == AMDGPU::G_ATOMIC_CMPXCHG_WITH_SUCCESS || 9470 AMDGPU::isGenericAtomic(opcode)) { 9471 return InstructionUniformity::NeverUniform; 9472 } 9473 return InstructionUniformity::Default; 9474 } 9475 9476 InstructionUniformity 9477 SIInstrInfo::getInstructionUniformity(const MachineInstr &MI) const { 9478 9479 if (isNeverUniform(MI)) 9480 return InstructionUniformity::NeverUniform; 9481 9482 unsigned opcode = MI.getOpcode(); 9483 if (opcode == AMDGPU::V_READLANE_B32 || 9484 opcode == AMDGPU::V_READFIRSTLANE_B32 || 9485 opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR) 9486 return InstructionUniformity::AlwaysUniform; 9487 9488 if (isCopyInstr(MI)) { 9489 const MachineOperand &srcOp = MI.getOperand(1); 9490 if (srcOp.isReg() && srcOp.getReg().isPhysical()) { 9491 const TargetRegisterClass *regClass = 9492 RI.getPhysRegBaseClass(srcOp.getReg()); 9493 return RI.isSGPRClass(regClass) ? InstructionUniformity::AlwaysUniform 9494 : InstructionUniformity::NeverUniform; 9495 } 9496 return InstructionUniformity::Default; 9497 } 9498 9499 // GMIR handling 9500 if (MI.isPreISelOpcode()) 9501 return SIInstrInfo::getGenericInstructionUniformity(MI); 9502 9503 // Atomics are divergent because they are executed sequentially: when an 9504 // atomic operation refers to the same address in each thread, then each 9505 // thread after the first sees the value written by the previous thread as 9506 // original value. 9507 9508 if (isAtomic(MI)) 9509 return InstructionUniformity::NeverUniform; 9510 9511 // Loads from the private and flat address spaces are divergent, because 9512 // threads can execute the load instruction with the same inputs and get 9513 // different results. 9514 if (isFLAT(MI) && MI.mayLoad()) { 9515 if (MI.memoperands_empty()) 9516 return InstructionUniformity::NeverUniform; // conservative assumption 9517 9518 if (llvm::any_of(MI.memoperands(), [](const MachineMemOperand *mmo) { 9519 return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS || 9520 mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS; 9521 })) { 9522 // At least one MMO in a non-global address space. 9523 return InstructionUniformity::NeverUniform; 9524 } 9525 9526 return InstructionUniformity::Default; 9527 } 9528 9529 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 9530 const AMDGPURegisterBankInfo *RBI = ST.getRegBankInfo(); 9531 9532 // FIXME: It's conceptually broken to report this for an instruction, and not 9533 // a specific def operand. For inline asm in particular, there could be mixed 9534 // uniform and divergent results. 9535 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) { 9536 const MachineOperand &SrcOp = MI.getOperand(I); 9537 if (!SrcOp.isReg()) 9538 continue; 9539 9540 Register Reg = SrcOp.getReg(); 9541 if (!Reg || !SrcOp.readsReg()) 9542 continue; 9543 9544 // If RegBank is null, this is unassigned or an unallocatable special 9545 // register, which are all scalars. 9546 const RegisterBank *RegBank = RBI->getRegBank(Reg, MRI, RI); 9547 if (RegBank && RegBank->getID() != AMDGPU::SGPRRegBankID) 9548 return InstructionUniformity::NeverUniform; 9549 } 9550 9551 // TODO: Uniformity check condtions above can be rearranged for more 9552 // redability 9553 9554 // TODO: amdgcn.{ballot, [if]cmp} should be AlwaysUniform, but they are 9555 // currently turned into no-op COPYs by SelectionDAG ISel and are 9556 // therefore no longer recognizable. 9557 9558 return InstructionUniformity::Default; 9559 } 9560 9561 unsigned SIInstrInfo::getDSShaderTypeValue(const MachineFunction &MF) { 9562 switch (MF.getFunction().getCallingConv()) { 9563 case CallingConv::AMDGPU_PS: 9564 return 1; 9565 case CallingConv::AMDGPU_VS: 9566 return 2; 9567 case CallingConv::AMDGPU_GS: 9568 return 3; 9569 case CallingConv::AMDGPU_HS: 9570 case CallingConv::AMDGPU_LS: 9571 case CallingConv::AMDGPU_ES: 9572 report_fatal_error("ds_ordered_count unsupported for this calling conv"); 9573 case CallingConv::AMDGPU_CS: 9574 case CallingConv::AMDGPU_KERNEL: 9575 case CallingConv::C: 9576 case CallingConv::Fast: 9577 default: 9578 // Assume other calling conventions are various compute callable functions 9579 return 0; 9580 } 9581 } 9582 9583 bool SIInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg, 9584 Register &SrcReg2, int64_t &CmpMask, 9585 int64_t &CmpValue) const { 9586 if (!MI.getOperand(0).isReg() || MI.getOperand(0).getSubReg()) 9587 return false; 9588 9589 switch (MI.getOpcode()) { 9590 default: 9591 break; 9592 case AMDGPU::S_CMP_EQ_U32: 9593 case AMDGPU::S_CMP_EQ_I32: 9594 case AMDGPU::S_CMP_LG_U32: 9595 case AMDGPU::S_CMP_LG_I32: 9596 case AMDGPU::S_CMP_LT_U32: 9597 case AMDGPU::S_CMP_LT_I32: 9598 case AMDGPU::S_CMP_GT_U32: 9599 case AMDGPU::S_CMP_GT_I32: 9600 case AMDGPU::S_CMP_LE_U32: 9601 case AMDGPU::S_CMP_LE_I32: 9602 case AMDGPU::S_CMP_GE_U32: 9603 case AMDGPU::S_CMP_GE_I32: 9604 case AMDGPU::S_CMP_EQ_U64: 9605 case AMDGPU::S_CMP_LG_U64: 9606 SrcReg = MI.getOperand(0).getReg(); 9607 if (MI.getOperand(1).isReg()) { 9608 if (MI.getOperand(1).getSubReg()) 9609 return false; 9610 SrcReg2 = MI.getOperand(1).getReg(); 9611 CmpValue = 0; 9612 } else if (MI.getOperand(1).isImm()) { 9613 SrcReg2 = Register(); 9614 CmpValue = MI.getOperand(1).getImm(); 9615 } else { 9616 return false; 9617 } 9618 CmpMask = ~0; 9619 return true; 9620 case AMDGPU::S_CMPK_EQ_U32: 9621 case AMDGPU::S_CMPK_EQ_I32: 9622 case AMDGPU::S_CMPK_LG_U32: 9623 case AMDGPU::S_CMPK_LG_I32: 9624 case AMDGPU::S_CMPK_LT_U32: 9625 case AMDGPU::S_CMPK_LT_I32: 9626 case AMDGPU::S_CMPK_GT_U32: 9627 case AMDGPU::S_CMPK_GT_I32: 9628 case AMDGPU::S_CMPK_LE_U32: 9629 case AMDGPU::S_CMPK_LE_I32: 9630 case AMDGPU::S_CMPK_GE_U32: 9631 case AMDGPU::S_CMPK_GE_I32: 9632 SrcReg = MI.getOperand(0).getReg(); 9633 SrcReg2 = Register(); 9634 CmpValue = MI.getOperand(1).getImm(); 9635 CmpMask = ~0; 9636 return true; 9637 } 9638 9639 return false; 9640 } 9641 9642 bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, 9643 Register SrcReg2, int64_t CmpMask, 9644 int64_t CmpValue, 9645 const MachineRegisterInfo *MRI) const { 9646 if (!SrcReg || SrcReg.isPhysical()) 9647 return false; 9648 9649 if (SrcReg2 && !getFoldableImm(SrcReg2, *MRI, CmpValue)) 9650 return false; 9651 9652 const auto optimizeCmpAnd = [&CmpInstr, SrcReg, CmpValue, MRI, 9653 this](int64_t ExpectedValue, unsigned SrcSize, 9654 bool IsReversible, bool IsSigned) -> bool { 9655 // s_cmp_eq_u32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n 9656 // s_cmp_eq_i32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n 9657 // s_cmp_ge_u32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n 9658 // s_cmp_ge_i32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n 9659 // s_cmp_eq_u64 (s_and_b64 $src, 1 << n), 1 << n => s_and_b64 $src, 1 << n 9660 // s_cmp_lg_u32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n 9661 // s_cmp_lg_i32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n 9662 // s_cmp_gt_u32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n 9663 // s_cmp_gt_i32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n 9664 // s_cmp_lg_u64 (s_and_b64 $src, 1 << n), 0 => s_and_b64 $src, 1 << n 9665 // 9666 // Signed ge/gt are not used for the sign bit. 9667 // 9668 // If result of the AND is unused except in the compare: 9669 // s_and_b(32|64) $src, 1 << n => s_bitcmp1_b(32|64) $src, n 9670 // 9671 // s_cmp_eq_u32 (s_and_b32 $src, 1 << n), 0 => s_bitcmp0_b32 $src, n 9672 // s_cmp_eq_i32 (s_and_b32 $src, 1 << n), 0 => s_bitcmp0_b32 $src, n 9673 // s_cmp_eq_u64 (s_and_b64 $src, 1 << n), 0 => s_bitcmp0_b64 $src, n 9674 // s_cmp_lg_u32 (s_and_b32 $src, 1 << n), 1 << n => s_bitcmp0_b32 $src, n 9675 // s_cmp_lg_i32 (s_and_b32 $src, 1 << n), 1 << n => s_bitcmp0_b32 $src, n 9676 // s_cmp_lg_u64 (s_and_b64 $src, 1 << n), 1 << n => s_bitcmp0_b64 $src, n 9677 9678 MachineInstr *Def = MRI->getUniqueVRegDef(SrcReg); 9679 if (!Def || Def->getParent() != CmpInstr.getParent()) 9680 return false; 9681 9682 if (Def->getOpcode() != AMDGPU::S_AND_B32 && 9683 Def->getOpcode() != AMDGPU::S_AND_B64) 9684 return false; 9685 9686 int64_t Mask; 9687 const auto isMask = [&Mask, SrcSize](const MachineOperand *MO) -> bool { 9688 if (MO->isImm()) 9689 Mask = MO->getImm(); 9690 else if (!getFoldableImm(MO, Mask)) 9691 return false; 9692 Mask &= maxUIntN(SrcSize); 9693 return isPowerOf2_64(Mask); 9694 }; 9695 9696 MachineOperand *SrcOp = &Def->getOperand(1); 9697 if (isMask(SrcOp)) 9698 SrcOp = &Def->getOperand(2); 9699 else if (isMask(&Def->getOperand(2))) 9700 SrcOp = &Def->getOperand(1); 9701 else 9702 return false; 9703 9704 unsigned BitNo = llvm::countr_zero((uint64_t)Mask); 9705 if (IsSigned && BitNo == SrcSize - 1) 9706 return false; 9707 9708 ExpectedValue <<= BitNo; 9709 9710 bool IsReversedCC = false; 9711 if (CmpValue != ExpectedValue) { 9712 if (!IsReversible) 9713 return false; 9714 IsReversedCC = CmpValue == (ExpectedValue ^ Mask); 9715 if (!IsReversedCC) 9716 return false; 9717 } 9718 9719 Register DefReg = Def->getOperand(0).getReg(); 9720 if (IsReversedCC && !MRI->hasOneNonDBGUse(DefReg)) 9721 return false; 9722 9723 for (auto I = std::next(Def->getIterator()), E = CmpInstr.getIterator(); 9724 I != E; ++I) { 9725 if (I->modifiesRegister(AMDGPU::SCC, &RI) || 9726 I->killsRegister(AMDGPU::SCC, &RI)) 9727 return false; 9728 } 9729 9730 MachineOperand *SccDef = Def->findRegisterDefOperand(AMDGPU::SCC); 9731 SccDef->setIsDead(false); 9732 CmpInstr.eraseFromParent(); 9733 9734 if (!MRI->use_nodbg_empty(DefReg)) { 9735 assert(!IsReversedCC); 9736 return true; 9737 } 9738 9739 // Replace AND with unused result with a S_BITCMP. 9740 MachineBasicBlock *MBB = Def->getParent(); 9741 9742 unsigned NewOpc = (SrcSize == 32) ? IsReversedCC ? AMDGPU::S_BITCMP0_B32 9743 : AMDGPU::S_BITCMP1_B32 9744 : IsReversedCC ? AMDGPU::S_BITCMP0_B64 9745 : AMDGPU::S_BITCMP1_B64; 9746 9747 BuildMI(*MBB, Def, Def->getDebugLoc(), get(NewOpc)) 9748 .add(*SrcOp) 9749 .addImm(BitNo); 9750 Def->eraseFromParent(); 9751 9752 return true; 9753 }; 9754 9755 switch (CmpInstr.getOpcode()) { 9756 default: 9757 break; 9758 case AMDGPU::S_CMP_EQ_U32: 9759 case AMDGPU::S_CMP_EQ_I32: 9760 case AMDGPU::S_CMPK_EQ_U32: 9761 case AMDGPU::S_CMPK_EQ_I32: 9762 return optimizeCmpAnd(1, 32, true, false); 9763 case AMDGPU::S_CMP_GE_U32: 9764 case AMDGPU::S_CMPK_GE_U32: 9765 return optimizeCmpAnd(1, 32, false, false); 9766 case AMDGPU::S_CMP_GE_I32: 9767 case AMDGPU::S_CMPK_GE_I32: 9768 return optimizeCmpAnd(1, 32, false, true); 9769 case AMDGPU::S_CMP_EQ_U64: 9770 return optimizeCmpAnd(1, 64, true, false); 9771 case AMDGPU::S_CMP_LG_U32: 9772 case AMDGPU::S_CMP_LG_I32: 9773 case AMDGPU::S_CMPK_LG_U32: 9774 case AMDGPU::S_CMPK_LG_I32: 9775 return optimizeCmpAnd(0, 32, true, false); 9776 case AMDGPU::S_CMP_GT_U32: 9777 case AMDGPU::S_CMPK_GT_U32: 9778 return optimizeCmpAnd(0, 32, false, false); 9779 case AMDGPU::S_CMP_GT_I32: 9780 case AMDGPU::S_CMPK_GT_I32: 9781 return optimizeCmpAnd(0, 32, false, true); 9782 case AMDGPU::S_CMP_LG_U64: 9783 return optimizeCmpAnd(0, 64, true, false); 9784 } 9785 9786 return false; 9787 } 9788 9789 void SIInstrInfo::enforceOperandRCAlignment(MachineInstr &MI, 9790 unsigned OpName) const { 9791 if (!ST.needsAlignedVGPRs()) 9792 return; 9793 9794 int OpNo = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpName); 9795 if (OpNo < 0) 9796 return; 9797 MachineOperand &Op = MI.getOperand(OpNo); 9798 if (getOpSize(MI, OpNo) > 4) 9799 return; 9800 9801 // Add implicit aligned super-reg to force alignment on the data operand. 9802 const DebugLoc &DL = MI.getDebugLoc(); 9803 MachineBasicBlock *BB = MI.getParent(); 9804 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); 9805 Register DataReg = Op.getReg(); 9806 bool IsAGPR = RI.isAGPR(MRI, DataReg); 9807 Register Undef = MRI.createVirtualRegister( 9808 IsAGPR ? &AMDGPU::AGPR_32RegClass : &AMDGPU::VGPR_32RegClass); 9809 BuildMI(*BB, MI, DL, get(AMDGPU::IMPLICIT_DEF), Undef); 9810 Register NewVR = 9811 MRI.createVirtualRegister(IsAGPR ? &AMDGPU::AReg_64_Align2RegClass 9812 : &AMDGPU::VReg_64_Align2RegClass); 9813 BuildMI(*BB, MI, DL, get(AMDGPU::REG_SEQUENCE), NewVR) 9814 .addReg(DataReg, 0, Op.getSubReg()) 9815 .addImm(AMDGPU::sub0) 9816 .addReg(Undef) 9817 .addImm(AMDGPU::sub1); 9818 Op.setReg(NewVR); 9819 Op.setSubReg(AMDGPU::sub0); 9820 MI.addOperand(MachineOperand::CreateReg(NewVR, false, true)); 9821 } 9822