1 //===- SIInstrInfo.cpp - SI Instruction Information ----------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// SI Implementation of TargetInstrInfo. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "SIInstrInfo.h" 15 #include "AMDGPU.h" 16 #include "AMDGPUInstrInfo.h" 17 #include "GCNHazardRecognizer.h" 18 #include "GCNSubtarget.h" 19 #include "SIMachineFunctionInfo.h" 20 #include "Utils/AMDGPUBaseInfo.h" 21 #include "llvm/Analysis/ValueTracking.h" 22 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" 23 #include "llvm/CodeGen/LiveIntervals.h" 24 #include "llvm/CodeGen/LiveVariables.h" 25 #include "llvm/CodeGen/MachineDominators.h" 26 #include "llvm/CodeGen/MachineFrameInfo.h" 27 #include "llvm/CodeGen/MachineScheduler.h" 28 #include "llvm/CodeGen/RegisterScavenging.h" 29 #include "llvm/CodeGen/ScheduleDAG.h" 30 #include "llvm/IR/DiagnosticInfo.h" 31 #include "llvm/IR/IntrinsicsAMDGPU.h" 32 #include "llvm/MC/MCContext.h" 33 #include "llvm/Support/CommandLine.h" 34 #include "llvm/Target/TargetMachine.h" 35 36 using namespace llvm; 37 38 #define DEBUG_TYPE "si-instr-info" 39 40 #define GET_INSTRINFO_CTOR_DTOR 41 #include "AMDGPUGenInstrInfo.inc" 42 43 namespace llvm { 44 namespace AMDGPU { 45 #define GET_D16ImageDimIntrinsics_IMPL 46 #define GET_ImageDimIntrinsicTable_IMPL 47 #define GET_RsrcIntrinsics_IMPL 48 #include "AMDGPUGenSearchableTables.inc" 49 } 50 } 51 52 53 // Must be at least 4 to be able to branch over minimum unconditional branch 54 // code. This is only for making it possible to write reasonably small tests for 55 // long branches. 56 static cl::opt<unsigned> 57 BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16), 58 cl::desc("Restrict range of branch instructions (DEBUG)")); 59 60 static cl::opt<bool> Fix16BitCopies( 61 "amdgpu-fix-16-bit-physreg-copies", 62 cl::desc("Fix copies between 32 and 16 bit registers by extending to 32 bit"), 63 cl::init(true), 64 cl::ReallyHidden); 65 66 SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST) 67 : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN), 68 RI(ST), ST(ST) { 69 SchedModel.init(&ST); 70 } 71 72 //===----------------------------------------------------------------------===// 73 // TargetInstrInfo callbacks 74 //===----------------------------------------------------------------------===// 75 76 static unsigned getNumOperandsNoGlue(SDNode *Node) { 77 unsigned N = Node->getNumOperands(); 78 while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue) 79 --N; 80 return N; 81 } 82 83 /// Returns true if both nodes have the same value for the given 84 /// operand \p Op, or if both nodes do not have this operand. 85 static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) { 86 unsigned Opc0 = N0->getMachineOpcode(); 87 unsigned Opc1 = N1->getMachineOpcode(); 88 89 int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName); 90 int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName); 91 92 if (Op0Idx == -1 && Op1Idx == -1) 93 return true; 94 95 96 if ((Op0Idx == -1 && Op1Idx != -1) || 97 (Op1Idx == -1 && Op0Idx != -1)) 98 return false; 99 100 // getNamedOperandIdx returns the index for the MachineInstr's operands, 101 // which includes the result as the first operand. We are indexing into the 102 // MachineSDNode's operands, so we need to skip the result operand to get 103 // the real index. 104 --Op0Idx; 105 --Op1Idx; 106 107 return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx); 108 } 109 110 static bool canRemat(const MachineInstr &MI) { 111 112 if (SIInstrInfo::isVOP1(MI) || SIInstrInfo::isVOP2(MI) || 113 SIInstrInfo::isVOP3(MI) || SIInstrInfo::isSDWA(MI) || 114 SIInstrInfo::isSALU(MI)) 115 return true; 116 117 if (SIInstrInfo::isSMRD(MI)) { 118 return !MI.memoperands_empty() && 119 llvm::all_of(MI.memoperands(), [](const MachineMemOperand *MMO) { 120 return MMO->isLoad() && MMO->isInvariant(); 121 }); 122 } 123 124 return false; 125 } 126 127 bool SIInstrInfo::isReallyTriviallyReMaterializable( 128 const MachineInstr &MI) const { 129 130 if (canRemat(MI)) { 131 // Normally VALU use of exec would block the rematerialization, but that 132 // is OK in this case to have an implicit exec read as all VALU do. 133 // We really want all of the generic logic for this except for this. 134 135 // Another potential implicit use is mode register. The core logic of 136 // the RA will not attempt rematerialization if mode is set anywhere 137 // in the function, otherwise it is safe since mode is not changed. 138 139 // There is difference to generic method which does not allow 140 // rematerialization if there are virtual register uses. We allow this, 141 // therefore this method includes SOP instructions as well. 142 if (!MI.hasImplicitDef() && 143 MI.getNumImplicitOperands() == MI.getDesc().implicit_uses().size() && 144 !MI.mayRaiseFPException()) 145 return true; 146 } 147 148 return TargetInstrInfo::isReallyTriviallyReMaterializable(MI); 149 } 150 151 // Returns true if the scalar result of a VALU instruction depends on exec. 152 static bool resultDependsOnExec(const MachineInstr &MI) { 153 // Ignore comparisons which are only used masked with exec. 154 // This allows some hoisting/sinking of VALU comparisons. 155 if (MI.isCompare()) { 156 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 157 Register DstReg = MI.getOperand(0).getReg(); 158 if (!DstReg.isVirtual()) 159 return true; 160 for (MachineInstr &Use : MRI.use_nodbg_instructions(DstReg)) { 161 switch (Use.getOpcode()) { 162 case AMDGPU::S_AND_SAVEEXEC_B32: 163 case AMDGPU::S_AND_SAVEEXEC_B64: 164 break; 165 case AMDGPU::S_AND_B32: 166 case AMDGPU::S_AND_B64: 167 if (!Use.readsRegister(AMDGPU::EXEC)) 168 return true; 169 break; 170 default: 171 return true; 172 } 173 } 174 return false; 175 } 176 177 switch (MI.getOpcode()) { 178 default: 179 break; 180 case AMDGPU::V_READFIRSTLANE_B32: 181 return true; 182 } 183 184 return false; 185 } 186 187 bool SIInstrInfo::isIgnorableUse(const MachineOperand &MO) const { 188 // Any implicit use of exec by VALU is not a real register read. 189 return MO.getReg() == AMDGPU::EXEC && MO.isImplicit() && 190 isVALU(*MO.getParent()) && !resultDependsOnExec(*MO.getParent()); 191 } 192 193 bool SIInstrInfo::isSafeToSink(MachineInstr &MI, 194 MachineBasicBlock *SuccToSinkTo, 195 MachineCycleInfo *CI) const { 196 // Allow sinking if MI edits lane mask (divergent i1 in sgpr). 197 if (MI.getOpcode() == AMDGPU::SI_IF_BREAK) 198 return true; 199 200 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo(); 201 // Check if sinking of MI would create temporal divergent use. 202 for (auto Op : MI.uses()) { 203 if (Op.isReg() && Op.getReg().isVirtual() && 204 RI.isSGPRClass(MRI.getRegClass(Op.getReg()))) { 205 MachineInstr *SgprDef = MRI.getVRegDef(Op.getReg()); 206 207 // SgprDef defined inside cycle 208 MachineCycle *FromCycle = CI->getCycle(SgprDef->getParent()); 209 if (FromCycle == nullptr) 210 continue; 211 212 MachineCycle *ToCycle = CI->getCycle(SuccToSinkTo); 213 // Check if there is a FromCycle that contains SgprDef's basic block but 214 // does not contain SuccToSinkTo and also has divergent exit condition. 215 while (FromCycle && !FromCycle->contains(ToCycle)) { 216 // After structurize-cfg, there should be exactly one cycle exit. 217 SmallVector<MachineBasicBlock *, 1> ExitBlocks; 218 FromCycle->getExitBlocks(ExitBlocks); 219 assert(ExitBlocks.size() == 1); 220 assert(ExitBlocks[0]->getSinglePredecessor()); 221 222 // FromCycle has divergent exit condition. 223 if (hasDivergentBranch(ExitBlocks[0]->getSinglePredecessor())) { 224 return false; 225 } 226 227 FromCycle = FromCycle->getParentCycle(); 228 } 229 } 230 } 231 232 return true; 233 } 234 235 bool SIInstrInfo::areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1, 236 int64_t &Offset0, 237 int64_t &Offset1) const { 238 if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode()) 239 return false; 240 241 unsigned Opc0 = Load0->getMachineOpcode(); 242 unsigned Opc1 = Load1->getMachineOpcode(); 243 244 // Make sure both are actually loads. 245 if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad()) 246 return false; 247 248 // A mayLoad instruction without a def is not a load. Likely a prefetch. 249 if (!get(Opc0).getNumDefs() || !get(Opc1).getNumDefs()) 250 return false; 251 252 if (isDS(Opc0) && isDS(Opc1)) { 253 254 // FIXME: Handle this case: 255 if (getNumOperandsNoGlue(Load0) != getNumOperandsNoGlue(Load1)) 256 return false; 257 258 // Check base reg. 259 if (Load0->getOperand(0) != Load1->getOperand(0)) 260 return false; 261 262 // Skip read2 / write2 variants for simplicity. 263 // TODO: We should report true if the used offsets are adjacent (excluded 264 // st64 versions). 265 int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset); 266 int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset); 267 if (Offset0Idx == -1 || Offset1Idx == -1) 268 return false; 269 270 // XXX - be careful of dataless loads 271 // getNamedOperandIdx returns the index for MachineInstrs. Since they 272 // include the output in the operand list, but SDNodes don't, we need to 273 // subtract the index by one. 274 Offset0Idx -= get(Opc0).NumDefs; 275 Offset1Idx -= get(Opc1).NumDefs; 276 Offset0 = Load0->getConstantOperandVal(Offset0Idx); 277 Offset1 = Load1->getConstantOperandVal(Offset1Idx); 278 return true; 279 } 280 281 if (isSMRD(Opc0) && isSMRD(Opc1)) { 282 // Skip time and cache invalidation instructions. 283 if (!AMDGPU::hasNamedOperand(Opc0, AMDGPU::OpName::sbase) || 284 !AMDGPU::hasNamedOperand(Opc1, AMDGPU::OpName::sbase)) 285 return false; 286 287 unsigned NumOps = getNumOperandsNoGlue(Load0); 288 if (NumOps != getNumOperandsNoGlue(Load1)) 289 return false; 290 291 // Check base reg. 292 if (Load0->getOperand(0) != Load1->getOperand(0)) 293 return false; 294 295 // Match register offsets, if both register and immediate offsets present. 296 assert(NumOps == 4 || NumOps == 5); 297 if (NumOps == 5 && Load0->getOperand(1) != Load1->getOperand(1)) 298 return false; 299 300 const ConstantSDNode *Load0Offset = 301 dyn_cast<ConstantSDNode>(Load0->getOperand(NumOps - 3)); 302 const ConstantSDNode *Load1Offset = 303 dyn_cast<ConstantSDNode>(Load1->getOperand(NumOps - 3)); 304 305 if (!Load0Offset || !Load1Offset) 306 return false; 307 308 Offset0 = Load0Offset->getZExtValue(); 309 Offset1 = Load1Offset->getZExtValue(); 310 return true; 311 } 312 313 // MUBUF and MTBUF can access the same addresses. 314 if ((isMUBUF(Opc0) || isMTBUF(Opc0)) && (isMUBUF(Opc1) || isMTBUF(Opc1))) { 315 316 // MUBUF and MTBUF have vaddr at different indices. 317 if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) || 318 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) || 319 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc)) 320 return false; 321 322 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset); 323 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset); 324 325 if (OffIdx0 == -1 || OffIdx1 == -1) 326 return false; 327 328 // getNamedOperandIdx returns the index for MachineInstrs. Since they 329 // include the output in the operand list, but SDNodes don't, we need to 330 // subtract the index by one. 331 OffIdx0 -= get(Opc0).NumDefs; 332 OffIdx1 -= get(Opc1).NumDefs; 333 334 SDValue Off0 = Load0->getOperand(OffIdx0); 335 SDValue Off1 = Load1->getOperand(OffIdx1); 336 337 // The offset might be a FrameIndexSDNode. 338 if (!isa<ConstantSDNode>(Off0) || !isa<ConstantSDNode>(Off1)) 339 return false; 340 341 Offset0 = Off0->getAsZExtVal(); 342 Offset1 = Off1->getAsZExtVal(); 343 return true; 344 } 345 346 return false; 347 } 348 349 static bool isStride64(unsigned Opc) { 350 switch (Opc) { 351 case AMDGPU::DS_READ2ST64_B32: 352 case AMDGPU::DS_READ2ST64_B64: 353 case AMDGPU::DS_WRITE2ST64_B32: 354 case AMDGPU::DS_WRITE2ST64_B64: 355 return true; 356 default: 357 return false; 358 } 359 } 360 361 bool SIInstrInfo::getMemOperandsWithOffsetWidth( 362 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps, 363 int64_t &Offset, bool &OffsetIsScalable, unsigned &Width, 364 const TargetRegisterInfo *TRI) const { 365 if (!LdSt.mayLoadOrStore()) 366 return false; 367 368 unsigned Opc = LdSt.getOpcode(); 369 OffsetIsScalable = false; 370 const MachineOperand *BaseOp, *OffsetOp; 371 int DataOpIdx; 372 373 if (isDS(LdSt)) { 374 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr); 375 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset); 376 if (OffsetOp) { 377 // Normal, single offset LDS instruction. 378 if (!BaseOp) { 379 // DS_CONSUME/DS_APPEND use M0 for the base address. 380 // TODO: find the implicit use operand for M0 and use that as BaseOp? 381 return false; 382 } 383 BaseOps.push_back(BaseOp); 384 Offset = OffsetOp->getImm(); 385 // Get appropriate operand, and compute width accordingly. 386 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 387 if (DataOpIdx == -1) 388 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); 389 Width = getOpSize(LdSt, DataOpIdx); 390 } else { 391 // The 2 offset instructions use offset0 and offset1 instead. We can treat 392 // these as a load with a single offset if the 2 offsets are consecutive. 393 // We will use this for some partially aligned loads. 394 const MachineOperand *Offset0Op = 395 getNamedOperand(LdSt, AMDGPU::OpName::offset0); 396 const MachineOperand *Offset1Op = 397 getNamedOperand(LdSt, AMDGPU::OpName::offset1); 398 399 unsigned Offset0 = Offset0Op->getImm() & 0xff; 400 unsigned Offset1 = Offset1Op->getImm() & 0xff; 401 if (Offset0 + 1 != Offset1) 402 return false; 403 404 // Each of these offsets is in element sized units, so we need to convert 405 // to bytes of the individual reads. 406 407 unsigned EltSize; 408 if (LdSt.mayLoad()) 409 EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, 0)) / 16; 410 else { 411 assert(LdSt.mayStore()); 412 int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); 413 EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, Data0Idx)) / 8; 414 } 415 416 if (isStride64(Opc)) 417 EltSize *= 64; 418 419 BaseOps.push_back(BaseOp); 420 Offset = EltSize * Offset0; 421 // Get appropriate operand(s), and compute width accordingly. 422 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 423 if (DataOpIdx == -1) { 424 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); 425 Width = getOpSize(LdSt, DataOpIdx); 426 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1); 427 Width += getOpSize(LdSt, DataOpIdx); 428 } else { 429 Width = getOpSize(LdSt, DataOpIdx); 430 } 431 } 432 return true; 433 } 434 435 if (isMUBUF(LdSt) || isMTBUF(LdSt)) { 436 const MachineOperand *RSrc = getNamedOperand(LdSt, AMDGPU::OpName::srsrc); 437 if (!RSrc) // e.g. BUFFER_WBINVL1_VOL 438 return false; 439 BaseOps.push_back(RSrc); 440 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr); 441 if (BaseOp && !BaseOp->isFI()) 442 BaseOps.push_back(BaseOp); 443 const MachineOperand *OffsetImm = 444 getNamedOperand(LdSt, AMDGPU::OpName::offset); 445 Offset = OffsetImm->getImm(); 446 const MachineOperand *SOffset = 447 getNamedOperand(LdSt, AMDGPU::OpName::soffset); 448 if (SOffset) { 449 if (SOffset->isReg()) 450 BaseOps.push_back(SOffset); 451 else 452 Offset += SOffset->getImm(); 453 } 454 // Get appropriate operand, and compute width accordingly. 455 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 456 if (DataOpIdx == -1) 457 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); 458 if (DataOpIdx == -1) // LDS DMA 459 return false; 460 Width = getOpSize(LdSt, DataOpIdx); 461 return true; 462 } 463 464 if (isMIMG(LdSt)) { 465 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc); 466 BaseOps.push_back(&LdSt.getOperand(SRsrcIdx)); 467 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0); 468 if (VAddr0Idx >= 0) { 469 // GFX10 possible NSA encoding. 470 for (int I = VAddr0Idx; I < SRsrcIdx; ++I) 471 BaseOps.push_back(&LdSt.getOperand(I)); 472 } else { 473 BaseOps.push_back(getNamedOperand(LdSt, AMDGPU::OpName::vaddr)); 474 } 475 Offset = 0; 476 // Get appropriate operand, and compute width accordingly. 477 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); 478 Width = getOpSize(LdSt, DataOpIdx); 479 return true; 480 } 481 482 if (isSMRD(LdSt)) { 483 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::sbase); 484 if (!BaseOp) // e.g. S_MEMTIME 485 return false; 486 BaseOps.push_back(BaseOp); 487 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset); 488 Offset = OffsetOp ? OffsetOp->getImm() : 0; 489 // Get appropriate operand, and compute width accordingly. 490 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::sdst); 491 if (DataOpIdx == -1) 492 return false; 493 Width = getOpSize(LdSt, DataOpIdx); 494 return true; 495 } 496 497 if (isFLAT(LdSt)) { 498 // Instructions have either vaddr or saddr or both or none. 499 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr); 500 if (BaseOp) 501 BaseOps.push_back(BaseOp); 502 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr); 503 if (BaseOp) 504 BaseOps.push_back(BaseOp); 505 Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm(); 506 // Get appropriate operand, and compute width accordingly. 507 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 508 if (DataOpIdx == -1) 509 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); 510 if (DataOpIdx == -1) // LDS DMA 511 return false; 512 Width = getOpSize(LdSt, DataOpIdx); 513 return true; 514 } 515 516 return false; 517 } 518 519 static bool memOpsHaveSameBasePtr(const MachineInstr &MI1, 520 ArrayRef<const MachineOperand *> BaseOps1, 521 const MachineInstr &MI2, 522 ArrayRef<const MachineOperand *> BaseOps2) { 523 // Only examine the first "base" operand of each instruction, on the 524 // assumption that it represents the real base address of the memory access. 525 // Other operands are typically offsets or indices from this base address. 526 if (BaseOps1.front()->isIdenticalTo(*BaseOps2.front())) 527 return true; 528 529 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand()) 530 return false; 531 532 auto MO1 = *MI1.memoperands_begin(); 533 auto MO2 = *MI2.memoperands_begin(); 534 if (MO1->getAddrSpace() != MO2->getAddrSpace()) 535 return false; 536 537 auto Base1 = MO1->getValue(); 538 auto Base2 = MO2->getValue(); 539 if (!Base1 || !Base2) 540 return false; 541 Base1 = getUnderlyingObject(Base1); 542 Base2 = getUnderlyingObject(Base2); 543 544 if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2)) 545 return false; 546 547 return Base1 == Base2; 548 } 549 550 bool SIInstrInfo::shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1, 551 int64_t Offset1, bool OffsetIsScalable1, 552 ArrayRef<const MachineOperand *> BaseOps2, 553 int64_t Offset2, bool OffsetIsScalable2, 554 unsigned ClusterSize, 555 unsigned NumBytes) const { 556 // If the mem ops (to be clustered) do not have the same base ptr, then they 557 // should not be clustered 558 if (!BaseOps1.empty() && !BaseOps2.empty()) { 559 const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent(); 560 const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent(); 561 if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOps1, SecondLdSt, BaseOps2)) 562 return false; 563 } else if (!BaseOps1.empty() || !BaseOps2.empty()) { 564 // If only one base op is empty, they do not have the same base ptr 565 return false; 566 } 567 568 // In order to avoid register pressure, on an average, the number of DWORDS 569 // loaded together by all clustered mem ops should not exceed 8. This is an 570 // empirical value based on certain observations and performance related 571 // experiments. 572 // The good thing about this heuristic is - it avoids clustering of too many 573 // sub-word loads, and also avoids clustering of wide loads. Below is the 574 // brief summary of how the heuristic behaves for various `LoadSize`. 575 // (1) 1 <= LoadSize <= 4: cluster at max 8 mem ops 576 // (2) 5 <= LoadSize <= 8: cluster at max 4 mem ops 577 // (3) 9 <= LoadSize <= 12: cluster at max 2 mem ops 578 // (4) 13 <= LoadSize <= 16: cluster at max 2 mem ops 579 // (5) LoadSize >= 17: do not cluster 580 const unsigned LoadSize = NumBytes / ClusterSize; 581 const unsigned NumDWORDs = ((LoadSize + 3) / 4) * ClusterSize; 582 return NumDWORDs <= 8; 583 } 584 585 // FIXME: This behaves strangely. If, for example, you have 32 load + stores, 586 // the first 16 loads will be interleaved with the stores, and the next 16 will 587 // be clustered as expected. It should really split into 2 16 store batches. 588 // 589 // Loads are clustered until this returns false, rather than trying to schedule 590 // groups of stores. This also means we have to deal with saying different 591 // address space loads should be clustered, and ones which might cause bank 592 // conflicts. 593 // 594 // This might be deprecated so it might not be worth that much effort to fix. 595 bool SIInstrInfo::shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1, 596 int64_t Offset0, int64_t Offset1, 597 unsigned NumLoads) const { 598 assert(Offset1 > Offset0 && 599 "Second offset should be larger than first offset!"); 600 // If we have less than 16 loads in a row, and the offsets are within 64 601 // bytes, then schedule together. 602 603 // A cacheline is 64 bytes (for global memory). 604 return (NumLoads <= 16 && (Offset1 - Offset0) < 64); 605 } 606 607 static void reportIllegalCopy(const SIInstrInfo *TII, MachineBasicBlock &MBB, 608 MachineBasicBlock::iterator MI, 609 const DebugLoc &DL, MCRegister DestReg, 610 MCRegister SrcReg, bool KillSrc, 611 const char *Msg = "illegal VGPR to SGPR copy") { 612 MachineFunction *MF = MBB.getParent(); 613 DiagnosticInfoUnsupported IllegalCopy(MF->getFunction(), Msg, DL, DS_Error); 614 LLVMContext &C = MF->getFunction().getContext(); 615 C.diagnose(IllegalCopy); 616 617 BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg) 618 .addReg(SrcReg, getKillRegState(KillSrc)); 619 } 620 621 /// Handle copying from SGPR to AGPR, or from AGPR to AGPR on GFX908. It is not 622 /// possible to have a direct copy in these cases on GFX908, so an intermediate 623 /// VGPR copy is required. 624 static void indirectCopyToAGPR(const SIInstrInfo &TII, 625 MachineBasicBlock &MBB, 626 MachineBasicBlock::iterator MI, 627 const DebugLoc &DL, MCRegister DestReg, 628 MCRegister SrcReg, bool KillSrc, 629 RegScavenger &RS, bool RegsOverlap, 630 Register ImpDefSuperReg = Register(), 631 Register ImpUseSuperReg = Register()) { 632 assert((TII.getSubtarget().hasMAIInsts() && 633 !TII.getSubtarget().hasGFX90AInsts()) && 634 "Expected GFX908 subtarget."); 635 636 assert((AMDGPU::SReg_32RegClass.contains(SrcReg) || 637 AMDGPU::AGPR_32RegClass.contains(SrcReg)) && 638 "Source register of the copy should be either an SGPR or an AGPR."); 639 640 assert(AMDGPU::AGPR_32RegClass.contains(DestReg) && 641 "Destination register of the copy should be an AGPR."); 642 643 const SIRegisterInfo &RI = TII.getRegisterInfo(); 644 645 // First try to find defining accvgpr_write to avoid temporary registers. 646 // In the case of copies of overlapping AGPRs, we conservatively do not 647 // reuse previous accvgpr_writes. Otherwise, we may incorrectly pick up 648 // an accvgpr_write used for this same copy due to implicit-defs 649 if (!RegsOverlap) { 650 for (auto Def = MI, E = MBB.begin(); Def != E; ) { 651 --Def; 652 653 if (!Def->modifiesRegister(SrcReg, &RI)) 654 continue; 655 656 if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 || 657 Def->getOperand(0).getReg() != SrcReg) 658 break; 659 660 MachineOperand &DefOp = Def->getOperand(1); 661 assert(DefOp.isReg() || DefOp.isImm()); 662 663 if (DefOp.isReg()) { 664 bool SafeToPropagate = true; 665 // Check that register source operand is not clobbered before MI. 666 // Immediate operands are always safe to propagate. 667 for (auto I = Def; I != MI && SafeToPropagate; ++I) 668 if (I->modifiesRegister(DefOp.getReg(), &RI)) 669 SafeToPropagate = false; 670 671 if (!SafeToPropagate) 672 break; 673 674 DefOp.setIsKill(false); 675 } 676 677 MachineInstrBuilder Builder = 678 BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) 679 .add(DefOp); 680 if (ImpDefSuperReg) 681 Builder.addReg(ImpDefSuperReg, RegState::Define | RegState::Implicit); 682 683 if (ImpUseSuperReg) { 684 Builder.addReg(ImpUseSuperReg, 685 getKillRegState(KillSrc) | RegState::Implicit); 686 } 687 688 return; 689 } 690 } 691 692 RS.enterBasicBlockEnd(MBB); 693 RS.backward(std::next(MI)); 694 695 // Ideally we want to have three registers for a long reg_sequence copy 696 // to hide 2 waitstates between v_mov_b32 and accvgpr_write. 697 unsigned MaxVGPRs = RI.getRegPressureLimit(&AMDGPU::VGPR_32RegClass, 698 *MBB.getParent()); 699 700 // Registers in the sequence are allocated contiguously so we can just 701 // use register number to pick one of three round-robin temps. 702 unsigned RegNo = (DestReg - AMDGPU::AGPR0) % 3; 703 Register Tmp = 704 MBB.getParent()->getInfo<SIMachineFunctionInfo>()->getVGPRForAGPRCopy(); 705 assert(MBB.getParent()->getRegInfo().isReserved(Tmp) && 706 "VGPR used for an intermediate copy should have been reserved."); 707 708 // Only loop through if there are any free registers left. We don't want to 709 // spill. 710 while (RegNo--) { 711 Register Tmp2 = RS.scavengeRegisterBackwards(AMDGPU::VGPR_32RegClass, MI, 712 /* RestoreAfter */ false, 0, 713 /* AllowSpill */ false); 714 if (!Tmp2 || RI.getHWRegIndex(Tmp2) >= MaxVGPRs) 715 break; 716 Tmp = Tmp2; 717 RS.setRegUsed(Tmp); 718 } 719 720 // Insert copy to temporary VGPR. 721 unsigned TmpCopyOp = AMDGPU::V_MOV_B32_e32; 722 if (AMDGPU::AGPR_32RegClass.contains(SrcReg)) { 723 TmpCopyOp = AMDGPU::V_ACCVGPR_READ_B32_e64; 724 } else { 725 assert(AMDGPU::SReg_32RegClass.contains(SrcReg)); 726 } 727 728 MachineInstrBuilder UseBuilder = BuildMI(MBB, MI, DL, TII.get(TmpCopyOp), Tmp) 729 .addReg(SrcReg, getKillRegState(KillSrc)); 730 if (ImpUseSuperReg) { 731 UseBuilder.addReg(ImpUseSuperReg, 732 getKillRegState(KillSrc) | RegState::Implicit); 733 } 734 735 MachineInstrBuilder DefBuilder 736 = BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) 737 .addReg(Tmp, RegState::Kill); 738 739 if (ImpDefSuperReg) 740 DefBuilder.addReg(ImpDefSuperReg, RegState::Define | RegState::Implicit); 741 } 742 743 static void expandSGPRCopy(const SIInstrInfo &TII, MachineBasicBlock &MBB, 744 MachineBasicBlock::iterator MI, const DebugLoc &DL, 745 MCRegister DestReg, MCRegister SrcReg, bool KillSrc, 746 const TargetRegisterClass *RC, bool Forward) { 747 const SIRegisterInfo &RI = TII.getRegisterInfo(); 748 ArrayRef<int16_t> BaseIndices = RI.getRegSplitParts(RC, 4); 749 MachineBasicBlock::iterator I = MI; 750 MachineInstr *FirstMI = nullptr, *LastMI = nullptr; 751 752 for (unsigned Idx = 0; Idx < BaseIndices.size(); ++Idx) { 753 int16_t SubIdx = BaseIndices[Idx]; 754 Register DestSubReg = RI.getSubReg(DestReg, SubIdx); 755 Register SrcSubReg = RI.getSubReg(SrcReg, SubIdx); 756 assert(DestSubReg && SrcSubReg && "Failed to find subregs!"); 757 unsigned Opcode = AMDGPU::S_MOV_B32; 758 759 // Is SGPR aligned? If so try to combine with next. 760 bool AlignedDest = ((DestSubReg - AMDGPU::SGPR0) % 2) == 0; 761 bool AlignedSrc = ((SrcSubReg - AMDGPU::SGPR0) % 2) == 0; 762 if (AlignedDest && AlignedSrc && (Idx + 1 < BaseIndices.size())) { 763 // Can use SGPR64 copy 764 unsigned Channel = RI.getChannelFromSubReg(SubIdx); 765 SubIdx = RI.getSubRegFromChannel(Channel, 2); 766 DestSubReg = RI.getSubReg(DestReg, SubIdx); 767 SrcSubReg = RI.getSubReg(SrcReg, SubIdx); 768 assert(DestSubReg && SrcSubReg && "Failed to find subregs!"); 769 Opcode = AMDGPU::S_MOV_B64; 770 Idx++; 771 } 772 773 LastMI = BuildMI(MBB, I, DL, TII.get(Opcode), DestSubReg) 774 .addReg(SrcSubReg) 775 .addReg(SrcReg, RegState::Implicit); 776 777 if (!FirstMI) 778 FirstMI = LastMI; 779 780 if (!Forward) 781 I--; 782 } 783 784 assert(FirstMI && LastMI); 785 if (!Forward) 786 std::swap(FirstMI, LastMI); 787 788 FirstMI->addOperand( 789 MachineOperand::CreateReg(DestReg, true /*IsDef*/, true /*IsImp*/)); 790 791 if (KillSrc) 792 LastMI->addRegisterKilled(SrcReg, &RI); 793 } 794 795 void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB, 796 MachineBasicBlock::iterator MI, 797 const DebugLoc &DL, MCRegister DestReg, 798 MCRegister SrcReg, bool KillSrc) const { 799 const TargetRegisterClass *RC = RI.getPhysRegBaseClass(DestReg); 800 unsigned Size = RI.getRegSizeInBits(*RC); 801 const TargetRegisterClass *SrcRC = RI.getPhysRegBaseClass(SrcReg); 802 unsigned SrcSize = RI.getRegSizeInBits(*SrcRC); 803 804 // The rest of copyPhysReg assumes Src and Dst size are the same size. 805 // TODO-GFX11_16BIT If all true 16 bit instruction patterns are completed can 806 // we remove Fix16BitCopies and this code block? 807 if (Fix16BitCopies) { 808 if (((Size == 16) != (SrcSize == 16))) { 809 // Non-VGPR Src and Dst will later be expanded back to 32 bits. 810 assert(ST.hasTrue16BitInsts()); 811 MCRegister &RegToFix = (Size == 32) ? DestReg : SrcReg; 812 MCRegister SubReg = RI.getSubReg(RegToFix, AMDGPU::lo16); 813 RegToFix = SubReg; 814 815 if (DestReg == SrcReg) { 816 // Identity copy. Insert empty bundle since ExpandPostRA expects an 817 // instruction here. 818 BuildMI(MBB, MI, DL, get(AMDGPU::BUNDLE)); 819 return; 820 } 821 RC = RI.getPhysRegBaseClass(DestReg); 822 Size = RI.getRegSizeInBits(*RC); 823 SrcRC = RI.getPhysRegBaseClass(SrcReg); 824 SrcSize = RI.getRegSizeInBits(*SrcRC); 825 } 826 } 827 828 if (RC == &AMDGPU::VGPR_32RegClass) { 829 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) || 830 AMDGPU::SReg_32RegClass.contains(SrcReg) || 831 AMDGPU::AGPR_32RegClass.contains(SrcReg)); 832 unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ? 833 AMDGPU::V_ACCVGPR_READ_B32_e64 : AMDGPU::V_MOV_B32_e32; 834 BuildMI(MBB, MI, DL, get(Opc), DestReg) 835 .addReg(SrcReg, getKillRegState(KillSrc)); 836 return; 837 } 838 839 if (RC == &AMDGPU::SReg_32_XM0RegClass || 840 RC == &AMDGPU::SReg_32RegClass) { 841 if (SrcReg == AMDGPU::SCC) { 842 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg) 843 .addImm(1) 844 .addImm(0); 845 return; 846 } 847 848 if (DestReg == AMDGPU::VCC_LO) { 849 if (AMDGPU::SReg_32RegClass.contains(SrcReg)) { 850 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), AMDGPU::VCC_LO) 851 .addReg(SrcReg, getKillRegState(KillSrc)); 852 } else { 853 // FIXME: Hack until VReg_1 removed. 854 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg)); 855 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32)) 856 .addImm(0) 857 .addReg(SrcReg, getKillRegState(KillSrc)); 858 } 859 860 return; 861 } 862 863 if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) { 864 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc); 865 return; 866 } 867 868 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg) 869 .addReg(SrcReg, getKillRegState(KillSrc)); 870 return; 871 } 872 873 if (RC == &AMDGPU::SReg_64RegClass) { 874 if (SrcReg == AMDGPU::SCC) { 875 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B64), DestReg) 876 .addImm(1) 877 .addImm(0); 878 return; 879 } 880 881 if (DestReg == AMDGPU::VCC) { 882 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) { 883 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC) 884 .addReg(SrcReg, getKillRegState(KillSrc)); 885 } else { 886 // FIXME: Hack until VReg_1 removed. 887 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg)); 888 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32)) 889 .addImm(0) 890 .addReg(SrcReg, getKillRegState(KillSrc)); 891 } 892 893 return; 894 } 895 896 if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) { 897 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc); 898 return; 899 } 900 901 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg) 902 .addReg(SrcReg, getKillRegState(KillSrc)); 903 return; 904 } 905 906 if (DestReg == AMDGPU::SCC) { 907 // Copying 64-bit or 32-bit sources to SCC barely makes sense, 908 // but SelectionDAG emits such copies for i1 sources. 909 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) { 910 // This copy can only be produced by patterns 911 // with explicit SCC, which are known to be enabled 912 // only for subtargets with S_CMP_LG_U64 present. 913 assert(ST.hasScalarCompareEq64()); 914 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U64)) 915 .addReg(SrcReg, getKillRegState(KillSrc)) 916 .addImm(0); 917 } else { 918 assert(AMDGPU::SReg_32RegClass.contains(SrcReg)); 919 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32)) 920 .addReg(SrcReg, getKillRegState(KillSrc)) 921 .addImm(0); 922 } 923 924 return; 925 } 926 927 if (RC == &AMDGPU::AGPR_32RegClass) { 928 if (AMDGPU::VGPR_32RegClass.contains(SrcReg) || 929 (ST.hasGFX90AInsts() && AMDGPU::SReg_32RegClass.contains(SrcReg))) { 930 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) 931 .addReg(SrcReg, getKillRegState(KillSrc)); 932 return; 933 } 934 935 if (AMDGPU::AGPR_32RegClass.contains(SrcReg) && ST.hasGFX90AInsts()) { 936 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_MOV_B32), DestReg) 937 .addReg(SrcReg, getKillRegState(KillSrc)); 938 return; 939 } 940 941 // FIXME: Pass should maintain scavenger to avoid scan through the block on 942 // every AGPR spill. 943 RegScavenger RS; 944 const bool Overlap = RI.regsOverlap(SrcReg, DestReg); 945 indirectCopyToAGPR(*this, MBB, MI, DL, DestReg, SrcReg, KillSrc, RS, Overlap); 946 return; 947 } 948 949 if (Size == 16) { 950 assert(AMDGPU::VGPR_16RegClass.contains(SrcReg) || 951 AMDGPU::SReg_LO16RegClass.contains(SrcReg) || 952 AMDGPU::AGPR_LO16RegClass.contains(SrcReg)); 953 954 bool IsSGPRDst = AMDGPU::SReg_LO16RegClass.contains(DestReg); 955 bool IsSGPRSrc = AMDGPU::SReg_LO16RegClass.contains(SrcReg); 956 bool IsAGPRDst = AMDGPU::AGPR_LO16RegClass.contains(DestReg); 957 bool IsAGPRSrc = AMDGPU::AGPR_LO16RegClass.contains(SrcReg); 958 bool DstLow = !AMDGPU::isHi(DestReg, RI); 959 bool SrcLow = !AMDGPU::isHi(SrcReg, RI); 960 MCRegister NewDestReg = RI.get32BitRegister(DestReg); 961 MCRegister NewSrcReg = RI.get32BitRegister(SrcReg); 962 963 if (IsSGPRDst) { 964 if (!IsSGPRSrc) { 965 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc); 966 return; 967 } 968 969 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), NewDestReg) 970 .addReg(NewSrcReg, getKillRegState(KillSrc)); 971 return; 972 } 973 974 if (IsAGPRDst || IsAGPRSrc) { 975 if (!DstLow || !SrcLow) { 976 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc, 977 "Cannot use hi16 subreg with an AGPR!"); 978 } 979 980 copyPhysReg(MBB, MI, DL, NewDestReg, NewSrcReg, KillSrc); 981 return; 982 } 983 984 if (ST.hasTrue16BitInsts()) { 985 if (IsSGPRSrc) { 986 assert(SrcLow); 987 SrcReg = NewSrcReg; 988 } 989 // Use the smaller instruction encoding if possible. 990 if (AMDGPU::VGPR_16_Lo128RegClass.contains(DestReg) && 991 (IsSGPRSrc || AMDGPU::VGPR_16_Lo128RegClass.contains(SrcReg))) { 992 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B16_t16_e32), DestReg) 993 .addReg(SrcReg); 994 } else { 995 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B16_t16_e64), DestReg) 996 .addImm(0) // src0_modifiers 997 .addReg(SrcReg) 998 .addImm(0); // op_sel 999 } 1000 return; 1001 } 1002 1003 if (IsSGPRSrc && !ST.hasSDWAScalar()) { 1004 if (!DstLow || !SrcLow) { 1005 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc, 1006 "Cannot use hi16 subreg on VI!"); 1007 } 1008 1009 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), NewDestReg) 1010 .addReg(NewSrcReg, getKillRegState(KillSrc)); 1011 return; 1012 } 1013 1014 auto MIB = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_sdwa), NewDestReg) 1015 .addImm(0) // src0_modifiers 1016 .addReg(NewSrcReg) 1017 .addImm(0) // clamp 1018 .addImm(DstLow ? AMDGPU::SDWA::SdwaSel::WORD_0 1019 : AMDGPU::SDWA::SdwaSel::WORD_1) 1020 .addImm(AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE) 1021 .addImm(SrcLow ? AMDGPU::SDWA::SdwaSel::WORD_0 1022 : AMDGPU::SDWA::SdwaSel::WORD_1) 1023 .addReg(NewDestReg, RegState::Implicit | RegState::Undef); 1024 // First implicit operand is $exec. 1025 MIB->tieOperands(0, MIB->getNumOperands() - 1); 1026 return; 1027 } 1028 1029 if (RC == RI.getVGPR64Class() && (SrcRC == RC || RI.isSGPRClass(SrcRC))) { 1030 if (ST.hasMovB64()) { 1031 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_e32), DestReg) 1032 .addReg(SrcReg, getKillRegState(KillSrc)); 1033 return; 1034 } 1035 if (ST.hasPkMovB32()) { 1036 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DestReg) 1037 .addImm(SISrcMods::OP_SEL_1) 1038 .addReg(SrcReg) 1039 .addImm(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1) 1040 .addReg(SrcReg) 1041 .addImm(0) // op_sel_lo 1042 .addImm(0) // op_sel_hi 1043 .addImm(0) // neg_lo 1044 .addImm(0) // neg_hi 1045 .addImm(0) // clamp 1046 .addReg(SrcReg, getKillRegState(KillSrc) | RegState::Implicit); 1047 return; 1048 } 1049 } 1050 1051 const bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg); 1052 if (RI.isSGPRClass(RC)) { 1053 if (!RI.isSGPRClass(SrcRC)) { 1054 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc); 1055 return; 1056 } 1057 const bool CanKillSuperReg = KillSrc && !RI.regsOverlap(SrcReg, DestReg); 1058 expandSGPRCopy(*this, MBB, MI, DL, DestReg, SrcReg, CanKillSuperReg, RC, 1059 Forward); 1060 return; 1061 } 1062 1063 unsigned EltSize = 4; 1064 unsigned Opcode = AMDGPU::V_MOV_B32_e32; 1065 if (RI.isAGPRClass(RC)) { 1066 if (ST.hasGFX90AInsts() && RI.isAGPRClass(SrcRC)) 1067 Opcode = AMDGPU::V_ACCVGPR_MOV_B32; 1068 else if (RI.hasVGPRs(SrcRC) || 1069 (ST.hasGFX90AInsts() && RI.isSGPRClass(SrcRC))) 1070 Opcode = AMDGPU::V_ACCVGPR_WRITE_B32_e64; 1071 else 1072 Opcode = AMDGPU::INSTRUCTION_LIST_END; 1073 } else if (RI.hasVGPRs(RC) && RI.isAGPRClass(SrcRC)) { 1074 Opcode = AMDGPU::V_ACCVGPR_READ_B32_e64; 1075 } else if ((Size % 64 == 0) && RI.hasVGPRs(RC) && 1076 (RI.isProperlyAlignedRC(*RC) && 1077 (SrcRC == RC || RI.isSGPRClass(SrcRC)))) { 1078 // TODO: In 96-bit case, could do a 64-bit mov and then a 32-bit mov. 1079 if (ST.hasMovB64()) { 1080 Opcode = AMDGPU::V_MOV_B64_e32; 1081 EltSize = 8; 1082 } else if (ST.hasPkMovB32()) { 1083 Opcode = AMDGPU::V_PK_MOV_B32; 1084 EltSize = 8; 1085 } 1086 } 1087 1088 // For the cases where we need an intermediate instruction/temporary register 1089 // (destination is an AGPR), we need a scavenger. 1090 // 1091 // FIXME: The pass should maintain this for us so we don't have to re-scan the 1092 // whole block for every handled copy. 1093 std::unique_ptr<RegScavenger> RS; 1094 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) 1095 RS.reset(new RegScavenger()); 1096 1097 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize); 1098 1099 // If there is an overlap, we can't kill the super-register on the last 1100 // instruction, since it will also kill the components made live by this def. 1101 const bool Overlap = RI.regsOverlap(SrcReg, DestReg); 1102 const bool CanKillSuperReg = KillSrc && !Overlap; 1103 1104 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) { 1105 unsigned SubIdx; 1106 if (Forward) 1107 SubIdx = SubIndices[Idx]; 1108 else 1109 SubIdx = SubIndices[SubIndices.size() - Idx - 1]; 1110 Register DestSubReg = RI.getSubReg(DestReg, SubIdx); 1111 Register SrcSubReg = RI.getSubReg(SrcReg, SubIdx); 1112 assert(DestSubReg && SrcSubReg && "Failed to find subregs!"); 1113 1114 bool IsFirstSubreg = Idx == 0; 1115 bool UseKill = CanKillSuperReg && Idx == SubIndices.size() - 1; 1116 1117 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) { 1118 Register ImpDefSuper = IsFirstSubreg ? Register(DestReg) : Register(); 1119 Register ImpUseSuper = SrcReg; 1120 indirectCopyToAGPR(*this, MBB, MI, DL, DestSubReg, SrcSubReg, UseKill, 1121 *RS, Overlap, ImpDefSuper, ImpUseSuper); 1122 } else if (Opcode == AMDGPU::V_PK_MOV_B32) { 1123 MachineInstrBuilder MIB = 1124 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DestSubReg) 1125 .addImm(SISrcMods::OP_SEL_1) 1126 .addReg(SrcSubReg) 1127 .addImm(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1) 1128 .addReg(SrcSubReg) 1129 .addImm(0) // op_sel_lo 1130 .addImm(0) // op_sel_hi 1131 .addImm(0) // neg_lo 1132 .addImm(0) // neg_hi 1133 .addImm(0) // clamp 1134 .addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit); 1135 if (IsFirstSubreg) 1136 MIB.addReg(DestReg, RegState::Define | RegState::Implicit); 1137 } else { 1138 MachineInstrBuilder Builder = 1139 BuildMI(MBB, MI, DL, get(Opcode), DestSubReg).addReg(SrcSubReg); 1140 if (IsFirstSubreg) 1141 Builder.addReg(DestReg, RegState::Define | RegState::Implicit); 1142 1143 Builder.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit); 1144 } 1145 } 1146 } 1147 1148 int SIInstrInfo::commuteOpcode(unsigned Opcode) const { 1149 int NewOpc; 1150 1151 // Try to map original to commuted opcode 1152 NewOpc = AMDGPU::getCommuteRev(Opcode); 1153 if (NewOpc != -1) 1154 // Check if the commuted (REV) opcode exists on the target. 1155 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1; 1156 1157 // Try to map commuted to original opcode 1158 NewOpc = AMDGPU::getCommuteOrig(Opcode); 1159 if (NewOpc != -1) 1160 // Check if the original (non-REV) opcode exists on the target. 1161 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1; 1162 1163 return Opcode; 1164 } 1165 1166 void SIInstrInfo::materializeImmediate(MachineBasicBlock &MBB, 1167 MachineBasicBlock::iterator MI, 1168 const DebugLoc &DL, Register DestReg, 1169 int64_t Value) const { 1170 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 1171 const TargetRegisterClass *RegClass = MRI.getRegClass(DestReg); 1172 if (RegClass == &AMDGPU::SReg_32RegClass || 1173 RegClass == &AMDGPU::SGPR_32RegClass || 1174 RegClass == &AMDGPU::SReg_32_XM0RegClass || 1175 RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) { 1176 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg) 1177 .addImm(Value); 1178 return; 1179 } 1180 1181 if (RegClass == &AMDGPU::SReg_64RegClass || 1182 RegClass == &AMDGPU::SGPR_64RegClass || 1183 RegClass == &AMDGPU::SReg_64_XEXECRegClass) { 1184 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg) 1185 .addImm(Value); 1186 return; 1187 } 1188 1189 if (RegClass == &AMDGPU::VGPR_32RegClass) { 1190 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg) 1191 .addImm(Value); 1192 return; 1193 } 1194 if (RegClass->hasSuperClassEq(&AMDGPU::VReg_64RegClass)) { 1195 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg) 1196 .addImm(Value); 1197 return; 1198 } 1199 1200 unsigned EltSize = 4; 1201 unsigned Opcode = AMDGPU::V_MOV_B32_e32; 1202 if (RI.isSGPRClass(RegClass)) { 1203 if (RI.getRegSizeInBits(*RegClass) > 32) { 1204 Opcode = AMDGPU::S_MOV_B64; 1205 EltSize = 8; 1206 } else { 1207 Opcode = AMDGPU::S_MOV_B32; 1208 EltSize = 4; 1209 } 1210 } 1211 1212 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RegClass, EltSize); 1213 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) { 1214 int64_t IdxValue = Idx == 0 ? Value : 0; 1215 1216 MachineInstrBuilder Builder = BuildMI(MBB, MI, DL, 1217 get(Opcode), RI.getSubReg(DestReg, SubIndices[Idx])); 1218 Builder.addImm(IdxValue); 1219 } 1220 } 1221 1222 const TargetRegisterClass * 1223 SIInstrInfo::getPreferredSelectRegClass(unsigned Size) const { 1224 return &AMDGPU::VGPR_32RegClass; 1225 } 1226 1227 void SIInstrInfo::insertVectorSelect(MachineBasicBlock &MBB, 1228 MachineBasicBlock::iterator I, 1229 const DebugLoc &DL, Register DstReg, 1230 ArrayRef<MachineOperand> Cond, 1231 Register TrueReg, 1232 Register FalseReg) const { 1233 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 1234 const TargetRegisterClass *BoolXExecRC = 1235 RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 1236 assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass && 1237 "Not a VGPR32 reg"); 1238 1239 if (Cond.size() == 1) { 1240 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1241 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) 1242 .add(Cond[0]); 1243 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1244 .addImm(0) 1245 .addReg(FalseReg) 1246 .addImm(0) 1247 .addReg(TrueReg) 1248 .addReg(SReg); 1249 } else if (Cond.size() == 2) { 1250 assert(Cond[0].isImm() && "Cond[0] is not an immediate"); 1251 switch (Cond[0].getImm()) { 1252 case SIInstrInfo::SCC_TRUE: { 1253 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1254 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 1255 : AMDGPU::S_CSELECT_B64), SReg) 1256 .addImm(1) 1257 .addImm(0); 1258 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1259 .addImm(0) 1260 .addReg(FalseReg) 1261 .addImm(0) 1262 .addReg(TrueReg) 1263 .addReg(SReg); 1264 break; 1265 } 1266 case SIInstrInfo::SCC_FALSE: { 1267 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1268 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 1269 : AMDGPU::S_CSELECT_B64), SReg) 1270 .addImm(0) 1271 .addImm(1); 1272 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1273 .addImm(0) 1274 .addReg(FalseReg) 1275 .addImm(0) 1276 .addReg(TrueReg) 1277 .addReg(SReg); 1278 break; 1279 } 1280 case SIInstrInfo::VCCNZ: { 1281 MachineOperand RegOp = Cond[1]; 1282 RegOp.setImplicit(false); 1283 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1284 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) 1285 .add(RegOp); 1286 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1287 .addImm(0) 1288 .addReg(FalseReg) 1289 .addImm(0) 1290 .addReg(TrueReg) 1291 .addReg(SReg); 1292 break; 1293 } 1294 case SIInstrInfo::VCCZ: { 1295 MachineOperand RegOp = Cond[1]; 1296 RegOp.setImplicit(false); 1297 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1298 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) 1299 .add(RegOp); 1300 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1301 .addImm(0) 1302 .addReg(TrueReg) 1303 .addImm(0) 1304 .addReg(FalseReg) 1305 .addReg(SReg); 1306 break; 1307 } 1308 case SIInstrInfo::EXECNZ: { 1309 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1310 Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC()); 1311 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 1312 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2) 1313 .addImm(0); 1314 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 1315 : AMDGPU::S_CSELECT_B64), SReg) 1316 .addImm(1) 1317 .addImm(0); 1318 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1319 .addImm(0) 1320 .addReg(FalseReg) 1321 .addImm(0) 1322 .addReg(TrueReg) 1323 .addReg(SReg); 1324 break; 1325 } 1326 case SIInstrInfo::EXECZ: { 1327 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1328 Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC()); 1329 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 1330 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2) 1331 .addImm(0); 1332 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 1333 : AMDGPU::S_CSELECT_B64), SReg) 1334 .addImm(0) 1335 .addImm(1); 1336 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1337 .addImm(0) 1338 .addReg(FalseReg) 1339 .addImm(0) 1340 .addReg(TrueReg) 1341 .addReg(SReg); 1342 llvm_unreachable("Unhandled branch predicate EXECZ"); 1343 break; 1344 } 1345 default: 1346 llvm_unreachable("invalid branch predicate"); 1347 } 1348 } else { 1349 llvm_unreachable("Can only handle Cond size 1 or 2"); 1350 } 1351 } 1352 1353 Register SIInstrInfo::insertEQ(MachineBasicBlock *MBB, 1354 MachineBasicBlock::iterator I, 1355 const DebugLoc &DL, 1356 Register SrcReg, int Value) const { 1357 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 1358 Register Reg = MRI.createVirtualRegister(RI.getBoolRC()); 1359 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg) 1360 .addImm(Value) 1361 .addReg(SrcReg); 1362 1363 return Reg; 1364 } 1365 1366 Register SIInstrInfo::insertNE(MachineBasicBlock *MBB, 1367 MachineBasicBlock::iterator I, 1368 const DebugLoc &DL, 1369 Register SrcReg, int Value) const { 1370 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 1371 Register Reg = MRI.createVirtualRegister(RI.getBoolRC()); 1372 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg) 1373 .addImm(Value) 1374 .addReg(SrcReg); 1375 1376 return Reg; 1377 } 1378 1379 unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const { 1380 1381 if (RI.isAGPRClass(DstRC)) 1382 return AMDGPU::COPY; 1383 if (RI.getRegSizeInBits(*DstRC) == 16) { 1384 // Assume hi bits are unneeded. Only _e64 true16 instructions are legal 1385 // before RA. 1386 return RI.isSGPRClass(DstRC) ? AMDGPU::COPY : AMDGPU::V_MOV_B16_t16_e64; 1387 } else if (RI.getRegSizeInBits(*DstRC) == 32) { 1388 return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; 1389 } else if (RI.getRegSizeInBits(*DstRC) == 64 && RI.isSGPRClass(DstRC)) { 1390 return AMDGPU::S_MOV_B64; 1391 } else if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC)) { 1392 return AMDGPU::V_MOV_B64_PSEUDO; 1393 } 1394 return AMDGPU::COPY; 1395 } 1396 1397 const MCInstrDesc & 1398 SIInstrInfo::getIndirectGPRIDXPseudo(unsigned VecSize, 1399 bool IsIndirectSrc) const { 1400 if (IsIndirectSrc) { 1401 if (VecSize <= 32) // 4 bytes 1402 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1); 1403 if (VecSize <= 64) // 8 bytes 1404 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2); 1405 if (VecSize <= 96) // 12 bytes 1406 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3); 1407 if (VecSize <= 128) // 16 bytes 1408 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4); 1409 if (VecSize <= 160) // 20 bytes 1410 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5); 1411 if (VecSize <= 256) // 32 bytes 1412 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8); 1413 if (VecSize <= 288) // 36 bytes 1414 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9); 1415 if (VecSize <= 320) // 40 bytes 1416 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10); 1417 if (VecSize <= 352) // 44 bytes 1418 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11); 1419 if (VecSize <= 384) // 48 bytes 1420 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12); 1421 if (VecSize <= 512) // 64 bytes 1422 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16); 1423 if (VecSize <= 1024) // 128 bytes 1424 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32); 1425 1426 llvm_unreachable("unsupported size for IndirectRegReadGPRIDX pseudos"); 1427 } 1428 1429 if (VecSize <= 32) // 4 bytes 1430 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1); 1431 if (VecSize <= 64) // 8 bytes 1432 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2); 1433 if (VecSize <= 96) // 12 bytes 1434 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3); 1435 if (VecSize <= 128) // 16 bytes 1436 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4); 1437 if (VecSize <= 160) // 20 bytes 1438 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5); 1439 if (VecSize <= 256) // 32 bytes 1440 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8); 1441 if (VecSize <= 288) // 36 bytes 1442 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9); 1443 if (VecSize <= 320) // 40 bytes 1444 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10); 1445 if (VecSize <= 352) // 44 bytes 1446 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11); 1447 if (VecSize <= 384) // 48 bytes 1448 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12); 1449 if (VecSize <= 512) // 64 bytes 1450 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16); 1451 if (VecSize <= 1024) // 128 bytes 1452 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32); 1453 1454 llvm_unreachable("unsupported size for IndirectRegWriteGPRIDX pseudos"); 1455 } 1456 1457 static unsigned getIndirectVGPRWriteMovRelPseudoOpc(unsigned VecSize) { 1458 if (VecSize <= 32) // 4 bytes 1459 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1; 1460 if (VecSize <= 64) // 8 bytes 1461 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2; 1462 if (VecSize <= 96) // 12 bytes 1463 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3; 1464 if (VecSize <= 128) // 16 bytes 1465 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4; 1466 if (VecSize <= 160) // 20 bytes 1467 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5; 1468 if (VecSize <= 256) // 32 bytes 1469 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8; 1470 if (VecSize <= 288) // 36 bytes 1471 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9; 1472 if (VecSize <= 320) // 40 bytes 1473 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10; 1474 if (VecSize <= 352) // 44 bytes 1475 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11; 1476 if (VecSize <= 384) // 48 bytes 1477 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12; 1478 if (VecSize <= 512) // 64 bytes 1479 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16; 1480 if (VecSize <= 1024) // 128 bytes 1481 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32; 1482 1483 llvm_unreachable("unsupported size for IndirectRegWrite pseudos"); 1484 } 1485 1486 static unsigned getIndirectSGPRWriteMovRelPseudo32(unsigned VecSize) { 1487 if (VecSize <= 32) // 4 bytes 1488 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1; 1489 if (VecSize <= 64) // 8 bytes 1490 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2; 1491 if (VecSize <= 96) // 12 bytes 1492 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3; 1493 if (VecSize <= 128) // 16 bytes 1494 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4; 1495 if (VecSize <= 160) // 20 bytes 1496 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5; 1497 if (VecSize <= 256) // 32 bytes 1498 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8; 1499 if (VecSize <= 288) // 36 bytes 1500 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9; 1501 if (VecSize <= 320) // 40 bytes 1502 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10; 1503 if (VecSize <= 352) // 44 bytes 1504 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11; 1505 if (VecSize <= 384) // 48 bytes 1506 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12; 1507 if (VecSize <= 512) // 64 bytes 1508 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16; 1509 if (VecSize <= 1024) // 128 bytes 1510 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32; 1511 1512 llvm_unreachable("unsupported size for IndirectRegWrite pseudos"); 1513 } 1514 1515 static unsigned getIndirectSGPRWriteMovRelPseudo64(unsigned VecSize) { 1516 if (VecSize <= 64) // 8 bytes 1517 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1; 1518 if (VecSize <= 128) // 16 bytes 1519 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2; 1520 if (VecSize <= 256) // 32 bytes 1521 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4; 1522 if (VecSize <= 512) // 64 bytes 1523 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8; 1524 if (VecSize <= 1024) // 128 bytes 1525 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16; 1526 1527 llvm_unreachable("unsupported size for IndirectRegWrite pseudos"); 1528 } 1529 1530 const MCInstrDesc & 1531 SIInstrInfo::getIndirectRegWriteMovRelPseudo(unsigned VecSize, unsigned EltSize, 1532 bool IsSGPR) const { 1533 if (IsSGPR) { 1534 switch (EltSize) { 1535 case 32: 1536 return get(getIndirectSGPRWriteMovRelPseudo32(VecSize)); 1537 case 64: 1538 return get(getIndirectSGPRWriteMovRelPseudo64(VecSize)); 1539 default: 1540 llvm_unreachable("invalid reg indexing elt size"); 1541 } 1542 } 1543 1544 assert(EltSize == 32 && "invalid reg indexing elt size"); 1545 return get(getIndirectVGPRWriteMovRelPseudoOpc(VecSize)); 1546 } 1547 1548 static unsigned getSGPRSpillSaveOpcode(unsigned Size) { 1549 switch (Size) { 1550 case 4: 1551 return AMDGPU::SI_SPILL_S32_SAVE; 1552 case 8: 1553 return AMDGPU::SI_SPILL_S64_SAVE; 1554 case 12: 1555 return AMDGPU::SI_SPILL_S96_SAVE; 1556 case 16: 1557 return AMDGPU::SI_SPILL_S128_SAVE; 1558 case 20: 1559 return AMDGPU::SI_SPILL_S160_SAVE; 1560 case 24: 1561 return AMDGPU::SI_SPILL_S192_SAVE; 1562 case 28: 1563 return AMDGPU::SI_SPILL_S224_SAVE; 1564 case 32: 1565 return AMDGPU::SI_SPILL_S256_SAVE; 1566 case 36: 1567 return AMDGPU::SI_SPILL_S288_SAVE; 1568 case 40: 1569 return AMDGPU::SI_SPILL_S320_SAVE; 1570 case 44: 1571 return AMDGPU::SI_SPILL_S352_SAVE; 1572 case 48: 1573 return AMDGPU::SI_SPILL_S384_SAVE; 1574 case 64: 1575 return AMDGPU::SI_SPILL_S512_SAVE; 1576 case 128: 1577 return AMDGPU::SI_SPILL_S1024_SAVE; 1578 default: 1579 llvm_unreachable("unknown register size"); 1580 } 1581 } 1582 1583 static unsigned getVGPRSpillSaveOpcode(unsigned Size) { 1584 switch (Size) { 1585 case 4: 1586 return AMDGPU::SI_SPILL_V32_SAVE; 1587 case 8: 1588 return AMDGPU::SI_SPILL_V64_SAVE; 1589 case 12: 1590 return AMDGPU::SI_SPILL_V96_SAVE; 1591 case 16: 1592 return AMDGPU::SI_SPILL_V128_SAVE; 1593 case 20: 1594 return AMDGPU::SI_SPILL_V160_SAVE; 1595 case 24: 1596 return AMDGPU::SI_SPILL_V192_SAVE; 1597 case 28: 1598 return AMDGPU::SI_SPILL_V224_SAVE; 1599 case 32: 1600 return AMDGPU::SI_SPILL_V256_SAVE; 1601 case 36: 1602 return AMDGPU::SI_SPILL_V288_SAVE; 1603 case 40: 1604 return AMDGPU::SI_SPILL_V320_SAVE; 1605 case 44: 1606 return AMDGPU::SI_SPILL_V352_SAVE; 1607 case 48: 1608 return AMDGPU::SI_SPILL_V384_SAVE; 1609 case 64: 1610 return AMDGPU::SI_SPILL_V512_SAVE; 1611 case 128: 1612 return AMDGPU::SI_SPILL_V1024_SAVE; 1613 default: 1614 llvm_unreachable("unknown register size"); 1615 } 1616 } 1617 1618 static unsigned getAGPRSpillSaveOpcode(unsigned Size) { 1619 switch (Size) { 1620 case 4: 1621 return AMDGPU::SI_SPILL_A32_SAVE; 1622 case 8: 1623 return AMDGPU::SI_SPILL_A64_SAVE; 1624 case 12: 1625 return AMDGPU::SI_SPILL_A96_SAVE; 1626 case 16: 1627 return AMDGPU::SI_SPILL_A128_SAVE; 1628 case 20: 1629 return AMDGPU::SI_SPILL_A160_SAVE; 1630 case 24: 1631 return AMDGPU::SI_SPILL_A192_SAVE; 1632 case 28: 1633 return AMDGPU::SI_SPILL_A224_SAVE; 1634 case 32: 1635 return AMDGPU::SI_SPILL_A256_SAVE; 1636 case 36: 1637 return AMDGPU::SI_SPILL_A288_SAVE; 1638 case 40: 1639 return AMDGPU::SI_SPILL_A320_SAVE; 1640 case 44: 1641 return AMDGPU::SI_SPILL_A352_SAVE; 1642 case 48: 1643 return AMDGPU::SI_SPILL_A384_SAVE; 1644 case 64: 1645 return AMDGPU::SI_SPILL_A512_SAVE; 1646 case 128: 1647 return AMDGPU::SI_SPILL_A1024_SAVE; 1648 default: 1649 llvm_unreachable("unknown register size"); 1650 } 1651 } 1652 1653 static unsigned getAVSpillSaveOpcode(unsigned Size) { 1654 switch (Size) { 1655 case 4: 1656 return AMDGPU::SI_SPILL_AV32_SAVE; 1657 case 8: 1658 return AMDGPU::SI_SPILL_AV64_SAVE; 1659 case 12: 1660 return AMDGPU::SI_SPILL_AV96_SAVE; 1661 case 16: 1662 return AMDGPU::SI_SPILL_AV128_SAVE; 1663 case 20: 1664 return AMDGPU::SI_SPILL_AV160_SAVE; 1665 case 24: 1666 return AMDGPU::SI_SPILL_AV192_SAVE; 1667 case 28: 1668 return AMDGPU::SI_SPILL_AV224_SAVE; 1669 case 32: 1670 return AMDGPU::SI_SPILL_AV256_SAVE; 1671 case 36: 1672 return AMDGPU::SI_SPILL_AV288_SAVE; 1673 case 40: 1674 return AMDGPU::SI_SPILL_AV320_SAVE; 1675 case 44: 1676 return AMDGPU::SI_SPILL_AV352_SAVE; 1677 case 48: 1678 return AMDGPU::SI_SPILL_AV384_SAVE; 1679 case 64: 1680 return AMDGPU::SI_SPILL_AV512_SAVE; 1681 case 128: 1682 return AMDGPU::SI_SPILL_AV1024_SAVE; 1683 default: 1684 llvm_unreachable("unknown register size"); 1685 } 1686 } 1687 1688 static unsigned getWWMRegSpillSaveOpcode(unsigned Size, 1689 bool IsVectorSuperClass) { 1690 // Currently, there is only 32-bit WWM register spills needed. 1691 if (Size != 4) 1692 llvm_unreachable("unknown wwm register spill size"); 1693 1694 if (IsVectorSuperClass) 1695 return AMDGPU::SI_SPILL_WWM_AV32_SAVE; 1696 1697 return AMDGPU::SI_SPILL_WWM_V32_SAVE; 1698 } 1699 1700 static unsigned getVectorRegSpillSaveOpcode(Register Reg, 1701 const TargetRegisterClass *RC, 1702 unsigned Size, 1703 const SIRegisterInfo &TRI, 1704 const SIMachineFunctionInfo &MFI) { 1705 bool IsVectorSuperClass = TRI.isVectorSuperClass(RC); 1706 1707 // Choose the right opcode if spilling a WWM register. 1708 if (MFI.checkFlag(Reg, AMDGPU::VirtRegFlag::WWM_REG)) 1709 return getWWMRegSpillSaveOpcode(Size, IsVectorSuperClass); 1710 1711 if (IsVectorSuperClass) 1712 return getAVSpillSaveOpcode(Size); 1713 1714 return TRI.isAGPRClass(RC) ? getAGPRSpillSaveOpcode(Size) 1715 : getVGPRSpillSaveOpcode(Size); 1716 } 1717 1718 void SIInstrInfo::storeRegToStackSlot( 1719 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg, 1720 bool isKill, int FrameIndex, const TargetRegisterClass *RC, 1721 const TargetRegisterInfo *TRI, Register VReg) const { 1722 MachineFunction *MF = MBB.getParent(); 1723 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 1724 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 1725 const DebugLoc &DL = MBB.findDebugLoc(MI); 1726 1727 MachinePointerInfo PtrInfo 1728 = MachinePointerInfo::getFixedStack(*MF, FrameIndex); 1729 MachineMemOperand *MMO = MF->getMachineMemOperand( 1730 PtrInfo, MachineMemOperand::MOStore, FrameInfo.getObjectSize(FrameIndex), 1731 FrameInfo.getObjectAlign(FrameIndex)); 1732 unsigned SpillSize = TRI->getSpillSize(*RC); 1733 1734 MachineRegisterInfo &MRI = MF->getRegInfo(); 1735 if (RI.isSGPRClass(RC)) { 1736 MFI->setHasSpilledSGPRs(); 1737 assert(SrcReg != AMDGPU::M0 && "m0 should not be spilled"); 1738 assert(SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI && 1739 SrcReg != AMDGPU::EXEC && "exec should not be spilled"); 1740 1741 // We are only allowed to create one new instruction when spilling 1742 // registers, so we need to use pseudo instruction for spilling SGPRs. 1743 const MCInstrDesc &OpDesc = get(getSGPRSpillSaveOpcode(SpillSize)); 1744 1745 // The SGPR spill/restore instructions only work on number sgprs, so we need 1746 // to make sure we are using the correct register class. 1747 if (SrcReg.isVirtual() && SpillSize == 4) { 1748 MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); 1749 } 1750 1751 BuildMI(MBB, MI, DL, OpDesc) 1752 .addReg(SrcReg, getKillRegState(isKill)) // data 1753 .addFrameIndex(FrameIndex) // addr 1754 .addMemOperand(MMO) 1755 .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit); 1756 1757 if (RI.spillSGPRToVGPR()) 1758 FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill); 1759 return; 1760 } 1761 1762 unsigned Opcode = getVectorRegSpillSaveOpcode(VReg ? VReg : SrcReg, RC, 1763 SpillSize, RI, *MFI); 1764 MFI->setHasSpilledVGPRs(); 1765 1766 BuildMI(MBB, MI, DL, get(Opcode)) 1767 .addReg(SrcReg, getKillRegState(isKill)) // data 1768 .addFrameIndex(FrameIndex) // addr 1769 .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset 1770 .addImm(0) // offset 1771 .addMemOperand(MMO); 1772 } 1773 1774 static unsigned getSGPRSpillRestoreOpcode(unsigned Size) { 1775 switch (Size) { 1776 case 4: 1777 return AMDGPU::SI_SPILL_S32_RESTORE; 1778 case 8: 1779 return AMDGPU::SI_SPILL_S64_RESTORE; 1780 case 12: 1781 return AMDGPU::SI_SPILL_S96_RESTORE; 1782 case 16: 1783 return AMDGPU::SI_SPILL_S128_RESTORE; 1784 case 20: 1785 return AMDGPU::SI_SPILL_S160_RESTORE; 1786 case 24: 1787 return AMDGPU::SI_SPILL_S192_RESTORE; 1788 case 28: 1789 return AMDGPU::SI_SPILL_S224_RESTORE; 1790 case 32: 1791 return AMDGPU::SI_SPILL_S256_RESTORE; 1792 case 36: 1793 return AMDGPU::SI_SPILL_S288_RESTORE; 1794 case 40: 1795 return AMDGPU::SI_SPILL_S320_RESTORE; 1796 case 44: 1797 return AMDGPU::SI_SPILL_S352_RESTORE; 1798 case 48: 1799 return AMDGPU::SI_SPILL_S384_RESTORE; 1800 case 64: 1801 return AMDGPU::SI_SPILL_S512_RESTORE; 1802 case 128: 1803 return AMDGPU::SI_SPILL_S1024_RESTORE; 1804 default: 1805 llvm_unreachable("unknown register size"); 1806 } 1807 } 1808 1809 static unsigned getVGPRSpillRestoreOpcode(unsigned Size) { 1810 switch (Size) { 1811 case 4: 1812 return AMDGPU::SI_SPILL_V32_RESTORE; 1813 case 8: 1814 return AMDGPU::SI_SPILL_V64_RESTORE; 1815 case 12: 1816 return AMDGPU::SI_SPILL_V96_RESTORE; 1817 case 16: 1818 return AMDGPU::SI_SPILL_V128_RESTORE; 1819 case 20: 1820 return AMDGPU::SI_SPILL_V160_RESTORE; 1821 case 24: 1822 return AMDGPU::SI_SPILL_V192_RESTORE; 1823 case 28: 1824 return AMDGPU::SI_SPILL_V224_RESTORE; 1825 case 32: 1826 return AMDGPU::SI_SPILL_V256_RESTORE; 1827 case 36: 1828 return AMDGPU::SI_SPILL_V288_RESTORE; 1829 case 40: 1830 return AMDGPU::SI_SPILL_V320_RESTORE; 1831 case 44: 1832 return AMDGPU::SI_SPILL_V352_RESTORE; 1833 case 48: 1834 return AMDGPU::SI_SPILL_V384_RESTORE; 1835 case 64: 1836 return AMDGPU::SI_SPILL_V512_RESTORE; 1837 case 128: 1838 return AMDGPU::SI_SPILL_V1024_RESTORE; 1839 default: 1840 llvm_unreachable("unknown register size"); 1841 } 1842 } 1843 1844 static unsigned getAGPRSpillRestoreOpcode(unsigned Size) { 1845 switch (Size) { 1846 case 4: 1847 return AMDGPU::SI_SPILL_A32_RESTORE; 1848 case 8: 1849 return AMDGPU::SI_SPILL_A64_RESTORE; 1850 case 12: 1851 return AMDGPU::SI_SPILL_A96_RESTORE; 1852 case 16: 1853 return AMDGPU::SI_SPILL_A128_RESTORE; 1854 case 20: 1855 return AMDGPU::SI_SPILL_A160_RESTORE; 1856 case 24: 1857 return AMDGPU::SI_SPILL_A192_RESTORE; 1858 case 28: 1859 return AMDGPU::SI_SPILL_A224_RESTORE; 1860 case 32: 1861 return AMDGPU::SI_SPILL_A256_RESTORE; 1862 case 36: 1863 return AMDGPU::SI_SPILL_A288_RESTORE; 1864 case 40: 1865 return AMDGPU::SI_SPILL_A320_RESTORE; 1866 case 44: 1867 return AMDGPU::SI_SPILL_A352_RESTORE; 1868 case 48: 1869 return AMDGPU::SI_SPILL_A384_RESTORE; 1870 case 64: 1871 return AMDGPU::SI_SPILL_A512_RESTORE; 1872 case 128: 1873 return AMDGPU::SI_SPILL_A1024_RESTORE; 1874 default: 1875 llvm_unreachable("unknown register size"); 1876 } 1877 } 1878 1879 static unsigned getAVSpillRestoreOpcode(unsigned Size) { 1880 switch (Size) { 1881 case 4: 1882 return AMDGPU::SI_SPILL_AV32_RESTORE; 1883 case 8: 1884 return AMDGPU::SI_SPILL_AV64_RESTORE; 1885 case 12: 1886 return AMDGPU::SI_SPILL_AV96_RESTORE; 1887 case 16: 1888 return AMDGPU::SI_SPILL_AV128_RESTORE; 1889 case 20: 1890 return AMDGPU::SI_SPILL_AV160_RESTORE; 1891 case 24: 1892 return AMDGPU::SI_SPILL_AV192_RESTORE; 1893 case 28: 1894 return AMDGPU::SI_SPILL_AV224_RESTORE; 1895 case 32: 1896 return AMDGPU::SI_SPILL_AV256_RESTORE; 1897 case 36: 1898 return AMDGPU::SI_SPILL_AV288_RESTORE; 1899 case 40: 1900 return AMDGPU::SI_SPILL_AV320_RESTORE; 1901 case 44: 1902 return AMDGPU::SI_SPILL_AV352_RESTORE; 1903 case 48: 1904 return AMDGPU::SI_SPILL_AV384_RESTORE; 1905 case 64: 1906 return AMDGPU::SI_SPILL_AV512_RESTORE; 1907 case 128: 1908 return AMDGPU::SI_SPILL_AV1024_RESTORE; 1909 default: 1910 llvm_unreachable("unknown register size"); 1911 } 1912 } 1913 1914 static unsigned getWWMRegSpillRestoreOpcode(unsigned Size, 1915 bool IsVectorSuperClass) { 1916 // Currently, there is only 32-bit WWM register spills needed. 1917 if (Size != 4) 1918 llvm_unreachable("unknown wwm register spill size"); 1919 1920 if (IsVectorSuperClass) 1921 return AMDGPU::SI_SPILL_WWM_AV32_RESTORE; 1922 1923 return AMDGPU::SI_SPILL_WWM_V32_RESTORE; 1924 } 1925 1926 static unsigned 1927 getVectorRegSpillRestoreOpcode(Register Reg, const TargetRegisterClass *RC, 1928 unsigned Size, const SIRegisterInfo &TRI, 1929 const SIMachineFunctionInfo &MFI) { 1930 bool IsVectorSuperClass = TRI.isVectorSuperClass(RC); 1931 1932 // Choose the right opcode if restoring a WWM register. 1933 if (MFI.checkFlag(Reg, AMDGPU::VirtRegFlag::WWM_REG)) 1934 return getWWMRegSpillRestoreOpcode(Size, IsVectorSuperClass); 1935 1936 if (IsVectorSuperClass) 1937 return getAVSpillRestoreOpcode(Size); 1938 1939 return TRI.isAGPRClass(RC) ? getAGPRSpillRestoreOpcode(Size) 1940 : getVGPRSpillRestoreOpcode(Size); 1941 } 1942 1943 void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, 1944 MachineBasicBlock::iterator MI, 1945 Register DestReg, int FrameIndex, 1946 const TargetRegisterClass *RC, 1947 const TargetRegisterInfo *TRI, 1948 Register VReg) const { 1949 MachineFunction *MF = MBB.getParent(); 1950 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 1951 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 1952 const DebugLoc &DL = MBB.findDebugLoc(MI); 1953 unsigned SpillSize = TRI->getSpillSize(*RC); 1954 1955 MachinePointerInfo PtrInfo 1956 = MachinePointerInfo::getFixedStack(*MF, FrameIndex); 1957 1958 MachineMemOperand *MMO = MF->getMachineMemOperand( 1959 PtrInfo, MachineMemOperand::MOLoad, FrameInfo.getObjectSize(FrameIndex), 1960 FrameInfo.getObjectAlign(FrameIndex)); 1961 1962 if (RI.isSGPRClass(RC)) { 1963 MFI->setHasSpilledSGPRs(); 1964 assert(DestReg != AMDGPU::M0 && "m0 should not be reloaded into"); 1965 assert(DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI && 1966 DestReg != AMDGPU::EXEC && "exec should not be spilled"); 1967 1968 // FIXME: Maybe this should not include a memoperand because it will be 1969 // lowered to non-memory instructions. 1970 const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(SpillSize)); 1971 if (DestReg.isVirtual() && SpillSize == 4) { 1972 MachineRegisterInfo &MRI = MF->getRegInfo(); 1973 MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); 1974 } 1975 1976 if (RI.spillSGPRToVGPR()) 1977 FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill); 1978 BuildMI(MBB, MI, DL, OpDesc, DestReg) 1979 .addFrameIndex(FrameIndex) // addr 1980 .addMemOperand(MMO) 1981 .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit); 1982 1983 return; 1984 } 1985 1986 unsigned Opcode = getVectorRegSpillRestoreOpcode(VReg ? VReg : DestReg, RC, 1987 SpillSize, RI, *MFI); 1988 BuildMI(MBB, MI, DL, get(Opcode), DestReg) 1989 .addFrameIndex(FrameIndex) // vaddr 1990 .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset 1991 .addImm(0) // offset 1992 .addMemOperand(MMO); 1993 } 1994 1995 void SIInstrInfo::insertNoop(MachineBasicBlock &MBB, 1996 MachineBasicBlock::iterator MI) const { 1997 insertNoops(MBB, MI, 1); 1998 } 1999 2000 void SIInstrInfo::insertNoops(MachineBasicBlock &MBB, 2001 MachineBasicBlock::iterator MI, 2002 unsigned Quantity) const { 2003 DebugLoc DL = MBB.findDebugLoc(MI); 2004 while (Quantity > 0) { 2005 unsigned Arg = std::min(Quantity, 8u); 2006 Quantity -= Arg; 2007 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP)).addImm(Arg - 1); 2008 } 2009 } 2010 2011 void SIInstrInfo::insertReturn(MachineBasicBlock &MBB) const { 2012 auto MF = MBB.getParent(); 2013 SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>(); 2014 2015 assert(Info->isEntryFunction()); 2016 2017 if (MBB.succ_empty()) { 2018 bool HasNoTerminator = MBB.getFirstTerminator() == MBB.end(); 2019 if (HasNoTerminator) { 2020 if (Info->returnsVoid()) { 2021 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::S_ENDPGM)).addImm(0); 2022 } else { 2023 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::SI_RETURN_TO_EPILOG)); 2024 } 2025 } 2026 } 2027 } 2028 2029 unsigned SIInstrInfo::getNumWaitStates(const MachineInstr &MI) { 2030 switch (MI.getOpcode()) { 2031 default: 2032 if (MI.isMetaInstruction()) 2033 return 0; 2034 return 1; // FIXME: Do wait states equal cycles? 2035 2036 case AMDGPU::S_NOP: 2037 return MI.getOperand(0).getImm() + 1; 2038 // SI_RETURN_TO_EPILOG is a fallthrough to code outside of the function. The 2039 // hazard, even if one exist, won't really be visible. Should we handle it? 2040 } 2041 } 2042 2043 bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const { 2044 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 2045 MachineBasicBlock &MBB = *MI.getParent(); 2046 DebugLoc DL = MBB.findDebugLoc(MI); 2047 switch (MI.getOpcode()) { 2048 default: return TargetInstrInfo::expandPostRAPseudo(MI); 2049 case AMDGPU::S_MOV_B64_term: 2050 // This is only a terminator to get the correct spill code placement during 2051 // register allocation. 2052 MI.setDesc(get(AMDGPU::S_MOV_B64)); 2053 break; 2054 2055 case AMDGPU::S_MOV_B32_term: 2056 // This is only a terminator to get the correct spill code placement during 2057 // register allocation. 2058 MI.setDesc(get(AMDGPU::S_MOV_B32)); 2059 break; 2060 2061 case AMDGPU::S_XOR_B64_term: 2062 // This is only a terminator to get the correct spill code placement during 2063 // register allocation. 2064 MI.setDesc(get(AMDGPU::S_XOR_B64)); 2065 break; 2066 2067 case AMDGPU::S_XOR_B32_term: 2068 // This is only a terminator to get the correct spill code placement during 2069 // register allocation. 2070 MI.setDesc(get(AMDGPU::S_XOR_B32)); 2071 break; 2072 case AMDGPU::S_OR_B64_term: 2073 // This is only a terminator to get the correct spill code placement during 2074 // register allocation. 2075 MI.setDesc(get(AMDGPU::S_OR_B64)); 2076 break; 2077 case AMDGPU::S_OR_B32_term: 2078 // This is only a terminator to get the correct spill code placement during 2079 // register allocation. 2080 MI.setDesc(get(AMDGPU::S_OR_B32)); 2081 break; 2082 2083 case AMDGPU::S_ANDN2_B64_term: 2084 // This is only a terminator to get the correct spill code placement during 2085 // register allocation. 2086 MI.setDesc(get(AMDGPU::S_ANDN2_B64)); 2087 break; 2088 2089 case AMDGPU::S_ANDN2_B32_term: 2090 // This is only a terminator to get the correct spill code placement during 2091 // register allocation. 2092 MI.setDesc(get(AMDGPU::S_ANDN2_B32)); 2093 break; 2094 2095 case AMDGPU::S_AND_B64_term: 2096 // This is only a terminator to get the correct spill code placement during 2097 // register allocation. 2098 MI.setDesc(get(AMDGPU::S_AND_B64)); 2099 break; 2100 2101 case AMDGPU::S_AND_B32_term: 2102 // This is only a terminator to get the correct spill code placement during 2103 // register allocation. 2104 MI.setDesc(get(AMDGPU::S_AND_B32)); 2105 break; 2106 2107 case AMDGPU::S_AND_SAVEEXEC_B64_term: 2108 // This is only a terminator to get the correct spill code placement during 2109 // register allocation. 2110 MI.setDesc(get(AMDGPU::S_AND_SAVEEXEC_B64)); 2111 break; 2112 2113 case AMDGPU::S_AND_SAVEEXEC_B32_term: 2114 // This is only a terminator to get the correct spill code placement during 2115 // register allocation. 2116 MI.setDesc(get(AMDGPU::S_AND_SAVEEXEC_B32)); 2117 break; 2118 2119 case AMDGPU::SI_SPILL_S32_TO_VGPR: 2120 MI.setDesc(get(AMDGPU::V_WRITELANE_B32)); 2121 break; 2122 2123 case AMDGPU::SI_RESTORE_S32_FROM_VGPR: 2124 MI.setDesc(get(AMDGPU::V_READLANE_B32)); 2125 break; 2126 2127 case AMDGPU::V_MOV_B64_PSEUDO: { 2128 Register Dst = MI.getOperand(0).getReg(); 2129 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0); 2130 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1); 2131 2132 const MachineOperand &SrcOp = MI.getOperand(1); 2133 // FIXME: Will this work for 64-bit floating point immediates? 2134 assert(!SrcOp.isFPImm()); 2135 if (ST.hasMovB64()) { 2136 MI.setDesc(get(AMDGPU::V_MOV_B64_e32)); 2137 if (SrcOp.isReg() || isInlineConstant(MI, 1) || 2138 isUInt<32>(SrcOp.getImm())) 2139 break; 2140 } 2141 if (SrcOp.isImm()) { 2142 APInt Imm(64, SrcOp.getImm()); 2143 APInt Lo(32, Imm.getLoBits(32).getZExtValue()); 2144 APInt Hi(32, Imm.getHiBits(32).getZExtValue()); 2145 if (ST.hasPkMovB32() && Lo == Hi && isInlineConstant(Lo)) { 2146 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst) 2147 .addImm(SISrcMods::OP_SEL_1) 2148 .addImm(Lo.getSExtValue()) 2149 .addImm(SISrcMods::OP_SEL_1) 2150 .addImm(Lo.getSExtValue()) 2151 .addImm(0) // op_sel_lo 2152 .addImm(0) // op_sel_hi 2153 .addImm(0) // neg_lo 2154 .addImm(0) // neg_hi 2155 .addImm(0); // clamp 2156 } else { 2157 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo) 2158 .addImm(Lo.getSExtValue()) 2159 .addReg(Dst, RegState::Implicit | RegState::Define); 2160 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi) 2161 .addImm(Hi.getSExtValue()) 2162 .addReg(Dst, RegState::Implicit | RegState::Define); 2163 } 2164 } else { 2165 assert(SrcOp.isReg()); 2166 if (ST.hasPkMovB32() && 2167 !RI.isAGPR(MBB.getParent()->getRegInfo(), SrcOp.getReg())) { 2168 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst) 2169 .addImm(SISrcMods::OP_SEL_1) // src0_mod 2170 .addReg(SrcOp.getReg()) 2171 .addImm(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1) // src1_mod 2172 .addReg(SrcOp.getReg()) 2173 .addImm(0) // op_sel_lo 2174 .addImm(0) // op_sel_hi 2175 .addImm(0) // neg_lo 2176 .addImm(0) // neg_hi 2177 .addImm(0); // clamp 2178 } else { 2179 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo) 2180 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0)) 2181 .addReg(Dst, RegState::Implicit | RegState::Define); 2182 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi) 2183 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1)) 2184 .addReg(Dst, RegState::Implicit | RegState::Define); 2185 } 2186 } 2187 MI.eraseFromParent(); 2188 break; 2189 } 2190 case AMDGPU::V_MOV_B64_DPP_PSEUDO: { 2191 expandMovDPP64(MI); 2192 break; 2193 } 2194 case AMDGPU::S_MOV_B64_IMM_PSEUDO: { 2195 const MachineOperand &SrcOp = MI.getOperand(1); 2196 assert(!SrcOp.isFPImm()); 2197 APInt Imm(64, SrcOp.getImm()); 2198 if (Imm.isIntN(32) || isInlineConstant(Imm)) { 2199 MI.setDesc(get(AMDGPU::S_MOV_B64)); 2200 break; 2201 } 2202 2203 Register Dst = MI.getOperand(0).getReg(); 2204 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0); 2205 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1); 2206 2207 APInt Lo(32, Imm.getLoBits(32).getZExtValue()); 2208 APInt Hi(32, Imm.getHiBits(32).getZExtValue()); 2209 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstLo) 2210 .addImm(Lo.getSExtValue()) 2211 .addReg(Dst, RegState::Implicit | RegState::Define); 2212 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstHi) 2213 .addImm(Hi.getSExtValue()) 2214 .addReg(Dst, RegState::Implicit | RegState::Define); 2215 MI.eraseFromParent(); 2216 break; 2217 } 2218 case AMDGPU::V_SET_INACTIVE_B32: { 2219 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64; 2220 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 2221 // FIXME: We may possibly optimize the COPY once we find ways to make LLVM 2222 // optimizations (mainly Register Coalescer) aware of WWM register liveness. 2223 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg()) 2224 .add(MI.getOperand(1)); 2225 auto FirstNot = BuildMI(MBB, MI, DL, get(NotOpc), Exec).addReg(Exec); 2226 FirstNot->addRegisterDead(AMDGPU::SCC, TRI); // SCC is overwritten 2227 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg()) 2228 .add(MI.getOperand(2)); 2229 BuildMI(MBB, MI, DL, get(NotOpc), Exec) 2230 .addReg(Exec); 2231 MI.eraseFromParent(); 2232 break; 2233 } 2234 case AMDGPU::V_SET_INACTIVE_B64: { 2235 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64; 2236 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 2237 MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), 2238 MI.getOperand(0).getReg()) 2239 .add(MI.getOperand(1)); 2240 expandPostRAPseudo(*Copy); 2241 auto FirstNot = BuildMI(MBB, MI, DL, get(NotOpc), Exec).addReg(Exec); 2242 FirstNot->addRegisterDead(AMDGPU::SCC, TRI); // SCC is overwritten 2243 Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), 2244 MI.getOperand(0).getReg()) 2245 .add(MI.getOperand(2)); 2246 expandPostRAPseudo(*Copy); 2247 BuildMI(MBB, MI, DL, get(NotOpc), Exec) 2248 .addReg(Exec); 2249 MI.eraseFromParent(); 2250 break; 2251 } 2252 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1: 2253 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2: 2254 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3: 2255 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4: 2256 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5: 2257 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8: 2258 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9: 2259 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10: 2260 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11: 2261 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12: 2262 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16: 2263 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32: 2264 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1: 2265 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2: 2266 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3: 2267 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4: 2268 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5: 2269 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8: 2270 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9: 2271 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10: 2272 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11: 2273 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12: 2274 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16: 2275 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32: 2276 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1: 2277 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2: 2278 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4: 2279 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8: 2280 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16: { 2281 const TargetRegisterClass *EltRC = getOpRegClass(MI, 2); 2282 2283 unsigned Opc; 2284 if (RI.hasVGPRs(EltRC)) { 2285 Opc = AMDGPU::V_MOVRELD_B32_e32; 2286 } else { 2287 Opc = RI.getRegSizeInBits(*EltRC) == 64 ? AMDGPU::S_MOVRELD_B64 2288 : AMDGPU::S_MOVRELD_B32; 2289 } 2290 2291 const MCInstrDesc &OpDesc = get(Opc); 2292 Register VecReg = MI.getOperand(0).getReg(); 2293 bool IsUndef = MI.getOperand(1).isUndef(); 2294 unsigned SubReg = MI.getOperand(3).getImm(); 2295 assert(VecReg == MI.getOperand(1).getReg()); 2296 2297 MachineInstrBuilder MIB = 2298 BuildMI(MBB, MI, DL, OpDesc) 2299 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef) 2300 .add(MI.getOperand(2)) 2301 .addReg(VecReg, RegState::ImplicitDefine) 2302 .addReg(VecReg, RegState::Implicit | (IsUndef ? RegState::Undef : 0)); 2303 2304 const int ImpDefIdx = 2305 OpDesc.getNumOperands() + OpDesc.implicit_uses().size(); 2306 const int ImpUseIdx = ImpDefIdx + 1; 2307 MIB->tieOperands(ImpDefIdx, ImpUseIdx); 2308 MI.eraseFromParent(); 2309 break; 2310 } 2311 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1: 2312 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2: 2313 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3: 2314 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4: 2315 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5: 2316 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8: 2317 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9: 2318 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10: 2319 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11: 2320 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12: 2321 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16: 2322 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32: { 2323 assert(ST.useVGPRIndexMode()); 2324 Register VecReg = MI.getOperand(0).getReg(); 2325 bool IsUndef = MI.getOperand(1).isUndef(); 2326 Register Idx = MI.getOperand(3).getReg(); 2327 Register SubReg = MI.getOperand(4).getImm(); 2328 2329 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON)) 2330 .addReg(Idx) 2331 .addImm(AMDGPU::VGPRIndexMode::DST_ENABLE); 2332 SetOn->getOperand(3).setIsUndef(); 2333 2334 const MCInstrDesc &OpDesc = get(AMDGPU::V_MOV_B32_indirect_write); 2335 MachineInstrBuilder MIB = 2336 BuildMI(MBB, MI, DL, OpDesc) 2337 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef) 2338 .add(MI.getOperand(2)) 2339 .addReg(VecReg, RegState::ImplicitDefine) 2340 .addReg(VecReg, 2341 RegState::Implicit | (IsUndef ? RegState::Undef : 0)); 2342 2343 const int ImpDefIdx = 2344 OpDesc.getNumOperands() + OpDesc.implicit_uses().size(); 2345 const int ImpUseIdx = ImpDefIdx + 1; 2346 MIB->tieOperands(ImpDefIdx, ImpUseIdx); 2347 2348 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF)); 2349 2350 finalizeBundle(MBB, SetOn->getIterator(), std::next(SetOff->getIterator())); 2351 2352 MI.eraseFromParent(); 2353 break; 2354 } 2355 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1: 2356 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2: 2357 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3: 2358 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4: 2359 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5: 2360 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8: 2361 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9: 2362 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10: 2363 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11: 2364 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12: 2365 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16: 2366 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32: { 2367 assert(ST.useVGPRIndexMode()); 2368 Register Dst = MI.getOperand(0).getReg(); 2369 Register VecReg = MI.getOperand(1).getReg(); 2370 bool IsUndef = MI.getOperand(1).isUndef(); 2371 Register Idx = MI.getOperand(2).getReg(); 2372 Register SubReg = MI.getOperand(3).getImm(); 2373 2374 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON)) 2375 .addReg(Idx) 2376 .addImm(AMDGPU::VGPRIndexMode::SRC0_ENABLE); 2377 SetOn->getOperand(3).setIsUndef(); 2378 2379 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_indirect_read)) 2380 .addDef(Dst) 2381 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef) 2382 .addReg(VecReg, RegState::Implicit | (IsUndef ? RegState::Undef : 0)); 2383 2384 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF)); 2385 2386 finalizeBundle(MBB, SetOn->getIterator(), std::next(SetOff->getIterator())); 2387 2388 MI.eraseFromParent(); 2389 break; 2390 } 2391 case AMDGPU::SI_PC_ADD_REL_OFFSET: { 2392 MachineFunction &MF = *MBB.getParent(); 2393 Register Reg = MI.getOperand(0).getReg(); 2394 Register RegLo = RI.getSubReg(Reg, AMDGPU::sub0); 2395 Register RegHi = RI.getSubReg(Reg, AMDGPU::sub1); 2396 MachineOperand OpLo = MI.getOperand(1); 2397 MachineOperand OpHi = MI.getOperand(2); 2398 2399 // Create a bundle so these instructions won't be re-ordered by the 2400 // post-RA scheduler. 2401 MIBundleBuilder Bundler(MBB, MI); 2402 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg)); 2403 2404 // What we want here is an offset from the value returned by s_getpc (which 2405 // is the address of the s_add_u32 instruction) to the global variable, but 2406 // since the encoding of $symbol starts 4 bytes after the start of the 2407 // s_add_u32 instruction, we end up with an offset that is 4 bytes too 2408 // small. This requires us to add 4 to the global variable offset in order 2409 // to compute the correct address. Similarly for the s_addc_u32 instruction, 2410 // the encoding of $symbol starts 12 bytes after the start of the s_add_u32 2411 // instruction. 2412 2413 if (OpLo.isGlobal()) 2414 OpLo.setOffset(OpLo.getOffset() + 4); 2415 Bundler.append( 2416 BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo).addReg(RegLo).add(OpLo)); 2417 2418 if (OpHi.isGlobal()) 2419 OpHi.setOffset(OpHi.getOffset() + 12); 2420 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi) 2421 .addReg(RegHi) 2422 .add(OpHi)); 2423 2424 finalizeBundle(MBB, Bundler.begin()); 2425 2426 MI.eraseFromParent(); 2427 break; 2428 } 2429 case AMDGPU::ENTER_STRICT_WWM: { 2430 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when 2431 // Whole Wave Mode is entered. 2432 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 2433 : AMDGPU::S_OR_SAVEEXEC_B64)); 2434 break; 2435 } 2436 case AMDGPU::ENTER_STRICT_WQM: { 2437 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when 2438 // STRICT_WQM is entered. 2439 const unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 2440 const unsigned WQMOp = ST.isWave32() ? AMDGPU::S_WQM_B32 : AMDGPU::S_WQM_B64; 2441 const unsigned MovOp = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; 2442 BuildMI(MBB, MI, DL, get(MovOp), MI.getOperand(0).getReg()).addReg(Exec); 2443 BuildMI(MBB, MI, DL, get(WQMOp), Exec).addReg(Exec); 2444 2445 MI.eraseFromParent(); 2446 break; 2447 } 2448 case AMDGPU::EXIT_STRICT_WWM: 2449 case AMDGPU::EXIT_STRICT_WQM: { 2450 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when 2451 // WWM/STICT_WQM is exited. 2452 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64)); 2453 break; 2454 } 2455 case AMDGPU::ENTER_PSEUDO_WM: 2456 case AMDGPU::EXIT_PSEUDO_WM: { 2457 // These do nothing. 2458 MI.eraseFromParent(); 2459 break; 2460 } 2461 case AMDGPU::SI_RETURN: { 2462 const MachineFunction *MF = MBB.getParent(); 2463 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); 2464 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 2465 // Hiding the return address use with SI_RETURN may lead to extra kills in 2466 // the function and missing live-ins. We are fine in practice because callee 2467 // saved register handling ensures the register value is restored before 2468 // RET, but we need the undef flag here to appease the MachineVerifier 2469 // liveness checks. 2470 MachineInstrBuilder MIB = 2471 BuildMI(MBB, MI, DL, get(AMDGPU::S_SETPC_B64_return)) 2472 .addReg(TRI->getReturnAddressReg(*MF), RegState::Undef); 2473 2474 MIB.copyImplicitOps(MI); 2475 MI.eraseFromParent(); 2476 break; 2477 } 2478 2479 case AMDGPU::S_MUL_U64_U32_PSEUDO: 2480 case AMDGPU::S_MUL_I64_I32_PSEUDO: 2481 MI.setDesc(get(AMDGPU::S_MUL_U64)); 2482 break; 2483 } 2484 return true; 2485 } 2486 2487 void SIInstrInfo::reMaterialize(MachineBasicBlock &MBB, 2488 MachineBasicBlock::iterator I, Register DestReg, 2489 unsigned SubIdx, const MachineInstr &Orig, 2490 const TargetRegisterInfo &RI) const { 2491 2492 // Try shrinking the instruction to remat only the part needed for current 2493 // context. 2494 // TODO: Handle more cases. 2495 unsigned Opcode = Orig.getOpcode(); 2496 switch (Opcode) { 2497 case AMDGPU::S_LOAD_DWORDX16_IMM: 2498 case AMDGPU::S_LOAD_DWORDX8_IMM: { 2499 if (SubIdx != 0) 2500 break; 2501 2502 if (I == MBB.end()) 2503 break; 2504 2505 if (I->isBundled()) 2506 break; 2507 2508 // Look for a single use of the register that is also a subreg. 2509 Register RegToFind = Orig.getOperand(0).getReg(); 2510 MachineOperand *UseMO = nullptr; 2511 for (auto &CandMO : I->operands()) { 2512 if (!CandMO.isReg() || CandMO.getReg() != RegToFind || CandMO.isDef()) 2513 continue; 2514 if (UseMO) { 2515 UseMO = nullptr; 2516 break; 2517 } 2518 UseMO = &CandMO; 2519 } 2520 if (!UseMO || UseMO->getSubReg() == AMDGPU::NoSubRegister) 2521 break; 2522 2523 unsigned Offset = RI.getSubRegIdxOffset(UseMO->getSubReg()); 2524 unsigned SubregSize = RI.getSubRegIdxSize(UseMO->getSubReg()); 2525 2526 MachineFunction *MF = MBB.getParent(); 2527 MachineRegisterInfo &MRI = MF->getRegInfo(); 2528 assert(MRI.use_nodbg_empty(DestReg) && "DestReg should have no users yet."); 2529 2530 unsigned NewOpcode = -1; 2531 if (SubregSize == 256) 2532 NewOpcode = AMDGPU::S_LOAD_DWORDX8_IMM; 2533 else if (SubregSize == 128) 2534 NewOpcode = AMDGPU::S_LOAD_DWORDX4_IMM; 2535 else 2536 break; 2537 2538 const MCInstrDesc &TID = get(NewOpcode); 2539 const TargetRegisterClass *NewRC = 2540 RI.getAllocatableClass(getRegClass(TID, 0, &RI, *MF)); 2541 MRI.setRegClass(DestReg, NewRC); 2542 2543 UseMO->setReg(DestReg); 2544 UseMO->setSubReg(AMDGPU::NoSubRegister); 2545 2546 // Use a smaller load with the desired size, possibly with updated offset. 2547 MachineInstr *MI = MF->CloneMachineInstr(&Orig); 2548 MI->setDesc(TID); 2549 MI->getOperand(0).setReg(DestReg); 2550 MI->getOperand(0).setSubReg(AMDGPU::NoSubRegister); 2551 if (Offset) { 2552 MachineOperand *OffsetMO = getNamedOperand(*MI, AMDGPU::OpName::offset); 2553 int64_t FinalOffset = OffsetMO->getImm() + Offset / 8; 2554 OffsetMO->setImm(FinalOffset); 2555 } 2556 SmallVector<MachineMemOperand *> NewMMOs; 2557 for (const MachineMemOperand *MemOp : Orig.memoperands()) 2558 NewMMOs.push_back(MF->getMachineMemOperand(MemOp, MemOp->getPointerInfo(), 2559 SubregSize / 8)); 2560 MI->setMemRefs(*MF, NewMMOs); 2561 2562 MBB.insert(I, MI); 2563 return; 2564 } 2565 2566 default: 2567 break; 2568 } 2569 2570 TargetInstrInfo::reMaterialize(MBB, I, DestReg, SubIdx, Orig, RI); 2571 } 2572 2573 std::pair<MachineInstr*, MachineInstr*> 2574 SIInstrInfo::expandMovDPP64(MachineInstr &MI) const { 2575 assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO); 2576 2577 if (ST.hasMovB64() && 2578 AMDGPU::isLegalDPALU_DPPControl( 2579 getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl)->getImm())) { 2580 MI.setDesc(get(AMDGPU::V_MOV_B64_dpp)); 2581 return std::pair(&MI, nullptr); 2582 } 2583 2584 MachineBasicBlock &MBB = *MI.getParent(); 2585 DebugLoc DL = MBB.findDebugLoc(MI); 2586 MachineFunction *MF = MBB.getParent(); 2587 MachineRegisterInfo &MRI = MF->getRegInfo(); 2588 Register Dst = MI.getOperand(0).getReg(); 2589 unsigned Part = 0; 2590 MachineInstr *Split[2]; 2591 2592 for (auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) { 2593 auto MovDPP = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_dpp)); 2594 if (Dst.isPhysical()) { 2595 MovDPP.addDef(RI.getSubReg(Dst, Sub)); 2596 } else { 2597 assert(MRI.isSSA()); 2598 auto Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 2599 MovDPP.addDef(Tmp); 2600 } 2601 2602 for (unsigned I = 1; I <= 2; ++I) { // old and src operands. 2603 const MachineOperand &SrcOp = MI.getOperand(I); 2604 assert(!SrcOp.isFPImm()); 2605 if (SrcOp.isImm()) { 2606 APInt Imm(64, SrcOp.getImm()); 2607 Imm.ashrInPlace(Part * 32); 2608 MovDPP.addImm(Imm.getLoBits(32).getZExtValue()); 2609 } else { 2610 assert(SrcOp.isReg()); 2611 Register Src = SrcOp.getReg(); 2612 if (Src.isPhysical()) 2613 MovDPP.addReg(RI.getSubReg(Src, Sub)); 2614 else 2615 MovDPP.addReg(Src, SrcOp.isUndef() ? RegState::Undef : 0, Sub); 2616 } 2617 } 2618 2619 for (const MachineOperand &MO : llvm::drop_begin(MI.explicit_operands(), 3)) 2620 MovDPP.addImm(MO.getImm()); 2621 2622 Split[Part] = MovDPP; 2623 ++Part; 2624 } 2625 2626 if (Dst.isVirtual()) 2627 BuildMI(MBB, MI, DL, get(AMDGPU::REG_SEQUENCE), Dst) 2628 .addReg(Split[0]->getOperand(0).getReg()) 2629 .addImm(AMDGPU::sub0) 2630 .addReg(Split[1]->getOperand(0).getReg()) 2631 .addImm(AMDGPU::sub1); 2632 2633 MI.eraseFromParent(); 2634 return std::pair(Split[0], Split[1]); 2635 } 2636 2637 std::optional<DestSourcePair> 2638 SIInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const { 2639 if (MI.getOpcode() == AMDGPU::WWM_COPY) 2640 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)}; 2641 2642 return std::nullopt; 2643 } 2644 2645 bool SIInstrInfo::swapSourceModifiers(MachineInstr &MI, 2646 MachineOperand &Src0, 2647 unsigned Src0OpName, 2648 MachineOperand &Src1, 2649 unsigned Src1OpName) const { 2650 MachineOperand *Src0Mods = getNamedOperand(MI, Src0OpName); 2651 if (!Src0Mods) 2652 return false; 2653 2654 MachineOperand *Src1Mods = getNamedOperand(MI, Src1OpName); 2655 assert(Src1Mods && 2656 "All commutable instructions have both src0 and src1 modifiers"); 2657 2658 int Src0ModsVal = Src0Mods->getImm(); 2659 int Src1ModsVal = Src1Mods->getImm(); 2660 2661 Src1Mods->setImm(Src0ModsVal); 2662 Src0Mods->setImm(Src1ModsVal); 2663 return true; 2664 } 2665 2666 static MachineInstr *swapRegAndNonRegOperand(MachineInstr &MI, 2667 MachineOperand &RegOp, 2668 MachineOperand &NonRegOp) { 2669 Register Reg = RegOp.getReg(); 2670 unsigned SubReg = RegOp.getSubReg(); 2671 bool IsKill = RegOp.isKill(); 2672 bool IsDead = RegOp.isDead(); 2673 bool IsUndef = RegOp.isUndef(); 2674 bool IsDebug = RegOp.isDebug(); 2675 2676 if (NonRegOp.isImm()) 2677 RegOp.ChangeToImmediate(NonRegOp.getImm()); 2678 else if (NonRegOp.isFI()) 2679 RegOp.ChangeToFrameIndex(NonRegOp.getIndex()); 2680 else if (NonRegOp.isGlobal()) { 2681 RegOp.ChangeToGA(NonRegOp.getGlobal(), NonRegOp.getOffset(), 2682 NonRegOp.getTargetFlags()); 2683 } else 2684 return nullptr; 2685 2686 // Make sure we don't reinterpret a subreg index in the target flags. 2687 RegOp.setTargetFlags(NonRegOp.getTargetFlags()); 2688 2689 NonRegOp.ChangeToRegister(Reg, false, false, IsKill, IsDead, IsUndef, IsDebug); 2690 NonRegOp.setSubReg(SubReg); 2691 2692 return &MI; 2693 } 2694 2695 MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI, 2696 unsigned Src0Idx, 2697 unsigned Src1Idx) const { 2698 assert(!NewMI && "this should never be used"); 2699 2700 unsigned Opc = MI.getOpcode(); 2701 int CommutedOpcode = commuteOpcode(Opc); 2702 if (CommutedOpcode == -1) 2703 return nullptr; 2704 2705 if (Src0Idx > Src1Idx) 2706 std::swap(Src0Idx, Src1Idx); 2707 2708 assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) == 2709 static_cast<int>(Src0Idx) && 2710 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) == 2711 static_cast<int>(Src1Idx) && 2712 "inconsistency with findCommutedOpIndices"); 2713 2714 MachineOperand &Src0 = MI.getOperand(Src0Idx); 2715 MachineOperand &Src1 = MI.getOperand(Src1Idx); 2716 2717 MachineInstr *CommutedMI = nullptr; 2718 if (Src0.isReg() && Src1.isReg()) { 2719 if (isOperandLegal(MI, Src1Idx, &Src0)) { 2720 // Be sure to copy the source modifiers to the right place. 2721 CommutedMI 2722 = TargetInstrInfo::commuteInstructionImpl(MI, NewMI, Src0Idx, Src1Idx); 2723 } 2724 2725 } else if (Src0.isReg() && !Src1.isReg()) { 2726 // src0 should always be able to support any operand type, so no need to 2727 // check operand legality. 2728 CommutedMI = swapRegAndNonRegOperand(MI, Src0, Src1); 2729 } else if (!Src0.isReg() && Src1.isReg()) { 2730 if (isOperandLegal(MI, Src1Idx, &Src0)) 2731 CommutedMI = swapRegAndNonRegOperand(MI, Src1, Src0); 2732 } else { 2733 // FIXME: Found two non registers to commute. This does happen. 2734 return nullptr; 2735 } 2736 2737 if (CommutedMI) { 2738 swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers, 2739 Src1, AMDGPU::OpName::src1_modifiers); 2740 2741 CommutedMI->setDesc(get(CommutedOpcode)); 2742 } 2743 2744 return CommutedMI; 2745 } 2746 2747 // This needs to be implemented because the source modifiers may be inserted 2748 // between the true commutable operands, and the base 2749 // TargetInstrInfo::commuteInstruction uses it. 2750 bool SIInstrInfo::findCommutedOpIndices(const MachineInstr &MI, 2751 unsigned &SrcOpIdx0, 2752 unsigned &SrcOpIdx1) const { 2753 return findCommutedOpIndices(MI.getDesc(), SrcOpIdx0, SrcOpIdx1); 2754 } 2755 2756 bool SIInstrInfo::findCommutedOpIndices(const MCInstrDesc &Desc, 2757 unsigned &SrcOpIdx0, 2758 unsigned &SrcOpIdx1) const { 2759 if (!Desc.isCommutable()) 2760 return false; 2761 2762 unsigned Opc = Desc.getOpcode(); 2763 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); 2764 if (Src0Idx == -1) 2765 return false; 2766 2767 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); 2768 if (Src1Idx == -1) 2769 return false; 2770 2771 return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx); 2772 } 2773 2774 bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp, 2775 int64_t BrOffset) const { 2776 // BranchRelaxation should never have to check s_setpc_b64 because its dest 2777 // block is unanalyzable. 2778 assert(BranchOp != AMDGPU::S_SETPC_B64); 2779 2780 // Convert to dwords. 2781 BrOffset /= 4; 2782 2783 // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is 2784 // from the next instruction. 2785 BrOffset -= 1; 2786 2787 return isIntN(BranchOffsetBits, BrOffset); 2788 } 2789 2790 MachineBasicBlock * 2791 SIInstrInfo::getBranchDestBlock(const MachineInstr &MI) const { 2792 return MI.getOperand(0).getMBB(); 2793 } 2794 2795 bool SIInstrInfo::hasDivergentBranch(const MachineBasicBlock *MBB) const { 2796 for (const MachineInstr &MI : MBB->terminators()) { 2797 if (MI.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO || 2798 MI.getOpcode() == AMDGPU::SI_IF || MI.getOpcode() == AMDGPU::SI_ELSE || 2799 MI.getOpcode() == AMDGPU::SI_LOOP) 2800 return true; 2801 } 2802 return false; 2803 } 2804 2805 void SIInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB, 2806 MachineBasicBlock &DestBB, 2807 MachineBasicBlock &RestoreBB, 2808 const DebugLoc &DL, int64_t BrOffset, 2809 RegScavenger *RS) const { 2810 assert(RS && "RegScavenger required for long branching"); 2811 assert(MBB.empty() && 2812 "new block should be inserted for expanding unconditional branch"); 2813 assert(MBB.pred_size() == 1); 2814 assert(RestoreBB.empty() && 2815 "restore block should be inserted for restoring clobbered registers"); 2816 2817 MachineFunction *MF = MBB.getParent(); 2818 MachineRegisterInfo &MRI = MF->getRegInfo(); 2819 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 2820 2821 // FIXME: Virtual register workaround for RegScavenger not working with empty 2822 // blocks. 2823 Register PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); 2824 2825 auto I = MBB.end(); 2826 2827 // We need to compute the offset relative to the instruction immediately after 2828 // s_getpc_b64. Insert pc arithmetic code before last terminator. 2829 MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg); 2830 2831 auto &MCCtx = MF->getContext(); 2832 MCSymbol *PostGetPCLabel = 2833 MCCtx.createTempSymbol("post_getpc", /*AlwaysAddSuffix=*/true); 2834 GetPC->setPostInstrSymbol(*MF, PostGetPCLabel); 2835 2836 MCSymbol *OffsetLo = 2837 MCCtx.createTempSymbol("offset_lo", /*AlwaysAddSuffix=*/true); 2838 MCSymbol *OffsetHi = 2839 MCCtx.createTempSymbol("offset_hi", /*AlwaysAddSuffix=*/true); 2840 BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32)) 2841 .addReg(PCReg, RegState::Define, AMDGPU::sub0) 2842 .addReg(PCReg, 0, AMDGPU::sub0) 2843 .addSym(OffsetLo, MO_FAR_BRANCH_OFFSET); 2844 BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32)) 2845 .addReg(PCReg, RegState::Define, AMDGPU::sub1) 2846 .addReg(PCReg, 0, AMDGPU::sub1) 2847 .addSym(OffsetHi, MO_FAR_BRANCH_OFFSET); 2848 2849 // Insert the indirect branch after the other terminator. 2850 BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64)) 2851 .addReg(PCReg); 2852 2853 // If a spill is needed for the pc register pair, we need to insert a spill 2854 // restore block right before the destination block, and insert a short branch 2855 // into the old destination block's fallthrough predecessor. 2856 // e.g.: 2857 // 2858 // s_cbranch_scc0 skip_long_branch: 2859 // 2860 // long_branch_bb: 2861 // spill s[8:9] 2862 // s_getpc_b64 s[8:9] 2863 // s_add_u32 s8, s8, restore_bb 2864 // s_addc_u32 s9, s9, 0 2865 // s_setpc_b64 s[8:9] 2866 // 2867 // skip_long_branch: 2868 // foo; 2869 // 2870 // ..... 2871 // 2872 // dest_bb_fallthrough_predecessor: 2873 // bar; 2874 // s_branch dest_bb 2875 // 2876 // restore_bb: 2877 // restore s[8:9] 2878 // fallthrough dest_bb 2879 /// 2880 // dest_bb: 2881 // buzz; 2882 2883 Register LongBranchReservedReg = MFI->getLongBranchReservedReg(); 2884 Register Scav; 2885 2886 // If we've previously reserved a register for long branches 2887 // avoid running the scavenger and just use those registers 2888 if (LongBranchReservedReg) { 2889 RS->enterBasicBlock(MBB); 2890 Scav = LongBranchReservedReg; 2891 } else { 2892 RS->enterBasicBlockEnd(MBB); 2893 Scav = RS->scavengeRegisterBackwards( 2894 AMDGPU::SReg_64RegClass, MachineBasicBlock::iterator(GetPC), 2895 /* RestoreAfter */ false, 0, /* AllowSpill */ false); 2896 } 2897 if (Scav) { 2898 RS->setRegUsed(Scav); 2899 MRI.replaceRegWith(PCReg, Scav); 2900 MRI.clearVirtRegs(); 2901 } else { 2902 // As SGPR needs VGPR to be spilled, we reuse the slot of temporary VGPR for 2903 // SGPR spill. 2904 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); 2905 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 2906 TRI->spillEmergencySGPR(GetPC, RestoreBB, AMDGPU::SGPR0_SGPR1, RS); 2907 MRI.replaceRegWith(PCReg, AMDGPU::SGPR0_SGPR1); 2908 MRI.clearVirtRegs(); 2909 } 2910 2911 MCSymbol *DestLabel = Scav ? DestBB.getSymbol() : RestoreBB.getSymbol(); 2912 // Now, the distance could be defined. 2913 auto *Offset = MCBinaryExpr::createSub( 2914 MCSymbolRefExpr::create(DestLabel, MCCtx), 2915 MCSymbolRefExpr::create(PostGetPCLabel, MCCtx), MCCtx); 2916 // Add offset assignments. 2917 auto *Mask = MCConstantExpr::create(0xFFFFFFFFULL, MCCtx); 2918 OffsetLo->setVariableValue(MCBinaryExpr::createAnd(Offset, Mask, MCCtx)); 2919 auto *ShAmt = MCConstantExpr::create(32, MCCtx); 2920 OffsetHi->setVariableValue(MCBinaryExpr::createAShr(Offset, ShAmt, MCCtx)); 2921 } 2922 2923 unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) { 2924 switch (Cond) { 2925 case SIInstrInfo::SCC_TRUE: 2926 return AMDGPU::S_CBRANCH_SCC1; 2927 case SIInstrInfo::SCC_FALSE: 2928 return AMDGPU::S_CBRANCH_SCC0; 2929 case SIInstrInfo::VCCNZ: 2930 return AMDGPU::S_CBRANCH_VCCNZ; 2931 case SIInstrInfo::VCCZ: 2932 return AMDGPU::S_CBRANCH_VCCZ; 2933 case SIInstrInfo::EXECNZ: 2934 return AMDGPU::S_CBRANCH_EXECNZ; 2935 case SIInstrInfo::EXECZ: 2936 return AMDGPU::S_CBRANCH_EXECZ; 2937 default: 2938 llvm_unreachable("invalid branch predicate"); 2939 } 2940 } 2941 2942 SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) { 2943 switch (Opcode) { 2944 case AMDGPU::S_CBRANCH_SCC0: 2945 return SCC_FALSE; 2946 case AMDGPU::S_CBRANCH_SCC1: 2947 return SCC_TRUE; 2948 case AMDGPU::S_CBRANCH_VCCNZ: 2949 return VCCNZ; 2950 case AMDGPU::S_CBRANCH_VCCZ: 2951 return VCCZ; 2952 case AMDGPU::S_CBRANCH_EXECNZ: 2953 return EXECNZ; 2954 case AMDGPU::S_CBRANCH_EXECZ: 2955 return EXECZ; 2956 default: 2957 return INVALID_BR; 2958 } 2959 } 2960 2961 bool SIInstrInfo::analyzeBranchImpl(MachineBasicBlock &MBB, 2962 MachineBasicBlock::iterator I, 2963 MachineBasicBlock *&TBB, 2964 MachineBasicBlock *&FBB, 2965 SmallVectorImpl<MachineOperand> &Cond, 2966 bool AllowModify) const { 2967 if (I->getOpcode() == AMDGPU::S_BRANCH) { 2968 // Unconditional Branch 2969 TBB = I->getOperand(0).getMBB(); 2970 return false; 2971 } 2972 2973 MachineBasicBlock *CondBB = nullptr; 2974 2975 if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { 2976 CondBB = I->getOperand(1).getMBB(); 2977 Cond.push_back(I->getOperand(0)); 2978 } else { 2979 BranchPredicate Pred = getBranchPredicate(I->getOpcode()); 2980 if (Pred == INVALID_BR) 2981 return true; 2982 2983 CondBB = I->getOperand(0).getMBB(); 2984 Cond.push_back(MachineOperand::CreateImm(Pred)); 2985 Cond.push_back(I->getOperand(1)); // Save the branch register. 2986 } 2987 ++I; 2988 2989 if (I == MBB.end()) { 2990 // Conditional branch followed by fall-through. 2991 TBB = CondBB; 2992 return false; 2993 } 2994 2995 if (I->getOpcode() == AMDGPU::S_BRANCH) { 2996 TBB = CondBB; 2997 FBB = I->getOperand(0).getMBB(); 2998 return false; 2999 } 3000 3001 return true; 3002 } 3003 3004 bool SIInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, 3005 MachineBasicBlock *&FBB, 3006 SmallVectorImpl<MachineOperand> &Cond, 3007 bool AllowModify) const { 3008 MachineBasicBlock::iterator I = MBB.getFirstTerminator(); 3009 auto E = MBB.end(); 3010 if (I == E) 3011 return false; 3012 3013 // Skip over the instructions that are artificially terminators for special 3014 // exec management. 3015 while (I != E && !I->isBranch() && !I->isReturn()) { 3016 switch (I->getOpcode()) { 3017 case AMDGPU::S_MOV_B64_term: 3018 case AMDGPU::S_XOR_B64_term: 3019 case AMDGPU::S_OR_B64_term: 3020 case AMDGPU::S_ANDN2_B64_term: 3021 case AMDGPU::S_AND_B64_term: 3022 case AMDGPU::S_AND_SAVEEXEC_B64_term: 3023 case AMDGPU::S_MOV_B32_term: 3024 case AMDGPU::S_XOR_B32_term: 3025 case AMDGPU::S_OR_B32_term: 3026 case AMDGPU::S_ANDN2_B32_term: 3027 case AMDGPU::S_AND_B32_term: 3028 case AMDGPU::S_AND_SAVEEXEC_B32_term: 3029 break; 3030 case AMDGPU::SI_IF: 3031 case AMDGPU::SI_ELSE: 3032 case AMDGPU::SI_KILL_I1_TERMINATOR: 3033 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR: 3034 // FIXME: It's messy that these need to be considered here at all. 3035 return true; 3036 default: 3037 llvm_unreachable("unexpected non-branch terminator inst"); 3038 } 3039 3040 ++I; 3041 } 3042 3043 if (I == E) 3044 return false; 3045 3046 return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify); 3047 } 3048 3049 unsigned SIInstrInfo::removeBranch(MachineBasicBlock &MBB, 3050 int *BytesRemoved) const { 3051 unsigned Count = 0; 3052 unsigned RemovedSize = 0; 3053 for (MachineInstr &MI : llvm::make_early_inc_range(MBB.terminators())) { 3054 // Skip over artificial terminators when removing instructions. 3055 if (MI.isBranch() || MI.isReturn()) { 3056 RemovedSize += getInstSizeInBytes(MI); 3057 MI.eraseFromParent(); 3058 ++Count; 3059 } 3060 } 3061 3062 if (BytesRemoved) 3063 *BytesRemoved = RemovedSize; 3064 3065 return Count; 3066 } 3067 3068 // Copy the flags onto the implicit condition register operand. 3069 static void preserveCondRegFlags(MachineOperand &CondReg, 3070 const MachineOperand &OrigCond) { 3071 CondReg.setIsUndef(OrigCond.isUndef()); 3072 CondReg.setIsKill(OrigCond.isKill()); 3073 } 3074 3075 unsigned SIInstrInfo::insertBranch(MachineBasicBlock &MBB, 3076 MachineBasicBlock *TBB, 3077 MachineBasicBlock *FBB, 3078 ArrayRef<MachineOperand> Cond, 3079 const DebugLoc &DL, 3080 int *BytesAdded) const { 3081 if (!FBB && Cond.empty()) { 3082 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH)) 3083 .addMBB(TBB); 3084 if (BytesAdded) 3085 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4; 3086 return 1; 3087 } 3088 3089 if(Cond.size() == 1 && Cond[0].isReg()) { 3090 BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO)) 3091 .add(Cond[0]) 3092 .addMBB(TBB); 3093 return 1; 3094 } 3095 3096 assert(TBB && Cond[0].isImm()); 3097 3098 unsigned Opcode 3099 = getBranchOpcode(static_cast<BranchPredicate>(Cond[0].getImm())); 3100 3101 if (!FBB) { 3102 MachineInstr *CondBr = 3103 BuildMI(&MBB, DL, get(Opcode)) 3104 .addMBB(TBB); 3105 3106 // Copy the flags onto the implicit condition register operand. 3107 preserveCondRegFlags(CondBr->getOperand(1), Cond[1]); 3108 fixImplicitOperands(*CondBr); 3109 3110 if (BytesAdded) 3111 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4; 3112 return 1; 3113 } 3114 3115 assert(TBB && FBB); 3116 3117 MachineInstr *CondBr = 3118 BuildMI(&MBB, DL, get(Opcode)) 3119 .addMBB(TBB); 3120 fixImplicitOperands(*CondBr); 3121 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH)) 3122 .addMBB(FBB); 3123 3124 MachineOperand &CondReg = CondBr->getOperand(1); 3125 CondReg.setIsUndef(Cond[1].isUndef()); 3126 CondReg.setIsKill(Cond[1].isKill()); 3127 3128 if (BytesAdded) 3129 *BytesAdded = ST.hasOffset3fBug() ? 16 : 8; 3130 3131 return 2; 3132 } 3133 3134 bool SIInstrInfo::reverseBranchCondition( 3135 SmallVectorImpl<MachineOperand> &Cond) const { 3136 if (Cond.size() != 2) { 3137 return true; 3138 } 3139 3140 if (Cond[0].isImm()) { 3141 Cond[0].setImm(-Cond[0].getImm()); 3142 return false; 3143 } 3144 3145 return true; 3146 } 3147 3148 bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB, 3149 ArrayRef<MachineOperand> Cond, 3150 Register DstReg, Register TrueReg, 3151 Register FalseReg, int &CondCycles, 3152 int &TrueCycles, int &FalseCycles) const { 3153 switch (Cond[0].getImm()) { 3154 case VCCNZ: 3155 case VCCZ: { 3156 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 3157 const TargetRegisterClass *RC = MRI.getRegClass(TrueReg); 3158 if (MRI.getRegClass(FalseReg) != RC) 3159 return false; 3160 3161 int NumInsts = AMDGPU::getRegBitWidth(*RC) / 32; 3162 CondCycles = TrueCycles = FalseCycles = NumInsts; // ??? 3163 3164 // Limit to equal cost for branch vs. N v_cndmask_b32s. 3165 return RI.hasVGPRs(RC) && NumInsts <= 6; 3166 } 3167 case SCC_TRUE: 3168 case SCC_FALSE: { 3169 // FIXME: We could insert for VGPRs if we could replace the original compare 3170 // with a vector one. 3171 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 3172 const TargetRegisterClass *RC = MRI.getRegClass(TrueReg); 3173 if (MRI.getRegClass(FalseReg) != RC) 3174 return false; 3175 3176 int NumInsts = AMDGPU::getRegBitWidth(*RC) / 32; 3177 3178 // Multiples of 8 can do s_cselect_b64 3179 if (NumInsts % 2 == 0) 3180 NumInsts /= 2; 3181 3182 CondCycles = TrueCycles = FalseCycles = NumInsts; // ??? 3183 return RI.isSGPRClass(RC); 3184 } 3185 default: 3186 return false; 3187 } 3188 } 3189 3190 void SIInstrInfo::insertSelect(MachineBasicBlock &MBB, 3191 MachineBasicBlock::iterator I, const DebugLoc &DL, 3192 Register DstReg, ArrayRef<MachineOperand> Cond, 3193 Register TrueReg, Register FalseReg) const { 3194 BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm()); 3195 if (Pred == VCCZ || Pred == SCC_FALSE) { 3196 Pred = static_cast<BranchPredicate>(-Pred); 3197 std::swap(TrueReg, FalseReg); 3198 } 3199 3200 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 3201 const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg); 3202 unsigned DstSize = RI.getRegSizeInBits(*DstRC); 3203 3204 if (DstSize == 32) { 3205 MachineInstr *Select; 3206 if (Pred == SCC_TRUE) { 3207 Select = BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B32), DstReg) 3208 .addReg(TrueReg) 3209 .addReg(FalseReg); 3210 } else { 3211 // Instruction's operands are backwards from what is expected. 3212 Select = BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e32), DstReg) 3213 .addReg(FalseReg) 3214 .addReg(TrueReg); 3215 } 3216 3217 preserveCondRegFlags(Select->getOperand(3), Cond[1]); 3218 return; 3219 } 3220 3221 if (DstSize == 64 && Pred == SCC_TRUE) { 3222 MachineInstr *Select = 3223 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg) 3224 .addReg(TrueReg) 3225 .addReg(FalseReg); 3226 3227 preserveCondRegFlags(Select->getOperand(3), Cond[1]); 3228 return; 3229 } 3230 3231 static const int16_t Sub0_15[] = { 3232 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, 3233 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7, 3234 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11, 3235 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15, 3236 }; 3237 3238 static const int16_t Sub0_15_64[] = { 3239 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3, 3240 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7, 3241 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11, 3242 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15, 3243 }; 3244 3245 unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32; 3246 const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass; 3247 const int16_t *SubIndices = Sub0_15; 3248 int NElts = DstSize / 32; 3249 3250 // 64-bit select is only available for SALU. 3251 // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit. 3252 if (Pred == SCC_TRUE) { 3253 if (NElts % 2) { 3254 SelOp = AMDGPU::S_CSELECT_B32; 3255 EltRC = &AMDGPU::SGPR_32RegClass; 3256 } else { 3257 SelOp = AMDGPU::S_CSELECT_B64; 3258 EltRC = &AMDGPU::SGPR_64RegClass; 3259 SubIndices = Sub0_15_64; 3260 NElts /= 2; 3261 } 3262 } 3263 3264 MachineInstrBuilder MIB = BuildMI( 3265 MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg); 3266 3267 I = MIB->getIterator(); 3268 3269 SmallVector<Register, 8> Regs; 3270 for (int Idx = 0; Idx != NElts; ++Idx) { 3271 Register DstElt = MRI.createVirtualRegister(EltRC); 3272 Regs.push_back(DstElt); 3273 3274 unsigned SubIdx = SubIndices[Idx]; 3275 3276 MachineInstr *Select; 3277 if (SelOp == AMDGPU::V_CNDMASK_B32_e32) { 3278 Select = 3279 BuildMI(MBB, I, DL, get(SelOp), DstElt) 3280 .addReg(FalseReg, 0, SubIdx) 3281 .addReg(TrueReg, 0, SubIdx); 3282 } else { 3283 Select = 3284 BuildMI(MBB, I, DL, get(SelOp), DstElt) 3285 .addReg(TrueReg, 0, SubIdx) 3286 .addReg(FalseReg, 0, SubIdx); 3287 } 3288 3289 preserveCondRegFlags(Select->getOperand(3), Cond[1]); 3290 fixImplicitOperands(*Select); 3291 3292 MIB.addReg(DstElt) 3293 .addImm(SubIdx); 3294 } 3295 } 3296 3297 bool SIInstrInfo::isFoldableCopy(const MachineInstr &MI) { 3298 switch (MI.getOpcode()) { 3299 case AMDGPU::V_MOV_B32_e32: 3300 case AMDGPU::V_MOV_B32_e64: 3301 case AMDGPU::V_MOV_B64_PSEUDO: 3302 case AMDGPU::V_MOV_B64_e32: 3303 case AMDGPU::V_MOV_B64_e64: 3304 case AMDGPU::S_MOV_B32: 3305 case AMDGPU::S_MOV_B64: 3306 case AMDGPU::S_MOV_B64_IMM_PSEUDO: 3307 case AMDGPU::COPY: 3308 case AMDGPU::WWM_COPY: 3309 case AMDGPU::V_ACCVGPR_WRITE_B32_e64: 3310 case AMDGPU::V_ACCVGPR_READ_B32_e64: 3311 case AMDGPU::V_ACCVGPR_MOV_B32: 3312 return true; 3313 default: 3314 return false; 3315 } 3316 } 3317 3318 static constexpr unsigned ModifierOpNames[] = { 3319 AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src1_modifiers, 3320 AMDGPU::OpName::src2_modifiers, AMDGPU::OpName::clamp, 3321 AMDGPU::OpName::omod, AMDGPU::OpName::op_sel}; 3322 3323 void SIInstrInfo::removeModOperands(MachineInstr &MI) const { 3324 unsigned Opc = MI.getOpcode(); 3325 for (unsigned Name : reverse(ModifierOpNames)) { 3326 int Idx = AMDGPU::getNamedOperandIdx(Opc, Name); 3327 if (Idx >= 0) 3328 MI.removeOperand(Idx); 3329 } 3330 } 3331 3332 bool SIInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, 3333 Register Reg, MachineRegisterInfo *MRI) const { 3334 if (!MRI->hasOneNonDBGUse(Reg)) 3335 return false; 3336 3337 switch (DefMI.getOpcode()) { 3338 default: 3339 return false; 3340 case AMDGPU::V_MOV_B64_e32: 3341 case AMDGPU::S_MOV_B64: 3342 case AMDGPU::V_MOV_B64_PSEUDO: 3343 case AMDGPU::S_MOV_B64_IMM_PSEUDO: 3344 case AMDGPU::V_MOV_B32_e32: 3345 case AMDGPU::S_MOV_B32: 3346 case AMDGPU::V_ACCVGPR_WRITE_B32_e64: 3347 break; 3348 } 3349 3350 const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0); 3351 assert(ImmOp); 3352 // FIXME: We could handle FrameIndex values here. 3353 if (!ImmOp->isImm()) 3354 return false; 3355 3356 auto getImmFor = [ImmOp](const MachineOperand &UseOp) -> int64_t { 3357 int64_t Imm = ImmOp->getImm(); 3358 switch (UseOp.getSubReg()) { 3359 default: 3360 return Imm; 3361 case AMDGPU::sub0: 3362 return Lo_32(Imm); 3363 case AMDGPU::sub1: 3364 return Hi_32(Imm); 3365 case AMDGPU::lo16: 3366 return APInt(16, Imm).getSExtValue(); 3367 case AMDGPU::hi16: 3368 return APInt(32, Imm).ashr(16).getSExtValue(); 3369 case AMDGPU::sub1_lo16: 3370 return APInt(16, Hi_32(Imm)).getSExtValue(); 3371 case AMDGPU::sub1_hi16: 3372 return APInt(32, Hi_32(Imm)).ashr(16).getSExtValue(); 3373 } 3374 }; 3375 3376 assert(!DefMI.getOperand(0).getSubReg() && "Expected SSA form"); 3377 3378 unsigned Opc = UseMI.getOpcode(); 3379 if (Opc == AMDGPU::COPY) { 3380 assert(!UseMI.getOperand(0).getSubReg() && "Expected SSA form"); 3381 3382 Register DstReg = UseMI.getOperand(0).getReg(); 3383 unsigned OpSize = getOpSize(UseMI, 0); 3384 bool Is16Bit = OpSize == 2; 3385 bool Is64Bit = OpSize == 8; 3386 bool isVGPRCopy = RI.isVGPR(*MRI, DstReg); 3387 unsigned NewOpc = isVGPRCopy ? Is64Bit ? AMDGPU::V_MOV_B64_PSEUDO 3388 : AMDGPU::V_MOV_B32_e32 3389 : Is64Bit ? AMDGPU::S_MOV_B64_IMM_PSEUDO 3390 : AMDGPU::S_MOV_B32; 3391 APInt Imm(Is64Bit ? 64 : 32, getImmFor(UseMI.getOperand(1))); 3392 3393 if (RI.isAGPR(*MRI, DstReg)) { 3394 if (Is64Bit || !isInlineConstant(Imm)) 3395 return false; 3396 NewOpc = AMDGPU::V_ACCVGPR_WRITE_B32_e64; 3397 } 3398 3399 if (Is16Bit) { 3400 if (isVGPRCopy) 3401 return false; // Do not clobber vgpr_hi16 3402 3403 if (DstReg.isVirtual() && UseMI.getOperand(0).getSubReg() != AMDGPU::lo16) 3404 return false; 3405 3406 UseMI.getOperand(0).setSubReg(0); 3407 if (DstReg.isPhysical()) { 3408 DstReg = RI.get32BitRegister(DstReg); 3409 UseMI.getOperand(0).setReg(DstReg); 3410 } 3411 assert(UseMI.getOperand(1).getReg().isVirtual()); 3412 } 3413 3414 const MCInstrDesc &NewMCID = get(NewOpc); 3415 if (DstReg.isPhysical() && 3416 !RI.getRegClass(NewMCID.operands()[0].RegClass)->contains(DstReg)) 3417 return false; 3418 3419 UseMI.setDesc(NewMCID); 3420 UseMI.getOperand(1).ChangeToImmediate(Imm.getSExtValue()); 3421 UseMI.addImplicitDefUseOperands(*UseMI.getParent()->getParent()); 3422 return true; 3423 } 3424 3425 if (Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 || 3426 Opc == AMDGPU::V_MAD_F16_e64 || Opc == AMDGPU::V_MAC_F16_e64 || 3427 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 || 3428 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64 || 3429 Opc == AMDGPU::V_FMAC_F16_t16_e64) { 3430 // Don't fold if we are using source or output modifiers. The new VOP2 3431 // instructions don't have them. 3432 if (hasAnyModifiersSet(UseMI)) 3433 return false; 3434 3435 // If this is a free constant, there's no reason to do this. 3436 // TODO: We could fold this here instead of letting SIFoldOperands do it 3437 // later. 3438 MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0); 3439 3440 // Any src operand can be used for the legality check. 3441 if (isInlineConstant(UseMI, *Src0, *ImmOp)) 3442 return false; 3443 3444 bool IsF32 = Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 || 3445 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64; 3446 bool IsFMA = 3447 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 || 3448 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64 || 3449 Opc == AMDGPU::V_FMAC_F16_t16_e64; 3450 MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1); 3451 MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2); 3452 3453 // Multiplied part is the constant: Use v_madmk_{f16, f32}. 3454 if ((Src0->isReg() && Src0->getReg() == Reg) || 3455 (Src1->isReg() && Src1->getReg() == Reg)) { 3456 MachineOperand *RegSrc = 3457 Src1->isReg() && Src1->getReg() == Reg ? Src0 : Src1; 3458 if (!RegSrc->isReg()) 3459 return false; 3460 if (RI.isSGPRClass(MRI->getRegClass(RegSrc->getReg())) && 3461 ST.getConstantBusLimit(Opc) < 2) 3462 return false; 3463 3464 if (!Src2->isReg() || RI.isSGPRClass(MRI->getRegClass(Src2->getReg()))) 3465 return false; 3466 3467 // If src2 is also a literal constant then we have to choose which one to 3468 // fold. In general it is better to choose madak so that the other literal 3469 // can be materialized in an sgpr instead of a vgpr: 3470 // s_mov_b32 s0, literal 3471 // v_madak_f32 v0, s0, v0, literal 3472 // Instead of: 3473 // v_mov_b32 v1, literal 3474 // v_madmk_f32 v0, v0, literal, v1 3475 MachineInstr *Def = MRI->getUniqueVRegDef(Src2->getReg()); 3476 if (Def && Def->isMoveImmediate() && 3477 !isInlineConstant(Def->getOperand(1))) 3478 return false; 3479 3480 unsigned NewOpc = 3481 IsFMA ? (IsF32 ? AMDGPU::V_FMAMK_F32 3482 : ST.hasTrue16BitInsts() ? AMDGPU::V_FMAMK_F16_t16 3483 : AMDGPU::V_FMAMK_F16) 3484 : (IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16); 3485 if (pseudoToMCOpcode(NewOpc) == -1) 3486 return false; 3487 3488 // V_FMAMK_F16_t16 takes VGPR_32_Lo128 operands, so the rewrite 3489 // would also require restricting their register classes. For now 3490 // just bail out. 3491 if (NewOpc == AMDGPU::V_FMAMK_F16_t16) 3492 return false; 3493 3494 const int64_t Imm = getImmFor(RegSrc == Src1 ? *Src0 : *Src1); 3495 3496 // FIXME: This would be a lot easier if we could return a new instruction 3497 // instead of having to modify in place. 3498 3499 Register SrcReg = RegSrc->getReg(); 3500 unsigned SrcSubReg = RegSrc->getSubReg(); 3501 Src0->setReg(SrcReg); 3502 Src0->setSubReg(SrcSubReg); 3503 Src0->setIsKill(RegSrc->isKill()); 3504 3505 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 || 3506 Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 || 3507 Opc == AMDGPU::V_FMAC_F16_e64) 3508 UseMI.untieRegOperand( 3509 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)); 3510 3511 Src1->ChangeToImmediate(Imm); 3512 3513 removeModOperands(UseMI); 3514 UseMI.setDesc(get(NewOpc)); 3515 3516 bool DeleteDef = MRI->use_nodbg_empty(Reg); 3517 if (DeleteDef) 3518 DefMI.eraseFromParent(); 3519 3520 return true; 3521 } 3522 3523 // Added part is the constant: Use v_madak_{f16, f32}. 3524 if (Src2->isReg() && Src2->getReg() == Reg) { 3525 if (ST.getConstantBusLimit(Opc) < 2) { 3526 // Not allowed to use constant bus for another operand. 3527 // We can however allow an inline immediate as src0. 3528 bool Src0Inlined = false; 3529 if (Src0->isReg()) { 3530 // Try to inline constant if possible. 3531 // If the Def moves immediate and the use is single 3532 // We are saving VGPR here. 3533 MachineInstr *Def = MRI->getUniqueVRegDef(Src0->getReg()); 3534 if (Def && Def->isMoveImmediate() && 3535 isInlineConstant(Def->getOperand(1)) && 3536 MRI->hasOneUse(Src0->getReg())) { 3537 Src0->ChangeToImmediate(Def->getOperand(1).getImm()); 3538 Src0Inlined = true; 3539 } else if (ST.getConstantBusLimit(Opc) <= 1 && 3540 RI.isSGPRReg(*MRI, Src0->getReg())) { 3541 return false; 3542 } 3543 // VGPR is okay as Src0 - fallthrough 3544 } 3545 3546 if (Src1->isReg() && !Src0Inlined) { 3547 // We have one slot for inlinable constant so far - try to fill it 3548 MachineInstr *Def = MRI->getUniqueVRegDef(Src1->getReg()); 3549 if (Def && Def->isMoveImmediate() && 3550 isInlineConstant(Def->getOperand(1)) && 3551 MRI->hasOneUse(Src1->getReg()) && commuteInstruction(UseMI)) 3552 Src0->ChangeToImmediate(Def->getOperand(1).getImm()); 3553 else if (RI.isSGPRReg(*MRI, Src1->getReg())) 3554 return false; 3555 // VGPR is okay as Src1 - fallthrough 3556 } 3557 } 3558 3559 unsigned NewOpc = 3560 IsFMA ? (IsF32 ? AMDGPU::V_FMAAK_F32 3561 : ST.hasTrue16BitInsts() ? AMDGPU::V_FMAAK_F16_t16 3562 : AMDGPU::V_FMAAK_F16) 3563 : (IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16); 3564 if (pseudoToMCOpcode(NewOpc) == -1) 3565 return false; 3566 3567 // V_FMAAK_F16_t16 takes VGPR_32_Lo128 operands, so the rewrite 3568 // would also require restricting their register classes. For now 3569 // just bail out. 3570 if (NewOpc == AMDGPU::V_FMAAK_F16_t16) 3571 return false; 3572 3573 // FIXME: This would be a lot easier if we could return a new instruction 3574 // instead of having to modify in place. 3575 3576 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 || 3577 Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 || 3578 Opc == AMDGPU::V_FMAC_F16_e64) 3579 UseMI.untieRegOperand( 3580 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)); 3581 3582 // ChangingToImmediate adds Src2 back to the instruction. 3583 Src2->ChangeToImmediate(getImmFor(*Src2)); 3584 3585 // These come before src2. 3586 removeModOperands(UseMI); 3587 UseMI.setDesc(get(NewOpc)); 3588 // It might happen that UseMI was commuted 3589 // and we now have SGPR as SRC1. If so 2 inlined 3590 // constant and SGPR are illegal. 3591 legalizeOperands(UseMI); 3592 3593 bool DeleteDef = MRI->use_nodbg_empty(Reg); 3594 if (DeleteDef) 3595 DefMI.eraseFromParent(); 3596 3597 return true; 3598 } 3599 } 3600 3601 return false; 3602 } 3603 3604 static bool 3605 memOpsHaveSameBaseOperands(ArrayRef<const MachineOperand *> BaseOps1, 3606 ArrayRef<const MachineOperand *> BaseOps2) { 3607 if (BaseOps1.size() != BaseOps2.size()) 3608 return false; 3609 for (size_t I = 0, E = BaseOps1.size(); I < E; ++I) { 3610 if (!BaseOps1[I]->isIdenticalTo(*BaseOps2[I])) 3611 return false; 3612 } 3613 return true; 3614 } 3615 3616 static bool offsetsDoNotOverlap(int WidthA, int OffsetA, 3617 int WidthB, int OffsetB) { 3618 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB; 3619 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA; 3620 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB; 3621 return LowOffset + LowWidth <= HighOffset; 3622 } 3623 3624 bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa, 3625 const MachineInstr &MIb) const { 3626 SmallVector<const MachineOperand *, 4> BaseOps0, BaseOps1; 3627 int64_t Offset0, Offset1; 3628 unsigned Dummy0, Dummy1; 3629 bool Offset0IsScalable, Offset1IsScalable; 3630 if (!getMemOperandsWithOffsetWidth(MIa, BaseOps0, Offset0, Offset0IsScalable, 3631 Dummy0, &RI) || 3632 !getMemOperandsWithOffsetWidth(MIb, BaseOps1, Offset1, Offset1IsScalable, 3633 Dummy1, &RI)) 3634 return false; 3635 3636 if (!memOpsHaveSameBaseOperands(BaseOps0, BaseOps1)) 3637 return false; 3638 3639 if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) { 3640 // FIXME: Handle ds_read2 / ds_write2. 3641 return false; 3642 } 3643 unsigned Width0 = MIa.memoperands().front()->getSize(); 3644 unsigned Width1 = MIb.memoperands().front()->getSize(); 3645 return offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1); 3646 } 3647 3648 bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, 3649 const MachineInstr &MIb) const { 3650 assert(MIa.mayLoadOrStore() && 3651 "MIa must load from or modify a memory location"); 3652 assert(MIb.mayLoadOrStore() && 3653 "MIb must load from or modify a memory location"); 3654 3655 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects()) 3656 return false; 3657 3658 // XXX - Can we relax this between address spaces? 3659 if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef()) 3660 return false; 3661 3662 if (isLDSDMA(MIa) || isLDSDMA(MIb)) 3663 return false; 3664 3665 // TODO: Should we check the address space from the MachineMemOperand? That 3666 // would allow us to distinguish objects we know don't alias based on the 3667 // underlying address space, even if it was lowered to a different one, 3668 // e.g. private accesses lowered to use MUBUF instructions on a scratch 3669 // buffer. 3670 if (isDS(MIa)) { 3671 if (isDS(MIb)) 3672 return checkInstOffsetsDoNotOverlap(MIa, MIb); 3673 3674 return !isFLAT(MIb) || isSegmentSpecificFLAT(MIb); 3675 } 3676 3677 if (isMUBUF(MIa) || isMTBUF(MIa)) { 3678 if (isMUBUF(MIb) || isMTBUF(MIb)) 3679 return checkInstOffsetsDoNotOverlap(MIa, MIb); 3680 3681 if (isFLAT(MIb)) 3682 return isFLATScratch(MIb); 3683 3684 return !isSMRD(MIb); 3685 } 3686 3687 if (isSMRD(MIa)) { 3688 if (isSMRD(MIb)) 3689 return checkInstOffsetsDoNotOverlap(MIa, MIb); 3690 3691 if (isFLAT(MIb)) 3692 return isFLATScratch(MIb); 3693 3694 return !isMUBUF(MIb) && !isMTBUF(MIb); 3695 } 3696 3697 if (isFLAT(MIa)) { 3698 if (isFLAT(MIb)) { 3699 if ((isFLATScratch(MIa) && isFLATGlobal(MIb)) || 3700 (isFLATGlobal(MIa) && isFLATScratch(MIb))) 3701 return true; 3702 3703 return checkInstOffsetsDoNotOverlap(MIa, MIb); 3704 } 3705 3706 return false; 3707 } 3708 3709 return false; 3710 } 3711 3712 static bool getFoldableImm(Register Reg, const MachineRegisterInfo &MRI, 3713 int64_t &Imm, MachineInstr **DefMI = nullptr) { 3714 if (Reg.isPhysical()) 3715 return false; 3716 auto *Def = MRI.getUniqueVRegDef(Reg); 3717 if (Def && SIInstrInfo::isFoldableCopy(*Def) && Def->getOperand(1).isImm()) { 3718 Imm = Def->getOperand(1).getImm(); 3719 if (DefMI) 3720 *DefMI = Def; 3721 return true; 3722 } 3723 return false; 3724 } 3725 3726 static bool getFoldableImm(const MachineOperand *MO, int64_t &Imm, 3727 MachineInstr **DefMI = nullptr) { 3728 if (!MO->isReg()) 3729 return false; 3730 const MachineFunction *MF = MO->getParent()->getParent()->getParent(); 3731 const MachineRegisterInfo &MRI = MF->getRegInfo(); 3732 return getFoldableImm(MO->getReg(), MRI, Imm, DefMI); 3733 } 3734 3735 static void updateLiveVariables(LiveVariables *LV, MachineInstr &MI, 3736 MachineInstr &NewMI) { 3737 if (LV) { 3738 unsigned NumOps = MI.getNumOperands(); 3739 for (unsigned I = 1; I < NumOps; ++I) { 3740 MachineOperand &Op = MI.getOperand(I); 3741 if (Op.isReg() && Op.isKill()) 3742 LV->replaceKillInstruction(Op.getReg(), MI, NewMI); 3743 } 3744 } 3745 } 3746 3747 MachineInstr *SIInstrInfo::convertToThreeAddress(MachineInstr &MI, 3748 LiveVariables *LV, 3749 LiveIntervals *LIS) const { 3750 MachineBasicBlock &MBB = *MI.getParent(); 3751 unsigned Opc = MI.getOpcode(); 3752 3753 // Handle MFMA. 3754 int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(Opc); 3755 if (NewMFMAOpc != -1) { 3756 MachineInstrBuilder MIB = 3757 BuildMI(MBB, MI, MI.getDebugLoc(), get(NewMFMAOpc)); 3758 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) 3759 MIB.add(MI.getOperand(I)); 3760 updateLiveVariables(LV, MI, *MIB); 3761 if (LIS) 3762 LIS->ReplaceMachineInstrInMaps(MI, *MIB); 3763 return MIB; 3764 } 3765 3766 if (SIInstrInfo::isWMMA(MI)) { 3767 unsigned NewOpc = AMDGPU::mapWMMA2AddrTo3AddrOpcode(MI.getOpcode()); 3768 MachineInstrBuilder MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3769 .setMIFlags(MI.getFlags()); 3770 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) 3771 MIB->addOperand(MI.getOperand(I)); 3772 3773 updateLiveVariables(LV, MI, *MIB); 3774 if (LIS) 3775 LIS->ReplaceMachineInstrInMaps(MI, *MIB); 3776 3777 return MIB; 3778 } 3779 3780 assert(Opc != AMDGPU::V_FMAC_F16_t16_e32 && 3781 "V_FMAC_F16_t16_e32 is not supported and not expected to be present " 3782 "pre-RA"); 3783 3784 // Handle MAC/FMAC. 3785 bool IsF16 = Opc == AMDGPU::V_MAC_F16_e32 || Opc == AMDGPU::V_MAC_F16_e64 || 3786 Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64 || 3787 Opc == AMDGPU::V_FMAC_F16_t16_e64; 3788 bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64 || 3789 Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 || 3790 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64 || 3791 Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64 || 3792 Opc == AMDGPU::V_FMAC_F16_t16_e64 || 3793 Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64; 3794 bool IsF64 = Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64; 3795 bool IsLegacy = Opc == AMDGPU::V_MAC_LEGACY_F32_e32 || 3796 Opc == AMDGPU::V_MAC_LEGACY_F32_e64 || 3797 Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 || 3798 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64; 3799 bool Src0Literal = false; 3800 3801 switch (Opc) { 3802 default: 3803 return nullptr; 3804 case AMDGPU::V_MAC_F16_e64: 3805 case AMDGPU::V_FMAC_F16_e64: 3806 case AMDGPU::V_FMAC_F16_t16_e64: 3807 case AMDGPU::V_MAC_F32_e64: 3808 case AMDGPU::V_MAC_LEGACY_F32_e64: 3809 case AMDGPU::V_FMAC_F32_e64: 3810 case AMDGPU::V_FMAC_LEGACY_F32_e64: 3811 case AMDGPU::V_FMAC_F64_e64: 3812 break; 3813 case AMDGPU::V_MAC_F16_e32: 3814 case AMDGPU::V_FMAC_F16_e32: 3815 case AMDGPU::V_MAC_F32_e32: 3816 case AMDGPU::V_MAC_LEGACY_F32_e32: 3817 case AMDGPU::V_FMAC_F32_e32: 3818 case AMDGPU::V_FMAC_LEGACY_F32_e32: 3819 case AMDGPU::V_FMAC_F64_e32: { 3820 int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), 3821 AMDGPU::OpName::src0); 3822 const MachineOperand *Src0 = &MI.getOperand(Src0Idx); 3823 if (!Src0->isReg() && !Src0->isImm()) 3824 return nullptr; 3825 3826 if (Src0->isImm() && !isInlineConstant(MI, Src0Idx, *Src0)) 3827 Src0Literal = true; 3828 3829 break; 3830 } 3831 } 3832 3833 MachineInstrBuilder MIB; 3834 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); 3835 const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0); 3836 const MachineOperand *Src0Mods = 3837 getNamedOperand(MI, AMDGPU::OpName::src0_modifiers); 3838 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); 3839 const MachineOperand *Src1Mods = 3840 getNamedOperand(MI, AMDGPU::OpName::src1_modifiers); 3841 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); 3842 const MachineOperand *Src2Mods = 3843 getNamedOperand(MI, AMDGPU::OpName::src2_modifiers); 3844 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp); 3845 const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod); 3846 const MachineOperand *OpSel = getNamedOperand(MI, AMDGPU::OpName::op_sel); 3847 3848 if (!Src0Mods && !Src1Mods && !Src2Mods && !Clamp && !Omod && !IsF64 && 3849 !IsLegacy && 3850 // If we have an SGPR input, we will violate the constant bus restriction. 3851 (ST.getConstantBusLimit(Opc) > 1 || !Src0->isReg() || 3852 !RI.isSGPRReg(MBB.getParent()->getRegInfo(), Src0->getReg()))) { 3853 MachineInstr *DefMI; 3854 const auto killDef = [&]() -> void { 3855 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 3856 // The only user is the instruction which will be killed. 3857 Register DefReg = DefMI->getOperand(0).getReg(); 3858 if (!MRI.hasOneNonDBGUse(DefReg)) 3859 return; 3860 // We cannot just remove the DefMI here, calling pass will crash. 3861 DefMI->setDesc(get(AMDGPU::IMPLICIT_DEF)); 3862 for (unsigned I = DefMI->getNumOperands() - 1; I != 0; --I) 3863 DefMI->removeOperand(I); 3864 if (LV) 3865 LV->getVarInfo(DefReg).AliveBlocks.clear(); 3866 }; 3867 3868 int64_t Imm; 3869 if (!Src0Literal && getFoldableImm(Src2, Imm, &DefMI)) { 3870 unsigned NewOpc = 3871 IsFMA ? (IsF16 ? (ST.hasTrue16BitInsts() ? AMDGPU::V_FMAAK_F16_t16 3872 : AMDGPU::V_FMAAK_F16) 3873 : AMDGPU::V_FMAAK_F32) 3874 : (IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32); 3875 if (pseudoToMCOpcode(NewOpc) != -1) { 3876 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3877 .add(*Dst) 3878 .add(*Src0) 3879 .add(*Src1) 3880 .addImm(Imm); 3881 updateLiveVariables(LV, MI, *MIB); 3882 if (LIS) 3883 LIS->ReplaceMachineInstrInMaps(MI, *MIB); 3884 killDef(); 3885 return MIB; 3886 } 3887 } 3888 unsigned NewOpc = 3889 IsFMA ? (IsF16 ? (ST.hasTrue16BitInsts() ? AMDGPU::V_FMAMK_F16_t16 3890 : AMDGPU::V_FMAMK_F16) 3891 : AMDGPU::V_FMAMK_F32) 3892 : (IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32); 3893 if (!Src0Literal && getFoldableImm(Src1, Imm, &DefMI)) { 3894 if (pseudoToMCOpcode(NewOpc) != -1) { 3895 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3896 .add(*Dst) 3897 .add(*Src0) 3898 .addImm(Imm) 3899 .add(*Src2); 3900 updateLiveVariables(LV, MI, *MIB); 3901 if (LIS) 3902 LIS->ReplaceMachineInstrInMaps(MI, *MIB); 3903 killDef(); 3904 return MIB; 3905 } 3906 } 3907 if (Src0Literal || getFoldableImm(Src0, Imm, &DefMI)) { 3908 if (Src0Literal) { 3909 Imm = Src0->getImm(); 3910 DefMI = nullptr; 3911 } 3912 if (pseudoToMCOpcode(NewOpc) != -1 && 3913 isOperandLegal( 3914 MI, AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::src0), 3915 Src1)) { 3916 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3917 .add(*Dst) 3918 .add(*Src1) 3919 .addImm(Imm) 3920 .add(*Src2); 3921 updateLiveVariables(LV, MI, *MIB); 3922 if (LIS) 3923 LIS->ReplaceMachineInstrInMaps(MI, *MIB); 3924 if (DefMI) 3925 killDef(); 3926 return MIB; 3927 } 3928 } 3929 } 3930 3931 // VOP2 mac/fmac with a literal operand cannot be converted to VOP3 mad/fma 3932 // if VOP3 does not allow a literal operand. 3933 if (Src0Literal && !ST.hasVOP3Literal()) 3934 return nullptr; 3935 3936 unsigned NewOpc = IsFMA ? IsF16 ? AMDGPU::V_FMA_F16_gfx9_e64 3937 : IsF64 ? AMDGPU::V_FMA_F64_e64 3938 : IsLegacy 3939 ? AMDGPU::V_FMA_LEGACY_F32_e64 3940 : AMDGPU::V_FMA_F32_e64 3941 : IsF16 ? AMDGPU::V_MAD_F16_e64 3942 : IsLegacy ? AMDGPU::V_MAD_LEGACY_F32_e64 3943 : AMDGPU::V_MAD_F32_e64; 3944 if (pseudoToMCOpcode(NewOpc) == -1) 3945 return nullptr; 3946 3947 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3948 .add(*Dst) 3949 .addImm(Src0Mods ? Src0Mods->getImm() : 0) 3950 .add(*Src0) 3951 .addImm(Src1Mods ? Src1Mods->getImm() : 0) 3952 .add(*Src1) 3953 .addImm(Src2Mods ? Src2Mods->getImm() : 0) 3954 .add(*Src2) 3955 .addImm(Clamp ? Clamp->getImm() : 0) 3956 .addImm(Omod ? Omod->getImm() : 0); 3957 if (AMDGPU::hasNamedOperand(NewOpc, AMDGPU::OpName::op_sel)) 3958 MIB.addImm(OpSel ? OpSel->getImm() : 0); 3959 updateLiveVariables(LV, MI, *MIB); 3960 if (LIS) 3961 LIS->ReplaceMachineInstrInMaps(MI, *MIB); 3962 return MIB; 3963 } 3964 3965 // It's not generally safe to move VALU instructions across these since it will 3966 // start using the register as a base index rather than directly. 3967 // XXX - Why isn't hasSideEffects sufficient for these? 3968 static bool changesVGPRIndexingMode(const MachineInstr &MI) { 3969 switch (MI.getOpcode()) { 3970 case AMDGPU::S_SET_GPR_IDX_ON: 3971 case AMDGPU::S_SET_GPR_IDX_MODE: 3972 case AMDGPU::S_SET_GPR_IDX_OFF: 3973 return true; 3974 default: 3975 return false; 3976 } 3977 } 3978 3979 bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI, 3980 const MachineBasicBlock *MBB, 3981 const MachineFunction &MF) const { 3982 // Skipping the check for SP writes in the base implementation. The reason it 3983 // was added was apparently due to compile time concerns. 3984 // 3985 // TODO: Do we really want this barrier? It triggers unnecessary hazard nops 3986 // but is probably avoidable. 3987 3988 // Copied from base implementation. 3989 // Terminators and labels can't be scheduled around. 3990 if (MI.isTerminator() || MI.isPosition()) 3991 return true; 3992 3993 // INLINEASM_BR can jump to another block 3994 if (MI.getOpcode() == TargetOpcode::INLINEASM_BR) 3995 return true; 3996 3997 if (MI.getOpcode() == AMDGPU::SCHED_BARRIER && MI.getOperand(0).getImm() == 0) 3998 return true; 3999 4000 // Target-independent instructions do not have an implicit-use of EXEC, even 4001 // when they operate on VGPRs. Treating EXEC modifications as scheduling 4002 // boundaries prevents incorrect movements of such instructions. 4003 return MI.modifiesRegister(AMDGPU::EXEC, &RI) || 4004 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 || 4005 MI.getOpcode() == AMDGPU::S_SETREG_B32 || 4006 MI.getOpcode() == AMDGPU::S_SETPRIO || 4007 changesVGPRIndexingMode(MI); 4008 } 4009 4010 bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const { 4011 return Opcode == AMDGPU::DS_ORDERED_COUNT || isGWS(Opcode); 4012 } 4013 4014 bool SIInstrInfo::modifiesModeRegister(const MachineInstr &MI) { 4015 // Skip the full operand and register alias search modifiesRegister 4016 // does. There's only a handful of instructions that touch this, it's only an 4017 // implicit def, and doesn't alias any other registers. 4018 return is_contained(MI.getDesc().implicit_defs(), AMDGPU::MODE); 4019 } 4020 4021 bool SIInstrInfo::hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const { 4022 unsigned Opcode = MI.getOpcode(); 4023 4024 if (MI.mayStore() && isSMRD(MI)) 4025 return true; // scalar store or atomic 4026 4027 // This will terminate the function when other lanes may need to continue. 4028 if (MI.isReturn()) 4029 return true; 4030 4031 // These instructions cause shader I/O that may cause hardware lockups 4032 // when executed with an empty EXEC mask. 4033 // 4034 // Note: exp with VM = DONE = 0 is automatically skipped by hardware when 4035 // EXEC = 0, but checking for that case here seems not worth it 4036 // given the typical code patterns. 4037 if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT || 4038 isEXP(Opcode) || 4039 Opcode == AMDGPU::DS_ORDERED_COUNT || Opcode == AMDGPU::S_TRAP || 4040 Opcode == AMDGPU::DS_GWS_INIT || Opcode == AMDGPU::DS_GWS_BARRIER) 4041 return true; 4042 4043 if (MI.isCall() || MI.isInlineAsm()) 4044 return true; // conservative assumption 4045 4046 // A mode change is a scalar operation that influences vector instructions. 4047 if (modifiesModeRegister(MI)) 4048 return true; 4049 4050 // These are like SALU instructions in terms of effects, so it's questionable 4051 // whether we should return true for those. 4052 // 4053 // However, executing them with EXEC = 0 causes them to operate on undefined 4054 // data, which we avoid by returning true here. 4055 if (Opcode == AMDGPU::V_READFIRSTLANE_B32 || 4056 Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32 || 4057 Opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR || 4058 Opcode == AMDGPU::SI_SPILL_S32_TO_VGPR) 4059 return true; 4060 4061 return false; 4062 } 4063 4064 bool SIInstrInfo::mayReadEXEC(const MachineRegisterInfo &MRI, 4065 const MachineInstr &MI) const { 4066 if (MI.isMetaInstruction()) 4067 return false; 4068 4069 // This won't read exec if this is an SGPR->SGPR copy. 4070 if (MI.isCopyLike()) { 4071 if (!RI.isSGPRReg(MRI, MI.getOperand(0).getReg())) 4072 return true; 4073 4074 // Make sure this isn't copying exec as a normal operand 4075 return MI.readsRegister(AMDGPU::EXEC, &RI); 4076 } 4077 4078 // Make a conservative assumption about the callee. 4079 if (MI.isCall()) 4080 return true; 4081 4082 // Be conservative with any unhandled generic opcodes. 4083 if (!isTargetSpecificOpcode(MI.getOpcode())) 4084 return true; 4085 4086 return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI); 4087 } 4088 4089 bool SIInstrInfo::isInlineConstant(const APInt &Imm) const { 4090 switch (Imm.getBitWidth()) { 4091 case 1: // This likely will be a condition code mask. 4092 return true; 4093 4094 case 32: 4095 return AMDGPU::isInlinableLiteral32(Imm.getSExtValue(), 4096 ST.hasInv2PiInlineImm()); 4097 case 64: 4098 return AMDGPU::isInlinableLiteral64(Imm.getSExtValue(), 4099 ST.hasInv2PiInlineImm()); 4100 case 16: 4101 return ST.has16BitInsts() && 4102 AMDGPU::isInlinableLiteral16(Imm.getSExtValue(), 4103 ST.hasInv2PiInlineImm()); 4104 default: 4105 llvm_unreachable("invalid bitwidth"); 4106 } 4107 } 4108 4109 bool SIInstrInfo::isInlineConstant(const MachineOperand &MO, 4110 uint8_t OperandType) const { 4111 assert(!MO.isReg() && "isInlineConstant called on register operand!"); 4112 if (!MO.isImm()) 4113 return false; 4114 4115 // MachineOperand provides no way to tell the true operand size, since it only 4116 // records a 64-bit value. We need to know the size to determine if a 32-bit 4117 // floating point immediate bit pattern is legal for an integer immediate. It 4118 // would be for any 32-bit integer operand, but would not be for a 64-bit one. 4119 4120 int64_t Imm = MO.getImm(); 4121 switch (OperandType) { 4122 case AMDGPU::OPERAND_REG_IMM_INT32: 4123 case AMDGPU::OPERAND_REG_IMM_FP32: 4124 case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED: 4125 case AMDGPU::OPERAND_REG_INLINE_C_INT32: 4126 case AMDGPU::OPERAND_REG_INLINE_C_FP32: 4127 case AMDGPU::OPERAND_REG_IMM_V2FP32: 4128 case AMDGPU::OPERAND_REG_INLINE_C_V2FP32: 4129 case AMDGPU::OPERAND_REG_IMM_V2INT32: 4130 case AMDGPU::OPERAND_REG_INLINE_C_V2INT32: 4131 case AMDGPU::OPERAND_REG_INLINE_AC_INT32: 4132 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: 4133 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32: { 4134 int32_t Trunc = static_cast<int32_t>(Imm); 4135 return AMDGPU::isInlinableLiteral32(Trunc, ST.hasInv2PiInlineImm()); 4136 } 4137 case AMDGPU::OPERAND_REG_IMM_INT64: 4138 case AMDGPU::OPERAND_REG_IMM_FP64: 4139 case AMDGPU::OPERAND_REG_INLINE_C_INT64: 4140 case AMDGPU::OPERAND_REG_INLINE_C_FP64: 4141 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: 4142 return AMDGPU::isInlinableLiteral64(MO.getImm(), 4143 ST.hasInv2PiInlineImm()); 4144 case AMDGPU::OPERAND_REG_IMM_INT16: 4145 case AMDGPU::OPERAND_REG_INLINE_C_INT16: 4146 case AMDGPU::OPERAND_REG_INLINE_AC_INT16: 4147 // We would expect inline immediates to not be concerned with an integer/fp 4148 // distinction. However, in the case of 16-bit integer operations, the 4149 // "floating point" values appear to not work. It seems read the low 16-bits 4150 // of 32-bit immediates, which happens to always work for the integer 4151 // values. 4152 // 4153 // See llvm bugzilla 46302. 4154 // 4155 // TODO: Theoretically we could use op-sel to use the high bits of the 4156 // 32-bit FP values. 4157 return AMDGPU::isInlinableIntLiteral(Imm); 4158 case AMDGPU::OPERAND_REG_IMM_V2INT16: 4159 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: 4160 case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16: 4161 return AMDGPU::isInlinableLiteralV2I16(Imm); 4162 case AMDGPU::OPERAND_REG_IMM_V2FP16: 4163 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: 4164 case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: 4165 return AMDGPU::isInlinableLiteralV2F16(Imm); 4166 case AMDGPU::OPERAND_REG_IMM_FP16: 4167 case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED: 4168 case AMDGPU::OPERAND_REG_INLINE_C_FP16: 4169 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: { 4170 if (isInt<16>(Imm) || isUInt<16>(Imm)) { 4171 // A few special case instructions have 16-bit operands on subtargets 4172 // where 16-bit instructions are not legal. 4173 // TODO: Do the 32-bit immediates work? We shouldn't really need to handle 4174 // constants in these cases 4175 int16_t Trunc = static_cast<int16_t>(Imm); 4176 return ST.has16BitInsts() && 4177 AMDGPU::isInlinableLiteral16(Trunc, ST.hasInv2PiInlineImm()); 4178 } 4179 4180 return false; 4181 } 4182 case AMDGPU::OPERAND_KIMM32: 4183 case AMDGPU::OPERAND_KIMM16: 4184 return false; 4185 case AMDGPU::OPERAND_INPUT_MODS: 4186 case MCOI::OPERAND_IMMEDIATE: 4187 // Always embedded in the instruction for free. 4188 return true; 4189 case MCOI::OPERAND_UNKNOWN: 4190 case MCOI::OPERAND_REGISTER: 4191 case MCOI::OPERAND_PCREL: 4192 case MCOI::OPERAND_GENERIC_0: 4193 case MCOI::OPERAND_GENERIC_1: 4194 case MCOI::OPERAND_GENERIC_2: 4195 case MCOI::OPERAND_GENERIC_3: 4196 case MCOI::OPERAND_GENERIC_4: 4197 case MCOI::OPERAND_GENERIC_5: 4198 // Just ignore anything else. 4199 return true; 4200 default: 4201 llvm_unreachable("invalid operand type"); 4202 } 4203 } 4204 4205 static bool compareMachineOp(const MachineOperand &Op0, 4206 const MachineOperand &Op1) { 4207 if (Op0.getType() != Op1.getType()) 4208 return false; 4209 4210 switch (Op0.getType()) { 4211 case MachineOperand::MO_Register: 4212 return Op0.getReg() == Op1.getReg(); 4213 case MachineOperand::MO_Immediate: 4214 return Op0.getImm() == Op1.getImm(); 4215 default: 4216 llvm_unreachable("Didn't expect to be comparing these operand types"); 4217 } 4218 } 4219 4220 bool SIInstrInfo::isImmOperandLegal(const MachineInstr &MI, unsigned OpNo, 4221 const MachineOperand &MO) const { 4222 const MCInstrDesc &InstDesc = MI.getDesc(); 4223 const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo]; 4224 4225 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal()); 4226 4227 if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE) 4228 return true; 4229 4230 if (OpInfo.RegClass < 0) 4231 return false; 4232 4233 if (MO.isImm() && isInlineConstant(MO, OpInfo)) { 4234 if (isMAI(MI) && ST.hasMFMAInlineLiteralBug() && 4235 OpNo ==(unsigned)AMDGPU::getNamedOperandIdx(MI.getOpcode(), 4236 AMDGPU::OpName::src2)) 4237 return false; 4238 return RI.opCanUseInlineConstant(OpInfo.OperandType); 4239 } 4240 4241 if (!RI.opCanUseLiteralConstant(OpInfo.OperandType)) 4242 return false; 4243 4244 if (!isVOP3(MI) || !AMDGPU::isSISrcOperand(InstDesc, OpNo)) 4245 return true; 4246 4247 return ST.hasVOP3Literal(); 4248 } 4249 4250 bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const { 4251 // GFX90A does not have V_MUL_LEGACY_F32_e32. 4252 if (Opcode == AMDGPU::V_MUL_LEGACY_F32_e64 && ST.hasGFX90AInsts()) 4253 return false; 4254 4255 int Op32 = AMDGPU::getVOPe32(Opcode); 4256 if (Op32 == -1) 4257 return false; 4258 4259 return pseudoToMCOpcode(Op32) != -1; 4260 } 4261 4262 bool SIInstrInfo::hasModifiers(unsigned Opcode) const { 4263 // The src0_modifier operand is present on all instructions 4264 // that have modifiers. 4265 4266 return AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src0_modifiers); 4267 } 4268 4269 bool SIInstrInfo::hasModifiersSet(const MachineInstr &MI, 4270 unsigned OpName) const { 4271 const MachineOperand *Mods = getNamedOperand(MI, OpName); 4272 return Mods && Mods->getImm(); 4273 } 4274 4275 bool SIInstrInfo::hasAnyModifiersSet(const MachineInstr &MI) const { 4276 return any_of(ModifierOpNames, 4277 [&](unsigned Name) { return hasModifiersSet(MI, Name); }); 4278 } 4279 4280 bool SIInstrInfo::canShrink(const MachineInstr &MI, 4281 const MachineRegisterInfo &MRI) const { 4282 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); 4283 // Can't shrink instruction with three operands. 4284 if (Src2) { 4285 switch (MI.getOpcode()) { 4286 default: return false; 4287 4288 case AMDGPU::V_ADDC_U32_e64: 4289 case AMDGPU::V_SUBB_U32_e64: 4290 case AMDGPU::V_SUBBREV_U32_e64: { 4291 const MachineOperand *Src1 4292 = getNamedOperand(MI, AMDGPU::OpName::src1); 4293 if (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg())) 4294 return false; 4295 // Additional verification is needed for sdst/src2. 4296 return true; 4297 } 4298 case AMDGPU::V_MAC_F16_e64: 4299 case AMDGPU::V_MAC_F32_e64: 4300 case AMDGPU::V_MAC_LEGACY_F32_e64: 4301 case AMDGPU::V_FMAC_F16_e64: 4302 case AMDGPU::V_FMAC_F16_t16_e64: 4303 case AMDGPU::V_FMAC_F32_e64: 4304 case AMDGPU::V_FMAC_F64_e64: 4305 case AMDGPU::V_FMAC_LEGACY_F32_e64: 4306 if (!Src2->isReg() || !RI.isVGPR(MRI, Src2->getReg()) || 4307 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers)) 4308 return false; 4309 break; 4310 4311 case AMDGPU::V_CNDMASK_B32_e64: 4312 break; 4313 } 4314 } 4315 4316 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); 4317 if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()) || 4318 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers))) 4319 return false; 4320 4321 // We don't need to check src0, all input types are legal, so just make sure 4322 // src0 isn't using any modifiers. 4323 if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers)) 4324 return false; 4325 4326 // Can it be shrunk to a valid 32 bit opcode? 4327 if (!hasVALU32BitEncoding(MI.getOpcode())) 4328 return false; 4329 4330 // Check output modifiers 4331 return !hasModifiersSet(MI, AMDGPU::OpName::omod) && 4332 !hasModifiersSet(MI, AMDGPU::OpName::clamp); 4333 } 4334 4335 // Set VCC operand with all flags from \p Orig, except for setting it as 4336 // implicit. 4337 static void copyFlagsToImplicitVCC(MachineInstr &MI, 4338 const MachineOperand &Orig) { 4339 4340 for (MachineOperand &Use : MI.implicit_operands()) { 4341 if (Use.isUse() && 4342 (Use.getReg() == AMDGPU::VCC || Use.getReg() == AMDGPU::VCC_LO)) { 4343 Use.setIsUndef(Orig.isUndef()); 4344 Use.setIsKill(Orig.isKill()); 4345 return; 4346 } 4347 } 4348 } 4349 4350 MachineInstr *SIInstrInfo::buildShrunkInst(MachineInstr &MI, 4351 unsigned Op32) const { 4352 MachineBasicBlock *MBB = MI.getParent(); 4353 MachineInstrBuilder Inst32 = 4354 BuildMI(*MBB, MI, MI.getDebugLoc(), get(Op32)) 4355 .setMIFlags(MI.getFlags()); 4356 4357 // Add the dst operand if the 32-bit encoding also has an explicit $vdst. 4358 // For VOPC instructions, this is replaced by an implicit def of vcc. 4359 if (AMDGPU::hasNamedOperand(Op32, AMDGPU::OpName::vdst)) { 4360 // dst 4361 Inst32.add(MI.getOperand(0)); 4362 } else if (AMDGPU::hasNamedOperand(Op32, AMDGPU::OpName::sdst)) { 4363 // VOPCX instructions won't be writing to an explicit dst, so this should 4364 // not fail for these instructions. 4365 assert(((MI.getOperand(0).getReg() == AMDGPU::VCC) || 4366 (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) && 4367 "Unexpected case"); 4368 } 4369 4370 Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0)); 4371 4372 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); 4373 if (Src1) 4374 Inst32.add(*Src1); 4375 4376 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); 4377 4378 if (Src2) { 4379 int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2); 4380 if (Op32Src2Idx != -1) { 4381 Inst32.add(*Src2); 4382 } else { 4383 // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is 4384 // replaced with an implicit read of vcc or vcc_lo. The implicit read 4385 // of vcc was already added during the initial BuildMI, but we 4386 // 1) may need to change vcc to vcc_lo to preserve the original register 4387 // 2) have to preserve the original flags. 4388 fixImplicitOperands(*Inst32); 4389 copyFlagsToImplicitVCC(*Inst32, *Src2); 4390 } 4391 } 4392 4393 return Inst32; 4394 } 4395 4396 bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo &MRI, 4397 const MachineOperand &MO, 4398 const MCOperandInfo &OpInfo) const { 4399 // Literal constants use the constant bus. 4400 if (!MO.isReg()) 4401 return !isInlineConstant(MO, OpInfo); 4402 4403 if (!MO.isUse()) 4404 return false; 4405 4406 if (MO.getReg().isVirtual()) 4407 return RI.isSGPRClass(MRI.getRegClass(MO.getReg())); 4408 4409 // Null is free 4410 if (MO.getReg() == AMDGPU::SGPR_NULL || MO.getReg() == AMDGPU::SGPR_NULL64) 4411 return false; 4412 4413 // SGPRs use the constant bus 4414 if (MO.isImplicit()) { 4415 return MO.getReg() == AMDGPU::M0 || 4416 MO.getReg() == AMDGPU::VCC || 4417 MO.getReg() == AMDGPU::VCC_LO; 4418 } else { 4419 return AMDGPU::SReg_32RegClass.contains(MO.getReg()) || 4420 AMDGPU::SReg_64RegClass.contains(MO.getReg()); 4421 } 4422 } 4423 4424 static Register findImplicitSGPRRead(const MachineInstr &MI) { 4425 for (const MachineOperand &MO : MI.implicit_operands()) { 4426 // We only care about reads. 4427 if (MO.isDef()) 4428 continue; 4429 4430 switch (MO.getReg()) { 4431 case AMDGPU::VCC: 4432 case AMDGPU::VCC_LO: 4433 case AMDGPU::VCC_HI: 4434 case AMDGPU::M0: 4435 case AMDGPU::FLAT_SCR: 4436 return MO.getReg(); 4437 4438 default: 4439 break; 4440 } 4441 } 4442 4443 return Register(); 4444 } 4445 4446 static bool shouldReadExec(const MachineInstr &MI) { 4447 if (SIInstrInfo::isVALU(MI)) { 4448 switch (MI.getOpcode()) { 4449 case AMDGPU::V_READLANE_B32: 4450 case AMDGPU::SI_RESTORE_S32_FROM_VGPR: 4451 case AMDGPU::V_WRITELANE_B32: 4452 case AMDGPU::SI_SPILL_S32_TO_VGPR: 4453 return false; 4454 } 4455 4456 return true; 4457 } 4458 4459 if (MI.isPreISelOpcode() || 4460 SIInstrInfo::isGenericOpcode(MI.getOpcode()) || 4461 SIInstrInfo::isSALU(MI) || 4462 SIInstrInfo::isSMRD(MI)) 4463 return false; 4464 4465 return true; 4466 } 4467 4468 static bool isSubRegOf(const SIRegisterInfo &TRI, 4469 const MachineOperand &SuperVec, 4470 const MachineOperand &SubReg) { 4471 if (SubReg.getReg().isPhysical()) 4472 return TRI.isSubRegister(SuperVec.getReg(), SubReg.getReg()); 4473 4474 return SubReg.getSubReg() != AMDGPU::NoSubRegister && 4475 SubReg.getReg() == SuperVec.getReg(); 4476 } 4477 4478 bool SIInstrInfo::verifyInstruction(const MachineInstr &MI, 4479 StringRef &ErrInfo) const { 4480 uint16_t Opcode = MI.getOpcode(); 4481 if (SIInstrInfo::isGenericOpcode(MI.getOpcode())) 4482 return true; 4483 4484 const MachineFunction *MF = MI.getParent()->getParent(); 4485 const MachineRegisterInfo &MRI = MF->getRegInfo(); 4486 4487 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0); 4488 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1); 4489 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2); 4490 int Src3Idx = -1; 4491 if (Src0Idx == -1) { 4492 // VOPD V_DUAL_* instructions use different operand names. 4493 Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0X); 4494 Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vsrc1X); 4495 Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0Y); 4496 Src3Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vsrc1Y); 4497 } 4498 4499 // Make sure the number of operands is correct. 4500 const MCInstrDesc &Desc = get(Opcode); 4501 if (!Desc.isVariadic() && 4502 Desc.getNumOperands() != MI.getNumExplicitOperands()) { 4503 ErrInfo = "Instruction has wrong number of operands."; 4504 return false; 4505 } 4506 4507 if (MI.isInlineAsm()) { 4508 // Verify register classes for inlineasm constraints. 4509 for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands(); 4510 I != E; ++I) { 4511 const TargetRegisterClass *RC = MI.getRegClassConstraint(I, this, &RI); 4512 if (!RC) 4513 continue; 4514 4515 const MachineOperand &Op = MI.getOperand(I); 4516 if (!Op.isReg()) 4517 continue; 4518 4519 Register Reg = Op.getReg(); 4520 if (!Reg.isVirtual() && !RC->contains(Reg)) { 4521 ErrInfo = "inlineasm operand has incorrect register class."; 4522 return false; 4523 } 4524 } 4525 4526 return true; 4527 } 4528 4529 if (isImage(MI) && MI.memoperands_empty() && MI.mayLoadOrStore()) { 4530 ErrInfo = "missing memory operand from image instruction."; 4531 return false; 4532 } 4533 4534 // Make sure the register classes are correct. 4535 for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) { 4536 const MachineOperand &MO = MI.getOperand(i); 4537 if (MO.isFPImm()) { 4538 ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast " 4539 "all fp values to integers."; 4540 return false; 4541 } 4542 4543 int RegClass = Desc.operands()[i].RegClass; 4544 4545 switch (Desc.operands()[i].OperandType) { 4546 case MCOI::OPERAND_REGISTER: 4547 if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) { 4548 ErrInfo = "Illegal immediate value for operand."; 4549 return false; 4550 } 4551 break; 4552 case AMDGPU::OPERAND_REG_IMM_INT32: 4553 case AMDGPU::OPERAND_REG_IMM_FP32: 4554 case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED: 4555 case AMDGPU::OPERAND_REG_IMM_V2FP32: 4556 break; 4557 case AMDGPU::OPERAND_REG_INLINE_C_INT32: 4558 case AMDGPU::OPERAND_REG_INLINE_C_FP32: 4559 case AMDGPU::OPERAND_REG_INLINE_C_INT64: 4560 case AMDGPU::OPERAND_REG_INLINE_C_FP64: 4561 case AMDGPU::OPERAND_REG_INLINE_C_INT16: 4562 case AMDGPU::OPERAND_REG_INLINE_C_FP16: 4563 case AMDGPU::OPERAND_REG_INLINE_AC_INT32: 4564 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: 4565 case AMDGPU::OPERAND_REG_INLINE_AC_INT16: 4566 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: 4567 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: { 4568 if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, i))) { 4569 ErrInfo = "Illegal immediate value for operand."; 4570 return false; 4571 } 4572 break; 4573 } 4574 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32: 4575 if (!MI.getOperand(i).isImm() || !isInlineConstant(MI, i)) { 4576 ErrInfo = "Expected inline constant for operand."; 4577 return false; 4578 } 4579 break; 4580 case MCOI::OPERAND_IMMEDIATE: 4581 case AMDGPU::OPERAND_KIMM32: 4582 // Check if this operand is an immediate. 4583 // FrameIndex operands will be replaced by immediates, so they are 4584 // allowed. 4585 if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) { 4586 ErrInfo = "Expected immediate, but got non-immediate"; 4587 return false; 4588 } 4589 [[fallthrough]]; 4590 default: 4591 continue; 4592 } 4593 4594 if (!MO.isReg()) 4595 continue; 4596 Register Reg = MO.getReg(); 4597 if (!Reg) 4598 continue; 4599 4600 // FIXME: Ideally we would have separate instruction definitions with the 4601 // aligned register constraint. 4602 // FIXME: We do not verify inline asm operands, but custom inline asm 4603 // verification is broken anyway 4604 if (ST.needsAlignedVGPRs()) { 4605 const TargetRegisterClass *RC = RI.getRegClassForReg(MRI, Reg); 4606 if (RI.hasVectorRegisters(RC) && MO.getSubReg()) { 4607 const TargetRegisterClass *SubRC = 4608 RI.getSubRegisterClass(RC, MO.getSubReg()); 4609 RC = RI.getCompatibleSubRegClass(RC, SubRC, MO.getSubReg()); 4610 if (RC) 4611 RC = SubRC; 4612 } 4613 4614 // Check that this is the aligned version of the class. 4615 if (!RC || !RI.isProperlyAlignedRC(*RC)) { 4616 ErrInfo = "Subtarget requires even aligned vector registers"; 4617 return false; 4618 } 4619 } 4620 4621 if (RegClass != -1) { 4622 if (Reg.isVirtual()) 4623 continue; 4624 4625 const TargetRegisterClass *RC = RI.getRegClass(RegClass); 4626 if (!RC->contains(Reg)) { 4627 ErrInfo = "Operand has incorrect register class."; 4628 return false; 4629 } 4630 } 4631 } 4632 4633 // Verify SDWA 4634 if (isSDWA(MI)) { 4635 if (!ST.hasSDWA()) { 4636 ErrInfo = "SDWA is not supported on this target"; 4637 return false; 4638 } 4639 4640 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst); 4641 4642 for (int OpIdx : {DstIdx, Src0Idx, Src1Idx, Src2Idx}) { 4643 if (OpIdx == -1) 4644 continue; 4645 const MachineOperand &MO = MI.getOperand(OpIdx); 4646 4647 if (!ST.hasSDWAScalar()) { 4648 // Only VGPRS on VI 4649 if (!MO.isReg() || !RI.hasVGPRs(RI.getRegClassForReg(MRI, MO.getReg()))) { 4650 ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI"; 4651 return false; 4652 } 4653 } else { 4654 // No immediates on GFX9 4655 if (!MO.isReg()) { 4656 ErrInfo = 4657 "Only reg allowed as operands in SDWA instructions on GFX9+"; 4658 return false; 4659 } 4660 } 4661 } 4662 4663 if (!ST.hasSDWAOmod()) { 4664 // No omod allowed on VI 4665 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod); 4666 if (OMod != nullptr && 4667 (!OMod->isImm() || OMod->getImm() != 0)) { 4668 ErrInfo = "OMod not allowed in SDWA instructions on VI"; 4669 return false; 4670 } 4671 } 4672 4673 uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode); 4674 if (isVOPC(BasicOpcode)) { 4675 if (!ST.hasSDWASdst() && DstIdx != -1) { 4676 // Only vcc allowed as dst on VI for VOPC 4677 const MachineOperand &Dst = MI.getOperand(DstIdx); 4678 if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) { 4679 ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI"; 4680 return false; 4681 } 4682 } else if (!ST.hasSDWAOutModsVOPC()) { 4683 // No clamp allowed on GFX9 for VOPC 4684 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp); 4685 if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) { 4686 ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI"; 4687 return false; 4688 } 4689 4690 // No omod allowed on GFX9 for VOPC 4691 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod); 4692 if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) { 4693 ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI"; 4694 return false; 4695 } 4696 } 4697 } 4698 4699 const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused); 4700 if (DstUnused && DstUnused->isImm() && 4701 DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) { 4702 const MachineOperand &Dst = MI.getOperand(DstIdx); 4703 if (!Dst.isReg() || !Dst.isTied()) { 4704 ErrInfo = "Dst register should have tied register"; 4705 return false; 4706 } 4707 4708 const MachineOperand &TiedMO = 4709 MI.getOperand(MI.findTiedOperandIdx(DstIdx)); 4710 if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) { 4711 ErrInfo = 4712 "Dst register should be tied to implicit use of preserved register"; 4713 return false; 4714 } else if (TiedMO.getReg().isPhysical() && 4715 Dst.getReg() != TiedMO.getReg()) { 4716 ErrInfo = "Dst register should use same physical register as preserved"; 4717 return false; 4718 } 4719 } 4720 } 4721 4722 // Verify MIMG / VIMAGE / VSAMPLE 4723 if (isImage(MI.getOpcode()) && !MI.mayStore()) { 4724 // Ensure that the return type used is large enough for all the options 4725 // being used TFE/LWE require an extra result register. 4726 const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask); 4727 if (DMask) { 4728 uint64_t DMaskImm = DMask->getImm(); 4729 uint32_t RegCount = 4730 isGather4(MI.getOpcode()) ? 4 : llvm::popcount(DMaskImm); 4731 const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe); 4732 const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe); 4733 const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16); 4734 4735 // Adjust for packed 16 bit values 4736 if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem()) 4737 RegCount = divideCeil(RegCount, 2); 4738 4739 // Adjust if using LWE or TFE 4740 if ((LWE && LWE->getImm()) || (TFE && TFE->getImm())) 4741 RegCount += 1; 4742 4743 const uint32_t DstIdx = 4744 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata); 4745 const MachineOperand &Dst = MI.getOperand(DstIdx); 4746 if (Dst.isReg()) { 4747 const TargetRegisterClass *DstRC = getOpRegClass(MI, DstIdx); 4748 uint32_t DstSize = RI.getRegSizeInBits(*DstRC) / 32; 4749 if (RegCount > DstSize) { 4750 ErrInfo = "Image instruction returns too many registers for dst " 4751 "register class"; 4752 return false; 4753 } 4754 } 4755 } 4756 } 4757 4758 // Verify VOP*. Ignore multiple sgpr operands on writelane. 4759 if (isVALU(MI) && Desc.getOpcode() != AMDGPU::V_WRITELANE_B32) { 4760 unsigned ConstantBusCount = 0; 4761 bool UsesLiteral = false; 4762 const MachineOperand *LiteralVal = nullptr; 4763 4764 int ImmIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm); 4765 if (ImmIdx != -1) { 4766 ++ConstantBusCount; 4767 UsesLiteral = true; 4768 LiteralVal = &MI.getOperand(ImmIdx); 4769 } 4770 4771 SmallVector<Register, 2> SGPRsUsed; 4772 Register SGPRUsed; 4773 4774 // Only look at the true operands. Only a real operand can use the constant 4775 // bus, and we don't want to check pseudo-operands like the source modifier 4776 // flags. 4777 for (int OpIdx : {Src0Idx, Src1Idx, Src2Idx, Src3Idx}) { 4778 if (OpIdx == -1) 4779 continue; 4780 const MachineOperand &MO = MI.getOperand(OpIdx); 4781 if (usesConstantBus(MRI, MO, MI.getDesc().operands()[OpIdx])) { 4782 if (MO.isReg()) { 4783 SGPRUsed = MO.getReg(); 4784 if (!llvm::is_contained(SGPRsUsed, SGPRUsed)) { 4785 ++ConstantBusCount; 4786 SGPRsUsed.push_back(SGPRUsed); 4787 } 4788 } else { 4789 if (!UsesLiteral) { 4790 ++ConstantBusCount; 4791 UsesLiteral = true; 4792 LiteralVal = &MO; 4793 } else if (!MO.isIdenticalTo(*LiteralVal)) { 4794 assert(isVOP2(MI) || isVOP3(MI)); 4795 ErrInfo = "VOP2/VOP3 instruction uses more than one literal"; 4796 return false; 4797 } 4798 } 4799 } 4800 } 4801 4802 SGPRUsed = findImplicitSGPRRead(MI); 4803 if (SGPRUsed) { 4804 // Implicit uses may safely overlap true operands 4805 if (llvm::all_of(SGPRsUsed, [this, SGPRUsed](unsigned SGPR) { 4806 return !RI.regsOverlap(SGPRUsed, SGPR); 4807 })) { 4808 ++ConstantBusCount; 4809 SGPRsUsed.push_back(SGPRUsed); 4810 } 4811 } 4812 4813 // v_writelane_b32 is an exception from constant bus restriction: 4814 // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const 4815 if (ConstantBusCount > ST.getConstantBusLimit(Opcode) && 4816 Opcode != AMDGPU::V_WRITELANE_B32) { 4817 ErrInfo = "VOP* instruction violates constant bus restriction"; 4818 return false; 4819 } 4820 4821 if (isVOP3(MI) && UsesLiteral && !ST.hasVOP3Literal()) { 4822 ErrInfo = "VOP3 instruction uses literal"; 4823 return false; 4824 } 4825 } 4826 4827 // Special case for writelane - this can break the multiple constant bus rule, 4828 // but still can't use more than one SGPR register 4829 if (Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) { 4830 unsigned SGPRCount = 0; 4831 Register SGPRUsed; 4832 4833 for (int OpIdx : {Src0Idx, Src1Idx}) { 4834 if (OpIdx == -1) 4835 break; 4836 4837 const MachineOperand &MO = MI.getOperand(OpIdx); 4838 4839 if (usesConstantBus(MRI, MO, MI.getDesc().operands()[OpIdx])) { 4840 if (MO.isReg() && MO.getReg() != AMDGPU::M0) { 4841 if (MO.getReg() != SGPRUsed) 4842 ++SGPRCount; 4843 SGPRUsed = MO.getReg(); 4844 } 4845 } 4846 if (SGPRCount > ST.getConstantBusLimit(Opcode)) { 4847 ErrInfo = "WRITELANE instruction violates constant bus restriction"; 4848 return false; 4849 } 4850 } 4851 } 4852 4853 // Verify misc. restrictions on specific instructions. 4854 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32_e64 || 4855 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64_e64) { 4856 const MachineOperand &Src0 = MI.getOperand(Src0Idx); 4857 const MachineOperand &Src1 = MI.getOperand(Src1Idx); 4858 const MachineOperand &Src2 = MI.getOperand(Src2Idx); 4859 if (Src0.isReg() && Src1.isReg() && Src2.isReg()) { 4860 if (!compareMachineOp(Src0, Src1) && 4861 !compareMachineOp(Src0, Src2)) { 4862 ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2"; 4863 return false; 4864 } 4865 } 4866 if ((getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm() & 4867 SISrcMods::ABS) || 4868 (getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm() & 4869 SISrcMods::ABS) || 4870 (getNamedOperand(MI, AMDGPU::OpName::src2_modifiers)->getImm() & 4871 SISrcMods::ABS)) { 4872 ErrInfo = "ABS not allowed in VOP3B instructions"; 4873 return false; 4874 } 4875 } 4876 4877 if (isSOP2(MI) || isSOPC(MI)) { 4878 const MachineOperand &Src0 = MI.getOperand(Src0Idx); 4879 const MachineOperand &Src1 = MI.getOperand(Src1Idx); 4880 4881 if (!Src0.isReg() && !Src1.isReg() && 4882 !isInlineConstant(Src0, Desc.operands()[Src0Idx]) && 4883 !isInlineConstant(Src1, Desc.operands()[Src1Idx]) && 4884 !Src0.isIdenticalTo(Src1)) { 4885 ErrInfo = "SOP2/SOPC instruction requires too many immediate constants"; 4886 return false; 4887 } 4888 } 4889 4890 if (isSOPK(MI)) { 4891 auto Op = getNamedOperand(MI, AMDGPU::OpName::simm16); 4892 if (Desc.isBranch()) { 4893 if (!Op->isMBB()) { 4894 ErrInfo = "invalid branch target for SOPK instruction"; 4895 return false; 4896 } 4897 } else { 4898 uint64_t Imm = Op->getImm(); 4899 if (sopkIsZext(MI)) { 4900 if (!isUInt<16>(Imm)) { 4901 ErrInfo = "invalid immediate for SOPK instruction"; 4902 return false; 4903 } 4904 } else { 4905 if (!isInt<16>(Imm)) { 4906 ErrInfo = "invalid immediate for SOPK instruction"; 4907 return false; 4908 } 4909 } 4910 } 4911 } 4912 4913 if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 || 4914 Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 || 4915 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 || 4916 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) { 4917 const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 || 4918 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64; 4919 4920 const unsigned StaticNumOps = 4921 Desc.getNumOperands() + Desc.implicit_uses().size(); 4922 const unsigned NumImplicitOps = IsDst ? 2 : 1; 4923 4924 // Allow additional implicit operands. This allows a fixup done by the post 4925 // RA scheduler where the main implicit operand is killed and implicit-defs 4926 // are added for sub-registers that remain live after this instruction. 4927 if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) { 4928 ErrInfo = "missing implicit register operands"; 4929 return false; 4930 } 4931 4932 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); 4933 if (IsDst) { 4934 if (!Dst->isUse()) { 4935 ErrInfo = "v_movreld_b32 vdst should be a use operand"; 4936 return false; 4937 } 4938 4939 unsigned UseOpIdx; 4940 if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) || 4941 UseOpIdx != StaticNumOps + 1) { 4942 ErrInfo = "movrel implicit operands should be tied"; 4943 return false; 4944 } 4945 } 4946 4947 const MachineOperand &Src0 = MI.getOperand(Src0Idx); 4948 const MachineOperand &ImpUse 4949 = MI.getOperand(StaticNumOps + NumImplicitOps - 1); 4950 if (!ImpUse.isReg() || !ImpUse.isUse() || 4951 !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) { 4952 ErrInfo = "src0 should be subreg of implicit vector use"; 4953 return false; 4954 } 4955 } 4956 4957 // Make sure we aren't losing exec uses in the td files. This mostly requires 4958 // being careful when using let Uses to try to add other use registers. 4959 if (shouldReadExec(MI)) { 4960 if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) { 4961 ErrInfo = "VALU instruction does not implicitly read exec mask"; 4962 return false; 4963 } 4964 } 4965 4966 if (isSMRD(MI)) { 4967 if (MI.mayStore() && 4968 ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS) { 4969 // The register offset form of scalar stores may only use m0 as the 4970 // soffset register. 4971 const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soffset); 4972 if (Soff && Soff->getReg() != AMDGPU::M0) { 4973 ErrInfo = "scalar stores must use m0 as offset register"; 4974 return false; 4975 } 4976 } 4977 } 4978 4979 if (isFLAT(MI) && !ST.hasFlatInstOffsets()) { 4980 const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset); 4981 if (Offset->getImm() != 0) { 4982 ErrInfo = "subtarget does not support offsets in flat instructions"; 4983 return false; 4984 } 4985 } 4986 4987 if (isDS(MI) && !ST.hasGDS()) { 4988 const MachineOperand *GDSOp = getNamedOperand(MI, AMDGPU::OpName::gds); 4989 if (GDSOp && GDSOp->getImm() != 0) { 4990 ErrInfo = "GDS is not supported on this subtarget"; 4991 return false; 4992 } 4993 } 4994 4995 if (isImage(MI)) { 4996 const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim); 4997 if (DimOp) { 4998 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode, 4999 AMDGPU::OpName::vaddr0); 5000 int RSrcOpName = 5001 isMIMG(MI) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc; 5002 int RsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, RSrcOpName); 5003 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opcode); 5004 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = 5005 AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode); 5006 const AMDGPU::MIMGDimInfo *Dim = 5007 AMDGPU::getMIMGDimInfoByEncoding(DimOp->getImm()); 5008 5009 if (!Dim) { 5010 ErrInfo = "dim is out of range"; 5011 return false; 5012 } 5013 5014 bool IsA16 = false; 5015 if (ST.hasR128A16()) { 5016 const MachineOperand *R128A16 = getNamedOperand(MI, AMDGPU::OpName::r128); 5017 IsA16 = R128A16->getImm() != 0; 5018 } else if (ST.hasA16()) { 5019 const MachineOperand *A16 = getNamedOperand(MI, AMDGPU::OpName::a16); 5020 IsA16 = A16->getImm() != 0; 5021 } 5022 5023 bool IsNSA = RsrcIdx - VAddr0Idx > 1; 5024 5025 unsigned AddrWords = 5026 AMDGPU::getAddrSizeMIMGOp(BaseOpcode, Dim, IsA16, ST.hasG16()); 5027 5028 unsigned VAddrWords; 5029 if (IsNSA) { 5030 VAddrWords = RsrcIdx - VAddr0Idx; 5031 if (ST.hasPartialNSAEncoding() && 5032 AddrWords > ST.getNSAMaxSize(isVSAMPLE(MI))) { 5033 unsigned LastVAddrIdx = RsrcIdx - 1; 5034 VAddrWords += getOpSize(MI, LastVAddrIdx) / 4 - 1; 5035 } 5036 } else { 5037 VAddrWords = getOpSize(MI, VAddr0Idx) / 4; 5038 if (AddrWords > 12) 5039 AddrWords = 16; 5040 } 5041 5042 if (VAddrWords != AddrWords) { 5043 LLVM_DEBUG(dbgs() << "bad vaddr size, expected " << AddrWords 5044 << " but got " << VAddrWords << "\n"); 5045 ErrInfo = "bad vaddr size"; 5046 return false; 5047 } 5048 } 5049 } 5050 5051 const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl); 5052 if (DppCt) { 5053 using namespace AMDGPU::DPP; 5054 5055 unsigned DC = DppCt->getImm(); 5056 if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 || 5057 DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST || 5058 (DC >= DppCtrl::DPP_UNUSED4_FIRST && DC <= DppCtrl::DPP_UNUSED4_LAST) || 5059 (DC >= DppCtrl::DPP_UNUSED5_FIRST && DC <= DppCtrl::DPP_UNUSED5_LAST) || 5060 (DC >= DppCtrl::DPP_UNUSED6_FIRST && DC <= DppCtrl::DPP_UNUSED6_LAST) || 5061 (DC >= DppCtrl::DPP_UNUSED7_FIRST && DC <= DppCtrl::DPP_UNUSED7_LAST) || 5062 (DC >= DppCtrl::DPP_UNUSED8_FIRST && DC <= DppCtrl::DPP_UNUSED8_LAST)) { 5063 ErrInfo = "Invalid dpp_ctrl value"; 5064 return false; 5065 } 5066 if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 && 5067 ST.getGeneration() >= AMDGPUSubtarget::GFX10) { 5068 ErrInfo = "Invalid dpp_ctrl value: " 5069 "wavefront shifts are not supported on GFX10+"; 5070 return false; 5071 } 5072 if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 && 5073 ST.getGeneration() >= AMDGPUSubtarget::GFX10) { 5074 ErrInfo = "Invalid dpp_ctrl value: " 5075 "broadcasts are not supported on GFX10+"; 5076 return false; 5077 } 5078 if (DC >= DppCtrl::ROW_SHARE_FIRST && DC <= DppCtrl::ROW_XMASK_LAST && 5079 ST.getGeneration() < AMDGPUSubtarget::GFX10) { 5080 if (DC >= DppCtrl::ROW_NEWBCAST_FIRST && 5081 DC <= DppCtrl::ROW_NEWBCAST_LAST && 5082 !ST.hasGFX90AInsts()) { 5083 ErrInfo = "Invalid dpp_ctrl value: " 5084 "row_newbroadcast/row_share is not supported before " 5085 "GFX90A/GFX10"; 5086 return false; 5087 } else if (DC > DppCtrl::ROW_NEWBCAST_LAST || !ST.hasGFX90AInsts()) { 5088 ErrInfo = "Invalid dpp_ctrl value: " 5089 "row_share and row_xmask are not supported before GFX10"; 5090 return false; 5091 } 5092 } 5093 5094 if (Opcode != AMDGPU::V_MOV_B64_DPP_PSEUDO && 5095 !AMDGPU::isLegalDPALU_DPPControl(DC) && AMDGPU::isDPALU_DPP(Desc)) { 5096 ErrInfo = "Invalid dpp_ctrl value: " 5097 "DP ALU dpp only support row_newbcast"; 5098 return false; 5099 } 5100 } 5101 5102 if ((MI.mayStore() || MI.mayLoad()) && !isVGPRSpill(MI)) { 5103 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); 5104 uint16_t DataNameIdx = isDS(Opcode) ? AMDGPU::OpName::data0 5105 : AMDGPU::OpName::vdata; 5106 const MachineOperand *Data = getNamedOperand(MI, DataNameIdx); 5107 const MachineOperand *Data2 = getNamedOperand(MI, AMDGPU::OpName::data1); 5108 if (Data && !Data->isReg()) 5109 Data = nullptr; 5110 5111 if (ST.hasGFX90AInsts()) { 5112 if (Dst && Data && 5113 (RI.isAGPR(MRI, Dst->getReg()) != RI.isAGPR(MRI, Data->getReg()))) { 5114 ErrInfo = "Invalid register class: " 5115 "vdata and vdst should be both VGPR or AGPR"; 5116 return false; 5117 } 5118 if (Data && Data2 && 5119 (RI.isAGPR(MRI, Data->getReg()) != RI.isAGPR(MRI, Data2->getReg()))) { 5120 ErrInfo = "Invalid register class: " 5121 "both data operands should be VGPR or AGPR"; 5122 return false; 5123 } 5124 } else { 5125 if ((Dst && RI.isAGPR(MRI, Dst->getReg())) || 5126 (Data && RI.isAGPR(MRI, Data->getReg())) || 5127 (Data2 && RI.isAGPR(MRI, Data2->getReg()))) { 5128 ErrInfo = "Invalid register class: " 5129 "agpr loads and stores not supported on this GPU"; 5130 return false; 5131 } 5132 } 5133 } 5134 5135 if (ST.needsAlignedVGPRs()) { 5136 const auto isAlignedReg = [&MI, &MRI, this](unsigned OpName) -> bool { 5137 const MachineOperand *Op = getNamedOperand(MI, OpName); 5138 if (!Op) 5139 return true; 5140 Register Reg = Op->getReg(); 5141 if (Reg.isPhysical()) 5142 return !(RI.getHWRegIndex(Reg) & 1); 5143 const TargetRegisterClass &RC = *MRI.getRegClass(Reg); 5144 return RI.getRegSizeInBits(RC) > 32 && RI.isProperlyAlignedRC(RC) && 5145 !(RI.getChannelFromSubReg(Op->getSubReg()) & 1); 5146 }; 5147 5148 if (MI.getOpcode() == AMDGPU::DS_GWS_INIT || 5149 MI.getOpcode() == AMDGPU::DS_GWS_SEMA_BR || 5150 MI.getOpcode() == AMDGPU::DS_GWS_BARRIER) { 5151 5152 if (!isAlignedReg(AMDGPU::OpName::data0)) { 5153 ErrInfo = "Subtarget requires even aligned vector registers " 5154 "for DS_GWS instructions"; 5155 return false; 5156 } 5157 } 5158 5159 if (isMIMG(MI)) { 5160 if (!isAlignedReg(AMDGPU::OpName::vaddr)) { 5161 ErrInfo = "Subtarget requires even aligned vector registers " 5162 "for vaddr operand of image instructions"; 5163 return false; 5164 } 5165 } 5166 } 5167 5168 if (MI.getOpcode() == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && 5169 !ST.hasGFX90AInsts()) { 5170 const MachineOperand *Src = getNamedOperand(MI, AMDGPU::OpName::src0); 5171 if (Src->isReg() && RI.isSGPRReg(MRI, Src->getReg())) { 5172 ErrInfo = "Invalid register class: " 5173 "v_accvgpr_write with an SGPR is not supported on this GPU"; 5174 return false; 5175 } 5176 } 5177 5178 if (Desc.getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS) { 5179 const MachineOperand &SrcOp = MI.getOperand(1); 5180 if (!SrcOp.isReg() || SrcOp.getReg().isVirtual()) { 5181 ErrInfo = "pseudo expects only physical SGPRs"; 5182 return false; 5183 } 5184 } 5185 5186 return true; 5187 } 5188 5189 // It is more readable to list mapped opcodes on the same line. 5190 // clang-format off 5191 5192 unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const { 5193 switch (MI.getOpcode()) { 5194 default: return AMDGPU::INSTRUCTION_LIST_END; 5195 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE; 5196 case AMDGPU::COPY: return AMDGPU::COPY; 5197 case AMDGPU::PHI: return AMDGPU::PHI; 5198 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG; 5199 case AMDGPU::WQM: return AMDGPU::WQM; 5200 case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM; 5201 case AMDGPU::STRICT_WWM: return AMDGPU::STRICT_WWM; 5202 case AMDGPU::STRICT_WQM: return AMDGPU::STRICT_WQM; 5203 case AMDGPU::S_MOV_B32: { 5204 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 5205 return MI.getOperand(1).isReg() || 5206 RI.isAGPR(MRI, MI.getOperand(0).getReg()) ? 5207 AMDGPU::COPY : AMDGPU::V_MOV_B32_e32; 5208 } 5209 case AMDGPU::S_ADD_I32: 5210 return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32; 5211 case AMDGPU::S_ADDC_U32: 5212 return AMDGPU::V_ADDC_U32_e32; 5213 case AMDGPU::S_SUB_I32: 5214 return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_CO_U32_e32; 5215 // FIXME: These are not consistently handled, and selected when the carry is 5216 // used. 5217 case AMDGPU::S_ADD_U32: 5218 return AMDGPU::V_ADD_CO_U32_e32; 5219 case AMDGPU::S_SUB_U32: 5220 return AMDGPU::V_SUB_CO_U32_e32; 5221 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32; 5222 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32_e64; 5223 case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32_e64; 5224 case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32_e64; 5225 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64; 5226 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64; 5227 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64; 5228 case AMDGPU::S_XNOR_B32: 5229 return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END; 5230 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64; 5231 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64; 5232 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64; 5233 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64; 5234 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32; 5235 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64_e64; 5236 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32; 5237 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64_e64; 5238 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32; 5239 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64_e64; 5240 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32_e64; 5241 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32_e64; 5242 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32_e64; 5243 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32_e64; 5244 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64; 5245 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32; 5246 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32; 5247 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32; 5248 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e64; 5249 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e64; 5250 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e64; 5251 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e64; 5252 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e64; 5253 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e64; 5254 case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e64; 5255 case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e64; 5256 case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e64; 5257 case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e64; 5258 case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e64; 5259 case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e64; 5260 case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e64; 5261 case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e64; 5262 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64; 5263 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32; 5264 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32; 5265 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64; 5266 case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ; 5267 case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ; 5268 case AMDGPU::S_CVT_F32_I32: return AMDGPU::V_CVT_F32_I32_e64; 5269 case AMDGPU::S_CVT_F32_U32: return AMDGPU::V_CVT_F32_U32_e64; 5270 case AMDGPU::S_CVT_I32_F32: return AMDGPU::V_CVT_I32_F32_e64; 5271 case AMDGPU::S_CVT_U32_F32: return AMDGPU::V_CVT_U32_F32_e64; 5272 case AMDGPU::S_CVT_F32_F16: return AMDGPU::V_CVT_F32_F16_t16_e64; 5273 case AMDGPU::S_CVT_HI_F32_F16: return AMDGPU::V_CVT_F32_F16_t16_e64; 5274 case AMDGPU::S_CVT_F16_F32: return AMDGPU::V_CVT_F16_F32_t16_e64; 5275 case AMDGPU::S_CEIL_F32: return AMDGPU::V_CEIL_F32_e64; 5276 case AMDGPU::S_FLOOR_F32: return AMDGPU::V_FLOOR_F32_e64; 5277 case AMDGPU::S_TRUNC_F32: return AMDGPU::V_TRUNC_F32_e64; 5278 case AMDGPU::S_RNDNE_F32: return AMDGPU::V_RNDNE_F32_e64; 5279 case AMDGPU::S_CEIL_F16: return AMDGPU::V_CEIL_F16_t16_e64; 5280 case AMDGPU::S_FLOOR_F16: return AMDGPU::V_FLOOR_F16_t16_e64; 5281 case AMDGPU::S_TRUNC_F16: return AMDGPU::V_TRUNC_F16_t16_e64; 5282 case AMDGPU::S_RNDNE_F16: return AMDGPU::V_RNDNE_F16_t16_e64; 5283 case AMDGPU::S_ADD_F32: return AMDGPU::V_ADD_F32_e64; 5284 case AMDGPU::S_SUB_F32: return AMDGPU::V_SUB_F32_e64; 5285 case AMDGPU::S_MIN_F32: return AMDGPU::V_MIN_F32_e64; 5286 case AMDGPU::S_MAX_F32: return AMDGPU::V_MAX_F32_e64; 5287 case AMDGPU::S_MINIMUM_F32: return AMDGPU::V_MINIMUM_F32_e64; 5288 case AMDGPU::S_MAXIMUM_F32: return AMDGPU::V_MAXIMUM_F32_e64; 5289 case AMDGPU::S_MUL_F32: return AMDGPU::V_MUL_F32_e64; 5290 case AMDGPU::S_ADD_F16: return AMDGPU::V_ADD_F16_fake16_e64; 5291 case AMDGPU::S_SUB_F16: return AMDGPU::V_SUB_F16_fake16_e64; 5292 case AMDGPU::S_MIN_F16: return AMDGPU::V_MIN_F16_fake16_e64; 5293 case AMDGPU::S_MAX_F16: return AMDGPU::V_MAX_F16_fake16_e64; 5294 case AMDGPU::S_MINIMUM_F16: return AMDGPU::V_MINIMUM_F16_e64; 5295 case AMDGPU::S_MAXIMUM_F16: return AMDGPU::V_MAXIMUM_F16_e64; 5296 case AMDGPU::S_MUL_F16: return AMDGPU::V_MUL_F16_fake16_e64; 5297 case AMDGPU::S_CVT_PK_RTZ_F16_F32: return AMDGPU::V_CVT_PKRTZ_F16_F32_e64; 5298 case AMDGPU::S_FMAC_F32: return AMDGPU::V_FMAC_F32_e64; 5299 case AMDGPU::S_FMAC_F16: return AMDGPU::V_FMAC_F16_t16_e64; 5300 case AMDGPU::S_FMAMK_F32: return AMDGPU::V_FMAMK_F32; 5301 case AMDGPU::S_FMAAK_F32: return AMDGPU::V_FMAAK_F32; 5302 case AMDGPU::S_CMP_LT_F32: return AMDGPU::V_CMP_LT_F32_e64; 5303 case AMDGPU::S_CMP_EQ_F32: return AMDGPU::V_CMP_EQ_F32_e64; 5304 case AMDGPU::S_CMP_LE_F32: return AMDGPU::V_CMP_LE_F32_e64; 5305 case AMDGPU::S_CMP_GT_F32: return AMDGPU::V_CMP_GT_F32_e64; 5306 case AMDGPU::S_CMP_LG_F32: return AMDGPU::V_CMP_LG_F32_e64; 5307 case AMDGPU::S_CMP_GE_F32: return AMDGPU::V_CMP_GE_F32_e64; 5308 case AMDGPU::S_CMP_O_F32: return AMDGPU::V_CMP_O_F32_e64; 5309 case AMDGPU::S_CMP_U_F32: return AMDGPU::V_CMP_U_F32_e64; 5310 case AMDGPU::S_CMP_NGE_F32: return AMDGPU::V_CMP_NGE_F32_e64; 5311 case AMDGPU::S_CMP_NLG_F32: return AMDGPU::V_CMP_NLG_F32_e64; 5312 case AMDGPU::S_CMP_NGT_F32: return AMDGPU::V_CMP_NGT_F32_e64; 5313 case AMDGPU::S_CMP_NLE_F32: return AMDGPU::V_CMP_NLE_F32_e64; 5314 case AMDGPU::S_CMP_NEQ_F32: return AMDGPU::V_CMP_NEQ_F32_e64; 5315 case AMDGPU::S_CMP_NLT_F32: return AMDGPU::V_CMP_NLT_F32_e64; 5316 case AMDGPU::S_CMP_LT_F16: return AMDGPU::V_CMP_LT_F16_t16_e64; 5317 case AMDGPU::S_CMP_EQ_F16: return AMDGPU::V_CMP_EQ_F16_t16_e64; 5318 case AMDGPU::S_CMP_LE_F16: return AMDGPU::V_CMP_LE_F16_t16_e64; 5319 case AMDGPU::S_CMP_GT_F16: return AMDGPU::V_CMP_GT_F16_t16_e64; 5320 case AMDGPU::S_CMP_LG_F16: return AMDGPU::V_CMP_LG_F16_t16_e64; 5321 case AMDGPU::S_CMP_GE_F16: return AMDGPU::V_CMP_GE_F16_t16_e64; 5322 case AMDGPU::S_CMP_O_F16: return AMDGPU::V_CMP_O_F16_t16_e64; 5323 case AMDGPU::S_CMP_U_F16: return AMDGPU::V_CMP_U_F16_t16_e64; 5324 case AMDGPU::S_CMP_NGE_F16: return AMDGPU::V_CMP_NGE_F16_t16_e64; 5325 case AMDGPU::S_CMP_NLG_F16: return AMDGPU::V_CMP_NLG_F16_t16_e64; 5326 case AMDGPU::S_CMP_NGT_F16: return AMDGPU::V_CMP_NGT_F16_t16_e64; 5327 case AMDGPU::S_CMP_NLE_F16: return AMDGPU::V_CMP_NLE_F16_t16_e64; 5328 case AMDGPU::S_CMP_NEQ_F16: return AMDGPU::V_CMP_NEQ_F16_t16_e64; 5329 case AMDGPU::S_CMP_NLT_F16: return AMDGPU::V_CMP_NLT_F16_t16_e64; 5330 case AMDGPU::V_S_EXP_F32_e64: return AMDGPU::V_EXP_F32_e64; 5331 case AMDGPU::V_S_EXP_F16_e64: return AMDGPU::V_EXP_F16_t16_e64; 5332 case AMDGPU::V_S_LOG_F32_e64: return AMDGPU::V_LOG_F32_e64; 5333 case AMDGPU::V_S_LOG_F16_e64: return AMDGPU::V_LOG_F16_t16_e64; 5334 case AMDGPU::V_S_RCP_F32_e64: return AMDGPU::V_RCP_F32_e64; 5335 case AMDGPU::V_S_RCP_F16_e64: return AMDGPU::V_RCP_F16_t16_e64; 5336 case AMDGPU::V_S_RSQ_F32_e64: return AMDGPU::V_RSQ_F32_e64; 5337 case AMDGPU::V_S_RSQ_F16_e64: return AMDGPU::V_RSQ_F16_t16_e64; 5338 case AMDGPU::V_S_SQRT_F32_e64: return AMDGPU::V_SQRT_F32_e64; 5339 case AMDGPU::V_S_SQRT_F16_e64: return AMDGPU::V_SQRT_F16_t16_e64; 5340 } 5341 llvm_unreachable( 5342 "Unexpected scalar opcode without corresponding vector one!"); 5343 } 5344 5345 // clang-format on 5346 5347 void SIInstrInfo::insertScratchExecCopy(MachineFunction &MF, 5348 MachineBasicBlock &MBB, 5349 MachineBasicBlock::iterator MBBI, 5350 const DebugLoc &DL, Register Reg, 5351 bool IsSCCLive, 5352 SlotIndexes *Indexes) const { 5353 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 5354 const SIInstrInfo *TII = ST.getInstrInfo(); 5355 bool IsWave32 = ST.isWave32(); 5356 if (IsSCCLive) { 5357 // Insert two move instructions, one to save the original value of EXEC and 5358 // the other to turn on all bits in EXEC. This is required as we can't use 5359 // the single instruction S_OR_SAVEEXEC that clobbers SCC. 5360 unsigned MovOpc = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; 5361 MCRegister Exec = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 5362 auto StoreExecMI = BuildMI(MBB, MBBI, DL, TII->get(MovOpc), Reg) 5363 .addReg(Exec, RegState::Kill); 5364 auto FlipExecMI = BuildMI(MBB, MBBI, DL, TII->get(MovOpc), Exec).addImm(-1); 5365 if (Indexes) { 5366 Indexes->insertMachineInstrInMaps(*StoreExecMI); 5367 Indexes->insertMachineInstrInMaps(*FlipExecMI); 5368 } 5369 } else { 5370 const unsigned OrSaveExec = 5371 IsWave32 ? AMDGPU::S_OR_SAVEEXEC_B32 : AMDGPU::S_OR_SAVEEXEC_B64; 5372 auto SaveExec = 5373 BuildMI(MBB, MBBI, DL, TII->get(OrSaveExec), Reg).addImm(-1); 5374 SaveExec->getOperand(3).setIsDead(); // Mark SCC as dead. 5375 if (Indexes) 5376 Indexes->insertMachineInstrInMaps(*SaveExec); 5377 } 5378 } 5379 5380 void SIInstrInfo::restoreExec(MachineFunction &MF, MachineBasicBlock &MBB, 5381 MachineBasicBlock::iterator MBBI, 5382 const DebugLoc &DL, Register Reg, 5383 SlotIndexes *Indexes) const { 5384 unsigned ExecMov = isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; 5385 MCRegister Exec = isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 5386 auto ExecRestoreMI = 5387 BuildMI(MBB, MBBI, DL, get(ExecMov), Exec).addReg(Reg, RegState::Kill); 5388 if (Indexes) 5389 Indexes->insertMachineInstrInMaps(*ExecRestoreMI); 5390 } 5391 5392 static const TargetRegisterClass * 5393 adjustAllocatableRegClass(const GCNSubtarget &ST, const SIRegisterInfo &RI, 5394 const MachineRegisterInfo &MRI, 5395 const MCInstrDesc &TID, unsigned RCID, 5396 bool IsAllocatable) { 5397 if ((IsAllocatable || !ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) && 5398 (((TID.mayLoad() || TID.mayStore()) && 5399 !(TID.TSFlags & SIInstrFlags::VGPRSpill)) || 5400 (TID.TSFlags & (SIInstrFlags::DS | SIInstrFlags::MIMG)))) { 5401 switch (RCID) { 5402 case AMDGPU::AV_32RegClassID: 5403 RCID = AMDGPU::VGPR_32RegClassID; 5404 break; 5405 case AMDGPU::AV_64RegClassID: 5406 RCID = AMDGPU::VReg_64RegClassID; 5407 break; 5408 case AMDGPU::AV_96RegClassID: 5409 RCID = AMDGPU::VReg_96RegClassID; 5410 break; 5411 case AMDGPU::AV_128RegClassID: 5412 RCID = AMDGPU::VReg_128RegClassID; 5413 break; 5414 case AMDGPU::AV_160RegClassID: 5415 RCID = AMDGPU::VReg_160RegClassID; 5416 break; 5417 case AMDGPU::AV_512RegClassID: 5418 RCID = AMDGPU::VReg_512RegClassID; 5419 break; 5420 default: 5421 break; 5422 } 5423 } 5424 5425 return RI.getProperlyAlignedRC(RI.getRegClass(RCID)); 5426 } 5427 5428 const TargetRegisterClass *SIInstrInfo::getRegClass(const MCInstrDesc &TID, 5429 unsigned OpNum, const TargetRegisterInfo *TRI, 5430 const MachineFunction &MF) 5431 const { 5432 if (OpNum >= TID.getNumOperands()) 5433 return nullptr; 5434 auto RegClass = TID.operands()[OpNum].RegClass; 5435 bool IsAllocatable = false; 5436 if (TID.TSFlags & (SIInstrFlags::DS | SIInstrFlags::FLAT)) { 5437 // vdst and vdata should be both VGPR or AGPR, same for the DS instructions 5438 // with two data operands. Request register class constrained to VGPR only 5439 // of both operands present as Machine Copy Propagation can not check this 5440 // constraint and possibly other passes too. 5441 // 5442 // The check is limited to FLAT and DS because atomics in non-flat encoding 5443 // have their vdst and vdata tied to be the same register. 5444 const int VDstIdx = AMDGPU::getNamedOperandIdx(TID.Opcode, 5445 AMDGPU::OpName::vdst); 5446 const int DataIdx = AMDGPU::getNamedOperandIdx(TID.Opcode, 5447 (TID.TSFlags & SIInstrFlags::DS) ? AMDGPU::OpName::data0 5448 : AMDGPU::OpName::vdata); 5449 if (DataIdx != -1) { 5450 IsAllocatable = VDstIdx != -1 || AMDGPU::hasNamedOperand( 5451 TID.Opcode, AMDGPU::OpName::data1); 5452 } 5453 } 5454 return adjustAllocatableRegClass(ST, RI, MF.getRegInfo(), TID, RegClass, 5455 IsAllocatable); 5456 } 5457 5458 const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI, 5459 unsigned OpNo) const { 5460 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 5461 const MCInstrDesc &Desc = get(MI.getOpcode()); 5462 if (MI.isVariadic() || OpNo >= Desc.getNumOperands() || 5463 Desc.operands()[OpNo].RegClass == -1) { 5464 Register Reg = MI.getOperand(OpNo).getReg(); 5465 5466 if (Reg.isVirtual()) 5467 return MRI.getRegClass(Reg); 5468 return RI.getPhysRegBaseClass(Reg); 5469 } 5470 5471 unsigned RCID = Desc.operands()[OpNo].RegClass; 5472 return adjustAllocatableRegClass(ST, RI, MRI, Desc, RCID, true); 5473 } 5474 5475 void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const { 5476 MachineBasicBlock::iterator I = MI; 5477 MachineBasicBlock *MBB = MI.getParent(); 5478 MachineOperand &MO = MI.getOperand(OpIdx); 5479 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 5480 unsigned RCID = get(MI.getOpcode()).operands()[OpIdx].RegClass; 5481 const TargetRegisterClass *RC = RI.getRegClass(RCID); 5482 unsigned Size = RI.getRegSizeInBits(*RC); 5483 unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO : AMDGPU::V_MOV_B32_e32; 5484 if (MO.isReg()) 5485 Opcode = AMDGPU::COPY; 5486 else if (RI.isSGPRClass(RC)) 5487 Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32; 5488 5489 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(RC); 5490 Register Reg = MRI.createVirtualRegister(VRC); 5491 DebugLoc DL = MBB->findDebugLoc(I); 5492 BuildMI(*MI.getParent(), I, DL, get(Opcode), Reg).add(MO); 5493 MO.ChangeToRegister(Reg, false); 5494 } 5495 5496 unsigned SIInstrInfo::buildExtractSubReg( 5497 MachineBasicBlock::iterator MI, MachineRegisterInfo &MRI, 5498 const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC, 5499 unsigned SubIdx, const TargetRegisterClass *SubRC) const { 5500 MachineBasicBlock *MBB = MI->getParent(); 5501 DebugLoc DL = MI->getDebugLoc(); 5502 Register SubReg = MRI.createVirtualRegister(SubRC); 5503 5504 if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) { 5505 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg) 5506 .addReg(SuperReg.getReg(), 0, SubIdx); 5507 return SubReg; 5508 } 5509 5510 // Just in case the super register is itself a sub-register, copy it to a new 5511 // value so we don't need to worry about merging its subreg index with the 5512 // SubIdx passed to this function. The register coalescer should be able to 5513 // eliminate this extra copy. 5514 Register NewSuperReg = MRI.createVirtualRegister(SuperRC); 5515 5516 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), NewSuperReg) 5517 .addReg(SuperReg.getReg(), 0, SuperReg.getSubReg()); 5518 5519 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg) 5520 .addReg(NewSuperReg, 0, SubIdx); 5521 5522 return SubReg; 5523 } 5524 5525 MachineOperand SIInstrInfo::buildExtractSubRegOrImm( 5526 MachineBasicBlock::iterator MII, MachineRegisterInfo &MRI, 5527 const MachineOperand &Op, const TargetRegisterClass *SuperRC, 5528 unsigned SubIdx, const TargetRegisterClass *SubRC) const { 5529 if (Op.isImm()) { 5530 if (SubIdx == AMDGPU::sub0) 5531 return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm())); 5532 if (SubIdx == AMDGPU::sub1) 5533 return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm() >> 32)); 5534 5535 llvm_unreachable("Unhandled register index for immediate"); 5536 } 5537 5538 unsigned SubReg = buildExtractSubReg(MII, MRI, Op, SuperRC, 5539 SubIdx, SubRC); 5540 return MachineOperand::CreateReg(SubReg, false); 5541 } 5542 5543 // Change the order of operands from (0, 1, 2) to (0, 2, 1) 5544 void SIInstrInfo::swapOperands(MachineInstr &Inst) const { 5545 assert(Inst.getNumExplicitOperands() == 3); 5546 MachineOperand Op1 = Inst.getOperand(1); 5547 Inst.removeOperand(1); 5548 Inst.addOperand(Op1); 5549 } 5550 5551 bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo &MRI, 5552 const MCOperandInfo &OpInfo, 5553 const MachineOperand &MO) const { 5554 if (!MO.isReg()) 5555 return false; 5556 5557 Register Reg = MO.getReg(); 5558 5559 const TargetRegisterClass *DRC = RI.getRegClass(OpInfo.RegClass); 5560 if (Reg.isPhysical()) 5561 return DRC->contains(Reg); 5562 5563 const TargetRegisterClass *RC = MRI.getRegClass(Reg); 5564 5565 if (MO.getSubReg()) { 5566 const MachineFunction *MF = MO.getParent()->getParent()->getParent(); 5567 const TargetRegisterClass *SuperRC = RI.getLargestLegalSuperClass(RC, *MF); 5568 if (!SuperRC) 5569 return false; 5570 5571 DRC = RI.getMatchingSuperRegClass(SuperRC, DRC, MO.getSubReg()); 5572 if (!DRC) 5573 return false; 5574 } 5575 return RC->hasSuperClassEq(DRC); 5576 } 5577 5578 bool SIInstrInfo::isLegalVSrcOperand(const MachineRegisterInfo &MRI, 5579 const MCOperandInfo &OpInfo, 5580 const MachineOperand &MO) const { 5581 if (MO.isReg()) 5582 return isLegalRegOperand(MRI, OpInfo, MO); 5583 5584 // Handle non-register types that are treated like immediates. 5585 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal()); 5586 return true; 5587 } 5588 5589 bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx, 5590 const MachineOperand *MO) const { 5591 const MachineFunction &MF = *MI.getParent()->getParent(); 5592 const MachineRegisterInfo &MRI = MF.getRegInfo(); 5593 const MCInstrDesc &InstDesc = MI.getDesc(); 5594 const MCOperandInfo &OpInfo = InstDesc.operands()[OpIdx]; 5595 const TargetRegisterClass *DefinedRC = 5596 OpInfo.RegClass != -1 ? RI.getRegClass(OpInfo.RegClass) : nullptr; 5597 if (!MO) 5598 MO = &MI.getOperand(OpIdx); 5599 5600 int ConstantBusLimit = ST.getConstantBusLimit(MI.getOpcode()); 5601 int LiteralLimit = !isVOP3(MI) || ST.hasVOP3Literal() ? 1 : 0; 5602 if (isVALU(MI) && usesConstantBus(MRI, *MO, OpInfo)) { 5603 if (!MO->isReg() && !isInlineConstant(*MO, OpInfo) && !LiteralLimit--) 5604 return false; 5605 5606 SmallDenseSet<RegSubRegPair> SGPRsUsed; 5607 if (MO->isReg()) 5608 SGPRsUsed.insert(RegSubRegPair(MO->getReg(), MO->getSubReg())); 5609 5610 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 5611 if (i == OpIdx) 5612 continue; 5613 const MachineOperand &Op = MI.getOperand(i); 5614 if (Op.isReg()) { 5615 RegSubRegPair SGPR(Op.getReg(), Op.getSubReg()); 5616 if (!SGPRsUsed.count(SGPR) && 5617 // FIXME: This can access off the end of the operands() array. 5618 usesConstantBus(MRI, Op, InstDesc.operands().begin()[i])) { 5619 if (--ConstantBusLimit <= 0) 5620 return false; 5621 SGPRsUsed.insert(SGPR); 5622 } 5623 } else if (AMDGPU::isSISrcOperand(InstDesc, i) && 5624 !isInlineConstant(Op, InstDesc.operands()[i])) { 5625 if (!LiteralLimit--) 5626 return false; 5627 if (--ConstantBusLimit <= 0) 5628 return false; 5629 } 5630 } 5631 } 5632 5633 if (MO->isReg()) { 5634 if (!DefinedRC) 5635 return OpInfo.OperandType == MCOI::OPERAND_UNKNOWN; 5636 if (!isLegalRegOperand(MRI, OpInfo, *MO)) 5637 return false; 5638 bool IsAGPR = RI.isAGPR(MRI, MO->getReg()); 5639 if (IsAGPR && !ST.hasMAIInsts()) 5640 return false; 5641 unsigned Opc = MI.getOpcode(); 5642 if (IsAGPR && 5643 (!ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) && 5644 (MI.mayLoad() || MI.mayStore() || isDS(Opc) || isMIMG(Opc))) 5645 return false; 5646 // Atomics should have both vdst and vdata either vgpr or agpr. 5647 const int VDstIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 5648 const int DataIdx = AMDGPU::getNamedOperandIdx(Opc, 5649 isDS(Opc) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata); 5650 if ((int)OpIdx == VDstIdx && DataIdx != -1 && 5651 MI.getOperand(DataIdx).isReg() && 5652 RI.isAGPR(MRI, MI.getOperand(DataIdx).getReg()) != IsAGPR) 5653 return false; 5654 if ((int)OpIdx == DataIdx) { 5655 if (VDstIdx != -1 && 5656 RI.isAGPR(MRI, MI.getOperand(VDstIdx).getReg()) != IsAGPR) 5657 return false; 5658 // DS instructions with 2 src operands also must have tied RC. 5659 const int Data1Idx = AMDGPU::getNamedOperandIdx(Opc, 5660 AMDGPU::OpName::data1); 5661 if (Data1Idx != -1 && MI.getOperand(Data1Idx).isReg() && 5662 RI.isAGPR(MRI, MI.getOperand(Data1Idx).getReg()) != IsAGPR) 5663 return false; 5664 } 5665 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && !ST.hasGFX90AInsts() && 5666 (int)OpIdx == AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) && 5667 RI.isSGPRReg(MRI, MO->getReg())) 5668 return false; 5669 return true; 5670 } 5671 5672 if (MO->isImm()) { 5673 uint64_t Imm = MO->getImm(); 5674 bool Is64BitFPOp = OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_FP64; 5675 bool Is64BitOp = Is64BitFPOp || 5676 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_INT64 || 5677 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2INT32 || 5678 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2FP32; 5679 if (Is64BitOp && 5680 !AMDGPU::isInlinableLiteral64(Imm, ST.hasInv2PiInlineImm())) { 5681 if (!AMDGPU::isValid32BitLiteral(Imm, Is64BitFPOp)) 5682 return false; 5683 5684 // FIXME: We can use sign extended 64-bit literals, but only for signed 5685 // operands. At the moment we do not know if an operand is signed. 5686 // Such operand will be encoded as its low 32 bits and then either 5687 // correctly sign extended or incorrectly zero extended by HW. 5688 if (!Is64BitFPOp && (int32_t)Imm < 0) 5689 return false; 5690 } 5691 } 5692 5693 // Handle non-register types that are treated like immediates. 5694 assert(MO->isImm() || MO->isTargetIndex() || MO->isFI() || MO->isGlobal()); 5695 5696 if (!DefinedRC) { 5697 // This operand expects an immediate. 5698 return true; 5699 } 5700 5701 return isImmOperandLegal(MI, OpIdx, *MO); 5702 } 5703 5704 void SIInstrInfo::legalizeOperandsVOP2(MachineRegisterInfo &MRI, 5705 MachineInstr &MI) const { 5706 unsigned Opc = MI.getOpcode(); 5707 const MCInstrDesc &InstrDesc = get(Opc); 5708 5709 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); 5710 MachineOperand &Src0 = MI.getOperand(Src0Idx); 5711 5712 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); 5713 MachineOperand &Src1 = MI.getOperand(Src1Idx); 5714 5715 // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32 5716 // we need to only have one constant bus use before GFX10. 5717 bool HasImplicitSGPR = findImplicitSGPRRead(MI); 5718 if (HasImplicitSGPR && ST.getConstantBusLimit(Opc) <= 1 && Src0.isReg() && 5719 RI.isSGPRReg(MRI, Src0.getReg())) 5720 legalizeOpWithMove(MI, Src0Idx); 5721 5722 // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for 5723 // both the value to write (src0) and lane select (src1). Fix up non-SGPR 5724 // src0/src1 with V_READFIRSTLANE. 5725 if (Opc == AMDGPU::V_WRITELANE_B32) { 5726 const DebugLoc &DL = MI.getDebugLoc(); 5727 if (Src0.isReg() && RI.isVGPR(MRI, Src0.getReg())) { 5728 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 5729 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 5730 .add(Src0); 5731 Src0.ChangeToRegister(Reg, false); 5732 } 5733 if (Src1.isReg() && RI.isVGPR(MRI, Src1.getReg())) { 5734 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 5735 const DebugLoc &DL = MI.getDebugLoc(); 5736 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 5737 .add(Src1); 5738 Src1.ChangeToRegister(Reg, false); 5739 } 5740 return; 5741 } 5742 5743 // No VOP2 instructions support AGPRs. 5744 if (Src0.isReg() && RI.isAGPR(MRI, Src0.getReg())) 5745 legalizeOpWithMove(MI, Src0Idx); 5746 5747 if (Src1.isReg() && RI.isAGPR(MRI, Src1.getReg())) 5748 legalizeOpWithMove(MI, Src1Idx); 5749 5750 // Special case: V_FMAC_F32 and V_FMAC_F16 have src2. 5751 if (Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F16_e32) { 5752 int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); 5753 if (!RI.isVGPR(MRI, MI.getOperand(Src2Idx).getReg())) 5754 legalizeOpWithMove(MI, Src2Idx); 5755 } 5756 5757 // VOP2 src0 instructions support all operand types, so we don't need to check 5758 // their legality. If src1 is already legal, we don't need to do anything. 5759 if (isLegalRegOperand(MRI, InstrDesc.operands()[Src1Idx], Src1)) 5760 return; 5761 5762 // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for 5763 // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane 5764 // select is uniform. 5765 if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() && 5766 RI.isVGPR(MRI, Src1.getReg())) { 5767 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 5768 const DebugLoc &DL = MI.getDebugLoc(); 5769 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 5770 .add(Src1); 5771 Src1.ChangeToRegister(Reg, false); 5772 return; 5773 } 5774 5775 // We do not use commuteInstruction here because it is too aggressive and will 5776 // commute if it is possible. We only want to commute here if it improves 5777 // legality. This can be called a fairly large number of times so don't waste 5778 // compile time pointlessly swapping and checking legality again. 5779 if (HasImplicitSGPR || !MI.isCommutable()) { 5780 legalizeOpWithMove(MI, Src1Idx); 5781 return; 5782 } 5783 5784 // If src0 can be used as src1, commuting will make the operands legal. 5785 // Otherwise we have to give up and insert a move. 5786 // 5787 // TODO: Other immediate-like operand kinds could be commuted if there was a 5788 // MachineOperand::ChangeTo* for them. 5789 if ((!Src1.isImm() && !Src1.isReg()) || 5790 !isLegalRegOperand(MRI, InstrDesc.operands()[Src1Idx], Src0)) { 5791 legalizeOpWithMove(MI, Src1Idx); 5792 return; 5793 } 5794 5795 int CommutedOpc = commuteOpcode(MI); 5796 if (CommutedOpc == -1) { 5797 legalizeOpWithMove(MI, Src1Idx); 5798 return; 5799 } 5800 5801 MI.setDesc(get(CommutedOpc)); 5802 5803 Register Src0Reg = Src0.getReg(); 5804 unsigned Src0SubReg = Src0.getSubReg(); 5805 bool Src0Kill = Src0.isKill(); 5806 5807 if (Src1.isImm()) 5808 Src0.ChangeToImmediate(Src1.getImm()); 5809 else if (Src1.isReg()) { 5810 Src0.ChangeToRegister(Src1.getReg(), false, false, Src1.isKill()); 5811 Src0.setSubReg(Src1.getSubReg()); 5812 } else 5813 llvm_unreachable("Should only have register or immediate operands"); 5814 5815 Src1.ChangeToRegister(Src0Reg, false, false, Src0Kill); 5816 Src1.setSubReg(Src0SubReg); 5817 fixImplicitOperands(MI); 5818 } 5819 5820 // Legalize VOP3 operands. All operand types are supported for any operand 5821 // but only one literal constant and only starting from GFX10. 5822 void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI, 5823 MachineInstr &MI) const { 5824 unsigned Opc = MI.getOpcode(); 5825 5826 int VOP3Idx[3] = { 5827 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0), 5828 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1), 5829 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2) 5830 }; 5831 5832 if (Opc == AMDGPU::V_PERMLANE16_B32_e64 || 5833 Opc == AMDGPU::V_PERMLANEX16_B32_e64) { 5834 // src1 and src2 must be scalar 5835 MachineOperand &Src1 = MI.getOperand(VOP3Idx[1]); 5836 MachineOperand &Src2 = MI.getOperand(VOP3Idx[2]); 5837 const DebugLoc &DL = MI.getDebugLoc(); 5838 if (Src1.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src1.getReg()))) { 5839 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 5840 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 5841 .add(Src1); 5842 Src1.ChangeToRegister(Reg, false); 5843 } 5844 if (Src2.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src2.getReg()))) { 5845 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 5846 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 5847 .add(Src2); 5848 Src2.ChangeToRegister(Reg, false); 5849 } 5850 } 5851 5852 // Find the one SGPR operand we are allowed to use. 5853 int ConstantBusLimit = ST.getConstantBusLimit(Opc); 5854 int LiteralLimit = ST.hasVOP3Literal() ? 1 : 0; 5855 SmallDenseSet<unsigned> SGPRsUsed; 5856 Register SGPRReg = findUsedSGPR(MI, VOP3Idx); 5857 if (SGPRReg) { 5858 SGPRsUsed.insert(SGPRReg); 5859 --ConstantBusLimit; 5860 } 5861 5862 for (int Idx : VOP3Idx) { 5863 if (Idx == -1) 5864 break; 5865 MachineOperand &MO = MI.getOperand(Idx); 5866 5867 if (!MO.isReg()) { 5868 if (isInlineConstant(MO, get(Opc).operands()[Idx])) 5869 continue; 5870 5871 if (LiteralLimit > 0 && ConstantBusLimit > 0) { 5872 --LiteralLimit; 5873 --ConstantBusLimit; 5874 continue; 5875 } 5876 5877 --LiteralLimit; 5878 --ConstantBusLimit; 5879 legalizeOpWithMove(MI, Idx); 5880 continue; 5881 } 5882 5883 if (RI.hasAGPRs(RI.getRegClassForReg(MRI, MO.getReg())) && 5884 !isOperandLegal(MI, Idx, &MO)) { 5885 legalizeOpWithMove(MI, Idx); 5886 continue; 5887 } 5888 5889 if (!RI.isSGPRClass(RI.getRegClassForReg(MRI, MO.getReg()))) 5890 continue; // VGPRs are legal 5891 5892 // We can use one SGPR in each VOP3 instruction prior to GFX10 5893 // and two starting from GFX10. 5894 if (SGPRsUsed.count(MO.getReg())) 5895 continue; 5896 if (ConstantBusLimit > 0) { 5897 SGPRsUsed.insert(MO.getReg()); 5898 --ConstantBusLimit; 5899 continue; 5900 } 5901 5902 // If we make it this far, then the operand is not legal and we must 5903 // legalize it. 5904 legalizeOpWithMove(MI, Idx); 5905 } 5906 5907 // Special case: V_FMAC_F32 and V_FMAC_F16 have src2 tied to vdst. 5908 if ((Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_e64) && 5909 !RI.isVGPR(MRI, MI.getOperand(VOP3Idx[2]).getReg())) 5910 legalizeOpWithMove(MI, VOP3Idx[2]); 5911 } 5912 5913 Register SIInstrInfo::readlaneVGPRToSGPR(Register SrcReg, MachineInstr &UseMI, 5914 MachineRegisterInfo &MRI) const { 5915 const TargetRegisterClass *VRC = MRI.getRegClass(SrcReg); 5916 const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC); 5917 Register DstReg = MRI.createVirtualRegister(SRC); 5918 unsigned SubRegs = RI.getRegSizeInBits(*VRC) / 32; 5919 5920 if (RI.hasAGPRs(VRC)) { 5921 VRC = RI.getEquivalentVGPRClass(VRC); 5922 Register NewSrcReg = MRI.createVirtualRegister(VRC); 5923 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), 5924 get(TargetOpcode::COPY), NewSrcReg) 5925 .addReg(SrcReg); 5926 SrcReg = NewSrcReg; 5927 } 5928 5929 if (SubRegs == 1) { 5930 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), 5931 get(AMDGPU::V_READFIRSTLANE_B32), DstReg) 5932 .addReg(SrcReg); 5933 return DstReg; 5934 } 5935 5936 SmallVector<Register, 8> SRegs; 5937 for (unsigned i = 0; i < SubRegs; ++i) { 5938 Register SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 5939 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), 5940 get(AMDGPU::V_READFIRSTLANE_B32), SGPR) 5941 .addReg(SrcReg, 0, RI.getSubRegFromChannel(i)); 5942 SRegs.push_back(SGPR); 5943 } 5944 5945 MachineInstrBuilder MIB = 5946 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), 5947 get(AMDGPU::REG_SEQUENCE), DstReg); 5948 for (unsigned i = 0; i < SubRegs; ++i) { 5949 MIB.addReg(SRegs[i]); 5950 MIB.addImm(RI.getSubRegFromChannel(i)); 5951 } 5952 return DstReg; 5953 } 5954 5955 void SIInstrInfo::legalizeOperandsSMRD(MachineRegisterInfo &MRI, 5956 MachineInstr &MI) const { 5957 5958 // If the pointer is store in VGPRs, then we need to move them to 5959 // SGPRs using v_readfirstlane. This is safe because we only select 5960 // loads with uniform pointers to SMRD instruction so we know the 5961 // pointer value is uniform. 5962 MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase); 5963 if (SBase && !RI.isSGPRClass(MRI.getRegClass(SBase->getReg()))) { 5964 Register SGPR = readlaneVGPRToSGPR(SBase->getReg(), MI, MRI); 5965 SBase->setReg(SGPR); 5966 } 5967 MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soffset); 5968 if (SOff && !RI.isSGPRClass(MRI.getRegClass(SOff->getReg()))) { 5969 Register SGPR = readlaneVGPRToSGPR(SOff->getReg(), MI, MRI); 5970 SOff->setReg(SGPR); 5971 } 5972 } 5973 5974 bool SIInstrInfo::moveFlatAddrToVGPR(MachineInstr &Inst) const { 5975 unsigned Opc = Inst.getOpcode(); 5976 int OldSAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr); 5977 if (OldSAddrIdx < 0) 5978 return false; 5979 5980 assert(isSegmentSpecificFLAT(Inst)); 5981 5982 int NewOpc = AMDGPU::getGlobalVaddrOp(Opc); 5983 if (NewOpc < 0) 5984 NewOpc = AMDGPU::getFlatScratchInstSVfromSS(Opc); 5985 if (NewOpc < 0) 5986 return false; 5987 5988 MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo(); 5989 MachineOperand &SAddr = Inst.getOperand(OldSAddrIdx); 5990 if (RI.isSGPRReg(MRI, SAddr.getReg())) 5991 return false; 5992 5993 int NewVAddrIdx = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr); 5994 if (NewVAddrIdx < 0) 5995 return false; 5996 5997 int OldVAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr); 5998 5999 // Check vaddr, it shall be zero or absent. 6000 MachineInstr *VAddrDef = nullptr; 6001 if (OldVAddrIdx >= 0) { 6002 MachineOperand &VAddr = Inst.getOperand(OldVAddrIdx); 6003 VAddrDef = MRI.getUniqueVRegDef(VAddr.getReg()); 6004 if (!VAddrDef || VAddrDef->getOpcode() != AMDGPU::V_MOV_B32_e32 || 6005 !VAddrDef->getOperand(1).isImm() || 6006 VAddrDef->getOperand(1).getImm() != 0) 6007 return false; 6008 } 6009 6010 const MCInstrDesc &NewDesc = get(NewOpc); 6011 Inst.setDesc(NewDesc); 6012 6013 // Callers expect iterator to be valid after this call, so modify the 6014 // instruction in place. 6015 if (OldVAddrIdx == NewVAddrIdx) { 6016 MachineOperand &NewVAddr = Inst.getOperand(NewVAddrIdx); 6017 // Clear use list from the old vaddr holding a zero register. 6018 MRI.removeRegOperandFromUseList(&NewVAddr); 6019 MRI.moveOperands(&NewVAddr, &SAddr, 1); 6020 Inst.removeOperand(OldSAddrIdx); 6021 // Update the use list with the pointer we have just moved from vaddr to 6022 // saddr position. Otherwise new vaddr will be missing from the use list. 6023 MRI.removeRegOperandFromUseList(&NewVAddr); 6024 MRI.addRegOperandToUseList(&NewVAddr); 6025 } else { 6026 assert(OldSAddrIdx == NewVAddrIdx); 6027 6028 if (OldVAddrIdx >= 0) { 6029 int NewVDstIn = AMDGPU::getNamedOperandIdx(NewOpc, 6030 AMDGPU::OpName::vdst_in); 6031 6032 // removeOperand doesn't try to fixup tied operand indexes at it goes, so 6033 // it asserts. Untie the operands for now and retie them afterwards. 6034 if (NewVDstIn != -1) { 6035 int OldVDstIn = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst_in); 6036 Inst.untieRegOperand(OldVDstIn); 6037 } 6038 6039 Inst.removeOperand(OldVAddrIdx); 6040 6041 if (NewVDstIn != -1) { 6042 int NewVDst = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vdst); 6043 Inst.tieOperands(NewVDst, NewVDstIn); 6044 } 6045 } 6046 } 6047 6048 if (VAddrDef && MRI.use_nodbg_empty(VAddrDef->getOperand(0).getReg())) 6049 VAddrDef->eraseFromParent(); 6050 6051 return true; 6052 } 6053 6054 // FIXME: Remove this when SelectionDAG is obsoleted. 6055 void SIInstrInfo::legalizeOperandsFLAT(MachineRegisterInfo &MRI, 6056 MachineInstr &MI) const { 6057 if (!isSegmentSpecificFLAT(MI)) 6058 return; 6059 6060 // Fixup SGPR operands in VGPRs. We only select these when the DAG divergence 6061 // thinks they are uniform, so a readfirstlane should be valid. 6062 MachineOperand *SAddr = getNamedOperand(MI, AMDGPU::OpName::saddr); 6063 if (!SAddr || RI.isSGPRClass(MRI.getRegClass(SAddr->getReg()))) 6064 return; 6065 6066 if (moveFlatAddrToVGPR(MI)) 6067 return; 6068 6069 Register ToSGPR = readlaneVGPRToSGPR(SAddr->getReg(), MI, MRI); 6070 SAddr->setReg(ToSGPR); 6071 } 6072 6073 void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB, 6074 MachineBasicBlock::iterator I, 6075 const TargetRegisterClass *DstRC, 6076 MachineOperand &Op, 6077 MachineRegisterInfo &MRI, 6078 const DebugLoc &DL) const { 6079 Register OpReg = Op.getReg(); 6080 unsigned OpSubReg = Op.getSubReg(); 6081 6082 const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg( 6083 RI.getRegClassForReg(MRI, OpReg), OpSubReg); 6084 6085 // Check if operand is already the correct register class. 6086 if (DstRC == OpRC) 6087 return; 6088 6089 Register DstReg = MRI.createVirtualRegister(DstRC); 6090 auto Copy = BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).add(Op); 6091 6092 Op.setReg(DstReg); 6093 Op.setSubReg(0); 6094 6095 MachineInstr *Def = MRI.getVRegDef(OpReg); 6096 if (!Def) 6097 return; 6098 6099 // Try to eliminate the copy if it is copying an immediate value. 6100 if (Def->isMoveImmediate() && DstRC != &AMDGPU::VReg_1RegClass) 6101 FoldImmediate(*Copy, *Def, OpReg, &MRI); 6102 6103 bool ImpDef = Def->isImplicitDef(); 6104 while (!ImpDef && Def && Def->isCopy()) { 6105 if (Def->getOperand(1).getReg().isPhysical()) 6106 break; 6107 Def = MRI.getUniqueVRegDef(Def->getOperand(1).getReg()); 6108 ImpDef = Def && Def->isImplicitDef(); 6109 } 6110 if (!RI.isSGPRClass(DstRC) && !Copy->readsRegister(AMDGPU::EXEC, &RI) && 6111 !ImpDef) 6112 Copy.addReg(AMDGPU::EXEC, RegState::Implicit); 6113 } 6114 6115 // Emit the actual waterfall loop, executing the wrapped instruction for each 6116 // unique value of \p ScalarOps across all lanes. In the best case we execute 1 6117 // iteration, in the worst case we execute 64 (once per lane). 6118 static void emitLoadScalarOpsFromVGPRLoop( 6119 const SIInstrInfo &TII, MachineRegisterInfo &MRI, MachineBasicBlock &OrigBB, 6120 MachineBasicBlock &LoopBB, MachineBasicBlock &BodyBB, const DebugLoc &DL, 6121 ArrayRef<MachineOperand *> ScalarOps) { 6122 MachineFunction &MF = *OrigBB.getParent(); 6123 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 6124 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 6125 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 6126 unsigned SaveExecOpc = 6127 ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64; 6128 unsigned XorTermOpc = 6129 ST.isWave32() ? AMDGPU::S_XOR_B32_term : AMDGPU::S_XOR_B64_term; 6130 unsigned AndOpc = 6131 ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64; 6132 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 6133 6134 MachineBasicBlock::iterator I = LoopBB.begin(); 6135 6136 SmallVector<Register, 8> ReadlanePieces; 6137 Register CondReg; 6138 6139 for (MachineOperand *ScalarOp : ScalarOps) { 6140 unsigned RegSize = TRI->getRegSizeInBits(ScalarOp->getReg(), MRI); 6141 unsigned NumSubRegs = RegSize / 32; 6142 Register VScalarOp = ScalarOp->getReg(); 6143 6144 if (NumSubRegs == 1) { 6145 Register CurReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 6146 6147 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurReg) 6148 .addReg(VScalarOp); 6149 6150 Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC); 6151 6152 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U32_e64), NewCondReg) 6153 .addReg(CurReg) 6154 .addReg(VScalarOp); 6155 6156 // Combine the comparison results with AND. 6157 if (!CondReg) // First. 6158 CondReg = NewCondReg; 6159 else { // If not the first, we create an AND. 6160 Register AndReg = MRI.createVirtualRegister(BoolXExecRC); 6161 BuildMI(LoopBB, I, DL, TII.get(AndOpc), AndReg) 6162 .addReg(CondReg) 6163 .addReg(NewCondReg); 6164 CondReg = AndReg; 6165 } 6166 6167 // Update ScalarOp operand to use the SGPR ScalarOp. 6168 ScalarOp->setReg(CurReg); 6169 ScalarOp->setIsKill(); 6170 } else { 6171 unsigned VScalarOpUndef = getUndefRegState(ScalarOp->isUndef()); 6172 assert(NumSubRegs % 2 == 0 && NumSubRegs <= 32 && 6173 "Unhandled register size"); 6174 6175 for (unsigned Idx = 0; Idx < NumSubRegs; Idx += 2) { 6176 Register CurRegLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 6177 Register CurRegHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 6178 6179 // Read the next variant <- also loop target. 6180 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegLo) 6181 .addReg(VScalarOp, VScalarOpUndef, TRI->getSubRegFromChannel(Idx)); 6182 6183 // Read the next variant <- also loop target. 6184 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegHi) 6185 .addReg(VScalarOp, VScalarOpUndef, 6186 TRI->getSubRegFromChannel(Idx + 1)); 6187 6188 ReadlanePieces.push_back(CurRegLo); 6189 ReadlanePieces.push_back(CurRegHi); 6190 6191 // Comparison is to be done as 64-bit. 6192 Register CurReg = MRI.createVirtualRegister(&AMDGPU::SGPR_64RegClass); 6193 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), CurReg) 6194 .addReg(CurRegLo) 6195 .addImm(AMDGPU::sub0) 6196 .addReg(CurRegHi) 6197 .addImm(AMDGPU::sub1); 6198 6199 Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC); 6200 auto Cmp = BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), 6201 NewCondReg) 6202 .addReg(CurReg); 6203 if (NumSubRegs <= 2) 6204 Cmp.addReg(VScalarOp); 6205 else 6206 Cmp.addReg(VScalarOp, VScalarOpUndef, 6207 TRI->getSubRegFromChannel(Idx, 2)); 6208 6209 // Combine the comparison results with AND. 6210 if (!CondReg) // First. 6211 CondReg = NewCondReg; 6212 else { // If not the first, we create an AND. 6213 Register AndReg = MRI.createVirtualRegister(BoolXExecRC); 6214 BuildMI(LoopBB, I, DL, TII.get(AndOpc), AndReg) 6215 .addReg(CondReg) 6216 .addReg(NewCondReg); 6217 CondReg = AndReg; 6218 } 6219 } // End for loop. 6220 6221 auto SScalarOpRC = 6222 TRI->getEquivalentSGPRClass(MRI.getRegClass(VScalarOp)); 6223 Register SScalarOp = MRI.createVirtualRegister(SScalarOpRC); 6224 6225 // Build scalar ScalarOp. 6226 auto Merge = 6227 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SScalarOp); 6228 unsigned Channel = 0; 6229 for (Register Piece : ReadlanePieces) { 6230 Merge.addReg(Piece).addImm(TRI->getSubRegFromChannel(Channel++)); 6231 } 6232 6233 // Update ScalarOp operand to use the SGPR ScalarOp. 6234 ScalarOp->setReg(SScalarOp); 6235 ScalarOp->setIsKill(); 6236 } 6237 } 6238 6239 Register SaveExec = MRI.createVirtualRegister(BoolXExecRC); 6240 MRI.setSimpleHint(SaveExec, CondReg); 6241 6242 // Update EXEC to matching lanes, saving original to SaveExec. 6243 BuildMI(LoopBB, I, DL, TII.get(SaveExecOpc), SaveExec) 6244 .addReg(CondReg, RegState::Kill); 6245 6246 // The original instruction is here; we insert the terminators after it. 6247 I = BodyBB.end(); 6248 6249 // Update EXEC, switch all done bits to 0 and all todo bits to 1. 6250 BuildMI(BodyBB, I, DL, TII.get(XorTermOpc), Exec) 6251 .addReg(Exec) 6252 .addReg(SaveExec); 6253 6254 BuildMI(BodyBB, I, DL, TII.get(AMDGPU::SI_WATERFALL_LOOP)).addMBB(&LoopBB); 6255 } 6256 6257 // Build a waterfall loop around \p MI, replacing the VGPR \p ScalarOp register 6258 // with SGPRs by iterating over all unique values across all lanes. 6259 // Returns the loop basic block that now contains \p MI. 6260 static MachineBasicBlock * 6261 loadMBUFScalarOperandsFromVGPR(const SIInstrInfo &TII, MachineInstr &MI, 6262 ArrayRef<MachineOperand *> ScalarOps, 6263 MachineDominatorTree *MDT, 6264 MachineBasicBlock::iterator Begin = nullptr, 6265 MachineBasicBlock::iterator End = nullptr) { 6266 MachineBasicBlock &MBB = *MI.getParent(); 6267 MachineFunction &MF = *MBB.getParent(); 6268 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 6269 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 6270 MachineRegisterInfo &MRI = MF.getRegInfo(); 6271 if (!Begin.isValid()) 6272 Begin = &MI; 6273 if (!End.isValid()) { 6274 End = &MI; 6275 ++End; 6276 } 6277 const DebugLoc &DL = MI.getDebugLoc(); 6278 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 6279 unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; 6280 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 6281 6282 // Save SCC. Waterfall Loop may overwrite SCC. 6283 Register SaveSCCReg; 6284 bool SCCNotDead = (MBB.computeRegisterLiveness(TRI, AMDGPU::SCC, MI, 30) != 6285 MachineBasicBlock::LQR_Dead); 6286 if (SCCNotDead) { 6287 SaveSCCReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 6288 BuildMI(MBB, Begin, DL, TII.get(AMDGPU::S_CSELECT_B32), SaveSCCReg) 6289 .addImm(1) 6290 .addImm(0); 6291 } 6292 6293 Register SaveExec = MRI.createVirtualRegister(BoolXExecRC); 6294 6295 // Save the EXEC mask 6296 BuildMI(MBB, Begin, DL, TII.get(MovExecOpc), SaveExec).addReg(Exec); 6297 6298 // Killed uses in the instruction we are waterfalling around will be 6299 // incorrect due to the added control-flow. 6300 MachineBasicBlock::iterator AfterMI = MI; 6301 ++AfterMI; 6302 for (auto I = Begin; I != AfterMI; I++) { 6303 for (auto &MO : I->all_uses()) 6304 MRI.clearKillFlags(MO.getReg()); 6305 } 6306 6307 // To insert the loop we need to split the block. Move everything after this 6308 // point to a new block, and insert a new empty block between the two. 6309 MachineBasicBlock *LoopBB = MF.CreateMachineBasicBlock(); 6310 MachineBasicBlock *BodyBB = MF.CreateMachineBasicBlock(); 6311 MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock(); 6312 MachineFunction::iterator MBBI(MBB); 6313 ++MBBI; 6314 6315 MF.insert(MBBI, LoopBB); 6316 MF.insert(MBBI, BodyBB); 6317 MF.insert(MBBI, RemainderBB); 6318 6319 LoopBB->addSuccessor(BodyBB); 6320 BodyBB->addSuccessor(LoopBB); 6321 BodyBB->addSuccessor(RemainderBB); 6322 6323 // Move Begin to MI to the BodyBB, and the remainder of the block to 6324 // RemainderBB. 6325 RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB); 6326 RemainderBB->splice(RemainderBB->begin(), &MBB, End, MBB.end()); 6327 BodyBB->splice(BodyBB->begin(), &MBB, Begin, MBB.end()); 6328 6329 MBB.addSuccessor(LoopBB); 6330 6331 // Update dominators. We know that MBB immediately dominates LoopBB, that 6332 // LoopBB immediately dominates BodyBB, and BodyBB immediately dominates 6333 // RemainderBB. RemainderBB immediately dominates all of the successors 6334 // transferred to it from MBB that MBB used to properly dominate. 6335 if (MDT) { 6336 MDT->addNewBlock(LoopBB, &MBB); 6337 MDT->addNewBlock(BodyBB, LoopBB); 6338 MDT->addNewBlock(RemainderBB, BodyBB); 6339 for (auto &Succ : RemainderBB->successors()) { 6340 if (MDT->properlyDominates(&MBB, Succ)) { 6341 MDT->changeImmediateDominator(Succ, RemainderBB); 6342 } 6343 } 6344 } 6345 6346 emitLoadScalarOpsFromVGPRLoop(TII, MRI, MBB, *LoopBB, *BodyBB, DL, ScalarOps); 6347 6348 MachineBasicBlock::iterator First = RemainderBB->begin(); 6349 // Restore SCC 6350 if (SCCNotDead) { 6351 BuildMI(*RemainderBB, First, DL, TII.get(AMDGPU::S_CMP_LG_U32)) 6352 .addReg(SaveSCCReg, RegState::Kill) 6353 .addImm(0); 6354 } 6355 6356 // Restore the EXEC mask 6357 BuildMI(*RemainderBB, First, DL, TII.get(MovExecOpc), Exec).addReg(SaveExec); 6358 return BodyBB; 6359 } 6360 6361 // Extract pointer from Rsrc and return a zero-value Rsrc replacement. 6362 static std::tuple<unsigned, unsigned> 6363 extractRsrcPtr(const SIInstrInfo &TII, MachineInstr &MI, MachineOperand &Rsrc) { 6364 MachineBasicBlock &MBB = *MI.getParent(); 6365 MachineFunction &MF = *MBB.getParent(); 6366 MachineRegisterInfo &MRI = MF.getRegInfo(); 6367 6368 // Extract the ptr from the resource descriptor. 6369 unsigned RsrcPtr = 6370 TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass, 6371 AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass); 6372 6373 // Create an empty resource descriptor 6374 Register Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); 6375 Register SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 6376 Register SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 6377 Register NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass); 6378 uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat(); 6379 6380 // Zero64 = 0 6381 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64) 6382 .addImm(0); 6383 6384 // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0} 6385 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo) 6386 .addImm(RsrcDataFormat & 0xFFFFFFFF); 6387 6388 // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32} 6389 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi) 6390 .addImm(RsrcDataFormat >> 32); 6391 6392 // NewSRsrc = {Zero64, SRsrcFormat} 6393 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc) 6394 .addReg(Zero64) 6395 .addImm(AMDGPU::sub0_sub1) 6396 .addReg(SRsrcFormatLo) 6397 .addImm(AMDGPU::sub2) 6398 .addReg(SRsrcFormatHi) 6399 .addImm(AMDGPU::sub3); 6400 6401 return std::tuple(RsrcPtr, NewSRsrc); 6402 } 6403 6404 MachineBasicBlock * 6405 SIInstrInfo::legalizeOperands(MachineInstr &MI, 6406 MachineDominatorTree *MDT) const { 6407 MachineFunction &MF = *MI.getParent()->getParent(); 6408 MachineRegisterInfo &MRI = MF.getRegInfo(); 6409 MachineBasicBlock *CreatedBB = nullptr; 6410 6411 // Legalize VOP2 6412 if (isVOP2(MI) || isVOPC(MI)) { 6413 legalizeOperandsVOP2(MRI, MI); 6414 return CreatedBB; 6415 } 6416 6417 // Legalize VOP3 6418 if (isVOP3(MI)) { 6419 legalizeOperandsVOP3(MRI, MI); 6420 return CreatedBB; 6421 } 6422 6423 // Legalize SMRD 6424 if (isSMRD(MI)) { 6425 legalizeOperandsSMRD(MRI, MI); 6426 return CreatedBB; 6427 } 6428 6429 // Legalize FLAT 6430 if (isFLAT(MI)) { 6431 legalizeOperandsFLAT(MRI, MI); 6432 return CreatedBB; 6433 } 6434 6435 // Legalize REG_SEQUENCE and PHI 6436 // The register class of the operands much be the same type as the register 6437 // class of the output. 6438 if (MI.getOpcode() == AMDGPU::PHI) { 6439 const TargetRegisterClass *RC = nullptr, *SRC = nullptr, *VRC = nullptr; 6440 for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) { 6441 if (!MI.getOperand(i).isReg() || !MI.getOperand(i).getReg().isVirtual()) 6442 continue; 6443 const TargetRegisterClass *OpRC = 6444 MRI.getRegClass(MI.getOperand(i).getReg()); 6445 if (RI.hasVectorRegisters(OpRC)) { 6446 VRC = OpRC; 6447 } else { 6448 SRC = OpRC; 6449 } 6450 } 6451 6452 // If any of the operands are VGPR registers, then they all most be 6453 // otherwise we will create illegal VGPR->SGPR copies when legalizing 6454 // them. 6455 if (VRC || !RI.isSGPRClass(getOpRegClass(MI, 0))) { 6456 if (!VRC) { 6457 assert(SRC); 6458 if (getOpRegClass(MI, 0) == &AMDGPU::VReg_1RegClass) { 6459 VRC = &AMDGPU::VReg_1RegClass; 6460 } else 6461 VRC = RI.isAGPRClass(getOpRegClass(MI, 0)) 6462 ? RI.getEquivalentAGPRClass(SRC) 6463 : RI.getEquivalentVGPRClass(SRC); 6464 } else { 6465 VRC = RI.isAGPRClass(getOpRegClass(MI, 0)) 6466 ? RI.getEquivalentAGPRClass(VRC) 6467 : RI.getEquivalentVGPRClass(VRC); 6468 } 6469 RC = VRC; 6470 } else { 6471 RC = SRC; 6472 } 6473 6474 // Update all the operands so they have the same type. 6475 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { 6476 MachineOperand &Op = MI.getOperand(I); 6477 if (!Op.isReg() || !Op.getReg().isVirtual()) 6478 continue; 6479 6480 // MI is a PHI instruction. 6481 MachineBasicBlock *InsertBB = MI.getOperand(I + 1).getMBB(); 6482 MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator(); 6483 6484 // Avoid creating no-op copies with the same src and dst reg class. These 6485 // confuse some of the machine passes. 6486 legalizeGenericOperand(*InsertBB, Insert, RC, Op, MRI, MI.getDebugLoc()); 6487 } 6488 } 6489 6490 // REG_SEQUENCE doesn't really require operand legalization, but if one has a 6491 // VGPR dest type and SGPR sources, insert copies so all operands are 6492 // VGPRs. This seems to help operand folding / the register coalescer. 6493 if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) { 6494 MachineBasicBlock *MBB = MI.getParent(); 6495 const TargetRegisterClass *DstRC = getOpRegClass(MI, 0); 6496 if (RI.hasVGPRs(DstRC)) { 6497 // Update all the operands so they are VGPR register classes. These may 6498 // not be the same register class because REG_SEQUENCE supports mixing 6499 // subregister index types e.g. sub0_sub1 + sub2 + sub3 6500 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { 6501 MachineOperand &Op = MI.getOperand(I); 6502 if (!Op.isReg() || !Op.getReg().isVirtual()) 6503 continue; 6504 6505 const TargetRegisterClass *OpRC = MRI.getRegClass(Op.getReg()); 6506 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(OpRC); 6507 if (VRC == OpRC) 6508 continue; 6509 6510 legalizeGenericOperand(*MBB, MI, VRC, Op, MRI, MI.getDebugLoc()); 6511 Op.setIsKill(); 6512 } 6513 } 6514 6515 return CreatedBB; 6516 } 6517 6518 // Legalize INSERT_SUBREG 6519 // src0 must have the same register class as dst 6520 if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) { 6521 Register Dst = MI.getOperand(0).getReg(); 6522 Register Src0 = MI.getOperand(1).getReg(); 6523 const TargetRegisterClass *DstRC = MRI.getRegClass(Dst); 6524 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0); 6525 if (DstRC != Src0RC) { 6526 MachineBasicBlock *MBB = MI.getParent(); 6527 MachineOperand &Op = MI.getOperand(1); 6528 legalizeGenericOperand(*MBB, MI, DstRC, Op, MRI, MI.getDebugLoc()); 6529 } 6530 return CreatedBB; 6531 } 6532 6533 // Legalize SI_INIT_M0 6534 if (MI.getOpcode() == AMDGPU::SI_INIT_M0) { 6535 MachineOperand &Src = MI.getOperand(0); 6536 if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg()))) 6537 Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI)); 6538 return CreatedBB; 6539 } 6540 6541 // Legalize S_BITREPLICATE, S_QUADMASK and S_WQM 6542 if (MI.getOpcode() == AMDGPU::S_BITREPLICATE_B64_B32 || 6543 MI.getOpcode() == AMDGPU::S_QUADMASK_B32 || 6544 MI.getOpcode() == AMDGPU::S_QUADMASK_B64 || 6545 MI.getOpcode() == AMDGPU::S_WQM_B32 || 6546 MI.getOpcode() == AMDGPU::S_WQM_B64) { 6547 MachineOperand &Src = MI.getOperand(1); 6548 if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg()))) 6549 Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI)); 6550 return CreatedBB; 6551 } 6552 6553 // Legalize MIMG/VIMAGE/VSAMPLE and MUBUF/MTBUF for shaders. 6554 // 6555 // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via 6556 // scratch memory access. In both cases, the legalization never involves 6557 // conversion to the addr64 form. 6558 if (isImage(MI) || (AMDGPU::isGraphics(MF.getFunction().getCallingConv()) && 6559 (isMUBUF(MI) || isMTBUF(MI)))) { 6560 int RSrcOpName = (isVIMAGE(MI) || isVSAMPLE(MI)) ? AMDGPU::OpName::rsrc 6561 : AMDGPU::OpName::srsrc; 6562 MachineOperand *SRsrc = getNamedOperand(MI, RSrcOpName); 6563 if (SRsrc && !RI.isSGPRClass(MRI.getRegClass(SRsrc->getReg()))) 6564 CreatedBB = loadMBUFScalarOperandsFromVGPR(*this, MI, {SRsrc}, MDT); 6565 6566 int SampOpName = isMIMG(MI) ? AMDGPU::OpName::ssamp : AMDGPU::OpName::samp; 6567 MachineOperand *SSamp = getNamedOperand(MI, SampOpName); 6568 if (SSamp && !RI.isSGPRClass(MRI.getRegClass(SSamp->getReg()))) 6569 CreatedBB = loadMBUFScalarOperandsFromVGPR(*this, MI, {SSamp}, MDT); 6570 6571 return CreatedBB; 6572 } 6573 6574 // Legalize SI_CALL 6575 if (MI.getOpcode() == AMDGPU::SI_CALL_ISEL) { 6576 MachineOperand *Dest = &MI.getOperand(0); 6577 if (!RI.isSGPRClass(MRI.getRegClass(Dest->getReg()))) { 6578 // Move everything between ADJCALLSTACKUP and ADJCALLSTACKDOWN and 6579 // following copies, we also need to move copies from and to physical 6580 // registers into the loop block. 6581 unsigned FrameSetupOpcode = getCallFrameSetupOpcode(); 6582 unsigned FrameDestroyOpcode = getCallFrameDestroyOpcode(); 6583 6584 // Also move the copies to physical registers into the loop block 6585 MachineBasicBlock &MBB = *MI.getParent(); 6586 MachineBasicBlock::iterator Start(&MI); 6587 while (Start->getOpcode() != FrameSetupOpcode) 6588 --Start; 6589 MachineBasicBlock::iterator End(&MI); 6590 while (End->getOpcode() != FrameDestroyOpcode) 6591 ++End; 6592 // Also include following copies of the return value 6593 ++End; 6594 while (End != MBB.end() && End->isCopy() && End->getOperand(1).isReg() && 6595 MI.definesRegister(End->getOperand(1).getReg())) 6596 ++End; 6597 CreatedBB = 6598 loadMBUFScalarOperandsFromVGPR(*this, MI, {Dest}, MDT, Start, End); 6599 } 6600 } 6601 6602 // Legalize s_sleep_var. 6603 if (MI.getOpcode() == AMDGPU::S_SLEEP_VAR) { 6604 const DebugLoc &DL = MI.getDebugLoc(); 6605 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 6606 int Src0Idx = 6607 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0); 6608 MachineOperand &Src0 = MI.getOperand(Src0Idx); 6609 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 6610 .add(Src0); 6611 Src0.ChangeToRegister(Reg, false); 6612 return nullptr; 6613 } 6614 6615 // Legalize MUBUF instructions. 6616 bool isSoffsetLegal = true; 6617 int SoffsetIdx = 6618 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::soffset); 6619 if (SoffsetIdx != -1) { 6620 MachineOperand *Soffset = &MI.getOperand(SoffsetIdx); 6621 if (Soffset->isReg() && Soffset->getReg().isVirtual() && 6622 !RI.isSGPRClass(MRI.getRegClass(Soffset->getReg()))) { 6623 isSoffsetLegal = false; 6624 } 6625 } 6626 6627 bool isRsrcLegal = true; 6628 int RsrcIdx = 6629 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc); 6630 if (RsrcIdx != -1) { 6631 MachineOperand *Rsrc = &MI.getOperand(RsrcIdx); 6632 if (Rsrc->isReg() && !RI.isSGPRClass(MRI.getRegClass(Rsrc->getReg()))) { 6633 isRsrcLegal = false; 6634 } 6635 } 6636 6637 // The operands are legal. 6638 if (isRsrcLegal && isSoffsetLegal) 6639 return CreatedBB; 6640 6641 if (!isRsrcLegal) { 6642 // Legalize a VGPR Rsrc 6643 // 6644 // If the instruction is _ADDR64, we can avoid a waterfall by extracting 6645 // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using 6646 // a zero-value SRsrc. 6647 // 6648 // If the instruction is _OFFSET (both idxen and offen disabled), and we 6649 // support ADDR64 instructions, we can convert to ADDR64 and do the same as 6650 // above. 6651 // 6652 // Otherwise we are on non-ADDR64 hardware, and/or we have 6653 // idxen/offen/bothen and we fall back to a waterfall loop. 6654 6655 MachineOperand *Rsrc = &MI.getOperand(RsrcIdx); 6656 MachineBasicBlock &MBB = *MI.getParent(); 6657 6658 MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr); 6659 if (VAddr && AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -1) { 6660 // This is already an ADDR64 instruction so we need to add the pointer 6661 // extracted from the resource descriptor to the current value of VAddr. 6662 Register NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6663 Register NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6664 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 6665 6666 const auto *BoolXExecRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 6667 Register CondReg0 = MRI.createVirtualRegister(BoolXExecRC); 6668 Register CondReg1 = MRI.createVirtualRegister(BoolXExecRC); 6669 6670 unsigned RsrcPtr, NewSRsrc; 6671 std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc); 6672 6673 // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0 6674 const DebugLoc &DL = MI.getDebugLoc(); 6675 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_CO_U32_e64), NewVAddrLo) 6676 .addDef(CondReg0) 6677 .addReg(RsrcPtr, 0, AMDGPU::sub0) 6678 .addReg(VAddr->getReg(), 0, AMDGPU::sub0) 6679 .addImm(0); 6680 6681 // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1 6682 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e64), NewVAddrHi) 6683 .addDef(CondReg1, RegState::Dead) 6684 .addReg(RsrcPtr, 0, AMDGPU::sub1) 6685 .addReg(VAddr->getReg(), 0, AMDGPU::sub1) 6686 .addReg(CondReg0, RegState::Kill) 6687 .addImm(0); 6688 6689 // NewVaddr = {NewVaddrHi, NewVaddrLo} 6690 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr) 6691 .addReg(NewVAddrLo) 6692 .addImm(AMDGPU::sub0) 6693 .addReg(NewVAddrHi) 6694 .addImm(AMDGPU::sub1); 6695 6696 VAddr->setReg(NewVAddr); 6697 Rsrc->setReg(NewSRsrc); 6698 } else if (!VAddr && ST.hasAddr64()) { 6699 // This instructions is the _OFFSET variant, so we need to convert it to 6700 // ADDR64. 6701 assert(ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS && 6702 "FIXME: Need to emit flat atomics here"); 6703 6704 unsigned RsrcPtr, NewSRsrc; 6705 std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc); 6706 6707 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 6708 MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata); 6709 MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset); 6710 MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset); 6711 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode()); 6712 6713 // Atomics with return have an additional tied operand and are 6714 // missing some of the special bits. 6715 MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in); 6716 MachineInstr *Addr64; 6717 6718 if (!VDataIn) { 6719 // Regular buffer load / store. 6720 MachineInstrBuilder MIB = 6721 BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode)) 6722 .add(*VData) 6723 .addReg(NewVAddr) 6724 .addReg(NewSRsrc) 6725 .add(*SOffset) 6726 .add(*Offset); 6727 6728 if (const MachineOperand *CPol = 6729 getNamedOperand(MI, AMDGPU::OpName::cpol)) { 6730 MIB.addImm(CPol->getImm()); 6731 } 6732 6733 if (const MachineOperand *TFE = 6734 getNamedOperand(MI, AMDGPU::OpName::tfe)) { 6735 MIB.addImm(TFE->getImm()); 6736 } 6737 6738 MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::swz)); 6739 6740 MIB.cloneMemRefs(MI); 6741 Addr64 = MIB; 6742 } else { 6743 // Atomics with return. 6744 Addr64 = BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode)) 6745 .add(*VData) 6746 .add(*VDataIn) 6747 .addReg(NewVAddr) 6748 .addReg(NewSRsrc) 6749 .add(*SOffset) 6750 .add(*Offset) 6751 .addImm(getNamedImmOperand(MI, AMDGPU::OpName::cpol)) 6752 .cloneMemRefs(MI); 6753 } 6754 6755 MI.removeFromParent(); 6756 6757 // NewVaddr = {NewVaddrHi, NewVaddrLo} 6758 BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE), 6759 NewVAddr) 6760 .addReg(RsrcPtr, 0, AMDGPU::sub0) 6761 .addImm(AMDGPU::sub0) 6762 .addReg(RsrcPtr, 0, AMDGPU::sub1) 6763 .addImm(AMDGPU::sub1); 6764 } else { 6765 // Legalize a VGPR Rsrc and soffset together. 6766 if (!isSoffsetLegal) { 6767 MachineOperand *Soffset = getNamedOperand(MI, AMDGPU::OpName::soffset); 6768 CreatedBB = 6769 loadMBUFScalarOperandsFromVGPR(*this, MI, {Rsrc, Soffset}, MDT); 6770 return CreatedBB; 6771 } 6772 CreatedBB = loadMBUFScalarOperandsFromVGPR(*this, MI, {Rsrc}, MDT); 6773 return CreatedBB; 6774 } 6775 } 6776 6777 // Legalize a VGPR soffset. 6778 if (!isSoffsetLegal) { 6779 MachineOperand *Soffset = getNamedOperand(MI, AMDGPU::OpName::soffset); 6780 CreatedBB = loadMBUFScalarOperandsFromVGPR(*this, MI, {Soffset}, MDT); 6781 return CreatedBB; 6782 } 6783 return CreatedBB; 6784 } 6785 6786 void SIInstrWorklist::insert(MachineInstr *MI) { 6787 InstrList.insert(MI); 6788 // Add MBUF instructiosn to deferred list. 6789 int RsrcIdx = 6790 AMDGPU::getNamedOperandIdx(MI->getOpcode(), AMDGPU::OpName::srsrc); 6791 if (RsrcIdx != -1) { 6792 DeferredList.insert(MI); 6793 } 6794 } 6795 6796 bool SIInstrWorklist::isDeferred(MachineInstr *MI) { 6797 return DeferredList.contains(MI); 6798 } 6799 6800 void SIInstrInfo::moveToVALU(SIInstrWorklist &Worklist, 6801 MachineDominatorTree *MDT) const { 6802 6803 while (!Worklist.empty()) { 6804 MachineInstr &Inst = *Worklist.top(); 6805 Worklist.erase_top(); 6806 // Skip MachineInstr in the deferred list. 6807 if (Worklist.isDeferred(&Inst)) 6808 continue; 6809 moveToVALUImpl(Worklist, MDT, Inst); 6810 } 6811 6812 // Deferred list of instructions will be processed once 6813 // all the MachineInstr in the worklist are done. 6814 for (MachineInstr *Inst : Worklist.getDeferredList()) { 6815 moveToVALUImpl(Worklist, MDT, *Inst); 6816 assert(Worklist.empty() && 6817 "Deferred MachineInstr are not supposed to re-populate worklist"); 6818 } 6819 } 6820 6821 void SIInstrInfo::moveToVALUImpl(SIInstrWorklist &Worklist, 6822 MachineDominatorTree *MDT, 6823 MachineInstr &Inst) const { 6824 6825 MachineBasicBlock *MBB = Inst.getParent(); 6826 if (!MBB) 6827 return; 6828 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 6829 unsigned Opcode = Inst.getOpcode(); 6830 unsigned NewOpcode = getVALUOp(Inst); 6831 // Handle some special cases 6832 switch (Opcode) { 6833 default: 6834 break; 6835 case AMDGPU::S_ADD_U64_PSEUDO: 6836 NewOpcode = AMDGPU::V_ADD_U64_PSEUDO; 6837 break; 6838 case AMDGPU::S_SUB_U64_PSEUDO: 6839 NewOpcode = AMDGPU::V_SUB_U64_PSEUDO; 6840 break; 6841 case AMDGPU::S_ADD_I32: 6842 case AMDGPU::S_SUB_I32: { 6843 // FIXME: The u32 versions currently selected use the carry. 6844 bool Changed; 6845 MachineBasicBlock *CreatedBBTmp = nullptr; 6846 std::tie(Changed, CreatedBBTmp) = moveScalarAddSub(Worklist, Inst, MDT); 6847 if (Changed) 6848 return; 6849 6850 // Default handling 6851 break; 6852 } 6853 6854 case AMDGPU::S_MUL_U64: 6855 // Split s_mul_u64 in 32-bit vector multiplications. 6856 splitScalarSMulU64(Worklist, Inst, MDT); 6857 Inst.eraseFromParent(); 6858 return; 6859 6860 case AMDGPU::S_MUL_U64_U32_PSEUDO: 6861 case AMDGPU::S_MUL_I64_I32_PSEUDO: 6862 // This is a special case of s_mul_u64 where all the operands are either 6863 // zero extended or sign extended. 6864 splitScalarSMulPseudo(Worklist, Inst, MDT); 6865 Inst.eraseFromParent(); 6866 return; 6867 6868 case AMDGPU::S_AND_B64: 6869 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT); 6870 Inst.eraseFromParent(); 6871 return; 6872 6873 case AMDGPU::S_OR_B64: 6874 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT); 6875 Inst.eraseFromParent(); 6876 return; 6877 6878 case AMDGPU::S_XOR_B64: 6879 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT); 6880 Inst.eraseFromParent(); 6881 return; 6882 6883 case AMDGPU::S_NAND_B64: 6884 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT); 6885 Inst.eraseFromParent(); 6886 return; 6887 6888 case AMDGPU::S_NOR_B64: 6889 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT); 6890 Inst.eraseFromParent(); 6891 return; 6892 6893 case AMDGPU::S_XNOR_B64: 6894 if (ST.hasDLInsts()) 6895 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT); 6896 else 6897 splitScalar64BitXnor(Worklist, Inst, MDT); 6898 Inst.eraseFromParent(); 6899 return; 6900 6901 case AMDGPU::S_ANDN2_B64: 6902 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT); 6903 Inst.eraseFromParent(); 6904 return; 6905 6906 case AMDGPU::S_ORN2_B64: 6907 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT); 6908 Inst.eraseFromParent(); 6909 return; 6910 6911 case AMDGPU::S_BREV_B64: 6912 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_BREV_B32, true); 6913 Inst.eraseFromParent(); 6914 return; 6915 6916 case AMDGPU::S_NOT_B64: 6917 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32); 6918 Inst.eraseFromParent(); 6919 return; 6920 6921 case AMDGPU::S_BCNT1_I32_B64: 6922 splitScalar64BitBCNT(Worklist, Inst); 6923 Inst.eraseFromParent(); 6924 return; 6925 6926 case AMDGPU::S_BFE_I64: 6927 splitScalar64BitBFE(Worklist, Inst); 6928 Inst.eraseFromParent(); 6929 return; 6930 6931 case AMDGPU::S_FLBIT_I32_B64: 6932 splitScalar64BitCountOp(Worklist, Inst, AMDGPU::V_FFBH_U32_e32); 6933 Inst.eraseFromParent(); 6934 return; 6935 case AMDGPU::S_FF1_I32_B64: 6936 splitScalar64BitCountOp(Worklist, Inst, AMDGPU::V_FFBL_B32_e32); 6937 Inst.eraseFromParent(); 6938 return; 6939 6940 case AMDGPU::S_LSHL_B32: 6941 if (ST.hasOnlyRevVALUShifts()) { 6942 NewOpcode = AMDGPU::V_LSHLREV_B32_e64; 6943 swapOperands(Inst); 6944 } 6945 break; 6946 case AMDGPU::S_ASHR_I32: 6947 if (ST.hasOnlyRevVALUShifts()) { 6948 NewOpcode = AMDGPU::V_ASHRREV_I32_e64; 6949 swapOperands(Inst); 6950 } 6951 break; 6952 case AMDGPU::S_LSHR_B32: 6953 if (ST.hasOnlyRevVALUShifts()) { 6954 NewOpcode = AMDGPU::V_LSHRREV_B32_e64; 6955 swapOperands(Inst); 6956 } 6957 break; 6958 case AMDGPU::S_LSHL_B64: 6959 if (ST.hasOnlyRevVALUShifts()) { 6960 NewOpcode = ST.getGeneration() >= AMDGPUSubtarget::GFX12 6961 ? AMDGPU::V_LSHLREV_B64_pseudo_e64 6962 : AMDGPU::V_LSHLREV_B64_e64; 6963 swapOperands(Inst); 6964 } 6965 break; 6966 case AMDGPU::S_ASHR_I64: 6967 if (ST.hasOnlyRevVALUShifts()) { 6968 NewOpcode = AMDGPU::V_ASHRREV_I64_e64; 6969 swapOperands(Inst); 6970 } 6971 break; 6972 case AMDGPU::S_LSHR_B64: 6973 if (ST.hasOnlyRevVALUShifts()) { 6974 NewOpcode = AMDGPU::V_LSHRREV_B64_e64; 6975 swapOperands(Inst); 6976 } 6977 break; 6978 6979 case AMDGPU::S_ABS_I32: 6980 lowerScalarAbs(Worklist, Inst); 6981 Inst.eraseFromParent(); 6982 return; 6983 6984 case AMDGPU::S_CBRANCH_SCC0: 6985 case AMDGPU::S_CBRANCH_SCC1: { 6986 // Clear unused bits of vcc 6987 Register CondReg = Inst.getOperand(1).getReg(); 6988 bool IsSCC = CondReg == AMDGPU::SCC; 6989 Register VCC = RI.getVCC(); 6990 Register EXEC = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 6991 unsigned Opc = ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64; 6992 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(Opc), VCC) 6993 .addReg(EXEC) 6994 .addReg(IsSCC ? VCC : CondReg); 6995 Inst.removeOperand(1); 6996 } break; 6997 6998 case AMDGPU::S_BFE_U64: 6999 case AMDGPU::S_BFM_B64: 7000 llvm_unreachable("Moving this op to VALU not implemented"); 7001 7002 case AMDGPU::S_PACK_LL_B32_B16: 7003 case AMDGPU::S_PACK_LH_B32_B16: 7004 case AMDGPU::S_PACK_HL_B32_B16: 7005 case AMDGPU::S_PACK_HH_B32_B16: 7006 movePackToVALU(Worklist, MRI, Inst); 7007 Inst.eraseFromParent(); 7008 return; 7009 7010 case AMDGPU::S_XNOR_B32: 7011 lowerScalarXnor(Worklist, Inst); 7012 Inst.eraseFromParent(); 7013 return; 7014 7015 case AMDGPU::S_NAND_B32: 7016 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32); 7017 Inst.eraseFromParent(); 7018 return; 7019 7020 case AMDGPU::S_NOR_B32: 7021 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32); 7022 Inst.eraseFromParent(); 7023 return; 7024 7025 case AMDGPU::S_ANDN2_B32: 7026 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32); 7027 Inst.eraseFromParent(); 7028 return; 7029 7030 case AMDGPU::S_ORN2_B32: 7031 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32); 7032 Inst.eraseFromParent(); 7033 return; 7034 7035 // TODO: remove as soon as everything is ready 7036 // to replace VGPR to SGPR copy with V_READFIRSTLANEs. 7037 // S_ADD/SUB_CO_PSEUDO as well as S_UADDO/USUBO_PSEUDO 7038 // can only be selected from the uniform SDNode. 7039 case AMDGPU::S_ADD_CO_PSEUDO: 7040 case AMDGPU::S_SUB_CO_PSEUDO: { 7041 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO) 7042 ? AMDGPU::V_ADDC_U32_e64 7043 : AMDGPU::V_SUBB_U32_e64; 7044 const auto *CarryRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 7045 7046 Register CarryInReg = Inst.getOperand(4).getReg(); 7047 if (!MRI.constrainRegClass(CarryInReg, CarryRC)) { 7048 Register NewCarryReg = MRI.createVirtualRegister(CarryRC); 7049 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::COPY), NewCarryReg) 7050 .addReg(CarryInReg); 7051 } 7052 7053 Register CarryOutReg = Inst.getOperand(1).getReg(); 7054 7055 Register DestReg = MRI.createVirtualRegister(RI.getEquivalentVGPRClass( 7056 MRI.getRegClass(Inst.getOperand(0).getReg()))); 7057 MachineInstr *CarryOp = 7058 BuildMI(*MBB, &Inst, Inst.getDebugLoc(), get(Opc), DestReg) 7059 .addReg(CarryOutReg, RegState::Define) 7060 .add(Inst.getOperand(2)) 7061 .add(Inst.getOperand(3)) 7062 .addReg(CarryInReg) 7063 .addImm(0); 7064 legalizeOperands(*CarryOp); 7065 MRI.replaceRegWith(Inst.getOperand(0).getReg(), DestReg); 7066 addUsersToMoveToVALUWorklist(DestReg, MRI, Worklist); 7067 Inst.eraseFromParent(); 7068 } 7069 return; 7070 case AMDGPU::S_UADDO_PSEUDO: 7071 case AMDGPU::S_USUBO_PSEUDO: { 7072 const DebugLoc &DL = Inst.getDebugLoc(); 7073 MachineOperand &Dest0 = Inst.getOperand(0); 7074 MachineOperand &Dest1 = Inst.getOperand(1); 7075 MachineOperand &Src0 = Inst.getOperand(2); 7076 MachineOperand &Src1 = Inst.getOperand(3); 7077 7078 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_UADDO_PSEUDO) 7079 ? AMDGPU::V_ADD_CO_U32_e64 7080 : AMDGPU::V_SUB_CO_U32_e64; 7081 const TargetRegisterClass *NewRC = 7082 RI.getEquivalentVGPRClass(MRI.getRegClass(Dest0.getReg())); 7083 Register DestReg = MRI.createVirtualRegister(NewRC); 7084 MachineInstr *NewInstr = BuildMI(*MBB, &Inst, DL, get(Opc), DestReg) 7085 .addReg(Dest1.getReg(), RegState::Define) 7086 .add(Src0) 7087 .add(Src1) 7088 .addImm(0); // clamp bit 7089 7090 legalizeOperands(*NewInstr, MDT); 7091 MRI.replaceRegWith(Dest0.getReg(), DestReg); 7092 addUsersToMoveToVALUWorklist(NewInstr->getOperand(0).getReg(), MRI, 7093 Worklist); 7094 Inst.eraseFromParent(); 7095 } 7096 return; 7097 7098 case AMDGPU::S_CSELECT_B32: 7099 case AMDGPU::S_CSELECT_B64: 7100 lowerSelect(Worklist, Inst, MDT); 7101 Inst.eraseFromParent(); 7102 return; 7103 case AMDGPU::S_CMP_EQ_I32: 7104 case AMDGPU::S_CMP_LG_I32: 7105 case AMDGPU::S_CMP_GT_I32: 7106 case AMDGPU::S_CMP_GE_I32: 7107 case AMDGPU::S_CMP_LT_I32: 7108 case AMDGPU::S_CMP_LE_I32: 7109 case AMDGPU::S_CMP_EQ_U32: 7110 case AMDGPU::S_CMP_LG_U32: 7111 case AMDGPU::S_CMP_GT_U32: 7112 case AMDGPU::S_CMP_GE_U32: 7113 case AMDGPU::S_CMP_LT_U32: 7114 case AMDGPU::S_CMP_LE_U32: 7115 case AMDGPU::S_CMP_EQ_U64: 7116 case AMDGPU::S_CMP_LG_U64: 7117 case AMDGPU::S_CMP_LT_F32: 7118 case AMDGPU::S_CMP_EQ_F32: 7119 case AMDGPU::S_CMP_LE_F32: 7120 case AMDGPU::S_CMP_GT_F32: 7121 case AMDGPU::S_CMP_LG_F32: 7122 case AMDGPU::S_CMP_GE_F32: 7123 case AMDGPU::S_CMP_O_F32: 7124 case AMDGPU::S_CMP_U_F32: 7125 case AMDGPU::S_CMP_NGE_F32: 7126 case AMDGPU::S_CMP_NLG_F32: 7127 case AMDGPU::S_CMP_NGT_F32: 7128 case AMDGPU::S_CMP_NLE_F32: 7129 case AMDGPU::S_CMP_NEQ_F32: 7130 case AMDGPU::S_CMP_NLT_F32: 7131 case AMDGPU::S_CMP_LT_F16: 7132 case AMDGPU::S_CMP_EQ_F16: 7133 case AMDGPU::S_CMP_LE_F16: 7134 case AMDGPU::S_CMP_GT_F16: 7135 case AMDGPU::S_CMP_LG_F16: 7136 case AMDGPU::S_CMP_GE_F16: 7137 case AMDGPU::S_CMP_O_F16: 7138 case AMDGPU::S_CMP_U_F16: 7139 case AMDGPU::S_CMP_NGE_F16: 7140 case AMDGPU::S_CMP_NLG_F16: 7141 case AMDGPU::S_CMP_NGT_F16: 7142 case AMDGPU::S_CMP_NLE_F16: 7143 case AMDGPU::S_CMP_NEQ_F16: 7144 case AMDGPU::S_CMP_NLT_F16: { 7145 Register CondReg = MRI.createVirtualRegister(RI.getWaveMaskRegClass()); 7146 auto NewInstr = 7147 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(NewOpcode), CondReg) 7148 .setMIFlags(Inst.getFlags()); 7149 if (AMDGPU::getNamedOperandIdx(NewOpcode, 7150 AMDGPU::OpName::src0_modifiers) >= 0) { 7151 NewInstr 7152 .addImm(0) // src0_modifiers 7153 .add(Inst.getOperand(0)) // src0 7154 .addImm(0) // src1_modifiers 7155 .add(Inst.getOperand(1)) // src1 7156 .addImm(0); // clamp 7157 } else { 7158 NewInstr 7159 .add(Inst.getOperand(0)) 7160 .add(Inst.getOperand(1)); 7161 } 7162 legalizeOperands(*NewInstr, MDT); 7163 int SCCIdx = Inst.findRegisterDefOperandIdx(AMDGPU::SCC); 7164 MachineOperand SCCOp = Inst.getOperand(SCCIdx); 7165 addSCCDefUsersToVALUWorklist(SCCOp, Inst, Worklist, CondReg); 7166 Inst.eraseFromParent(); 7167 return; 7168 } 7169 case AMDGPU::S_CVT_HI_F32_F16: { 7170 const DebugLoc &DL = Inst.getDebugLoc(); 7171 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7172 Register NewDst = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7173 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg) 7174 .addImm(16) 7175 .add(Inst.getOperand(1)); 7176 BuildMI(*MBB, Inst, DL, get(NewOpcode), NewDst) 7177 .addImm(0) // src0_modifiers 7178 .addReg(TmpReg) 7179 .addImm(0) // clamp 7180 .addImm(0); // omod 7181 7182 MRI.replaceRegWith(Inst.getOperand(0).getReg(), NewDst); 7183 addUsersToMoveToVALUWorklist(NewDst, MRI, Worklist); 7184 Inst.eraseFromParent(); 7185 return; 7186 } 7187 case AMDGPU::S_MINIMUM_F32: 7188 case AMDGPU::S_MAXIMUM_F32: 7189 case AMDGPU::S_MINIMUM_F16: 7190 case AMDGPU::S_MAXIMUM_F16: { 7191 const DebugLoc &DL = Inst.getDebugLoc(); 7192 Register NewDst = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7193 MachineInstr *NewInstr = BuildMI(*MBB, Inst, DL, get(NewOpcode), NewDst) 7194 .addImm(0) // src0_modifiers 7195 .add(Inst.getOperand(1)) 7196 .addImm(0) // src1_modifiers 7197 .add(Inst.getOperand(2)) 7198 .addImm(0) // clamp 7199 .addImm(0); // omod 7200 MRI.replaceRegWith(Inst.getOperand(0).getReg(), NewDst); 7201 7202 legalizeOperands(*NewInstr, MDT); 7203 addUsersToMoveToVALUWorklist(NewDst, MRI, Worklist); 7204 Inst.eraseFromParent(); 7205 return; 7206 } 7207 } 7208 7209 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) { 7210 // We cannot move this instruction to the VALU, so we should try to 7211 // legalize its operands instead. 7212 legalizeOperands(Inst, MDT); 7213 return; 7214 } 7215 // Handle converting generic instructions like COPY-to-SGPR into 7216 // COPY-to-VGPR. 7217 if (NewOpcode == Opcode) { 7218 Register DstReg = Inst.getOperand(0).getReg(); 7219 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst); 7220 7221 // If it's a copy of a VGPR to a physical SGPR, insert a V_READFIRSTLANE and 7222 // hope for the best. 7223 if (Inst.isCopy() && DstReg.isPhysical() && 7224 RI.isVGPR(MRI, Inst.getOperand(1).getReg())) { 7225 // TODO: Only works for 32 bit registers. 7226 BuildMI(*Inst.getParent(), &Inst, Inst.getDebugLoc(), 7227 get(AMDGPU::V_READFIRSTLANE_B32), Inst.getOperand(0).getReg()) 7228 .add(Inst.getOperand(1)); 7229 Inst.eraseFromParent(); 7230 return; 7231 } 7232 7233 if (Inst.isCopy() && Inst.getOperand(1).getReg().isVirtual() && 7234 NewDstRC == RI.getRegClassForReg(MRI, Inst.getOperand(1).getReg())) { 7235 // Instead of creating a copy where src and dst are the same register 7236 // class, we just replace all uses of dst with src. These kinds of 7237 // copies interfere with the heuristics MachineSink uses to decide 7238 // whether or not to split a critical edge. Since the pass assumes 7239 // that copies will end up as machine instructions and not be 7240 // eliminated. 7241 addUsersToMoveToVALUWorklist(DstReg, MRI, Worklist); 7242 MRI.replaceRegWith(DstReg, Inst.getOperand(1).getReg()); 7243 MRI.clearKillFlags(Inst.getOperand(1).getReg()); 7244 Inst.getOperand(0).setReg(DstReg); 7245 // Make sure we don't leave around a dead VGPR->SGPR copy. Normally 7246 // these are deleted later, but at -O0 it would leave a suspicious 7247 // looking illegal copy of an undef register. 7248 for (unsigned I = Inst.getNumOperands() - 1; I != 0; --I) 7249 Inst.removeOperand(I); 7250 Inst.setDesc(get(AMDGPU::IMPLICIT_DEF)); 7251 return; 7252 } 7253 Register NewDstReg = MRI.createVirtualRegister(NewDstRC); 7254 MRI.replaceRegWith(DstReg, NewDstReg); 7255 legalizeOperands(Inst, MDT); 7256 addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist); 7257 return; 7258 } 7259 7260 // Use the new VALU Opcode. 7261 auto NewInstr = BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(NewOpcode)) 7262 .setMIFlags(Inst.getFlags()); 7263 if (isVOP3(NewOpcode) && !isVOP3(Opcode)) { 7264 // Intersperse VOP3 modifiers among the SALU operands. 7265 NewInstr->addOperand(Inst.getOperand(0)); 7266 if (AMDGPU::getNamedOperandIdx(NewOpcode, 7267 AMDGPU::OpName::src0_modifiers) >= 0) 7268 NewInstr.addImm(0); 7269 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::src0) >= 0) 7270 NewInstr->addOperand(Inst.getOperand(1)); 7271 7272 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) { 7273 // We are converting these to a BFE, so we need to add the missing 7274 // operands for the size and offset. 7275 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16; 7276 NewInstr.addImm(0); 7277 NewInstr.addImm(Size); 7278 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) { 7279 // The VALU version adds the second operand to the result, so insert an 7280 // extra 0 operand. 7281 NewInstr.addImm(0); 7282 } else if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) { 7283 const MachineOperand &OffsetWidthOp = Inst.getOperand(2); 7284 // If we need to move this to VGPRs, we need to unpack the second 7285 // operand back into the 2 separate ones for bit offset and width. 7286 assert(OffsetWidthOp.isImm() && 7287 "Scalar BFE is only implemented for constant width and offset"); 7288 uint32_t Imm = OffsetWidthOp.getImm(); 7289 7290 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0]. 7291 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16]. 7292 NewInstr.addImm(Offset); 7293 NewInstr.addImm(BitWidth); 7294 } else { 7295 if (AMDGPU::getNamedOperandIdx(NewOpcode, 7296 AMDGPU::OpName::src1_modifiers) >= 0) 7297 NewInstr.addImm(0); 7298 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::src1) >= 0) 7299 NewInstr->addOperand(Inst.getOperand(2)); 7300 if (AMDGPU::getNamedOperandIdx(NewOpcode, 7301 AMDGPU::OpName::src2_modifiers) >= 0) 7302 NewInstr.addImm(0); 7303 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::src2) >= 0) 7304 NewInstr->addOperand(Inst.getOperand(3)); 7305 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::clamp) >= 0) 7306 NewInstr.addImm(0); 7307 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::omod) >= 0) 7308 NewInstr.addImm(0); 7309 if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::op_sel) >= 0) 7310 NewInstr.addImm(0); 7311 } 7312 } else { 7313 // Just copy the SALU operands. 7314 for (const MachineOperand &Op : Inst.explicit_operands()) 7315 NewInstr->addOperand(Op); 7316 } 7317 7318 // Remove any references to SCC. Vector instructions can't read from it, and 7319 // We're just about to add the implicit use / defs of VCC, and we don't want 7320 // both. 7321 for (MachineOperand &Op : Inst.implicit_operands()) { 7322 if (Op.getReg() == AMDGPU::SCC) { 7323 // Only propagate through live-def of SCC. 7324 if (Op.isDef() && !Op.isDead()) 7325 addSCCDefUsersToVALUWorklist(Op, Inst, Worklist); 7326 if (Op.isUse()) 7327 addSCCDefsToVALUWorklist(NewInstr, Worklist); 7328 } 7329 } 7330 Inst.eraseFromParent(); 7331 Register NewDstReg; 7332 if (NewInstr->getOperand(0).isReg() && NewInstr->getOperand(0).isDef()) { 7333 Register DstReg = NewInstr->getOperand(0).getReg(); 7334 assert(DstReg.isVirtual()); 7335 // Update the destination register class. 7336 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(*NewInstr); 7337 assert(NewDstRC); 7338 NewDstReg = MRI.createVirtualRegister(NewDstRC); 7339 MRI.replaceRegWith(DstReg, NewDstReg); 7340 } 7341 fixImplicitOperands(*NewInstr); 7342 // Legalize the operands 7343 legalizeOperands(*NewInstr, MDT); 7344 if (NewDstReg) 7345 addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist); 7346 } 7347 7348 // Add/sub require special handling to deal with carry outs. 7349 std::pair<bool, MachineBasicBlock *> 7350 SIInstrInfo::moveScalarAddSub(SIInstrWorklist &Worklist, MachineInstr &Inst, 7351 MachineDominatorTree *MDT) const { 7352 if (ST.hasAddNoCarry()) { 7353 // Assume there is no user of scc since we don't select this in that case. 7354 // Since scc isn't used, it doesn't really matter if the i32 or u32 variant 7355 // is used. 7356 7357 MachineBasicBlock &MBB = *Inst.getParent(); 7358 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7359 7360 Register OldDstReg = Inst.getOperand(0).getReg(); 7361 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7362 7363 unsigned Opc = Inst.getOpcode(); 7364 assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32); 7365 7366 unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ? 7367 AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64; 7368 7369 assert(Inst.getOperand(3).getReg() == AMDGPU::SCC); 7370 Inst.removeOperand(3); 7371 7372 Inst.setDesc(get(NewOpc)); 7373 Inst.addOperand(MachineOperand::CreateImm(0)); // clamp bit 7374 Inst.addImplicitDefUseOperands(*MBB.getParent()); 7375 MRI.replaceRegWith(OldDstReg, ResultReg); 7376 MachineBasicBlock *NewBB = legalizeOperands(Inst, MDT); 7377 7378 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 7379 return std::pair(true, NewBB); 7380 } 7381 7382 return std::pair(false, nullptr); 7383 } 7384 7385 void SIInstrInfo::lowerSelect(SIInstrWorklist &Worklist, MachineInstr &Inst, 7386 MachineDominatorTree *MDT) const { 7387 7388 MachineBasicBlock &MBB = *Inst.getParent(); 7389 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7390 MachineBasicBlock::iterator MII = Inst; 7391 DebugLoc DL = Inst.getDebugLoc(); 7392 7393 MachineOperand &Dest = Inst.getOperand(0); 7394 MachineOperand &Src0 = Inst.getOperand(1); 7395 MachineOperand &Src1 = Inst.getOperand(2); 7396 MachineOperand &Cond = Inst.getOperand(3); 7397 7398 Register CondReg = Cond.getReg(); 7399 bool IsSCC = (CondReg == AMDGPU::SCC); 7400 7401 // If this is a trivial select where the condition is effectively not SCC 7402 // (CondReg is a source of copy to SCC), then the select is semantically 7403 // equivalent to copying CondReg. Hence, there is no need to create 7404 // V_CNDMASK, we can just use that and bail out. 7405 if (!IsSCC && Src0.isImm() && (Src0.getImm() == -1) && Src1.isImm() && 7406 (Src1.getImm() == 0)) { 7407 MRI.replaceRegWith(Dest.getReg(), CondReg); 7408 return; 7409 } 7410 7411 Register NewCondReg = CondReg; 7412 if (IsSCC) { 7413 const TargetRegisterClass *TC = 7414 RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 7415 NewCondReg = MRI.createVirtualRegister(TC); 7416 7417 // Now look for the closest SCC def if it is a copy 7418 // replacing the CondReg with the COPY source register 7419 bool CopyFound = false; 7420 for (MachineInstr &CandI : 7421 make_range(std::next(MachineBasicBlock::reverse_iterator(Inst)), 7422 Inst.getParent()->rend())) { 7423 if (CandI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != 7424 -1) { 7425 if (CandI.isCopy() && CandI.getOperand(0).getReg() == AMDGPU::SCC) { 7426 BuildMI(MBB, MII, DL, get(AMDGPU::COPY), NewCondReg) 7427 .addReg(CandI.getOperand(1).getReg()); 7428 CopyFound = true; 7429 } 7430 break; 7431 } 7432 } 7433 if (!CopyFound) { 7434 // SCC def is not a copy 7435 // Insert a trivial select instead of creating a copy, because a copy from 7436 // SCC would semantically mean just copying a single bit, but we may need 7437 // the result to be a vector condition mask that needs preserving. 7438 unsigned Opcode = (ST.getWavefrontSize() == 64) ? AMDGPU::S_CSELECT_B64 7439 : AMDGPU::S_CSELECT_B32; 7440 auto NewSelect = 7441 BuildMI(MBB, MII, DL, get(Opcode), NewCondReg).addImm(-1).addImm(0); 7442 NewSelect->getOperand(3).setIsUndef(Cond.isUndef()); 7443 } 7444 } 7445 7446 Register NewDestReg = MRI.createVirtualRegister( 7447 RI.getEquivalentVGPRClass(MRI.getRegClass(Dest.getReg()))); 7448 MachineInstr *NewInst; 7449 if (Inst.getOpcode() == AMDGPU::S_CSELECT_B32) { 7450 NewInst = BuildMI(MBB, MII, DL, get(AMDGPU::V_CNDMASK_B32_e64), NewDestReg) 7451 .addImm(0) 7452 .add(Src1) // False 7453 .addImm(0) 7454 .add(Src0) // True 7455 .addReg(NewCondReg); 7456 } else { 7457 NewInst = 7458 BuildMI(MBB, MII, DL, get(AMDGPU::V_CNDMASK_B64_PSEUDO), NewDestReg) 7459 .add(Src1) // False 7460 .add(Src0) // True 7461 .addReg(NewCondReg); 7462 } 7463 MRI.replaceRegWith(Dest.getReg(), NewDestReg); 7464 legalizeOperands(*NewInst, MDT); 7465 addUsersToMoveToVALUWorklist(NewDestReg, MRI, Worklist); 7466 } 7467 7468 void SIInstrInfo::lowerScalarAbs(SIInstrWorklist &Worklist, 7469 MachineInstr &Inst) const { 7470 MachineBasicBlock &MBB = *Inst.getParent(); 7471 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7472 MachineBasicBlock::iterator MII = Inst; 7473 DebugLoc DL = Inst.getDebugLoc(); 7474 7475 MachineOperand &Dest = Inst.getOperand(0); 7476 MachineOperand &Src = Inst.getOperand(1); 7477 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7478 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7479 7480 unsigned SubOp = ST.hasAddNoCarry() ? 7481 AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_CO_U32_e32; 7482 7483 BuildMI(MBB, MII, DL, get(SubOp), TmpReg) 7484 .addImm(0) 7485 .addReg(Src.getReg()); 7486 7487 BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg) 7488 .addReg(Src.getReg()) 7489 .addReg(TmpReg); 7490 7491 MRI.replaceRegWith(Dest.getReg(), ResultReg); 7492 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 7493 } 7494 7495 void SIInstrInfo::lowerScalarXnor(SIInstrWorklist &Worklist, 7496 MachineInstr &Inst) const { 7497 MachineBasicBlock &MBB = *Inst.getParent(); 7498 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7499 MachineBasicBlock::iterator MII = Inst; 7500 const DebugLoc &DL = Inst.getDebugLoc(); 7501 7502 MachineOperand &Dest = Inst.getOperand(0); 7503 MachineOperand &Src0 = Inst.getOperand(1); 7504 MachineOperand &Src1 = Inst.getOperand(2); 7505 7506 if (ST.hasDLInsts()) { 7507 Register NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7508 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL); 7509 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL); 7510 7511 BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest) 7512 .add(Src0) 7513 .add(Src1); 7514 7515 MRI.replaceRegWith(Dest.getReg(), NewDest); 7516 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist); 7517 } else { 7518 // Using the identity !(x ^ y) == (!x ^ y) == (x ^ !y), we can 7519 // invert either source and then perform the XOR. If either source is a 7520 // scalar register, then we can leave the inversion on the scalar unit to 7521 // achieve a better distribution of scalar and vector instructions. 7522 bool Src0IsSGPR = Src0.isReg() && 7523 RI.isSGPRClass(MRI.getRegClass(Src0.getReg())); 7524 bool Src1IsSGPR = Src1.isReg() && 7525 RI.isSGPRClass(MRI.getRegClass(Src1.getReg())); 7526 MachineInstr *Xor; 7527 Register Temp = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 7528 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 7529 7530 // Build a pair of scalar instructions and add them to the work list. 7531 // The next iteration over the work list will lower these to the vector 7532 // unit as necessary. 7533 if (Src0IsSGPR) { 7534 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src0); 7535 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest) 7536 .addReg(Temp) 7537 .add(Src1); 7538 } else if (Src1IsSGPR) { 7539 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src1); 7540 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest) 7541 .add(Src0) 7542 .addReg(Temp); 7543 } else { 7544 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), Temp) 7545 .add(Src0) 7546 .add(Src1); 7547 MachineInstr *Not = 7548 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest).addReg(Temp); 7549 Worklist.insert(Not); 7550 } 7551 7552 MRI.replaceRegWith(Dest.getReg(), NewDest); 7553 7554 Worklist.insert(Xor); 7555 7556 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist); 7557 } 7558 } 7559 7560 void SIInstrInfo::splitScalarNotBinop(SIInstrWorklist &Worklist, 7561 MachineInstr &Inst, 7562 unsigned Opcode) const { 7563 MachineBasicBlock &MBB = *Inst.getParent(); 7564 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7565 MachineBasicBlock::iterator MII = Inst; 7566 const DebugLoc &DL = Inst.getDebugLoc(); 7567 7568 MachineOperand &Dest = Inst.getOperand(0); 7569 MachineOperand &Src0 = Inst.getOperand(1); 7570 MachineOperand &Src1 = Inst.getOperand(2); 7571 7572 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 7573 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 7574 7575 MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), Interm) 7576 .add(Src0) 7577 .add(Src1); 7578 7579 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest) 7580 .addReg(Interm); 7581 7582 Worklist.insert(&Op); 7583 Worklist.insert(&Not); 7584 7585 MRI.replaceRegWith(Dest.getReg(), NewDest); 7586 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist); 7587 } 7588 7589 void SIInstrInfo::splitScalarBinOpN2(SIInstrWorklist &Worklist, 7590 MachineInstr &Inst, 7591 unsigned Opcode) const { 7592 MachineBasicBlock &MBB = *Inst.getParent(); 7593 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7594 MachineBasicBlock::iterator MII = Inst; 7595 const DebugLoc &DL = Inst.getDebugLoc(); 7596 7597 MachineOperand &Dest = Inst.getOperand(0); 7598 MachineOperand &Src0 = Inst.getOperand(1); 7599 MachineOperand &Src1 = Inst.getOperand(2); 7600 7601 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 7602 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 7603 7604 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Interm) 7605 .add(Src1); 7606 7607 MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), NewDest) 7608 .add(Src0) 7609 .addReg(Interm); 7610 7611 Worklist.insert(&Not); 7612 Worklist.insert(&Op); 7613 7614 MRI.replaceRegWith(Dest.getReg(), NewDest); 7615 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist); 7616 } 7617 7618 void SIInstrInfo::splitScalar64BitUnaryOp(SIInstrWorklist &Worklist, 7619 MachineInstr &Inst, unsigned Opcode, 7620 bool Swap) const { 7621 MachineBasicBlock &MBB = *Inst.getParent(); 7622 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7623 7624 MachineOperand &Dest = Inst.getOperand(0); 7625 MachineOperand &Src0 = Inst.getOperand(1); 7626 DebugLoc DL = Inst.getDebugLoc(); 7627 7628 MachineBasicBlock::iterator MII = Inst; 7629 7630 const MCInstrDesc &InstDesc = get(Opcode); 7631 const TargetRegisterClass *Src0RC = Src0.isReg() ? 7632 MRI.getRegClass(Src0.getReg()) : 7633 &AMDGPU::SGPR_32RegClass; 7634 7635 const TargetRegisterClass *Src0SubRC = 7636 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0); 7637 7638 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 7639 AMDGPU::sub0, Src0SubRC); 7640 7641 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg()); 7642 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC); 7643 const TargetRegisterClass *NewDestSubRC = 7644 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0); 7645 7646 Register DestSub0 = MRI.createVirtualRegister(NewDestSubRC); 7647 MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0).add(SrcReg0Sub0); 7648 7649 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 7650 AMDGPU::sub1, Src0SubRC); 7651 7652 Register DestSub1 = MRI.createVirtualRegister(NewDestSubRC); 7653 MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1).add(SrcReg0Sub1); 7654 7655 if (Swap) 7656 std::swap(DestSub0, DestSub1); 7657 7658 Register FullDestReg = MRI.createVirtualRegister(NewDestRC); 7659 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg) 7660 .addReg(DestSub0) 7661 .addImm(AMDGPU::sub0) 7662 .addReg(DestSub1) 7663 .addImm(AMDGPU::sub1); 7664 7665 MRI.replaceRegWith(Dest.getReg(), FullDestReg); 7666 7667 Worklist.insert(&LoHalf); 7668 Worklist.insert(&HiHalf); 7669 7670 // We don't need to legalizeOperands here because for a single operand, src0 7671 // will support any kind of input. 7672 7673 // Move all users of this moved value. 7674 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist); 7675 } 7676 7677 // There is not a vector equivalent of s_mul_u64. For this reason, we need to 7678 // split the s_mul_u64 in 32-bit vector multiplications. 7679 void SIInstrInfo::splitScalarSMulU64(SIInstrWorklist &Worklist, 7680 MachineInstr &Inst, 7681 MachineDominatorTree *MDT) const { 7682 MachineBasicBlock &MBB = *Inst.getParent(); 7683 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7684 7685 Register FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 7686 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7687 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7688 7689 MachineOperand &Dest = Inst.getOperand(0); 7690 MachineOperand &Src0 = Inst.getOperand(1); 7691 MachineOperand &Src1 = Inst.getOperand(2); 7692 const DebugLoc &DL = Inst.getDebugLoc(); 7693 MachineBasicBlock::iterator MII = Inst; 7694 7695 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0.getReg()); 7696 const TargetRegisterClass *Src1RC = MRI.getRegClass(Src1.getReg()); 7697 const TargetRegisterClass *Src0SubRC = 7698 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0); 7699 if (RI.isSGPRClass(Src0SubRC)) 7700 Src0SubRC = RI.getEquivalentVGPRClass(Src0SubRC); 7701 const TargetRegisterClass *Src1SubRC = 7702 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0); 7703 if (RI.isSGPRClass(Src1SubRC)) 7704 Src1SubRC = RI.getEquivalentVGPRClass(Src1SubRC); 7705 7706 // First, we extract the low 32-bit and high 32-bit values from each of the 7707 // operands. 7708 MachineOperand Op0L = 7709 buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC); 7710 MachineOperand Op1L = 7711 buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC); 7712 MachineOperand Op0H = 7713 buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, AMDGPU::sub1, Src0SubRC); 7714 MachineOperand Op1H = 7715 buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, AMDGPU::sub1, Src1SubRC); 7716 7717 // The multilication is done as follows: 7718 // 7719 // Op1H Op1L 7720 // * Op0H Op0L 7721 // -------------------- 7722 // Op1H*Op0L Op1L*Op0L 7723 // + Op1H*Op0H Op1L*Op0H 7724 // ----------------------------------------- 7725 // (Op1H*Op0L + Op1L*Op0H + carry) Op1L*Op0L 7726 // 7727 // We drop Op1H*Op0H because the result of the multiplication is a 64-bit 7728 // value and that would overflow. 7729 // The low 32-bit value is Op1L*Op0L. 7730 // The high 32-bit value is Op1H*Op0L + Op1L*Op0H + carry (from Op1L*Op0L). 7731 7732 Register Op1L_Op0H_Reg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7733 MachineInstr *Op1L_Op0H = 7734 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), Op1L_Op0H_Reg) 7735 .add(Op1L) 7736 .add(Op0H); 7737 7738 Register Op1H_Op0L_Reg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7739 MachineInstr *Op1H_Op0L = 7740 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), Op1H_Op0L_Reg) 7741 .add(Op1H) 7742 .add(Op0L); 7743 7744 Register CarryReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7745 MachineInstr *Carry = 7746 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_HI_U32_e64), CarryReg) 7747 .add(Op1L) 7748 .add(Op0L); 7749 7750 MachineInstr *LoHalf = 7751 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), DestSub0) 7752 .add(Op1L) 7753 .add(Op0L); 7754 7755 Register AddReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7756 MachineInstr *Add = BuildMI(MBB, MII, DL, get(AMDGPU::V_ADD_U32_e32), AddReg) 7757 .addReg(Op1L_Op0H_Reg) 7758 .addReg(Op1H_Op0L_Reg); 7759 7760 MachineInstr *HiHalf = 7761 BuildMI(MBB, MII, DL, get(AMDGPU::V_ADD_U32_e32), DestSub1) 7762 .addReg(AddReg) 7763 .addReg(CarryReg); 7764 7765 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg) 7766 .addReg(DestSub0) 7767 .addImm(AMDGPU::sub0) 7768 .addReg(DestSub1) 7769 .addImm(AMDGPU::sub1); 7770 7771 MRI.replaceRegWith(Dest.getReg(), FullDestReg); 7772 7773 // Try to legalize the operands in case we need to swap the order to keep it 7774 // valid. 7775 legalizeOperands(*Op1L_Op0H, MDT); 7776 legalizeOperands(*Op1H_Op0L, MDT); 7777 legalizeOperands(*Carry, MDT); 7778 legalizeOperands(*LoHalf, MDT); 7779 legalizeOperands(*Add, MDT); 7780 legalizeOperands(*HiHalf, MDT); 7781 7782 // Move all users of this moved value. 7783 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist); 7784 } 7785 7786 // Lower S_MUL_U64_U32_PSEUDO/S_MUL_I64_I32_PSEUDO in two 32-bit vector 7787 // multiplications. 7788 void SIInstrInfo::splitScalarSMulPseudo(SIInstrWorklist &Worklist, 7789 MachineInstr &Inst, 7790 MachineDominatorTree *MDT) const { 7791 MachineBasicBlock &MBB = *Inst.getParent(); 7792 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7793 7794 Register FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 7795 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7796 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7797 7798 MachineOperand &Dest = Inst.getOperand(0); 7799 MachineOperand &Src0 = Inst.getOperand(1); 7800 MachineOperand &Src1 = Inst.getOperand(2); 7801 const DebugLoc &DL = Inst.getDebugLoc(); 7802 MachineBasicBlock::iterator MII = Inst; 7803 7804 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0.getReg()); 7805 const TargetRegisterClass *Src1RC = MRI.getRegClass(Src1.getReg()); 7806 const TargetRegisterClass *Src0SubRC = 7807 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0); 7808 if (RI.isSGPRClass(Src0SubRC)) 7809 Src0SubRC = RI.getEquivalentVGPRClass(Src0SubRC); 7810 const TargetRegisterClass *Src1SubRC = 7811 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0); 7812 if (RI.isSGPRClass(Src1SubRC)) 7813 Src1SubRC = RI.getEquivalentVGPRClass(Src1SubRC); 7814 7815 // First, we extract the low 32-bit and high 32-bit values from each of the 7816 // operands. 7817 MachineOperand Op0L = 7818 buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC); 7819 MachineOperand Op1L = 7820 buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC); 7821 7822 unsigned Opc = Inst.getOpcode(); 7823 unsigned NewOpc = Opc == AMDGPU::S_MUL_U64_U32_PSEUDO 7824 ? AMDGPU::V_MUL_HI_U32_e64 7825 : AMDGPU::V_MUL_HI_I32_e64; 7826 MachineInstr *HiHalf = 7827 BuildMI(MBB, MII, DL, get(NewOpc), DestSub1).add(Op1L).add(Op0L); 7828 7829 MachineInstr *LoHalf = 7830 BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), DestSub0) 7831 .add(Op1L) 7832 .add(Op0L); 7833 7834 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg) 7835 .addReg(DestSub0) 7836 .addImm(AMDGPU::sub0) 7837 .addReg(DestSub1) 7838 .addImm(AMDGPU::sub1); 7839 7840 MRI.replaceRegWith(Dest.getReg(), FullDestReg); 7841 7842 // Try to legalize the operands in case we need to swap the order to keep it 7843 // valid. 7844 legalizeOperands(*HiHalf, MDT); 7845 legalizeOperands(*LoHalf, MDT); 7846 7847 // Move all users of this moved value. 7848 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist); 7849 } 7850 7851 void SIInstrInfo::splitScalar64BitBinaryOp(SIInstrWorklist &Worklist, 7852 MachineInstr &Inst, unsigned Opcode, 7853 MachineDominatorTree *MDT) const { 7854 MachineBasicBlock &MBB = *Inst.getParent(); 7855 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7856 7857 MachineOperand &Dest = Inst.getOperand(0); 7858 MachineOperand &Src0 = Inst.getOperand(1); 7859 MachineOperand &Src1 = Inst.getOperand(2); 7860 DebugLoc DL = Inst.getDebugLoc(); 7861 7862 MachineBasicBlock::iterator MII = Inst; 7863 7864 const MCInstrDesc &InstDesc = get(Opcode); 7865 const TargetRegisterClass *Src0RC = Src0.isReg() ? 7866 MRI.getRegClass(Src0.getReg()) : 7867 &AMDGPU::SGPR_32RegClass; 7868 7869 const TargetRegisterClass *Src0SubRC = 7870 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0); 7871 const TargetRegisterClass *Src1RC = Src1.isReg() ? 7872 MRI.getRegClass(Src1.getReg()) : 7873 &AMDGPU::SGPR_32RegClass; 7874 7875 const TargetRegisterClass *Src1SubRC = 7876 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0); 7877 7878 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 7879 AMDGPU::sub0, Src0SubRC); 7880 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, 7881 AMDGPU::sub0, Src1SubRC); 7882 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 7883 AMDGPU::sub1, Src0SubRC); 7884 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, 7885 AMDGPU::sub1, Src1SubRC); 7886 7887 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg()); 7888 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC); 7889 const TargetRegisterClass *NewDestSubRC = 7890 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0); 7891 7892 Register DestSub0 = MRI.createVirtualRegister(NewDestSubRC); 7893 MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0) 7894 .add(SrcReg0Sub0) 7895 .add(SrcReg1Sub0); 7896 7897 Register DestSub1 = MRI.createVirtualRegister(NewDestSubRC); 7898 MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1) 7899 .add(SrcReg0Sub1) 7900 .add(SrcReg1Sub1); 7901 7902 Register FullDestReg = MRI.createVirtualRegister(NewDestRC); 7903 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg) 7904 .addReg(DestSub0) 7905 .addImm(AMDGPU::sub0) 7906 .addReg(DestSub1) 7907 .addImm(AMDGPU::sub1); 7908 7909 MRI.replaceRegWith(Dest.getReg(), FullDestReg); 7910 7911 Worklist.insert(&LoHalf); 7912 Worklist.insert(&HiHalf); 7913 7914 // Move all users of this moved value. 7915 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist); 7916 } 7917 7918 void SIInstrInfo::splitScalar64BitXnor(SIInstrWorklist &Worklist, 7919 MachineInstr &Inst, 7920 MachineDominatorTree *MDT) const { 7921 MachineBasicBlock &MBB = *Inst.getParent(); 7922 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7923 7924 MachineOperand &Dest = Inst.getOperand(0); 7925 MachineOperand &Src0 = Inst.getOperand(1); 7926 MachineOperand &Src1 = Inst.getOperand(2); 7927 const DebugLoc &DL = Inst.getDebugLoc(); 7928 7929 MachineBasicBlock::iterator MII = Inst; 7930 7931 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg()); 7932 7933 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); 7934 7935 MachineOperand* Op0; 7936 MachineOperand* Op1; 7937 7938 if (Src0.isReg() && RI.isSGPRReg(MRI, Src0.getReg())) { 7939 Op0 = &Src0; 7940 Op1 = &Src1; 7941 } else { 7942 Op0 = &Src1; 7943 Op1 = &Src0; 7944 } 7945 7946 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B64), Interm) 7947 .add(*Op0); 7948 7949 Register NewDest = MRI.createVirtualRegister(DestRC); 7950 7951 MachineInstr &Xor = *BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B64), NewDest) 7952 .addReg(Interm) 7953 .add(*Op1); 7954 7955 MRI.replaceRegWith(Dest.getReg(), NewDest); 7956 7957 Worklist.insert(&Xor); 7958 } 7959 7960 void SIInstrInfo::splitScalar64BitBCNT(SIInstrWorklist &Worklist, 7961 MachineInstr &Inst) const { 7962 MachineBasicBlock &MBB = *Inst.getParent(); 7963 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7964 7965 MachineBasicBlock::iterator MII = Inst; 7966 const DebugLoc &DL = Inst.getDebugLoc(); 7967 7968 MachineOperand &Dest = Inst.getOperand(0); 7969 MachineOperand &Src = Inst.getOperand(1); 7970 7971 const MCInstrDesc &InstDesc = get(AMDGPU::V_BCNT_U32_B32_e64); 7972 const TargetRegisterClass *SrcRC = Src.isReg() ? 7973 MRI.getRegClass(Src.getReg()) : 7974 &AMDGPU::SGPR_32RegClass; 7975 7976 Register MidReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7977 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7978 7979 const TargetRegisterClass *SrcSubRC = 7980 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0); 7981 7982 MachineOperand SrcRegSub0 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, 7983 AMDGPU::sub0, SrcSubRC); 7984 MachineOperand SrcRegSub1 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, 7985 AMDGPU::sub1, SrcSubRC); 7986 7987 BuildMI(MBB, MII, DL, InstDesc, MidReg).add(SrcRegSub0).addImm(0); 7988 7989 BuildMI(MBB, MII, DL, InstDesc, ResultReg).add(SrcRegSub1).addReg(MidReg); 7990 7991 MRI.replaceRegWith(Dest.getReg(), ResultReg); 7992 7993 // We don't need to legalize operands here. src0 for either instruction can be 7994 // an SGPR, and the second input is unused or determined here. 7995 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 7996 } 7997 7998 void SIInstrInfo::splitScalar64BitBFE(SIInstrWorklist &Worklist, 7999 MachineInstr &Inst) const { 8000 MachineBasicBlock &MBB = *Inst.getParent(); 8001 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 8002 MachineBasicBlock::iterator MII = Inst; 8003 const DebugLoc &DL = Inst.getDebugLoc(); 8004 8005 MachineOperand &Dest = Inst.getOperand(0); 8006 uint32_t Imm = Inst.getOperand(2).getImm(); 8007 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0]. 8008 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16]. 8009 8010 (void) Offset; 8011 8012 // Only sext_inreg cases handled. 8013 assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 && 8014 Offset == 0 && "Not implemented"); 8015 8016 if (BitWidth < 32) { 8017 Register MidRegLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8018 Register MidRegHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8019 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 8020 8021 BuildMI(MBB, MII, DL, get(AMDGPU::V_BFE_I32_e64), MidRegLo) 8022 .addReg(Inst.getOperand(1).getReg(), 0, AMDGPU::sub0) 8023 .addImm(0) 8024 .addImm(BitWidth); 8025 8026 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e32), MidRegHi) 8027 .addImm(31) 8028 .addReg(MidRegLo); 8029 8030 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg) 8031 .addReg(MidRegLo) 8032 .addImm(AMDGPU::sub0) 8033 .addReg(MidRegHi) 8034 .addImm(AMDGPU::sub1); 8035 8036 MRI.replaceRegWith(Dest.getReg(), ResultReg); 8037 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 8038 return; 8039 } 8040 8041 MachineOperand &Src = Inst.getOperand(1); 8042 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8043 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 8044 8045 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e64), TmpReg) 8046 .addImm(31) 8047 .addReg(Src.getReg(), 0, AMDGPU::sub0); 8048 8049 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg) 8050 .addReg(Src.getReg(), 0, AMDGPU::sub0) 8051 .addImm(AMDGPU::sub0) 8052 .addReg(TmpReg) 8053 .addImm(AMDGPU::sub1); 8054 8055 MRI.replaceRegWith(Dest.getReg(), ResultReg); 8056 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 8057 } 8058 8059 void SIInstrInfo::splitScalar64BitCountOp(SIInstrWorklist &Worklist, 8060 MachineInstr &Inst, unsigned Opcode, 8061 MachineDominatorTree *MDT) const { 8062 // (S_FLBIT_I32_B64 hi:lo) -> 8063 // -> (umin (V_FFBH_U32_e32 hi), (uaddsat (V_FFBH_U32_e32 lo), 32)) 8064 // (S_FF1_I32_B64 hi:lo) -> 8065 // ->(umin (uaddsat (V_FFBL_B32_e32 hi), 32) (V_FFBL_B32_e32 lo)) 8066 8067 MachineBasicBlock &MBB = *Inst.getParent(); 8068 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 8069 MachineBasicBlock::iterator MII = Inst; 8070 const DebugLoc &DL = Inst.getDebugLoc(); 8071 8072 MachineOperand &Dest = Inst.getOperand(0); 8073 MachineOperand &Src = Inst.getOperand(1); 8074 8075 const MCInstrDesc &InstDesc = get(Opcode); 8076 8077 bool IsCtlz = Opcode == AMDGPU::V_FFBH_U32_e32; 8078 unsigned OpcodeAdd = 8079 ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32; 8080 8081 const TargetRegisterClass *SrcRC = 8082 Src.isReg() ? MRI.getRegClass(Src.getReg()) : &AMDGPU::SGPR_32RegClass; 8083 const TargetRegisterClass *SrcSubRC = 8084 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0); 8085 8086 MachineOperand SrcRegSub0 = 8087 buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, AMDGPU::sub0, SrcSubRC); 8088 MachineOperand SrcRegSub1 = 8089 buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, AMDGPU::sub1, SrcSubRC); 8090 8091 Register MidReg1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8092 Register MidReg2 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8093 Register MidReg3 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8094 Register MidReg4 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8095 8096 BuildMI(MBB, MII, DL, InstDesc, MidReg1).add(SrcRegSub0); 8097 8098 BuildMI(MBB, MII, DL, InstDesc, MidReg2).add(SrcRegSub1); 8099 8100 BuildMI(MBB, MII, DL, get(OpcodeAdd), MidReg3) 8101 .addReg(IsCtlz ? MidReg1 : MidReg2) 8102 .addImm(32) 8103 .addImm(1); // enable clamp 8104 8105 BuildMI(MBB, MII, DL, get(AMDGPU::V_MIN_U32_e64), MidReg4) 8106 .addReg(MidReg3) 8107 .addReg(IsCtlz ? MidReg2 : MidReg1); 8108 8109 MRI.replaceRegWith(Dest.getReg(), MidReg4); 8110 8111 addUsersToMoveToVALUWorklist(MidReg4, MRI, Worklist); 8112 } 8113 8114 void SIInstrInfo::addUsersToMoveToVALUWorklist( 8115 Register DstReg, MachineRegisterInfo &MRI, 8116 SIInstrWorklist &Worklist) const { 8117 for (MachineRegisterInfo::use_iterator I = MRI.use_begin(DstReg), 8118 E = MRI.use_end(); I != E;) { 8119 MachineInstr &UseMI = *I->getParent(); 8120 8121 unsigned OpNo = 0; 8122 8123 switch (UseMI.getOpcode()) { 8124 case AMDGPU::COPY: 8125 case AMDGPU::WQM: 8126 case AMDGPU::SOFT_WQM: 8127 case AMDGPU::STRICT_WWM: 8128 case AMDGPU::STRICT_WQM: 8129 case AMDGPU::REG_SEQUENCE: 8130 case AMDGPU::PHI: 8131 case AMDGPU::INSERT_SUBREG: 8132 break; 8133 default: 8134 OpNo = I.getOperandNo(); 8135 break; 8136 } 8137 8138 if (!RI.hasVectorRegisters(getOpRegClass(UseMI, OpNo))) { 8139 Worklist.insert(&UseMI); 8140 8141 do { 8142 ++I; 8143 } while (I != E && I->getParent() == &UseMI); 8144 } else { 8145 ++I; 8146 } 8147 } 8148 } 8149 8150 void SIInstrInfo::movePackToVALU(SIInstrWorklist &Worklist, 8151 MachineRegisterInfo &MRI, 8152 MachineInstr &Inst) const { 8153 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8154 MachineBasicBlock *MBB = Inst.getParent(); 8155 MachineOperand &Src0 = Inst.getOperand(1); 8156 MachineOperand &Src1 = Inst.getOperand(2); 8157 const DebugLoc &DL = Inst.getDebugLoc(); 8158 8159 switch (Inst.getOpcode()) { 8160 case AMDGPU::S_PACK_LL_B32_B16: { 8161 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8162 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8163 8164 // FIXME: Can do a lot better if we know the high bits of src0 or src1 are 8165 // 0. 8166 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) 8167 .addImm(0xffff); 8168 8169 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_B32_e64), TmpReg) 8170 .addReg(ImmReg, RegState::Kill) 8171 .add(Src0); 8172 8173 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32_e64), ResultReg) 8174 .add(Src1) 8175 .addImm(16) 8176 .addReg(TmpReg, RegState::Kill); 8177 break; 8178 } 8179 case AMDGPU::S_PACK_LH_B32_B16: { 8180 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8181 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) 8182 .addImm(0xffff); 8183 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_BFI_B32_e64), ResultReg) 8184 .addReg(ImmReg, RegState::Kill) 8185 .add(Src0) 8186 .add(Src1); 8187 break; 8188 } 8189 case AMDGPU::S_PACK_HL_B32_B16: { 8190 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8191 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg) 8192 .addImm(16) 8193 .add(Src0); 8194 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32_e64), ResultReg) 8195 .add(Src1) 8196 .addImm(16) 8197 .addReg(TmpReg, RegState::Kill); 8198 break; 8199 } 8200 case AMDGPU::S_PACK_HH_B32_B16: { 8201 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8202 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 8203 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg) 8204 .addImm(16) 8205 .add(Src0); 8206 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) 8207 .addImm(0xffff0000); 8208 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_OR_B32_e64), ResultReg) 8209 .add(Src1) 8210 .addReg(ImmReg, RegState::Kill) 8211 .addReg(TmpReg, RegState::Kill); 8212 break; 8213 } 8214 default: 8215 llvm_unreachable("unhandled s_pack_* instruction"); 8216 } 8217 8218 MachineOperand &Dest = Inst.getOperand(0); 8219 MRI.replaceRegWith(Dest.getReg(), ResultReg); 8220 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 8221 } 8222 8223 void SIInstrInfo::addSCCDefUsersToVALUWorklist(MachineOperand &Op, 8224 MachineInstr &SCCDefInst, 8225 SIInstrWorklist &Worklist, 8226 Register NewCond) const { 8227 8228 // Ensure that def inst defines SCC, which is still live. 8229 assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isDef() && 8230 !Op.isDead() && Op.getParent() == &SCCDefInst); 8231 SmallVector<MachineInstr *, 4> CopyToDelete; 8232 // This assumes that all the users of SCC are in the same block 8233 // as the SCC def. 8234 for (MachineInstr &MI : // Skip the def inst itself. 8235 make_range(std::next(MachineBasicBlock::iterator(SCCDefInst)), 8236 SCCDefInst.getParent()->end())) { 8237 // Check if SCC is used first. 8238 int SCCIdx = MI.findRegisterUseOperandIdx(AMDGPU::SCC, false, &RI); 8239 if (SCCIdx != -1) { 8240 if (MI.isCopy()) { 8241 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 8242 Register DestReg = MI.getOperand(0).getReg(); 8243 8244 MRI.replaceRegWith(DestReg, NewCond); 8245 CopyToDelete.push_back(&MI); 8246 } else { 8247 8248 if (NewCond.isValid()) 8249 MI.getOperand(SCCIdx).setReg(NewCond); 8250 8251 Worklist.insert(&MI); 8252 } 8253 } 8254 // Exit if we find another SCC def. 8255 if (MI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != -1) 8256 break; 8257 } 8258 for (auto &Copy : CopyToDelete) 8259 Copy->eraseFromParent(); 8260 } 8261 8262 // Instructions that use SCC may be converted to VALU instructions. When that 8263 // happens, the SCC register is changed to VCC_LO. The instruction that defines 8264 // SCC must be changed to an instruction that defines VCC. This function makes 8265 // sure that the instruction that defines SCC is added to the moveToVALU 8266 // worklist. 8267 void SIInstrInfo::addSCCDefsToVALUWorklist(MachineInstr *SCCUseInst, 8268 SIInstrWorklist &Worklist) const { 8269 // Look for a preceding instruction that either defines VCC or SCC. If VCC 8270 // then there is nothing to do because the defining instruction has been 8271 // converted to a VALU already. If SCC then that instruction needs to be 8272 // converted to a VALU. 8273 for (MachineInstr &MI : 8274 make_range(std::next(MachineBasicBlock::reverse_iterator(SCCUseInst)), 8275 SCCUseInst->getParent()->rend())) { 8276 if (MI.modifiesRegister(AMDGPU::VCC, &RI)) 8277 break; 8278 if (MI.definesRegister(AMDGPU::SCC, &RI)) { 8279 Worklist.insert(&MI); 8280 break; 8281 } 8282 } 8283 } 8284 8285 const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass( 8286 const MachineInstr &Inst) const { 8287 const TargetRegisterClass *NewDstRC = getOpRegClass(Inst, 0); 8288 8289 switch (Inst.getOpcode()) { 8290 // For target instructions, getOpRegClass just returns the virtual register 8291 // class associated with the operand, so we need to find an equivalent VGPR 8292 // register class in order to move the instruction to the VALU. 8293 case AMDGPU::COPY: 8294 case AMDGPU::PHI: 8295 case AMDGPU::REG_SEQUENCE: 8296 case AMDGPU::INSERT_SUBREG: 8297 case AMDGPU::WQM: 8298 case AMDGPU::SOFT_WQM: 8299 case AMDGPU::STRICT_WWM: 8300 case AMDGPU::STRICT_WQM: { 8301 const TargetRegisterClass *SrcRC = getOpRegClass(Inst, 1); 8302 if (RI.isAGPRClass(SrcRC)) { 8303 if (RI.isAGPRClass(NewDstRC)) 8304 return nullptr; 8305 8306 switch (Inst.getOpcode()) { 8307 case AMDGPU::PHI: 8308 case AMDGPU::REG_SEQUENCE: 8309 case AMDGPU::INSERT_SUBREG: 8310 NewDstRC = RI.getEquivalentAGPRClass(NewDstRC); 8311 break; 8312 default: 8313 NewDstRC = RI.getEquivalentVGPRClass(NewDstRC); 8314 } 8315 8316 if (!NewDstRC) 8317 return nullptr; 8318 } else { 8319 if (RI.isVGPRClass(NewDstRC) || NewDstRC == &AMDGPU::VReg_1RegClass) 8320 return nullptr; 8321 8322 NewDstRC = RI.getEquivalentVGPRClass(NewDstRC); 8323 if (!NewDstRC) 8324 return nullptr; 8325 } 8326 8327 return NewDstRC; 8328 } 8329 default: 8330 return NewDstRC; 8331 } 8332 } 8333 8334 // Find the one SGPR operand we are allowed to use. 8335 Register SIInstrInfo::findUsedSGPR(const MachineInstr &MI, 8336 int OpIndices[3]) const { 8337 const MCInstrDesc &Desc = MI.getDesc(); 8338 8339 // Find the one SGPR operand we are allowed to use. 8340 // 8341 // First we need to consider the instruction's operand requirements before 8342 // legalizing. Some operands are required to be SGPRs, such as implicit uses 8343 // of VCC, but we are still bound by the constant bus requirement to only use 8344 // one. 8345 // 8346 // If the operand's class is an SGPR, we can never move it. 8347 8348 Register SGPRReg = findImplicitSGPRRead(MI); 8349 if (SGPRReg) 8350 return SGPRReg; 8351 8352 Register UsedSGPRs[3] = {Register()}; 8353 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 8354 8355 for (unsigned i = 0; i < 3; ++i) { 8356 int Idx = OpIndices[i]; 8357 if (Idx == -1) 8358 break; 8359 8360 const MachineOperand &MO = MI.getOperand(Idx); 8361 if (!MO.isReg()) 8362 continue; 8363 8364 // Is this operand statically required to be an SGPR based on the operand 8365 // constraints? 8366 const TargetRegisterClass *OpRC = 8367 RI.getRegClass(Desc.operands()[Idx].RegClass); 8368 bool IsRequiredSGPR = RI.isSGPRClass(OpRC); 8369 if (IsRequiredSGPR) 8370 return MO.getReg(); 8371 8372 // If this could be a VGPR or an SGPR, Check the dynamic register class. 8373 Register Reg = MO.getReg(); 8374 const TargetRegisterClass *RegRC = MRI.getRegClass(Reg); 8375 if (RI.isSGPRClass(RegRC)) 8376 UsedSGPRs[i] = Reg; 8377 } 8378 8379 // We don't have a required SGPR operand, so we have a bit more freedom in 8380 // selecting operands to move. 8381 8382 // Try to select the most used SGPR. If an SGPR is equal to one of the 8383 // others, we choose that. 8384 // 8385 // e.g. 8386 // V_FMA_F32 v0, s0, s0, s0 -> No moves 8387 // V_FMA_F32 v0, s0, s1, s0 -> Move s1 8388 8389 // TODO: If some of the operands are 64-bit SGPRs and some 32, we should 8390 // prefer those. 8391 8392 if (UsedSGPRs[0]) { 8393 if (UsedSGPRs[0] == UsedSGPRs[1] || UsedSGPRs[0] == UsedSGPRs[2]) 8394 SGPRReg = UsedSGPRs[0]; 8395 } 8396 8397 if (!SGPRReg && UsedSGPRs[1]) { 8398 if (UsedSGPRs[1] == UsedSGPRs[2]) 8399 SGPRReg = UsedSGPRs[1]; 8400 } 8401 8402 return SGPRReg; 8403 } 8404 8405 MachineOperand *SIInstrInfo::getNamedOperand(MachineInstr &MI, 8406 unsigned OperandName) const { 8407 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OperandName); 8408 if (Idx == -1) 8409 return nullptr; 8410 8411 return &MI.getOperand(Idx); 8412 } 8413 8414 uint64_t SIInstrInfo::getDefaultRsrcDataFormat() const { 8415 if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) { 8416 int64_t Format = ST.getGeneration() >= AMDGPUSubtarget::GFX11 8417 ? (int64_t)AMDGPU::UfmtGFX11::UFMT_32_FLOAT 8418 : (int64_t)AMDGPU::UfmtGFX10::UFMT_32_FLOAT; 8419 return (Format << 44) | 8420 (1ULL << 56) | // RESOURCE_LEVEL = 1 8421 (3ULL << 60); // OOB_SELECT = 3 8422 } 8423 8424 uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT; 8425 if (ST.isAmdHsaOS()) { 8426 // Set ATC = 1. GFX9 doesn't have this bit. 8427 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) 8428 RsrcDataFormat |= (1ULL << 56); 8429 8430 // Set MTYPE = 2 (MTYPE_UC = uncached). GFX9 doesn't have this. 8431 // BTW, it disables TC L2 and therefore decreases performance. 8432 if (ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS) 8433 RsrcDataFormat |= (2ULL << 59); 8434 } 8435 8436 return RsrcDataFormat; 8437 } 8438 8439 uint64_t SIInstrInfo::getScratchRsrcWords23() const { 8440 uint64_t Rsrc23 = getDefaultRsrcDataFormat() | 8441 AMDGPU::RSRC_TID_ENABLE | 8442 0xffffffff; // Size; 8443 8444 // GFX9 doesn't have ELEMENT_SIZE. 8445 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) { 8446 uint64_t EltSizeValue = Log2_32(ST.getMaxPrivateElementSize(true)) - 1; 8447 Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT; 8448 } 8449 8450 // IndexStride = 64 / 32. 8451 uint64_t IndexStride = ST.getWavefrontSize() == 64 ? 3 : 2; 8452 Rsrc23 |= IndexStride << AMDGPU::RSRC_INDEX_STRIDE_SHIFT; 8453 8454 // If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17]. 8455 // Clear them unless we want a huge stride. 8456 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS && 8457 ST.getGeneration() <= AMDGPUSubtarget::GFX9) 8458 Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT; 8459 8460 return Rsrc23; 8461 } 8462 8463 bool SIInstrInfo::isLowLatencyInstruction(const MachineInstr &MI) const { 8464 unsigned Opc = MI.getOpcode(); 8465 8466 return isSMRD(Opc); 8467 } 8468 8469 bool SIInstrInfo::isHighLatencyDef(int Opc) const { 8470 return get(Opc).mayLoad() && 8471 (isMUBUF(Opc) || isMTBUF(Opc) || isMIMG(Opc) || isFLAT(Opc)); 8472 } 8473 8474 unsigned SIInstrInfo::isStackAccess(const MachineInstr &MI, 8475 int &FrameIndex) const { 8476 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::vaddr); 8477 if (!Addr || !Addr->isFI()) 8478 return Register(); 8479 8480 assert(!MI.memoperands_empty() && 8481 (*MI.memoperands_begin())->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS); 8482 8483 FrameIndex = Addr->getIndex(); 8484 return getNamedOperand(MI, AMDGPU::OpName::vdata)->getReg(); 8485 } 8486 8487 unsigned SIInstrInfo::isSGPRStackAccess(const MachineInstr &MI, 8488 int &FrameIndex) const { 8489 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::addr); 8490 assert(Addr && Addr->isFI()); 8491 FrameIndex = Addr->getIndex(); 8492 return getNamedOperand(MI, AMDGPU::OpName::data)->getReg(); 8493 } 8494 8495 unsigned SIInstrInfo::isLoadFromStackSlot(const MachineInstr &MI, 8496 int &FrameIndex) const { 8497 if (!MI.mayLoad()) 8498 return Register(); 8499 8500 if (isMUBUF(MI) || isVGPRSpill(MI)) 8501 return isStackAccess(MI, FrameIndex); 8502 8503 if (isSGPRSpill(MI)) 8504 return isSGPRStackAccess(MI, FrameIndex); 8505 8506 return Register(); 8507 } 8508 8509 unsigned SIInstrInfo::isStoreToStackSlot(const MachineInstr &MI, 8510 int &FrameIndex) const { 8511 if (!MI.mayStore()) 8512 return Register(); 8513 8514 if (isMUBUF(MI) || isVGPRSpill(MI)) 8515 return isStackAccess(MI, FrameIndex); 8516 8517 if (isSGPRSpill(MI)) 8518 return isSGPRStackAccess(MI, FrameIndex); 8519 8520 return Register(); 8521 } 8522 8523 unsigned SIInstrInfo::getInstBundleSize(const MachineInstr &MI) const { 8524 unsigned Size = 0; 8525 MachineBasicBlock::const_instr_iterator I = MI.getIterator(); 8526 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end(); 8527 while (++I != E && I->isInsideBundle()) { 8528 assert(!I->isBundle() && "No nested bundle!"); 8529 Size += getInstSizeInBytes(*I); 8530 } 8531 8532 return Size; 8533 } 8534 8535 unsigned SIInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { 8536 unsigned Opc = MI.getOpcode(); 8537 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opc); 8538 unsigned DescSize = Desc.getSize(); 8539 8540 // If we have a definitive size, we can use it. Otherwise we need to inspect 8541 // the operands to know the size. 8542 if (isFixedSize(MI)) { 8543 unsigned Size = DescSize; 8544 8545 // If we hit the buggy offset, an extra nop will be inserted in MC so 8546 // estimate the worst case. 8547 if (MI.isBranch() && ST.hasOffset3fBug()) 8548 Size += 4; 8549 8550 return Size; 8551 } 8552 8553 // Instructions may have a 32-bit literal encoded after them. Check 8554 // operands that could ever be literals. 8555 if (isVALU(MI) || isSALU(MI)) { 8556 if (isDPP(MI)) 8557 return DescSize; 8558 bool HasLiteral = false; 8559 for (int I = 0, E = MI.getNumExplicitOperands(); I != E; ++I) { 8560 const MachineOperand &Op = MI.getOperand(I); 8561 const MCOperandInfo &OpInfo = Desc.operands()[I]; 8562 if (!Op.isReg() && !isInlineConstant(Op, OpInfo)) { 8563 HasLiteral = true; 8564 break; 8565 } 8566 } 8567 return HasLiteral ? DescSize + 4 : DescSize; 8568 } 8569 8570 // Check whether we have extra NSA words. 8571 if (isMIMG(MI)) { 8572 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0); 8573 if (VAddr0Idx < 0) 8574 return 8; 8575 8576 int RSrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc); 8577 return 8 + 4 * ((RSrcIdx - VAddr0Idx + 2) / 4); 8578 } 8579 8580 switch (Opc) { 8581 case TargetOpcode::BUNDLE: 8582 return getInstBundleSize(MI); 8583 case TargetOpcode::INLINEASM: 8584 case TargetOpcode::INLINEASM_BR: { 8585 const MachineFunction *MF = MI.getParent()->getParent(); 8586 const char *AsmStr = MI.getOperand(0).getSymbolName(); 8587 return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo(), &ST); 8588 } 8589 default: 8590 if (MI.isMetaInstruction()) 8591 return 0; 8592 return DescSize; 8593 } 8594 } 8595 8596 bool SIInstrInfo::mayAccessFlatAddressSpace(const MachineInstr &MI) const { 8597 if (!isFLAT(MI)) 8598 return false; 8599 8600 if (MI.memoperands_empty()) 8601 return true; 8602 8603 for (const MachineMemOperand *MMO : MI.memoperands()) { 8604 if (MMO->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS) 8605 return true; 8606 } 8607 return false; 8608 } 8609 8610 bool SIInstrInfo::isNonUniformBranchInstr(MachineInstr &Branch) const { 8611 return Branch.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO; 8612 } 8613 8614 void SIInstrInfo::convertNonUniformIfRegion(MachineBasicBlock *IfEntry, 8615 MachineBasicBlock *IfEnd) const { 8616 MachineBasicBlock::iterator TI = IfEntry->getFirstTerminator(); 8617 assert(TI != IfEntry->end()); 8618 8619 MachineInstr *Branch = &(*TI); 8620 MachineFunction *MF = IfEntry->getParent(); 8621 MachineRegisterInfo &MRI = IfEntry->getParent()->getRegInfo(); 8622 8623 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { 8624 Register DstReg = MRI.createVirtualRegister(RI.getBoolRC()); 8625 MachineInstr *SIIF = 8626 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_IF), DstReg) 8627 .add(Branch->getOperand(0)) 8628 .add(Branch->getOperand(1)); 8629 MachineInstr *SIEND = 8630 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_END_CF)) 8631 .addReg(DstReg); 8632 8633 IfEntry->erase(TI); 8634 IfEntry->insert(IfEntry->end(), SIIF); 8635 IfEnd->insert(IfEnd->getFirstNonPHI(), SIEND); 8636 } 8637 } 8638 8639 void SIInstrInfo::convertNonUniformLoopRegion( 8640 MachineBasicBlock *LoopEntry, MachineBasicBlock *LoopEnd) const { 8641 MachineBasicBlock::iterator TI = LoopEnd->getFirstTerminator(); 8642 // We expect 2 terminators, one conditional and one unconditional. 8643 assert(TI != LoopEnd->end()); 8644 8645 MachineInstr *Branch = &(*TI); 8646 MachineFunction *MF = LoopEnd->getParent(); 8647 MachineRegisterInfo &MRI = LoopEnd->getParent()->getRegInfo(); 8648 8649 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { 8650 8651 Register DstReg = MRI.createVirtualRegister(RI.getBoolRC()); 8652 Register BackEdgeReg = MRI.createVirtualRegister(RI.getBoolRC()); 8653 MachineInstrBuilder HeaderPHIBuilder = 8654 BuildMI(*(MF), Branch->getDebugLoc(), get(TargetOpcode::PHI), DstReg); 8655 for (MachineBasicBlock *PMBB : LoopEntry->predecessors()) { 8656 if (PMBB == LoopEnd) { 8657 HeaderPHIBuilder.addReg(BackEdgeReg); 8658 } else { 8659 Register ZeroReg = MRI.createVirtualRegister(RI.getBoolRC()); 8660 materializeImmediate(*PMBB, PMBB->getFirstTerminator(), DebugLoc(), 8661 ZeroReg, 0); 8662 HeaderPHIBuilder.addReg(ZeroReg); 8663 } 8664 HeaderPHIBuilder.addMBB(PMBB); 8665 } 8666 MachineInstr *HeaderPhi = HeaderPHIBuilder; 8667 MachineInstr *SIIFBREAK = BuildMI(*(MF), Branch->getDebugLoc(), 8668 get(AMDGPU::SI_IF_BREAK), BackEdgeReg) 8669 .addReg(DstReg) 8670 .add(Branch->getOperand(0)); 8671 MachineInstr *SILOOP = 8672 BuildMI(*(MF), Branch->getDebugLoc(), get(AMDGPU::SI_LOOP)) 8673 .addReg(BackEdgeReg) 8674 .addMBB(LoopEntry); 8675 8676 LoopEntry->insert(LoopEntry->begin(), HeaderPhi); 8677 LoopEnd->erase(TI); 8678 LoopEnd->insert(LoopEnd->end(), SIIFBREAK); 8679 LoopEnd->insert(LoopEnd->end(), SILOOP); 8680 } 8681 } 8682 8683 ArrayRef<std::pair<int, const char *>> 8684 SIInstrInfo::getSerializableTargetIndices() const { 8685 static const std::pair<int, const char *> TargetIndices[] = { 8686 {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"}, 8687 {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"}, 8688 {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"}, 8689 {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"}, 8690 {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}}; 8691 return ArrayRef(TargetIndices); 8692 } 8693 8694 /// This is used by the post-RA scheduler (SchedulePostRAList.cpp). The 8695 /// post-RA version of misched uses CreateTargetMIHazardRecognizer. 8696 ScheduleHazardRecognizer * 8697 SIInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II, 8698 const ScheduleDAG *DAG) const { 8699 return new GCNHazardRecognizer(DAG->MF); 8700 } 8701 8702 /// This is the hazard recognizer used at -O0 by the PostRAHazardRecognizer 8703 /// pass. 8704 ScheduleHazardRecognizer * 8705 SIInstrInfo::CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const { 8706 return new GCNHazardRecognizer(MF); 8707 } 8708 8709 // Called during: 8710 // - pre-RA scheduling and post-RA scheduling 8711 ScheduleHazardRecognizer * 8712 SIInstrInfo::CreateTargetMIHazardRecognizer(const InstrItineraryData *II, 8713 const ScheduleDAGMI *DAG) const { 8714 // Borrowed from Arm Target 8715 // We would like to restrict this hazard recognizer to only 8716 // post-RA scheduling; we can tell that we're post-RA because we don't 8717 // track VRegLiveness. 8718 if (!DAG->hasVRegLiveness()) 8719 return new GCNHazardRecognizer(DAG->MF); 8720 return TargetInstrInfo::CreateTargetMIHazardRecognizer(II, DAG); 8721 } 8722 8723 std::pair<unsigned, unsigned> 8724 SIInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const { 8725 return std::pair(TF & MO_MASK, TF & ~MO_MASK); 8726 } 8727 8728 ArrayRef<std::pair<unsigned, const char *>> 8729 SIInstrInfo::getSerializableDirectMachineOperandTargetFlags() const { 8730 static const std::pair<unsigned, const char *> TargetFlags[] = { 8731 { MO_GOTPCREL, "amdgpu-gotprel" }, 8732 { MO_GOTPCREL32_LO, "amdgpu-gotprel32-lo" }, 8733 { MO_GOTPCREL32_HI, "amdgpu-gotprel32-hi" }, 8734 { MO_REL32_LO, "amdgpu-rel32-lo" }, 8735 { MO_REL32_HI, "amdgpu-rel32-hi" }, 8736 { MO_ABS32_LO, "amdgpu-abs32-lo" }, 8737 { MO_ABS32_HI, "amdgpu-abs32-hi" }, 8738 }; 8739 8740 return ArrayRef(TargetFlags); 8741 } 8742 8743 ArrayRef<std::pair<MachineMemOperand::Flags, const char *>> 8744 SIInstrInfo::getSerializableMachineMemOperandTargetFlags() const { 8745 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] = 8746 { 8747 {MONoClobber, "amdgpu-noclobber"}, 8748 }; 8749 8750 return ArrayRef(TargetFlags); 8751 } 8752 8753 unsigned SIInstrInfo::getLiveRangeSplitOpcode(Register SrcReg, 8754 const MachineFunction &MF) const { 8755 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); 8756 assert(SrcReg.isVirtual()); 8757 if (MFI->checkFlag(SrcReg, AMDGPU::VirtRegFlag::WWM_REG)) 8758 return AMDGPU::WWM_COPY; 8759 8760 return AMDGPU::COPY; 8761 } 8762 8763 bool SIInstrInfo::isBasicBlockPrologue(const MachineInstr &MI, 8764 Register Reg) const { 8765 // We need to handle instructions which may be inserted during register 8766 // allocation to handle the prolog. The initial prolog instruction may have 8767 // been separated from the start of the block by spills and copies inserted 8768 // needed by the prolog. However, the insertions for scalar registers can 8769 // always be placed at the BB top as they are independent of the exec mask 8770 // value. 8771 bool IsNullOrVectorRegister = true; 8772 if (Reg) { 8773 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 8774 IsNullOrVectorRegister = !RI.isSGPRClass(RI.getRegClassForReg(MRI, Reg)); 8775 } 8776 8777 uint16_t Opc = MI.getOpcode(); 8778 // FIXME: Copies inserted in the block prolog for live-range split should also 8779 // be included. 8780 return IsNullOrVectorRegister && 8781 (isSpillOpcode(Opc) || (!MI.isTerminator() && Opc != AMDGPU::COPY && 8782 MI.modifiesRegister(AMDGPU::EXEC, &RI))); 8783 } 8784 8785 MachineInstrBuilder 8786 SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB, 8787 MachineBasicBlock::iterator I, 8788 const DebugLoc &DL, 8789 Register DestReg) const { 8790 if (ST.hasAddNoCarry()) 8791 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e64), DestReg); 8792 8793 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 8794 Register UnusedCarry = MRI.createVirtualRegister(RI.getBoolRC()); 8795 MRI.setRegAllocationHint(UnusedCarry, 0, RI.getVCC()); 8796 8797 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg) 8798 .addReg(UnusedCarry, RegState::Define | RegState::Dead); 8799 } 8800 8801 MachineInstrBuilder SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB, 8802 MachineBasicBlock::iterator I, 8803 const DebugLoc &DL, 8804 Register DestReg, 8805 RegScavenger &RS) const { 8806 if (ST.hasAddNoCarry()) 8807 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e32), DestReg); 8808 8809 // If available, prefer to use vcc. 8810 Register UnusedCarry = !RS.isRegUsed(AMDGPU::VCC) 8811 ? Register(RI.getVCC()) 8812 : RS.scavengeRegisterBackwards( 8813 *RI.getBoolRC(), I, /* RestoreAfter */ false, 8814 0, /* AllowSpill */ false); 8815 8816 // TODO: Users need to deal with this. 8817 if (!UnusedCarry.isValid()) 8818 return MachineInstrBuilder(); 8819 8820 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg) 8821 .addReg(UnusedCarry, RegState::Define | RegState::Dead); 8822 } 8823 8824 bool SIInstrInfo::isKillTerminator(unsigned Opcode) { 8825 switch (Opcode) { 8826 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR: 8827 case AMDGPU::SI_KILL_I1_TERMINATOR: 8828 return true; 8829 default: 8830 return false; 8831 } 8832 } 8833 8834 const MCInstrDesc &SIInstrInfo::getKillTerminatorFromPseudo(unsigned Opcode) const { 8835 switch (Opcode) { 8836 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO: 8837 return get(AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR); 8838 case AMDGPU::SI_KILL_I1_PSEUDO: 8839 return get(AMDGPU::SI_KILL_I1_TERMINATOR); 8840 default: 8841 llvm_unreachable("invalid opcode, expected SI_KILL_*_PSEUDO"); 8842 } 8843 } 8844 8845 bool SIInstrInfo::isLegalMUBUFImmOffset(unsigned Imm) const { 8846 return Imm <= getMaxMUBUFImmOffset(ST); 8847 } 8848 8849 unsigned SIInstrInfo::getMaxMUBUFImmOffset(const GCNSubtarget &ST) { 8850 // GFX12 field is non-negative 24-bit signed byte offset. 8851 const unsigned OffsetBits = 8852 ST.getGeneration() >= AMDGPUSubtarget::GFX12 ? 23 : 12; 8853 return (1 << OffsetBits) - 1; 8854 } 8855 8856 void SIInstrInfo::fixImplicitOperands(MachineInstr &MI) const { 8857 if (!ST.isWave32()) 8858 return; 8859 8860 if (MI.isInlineAsm()) 8861 return; 8862 8863 for (auto &Op : MI.implicit_operands()) { 8864 if (Op.isReg() && Op.getReg() == AMDGPU::VCC) 8865 Op.setReg(AMDGPU::VCC_LO); 8866 } 8867 } 8868 8869 bool SIInstrInfo::isBufferSMRD(const MachineInstr &MI) const { 8870 if (!isSMRD(MI)) 8871 return false; 8872 8873 // Check that it is using a buffer resource. 8874 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::sbase); 8875 if (Idx == -1) // e.g. s_memtime 8876 return false; 8877 8878 const auto RCID = MI.getDesc().operands()[Idx].RegClass; 8879 return RI.getRegClass(RCID)->hasSubClassEq(&AMDGPU::SGPR_128RegClass); 8880 } 8881 8882 // Given Imm, split it into the values to put into the SOffset and ImmOffset 8883 // fields in an MUBUF instruction. Return false if it is not possible (due to a 8884 // hardware bug needing a workaround). 8885 // 8886 // The required alignment ensures that individual address components remain 8887 // aligned if they are aligned to begin with. It also ensures that additional 8888 // offsets within the given alignment can be added to the resulting ImmOffset. 8889 bool SIInstrInfo::splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, 8890 uint32_t &ImmOffset, Align Alignment) const { 8891 const uint32_t MaxOffset = SIInstrInfo::getMaxMUBUFImmOffset(ST); 8892 const uint32_t MaxImm = alignDown(MaxOffset, Alignment.value()); 8893 uint32_t Overflow = 0; 8894 8895 if (Imm > MaxImm) { 8896 if (Imm <= MaxImm + 64) { 8897 // Use an SOffset inline constant for 4..64 8898 Overflow = Imm - MaxImm; 8899 Imm = MaxImm; 8900 } else { 8901 // Try to keep the same value in SOffset for adjacent loads, so that 8902 // the corresponding register contents can be re-used. 8903 // 8904 // Load values with all low-bits (except for alignment bits) set into 8905 // SOffset, so that a larger range of values can be covered using 8906 // s_movk_i32. 8907 // 8908 // Atomic operations fail to work correctly when individual address 8909 // components are unaligned, even if their sum is aligned. 8910 uint32_t High = (Imm + Alignment.value()) & ~MaxOffset; 8911 uint32_t Low = (Imm + Alignment.value()) & MaxOffset; 8912 Imm = Low; 8913 Overflow = High - Alignment.value(); 8914 } 8915 } 8916 8917 if (Overflow > 0) { 8918 // There is a hardware bug in SI and CI which prevents address clamping in 8919 // MUBUF instructions from working correctly with SOffsets. The immediate 8920 // offset is unaffected. 8921 if (ST.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS) 8922 return false; 8923 8924 // It is not possible to set immediate in SOffset field on some targets. 8925 if (ST.hasRestrictedSOffset()) 8926 return false; 8927 } 8928 8929 ImmOffset = Imm; 8930 SOffset = Overflow; 8931 return true; 8932 } 8933 8934 // Depending on the used address space and instructions, some immediate offsets 8935 // are allowed and some are not. 8936 // In general, flat instruction offsets can only be non-negative, global and 8937 // scratch instruction offsets can also be negative. 8938 // 8939 // There are several bugs related to these offsets: 8940 // On gfx10.1, flat instructions that go into the global address space cannot 8941 // use an offset. 8942 // 8943 // For scratch instructions, the address can be either an SGPR or a VGPR. 8944 // The following offsets can be used, depending on the architecture (x means 8945 // cannot be used): 8946 // +----------------------------+------+------+ 8947 // | Address-Mode | SGPR | VGPR | 8948 // +----------------------------+------+------+ 8949 // | gfx9 | | | 8950 // | negative, 4-aligned offset | x | ok | 8951 // | negative, unaligned offset | x | ok | 8952 // +----------------------------+------+------+ 8953 // | gfx10 | | | 8954 // | negative, 4-aligned offset | ok | ok | 8955 // | negative, unaligned offset | ok | x | 8956 // +----------------------------+------+------+ 8957 // | gfx10.3 | | | 8958 // | negative, 4-aligned offset | ok | ok | 8959 // | negative, unaligned offset | ok | ok | 8960 // +----------------------------+------+------+ 8961 // 8962 // This function ignores the addressing mode, so if an offset cannot be used in 8963 // one addressing mode, it is considered illegal. 8964 bool SIInstrInfo::isLegalFLATOffset(int64_t Offset, unsigned AddrSpace, 8965 uint64_t FlatVariant) const { 8966 // TODO: Should 0 be special cased? 8967 if (!ST.hasFlatInstOffsets()) 8968 return false; 8969 8970 if (ST.hasFlatSegmentOffsetBug() && FlatVariant == SIInstrFlags::FLAT && 8971 (AddrSpace == AMDGPUAS::FLAT_ADDRESS || 8972 AddrSpace == AMDGPUAS::GLOBAL_ADDRESS)) 8973 return false; 8974 8975 if (ST.hasNegativeUnalignedScratchOffsetBug() && 8976 FlatVariant == SIInstrFlags::FlatScratch && Offset < 0 && 8977 (Offset % 4) != 0) { 8978 return false; 8979 } 8980 8981 bool AllowNegative = allowNegativeFlatOffset(FlatVariant); 8982 unsigned N = AMDGPU::getNumFlatOffsetBits(ST); 8983 return isIntN(N, Offset) && (AllowNegative || Offset >= 0); 8984 } 8985 8986 // See comment on SIInstrInfo::isLegalFLATOffset for what is legal and what not. 8987 std::pair<int64_t, int64_t> 8988 SIInstrInfo::splitFlatOffset(int64_t COffsetVal, unsigned AddrSpace, 8989 uint64_t FlatVariant) const { 8990 int64_t RemainderOffset = COffsetVal; 8991 int64_t ImmField = 0; 8992 8993 bool AllowNegative = allowNegativeFlatOffset(FlatVariant); 8994 const unsigned NumBits = AMDGPU::getNumFlatOffsetBits(ST) - 1; 8995 8996 if (AllowNegative) { 8997 // Use signed division by a power of two to truncate towards 0. 8998 int64_t D = 1LL << NumBits; 8999 RemainderOffset = (COffsetVal / D) * D; 9000 ImmField = COffsetVal - RemainderOffset; 9001 9002 if (ST.hasNegativeUnalignedScratchOffsetBug() && 9003 FlatVariant == SIInstrFlags::FlatScratch && ImmField < 0 && 9004 (ImmField % 4) != 0) { 9005 // Make ImmField a multiple of 4 9006 RemainderOffset += ImmField % 4; 9007 ImmField -= ImmField % 4; 9008 } 9009 } else if (COffsetVal >= 0) { 9010 ImmField = COffsetVal & maskTrailingOnes<uint64_t>(NumBits); 9011 RemainderOffset = COffsetVal - ImmField; 9012 } 9013 9014 assert(isLegalFLATOffset(ImmField, AddrSpace, FlatVariant)); 9015 assert(RemainderOffset + ImmField == COffsetVal); 9016 return {ImmField, RemainderOffset}; 9017 } 9018 9019 bool SIInstrInfo::allowNegativeFlatOffset(uint64_t FlatVariant) const { 9020 if (ST.hasNegativeScratchOffsetBug() && 9021 FlatVariant == SIInstrFlags::FlatScratch) 9022 return false; 9023 9024 return FlatVariant != SIInstrFlags::FLAT || AMDGPU::isGFX12Plus(ST); 9025 } 9026 9027 static unsigned subtargetEncodingFamily(const GCNSubtarget &ST) { 9028 switch (ST.getGeneration()) { 9029 default: 9030 break; 9031 case AMDGPUSubtarget::SOUTHERN_ISLANDS: 9032 case AMDGPUSubtarget::SEA_ISLANDS: 9033 return SIEncodingFamily::SI; 9034 case AMDGPUSubtarget::VOLCANIC_ISLANDS: 9035 case AMDGPUSubtarget::GFX9: 9036 return SIEncodingFamily::VI; 9037 case AMDGPUSubtarget::GFX10: 9038 return SIEncodingFamily::GFX10; 9039 case AMDGPUSubtarget::GFX11: 9040 return SIEncodingFamily::GFX11; 9041 case AMDGPUSubtarget::GFX12: 9042 return SIEncodingFamily::GFX12; 9043 } 9044 llvm_unreachable("Unknown subtarget generation!"); 9045 } 9046 9047 bool SIInstrInfo::isAsmOnlyOpcode(int MCOp) const { 9048 switch(MCOp) { 9049 // These opcodes use indirect register addressing so 9050 // they need special handling by codegen (currently missing). 9051 // Therefore it is too risky to allow these opcodes 9052 // to be selected by dpp combiner or sdwa peepholer. 9053 case AMDGPU::V_MOVRELS_B32_dpp_gfx10: 9054 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10: 9055 case AMDGPU::V_MOVRELD_B32_dpp_gfx10: 9056 case AMDGPU::V_MOVRELD_B32_sdwa_gfx10: 9057 case AMDGPU::V_MOVRELSD_B32_dpp_gfx10: 9058 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10: 9059 case AMDGPU::V_MOVRELSD_2_B32_dpp_gfx10: 9060 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10: 9061 return true; 9062 default: 9063 return false; 9064 } 9065 } 9066 9067 int SIInstrInfo::pseudoToMCOpcode(int Opcode) const { 9068 if (SIInstrInfo::isSoftWaitcnt(Opcode)) 9069 Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode); 9070 9071 unsigned Gen = subtargetEncodingFamily(ST); 9072 9073 if ((get(Opcode).TSFlags & SIInstrFlags::renamedInGFX9) != 0 && 9074 ST.getGeneration() == AMDGPUSubtarget::GFX9) 9075 Gen = SIEncodingFamily::GFX9; 9076 9077 // Adjust the encoding family to GFX80 for D16 buffer instructions when the 9078 // subtarget has UnpackedD16VMem feature. 9079 // TODO: remove this when we discard GFX80 encoding. 9080 if (ST.hasUnpackedD16VMem() && (get(Opcode).TSFlags & SIInstrFlags::D16Buf)) 9081 Gen = SIEncodingFamily::GFX80; 9082 9083 if (get(Opcode).TSFlags & SIInstrFlags::SDWA) { 9084 switch (ST.getGeneration()) { 9085 default: 9086 Gen = SIEncodingFamily::SDWA; 9087 break; 9088 case AMDGPUSubtarget::GFX9: 9089 Gen = SIEncodingFamily::SDWA9; 9090 break; 9091 case AMDGPUSubtarget::GFX10: 9092 Gen = SIEncodingFamily::SDWA10; 9093 break; 9094 } 9095 } 9096 9097 if (isMAI(Opcode)) { 9098 int MFMAOp = AMDGPU::getMFMAEarlyClobberOp(Opcode); 9099 if (MFMAOp != -1) 9100 Opcode = MFMAOp; 9101 } 9102 9103 int MCOp = AMDGPU::getMCOpcode(Opcode, Gen); 9104 9105 // TODO-GFX12: Remove this. 9106 // Hack to allow some GFX12 codegen tests to run before all the encodings are 9107 // implemented. 9108 if (MCOp == (uint16_t)-1 && Gen == SIEncodingFamily::GFX12) 9109 MCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX11); 9110 9111 // -1 means that Opcode is already a native instruction. 9112 if (MCOp == -1) 9113 return Opcode; 9114 9115 if (ST.hasGFX90AInsts()) { 9116 uint16_t NMCOp = (uint16_t)-1; 9117 if (ST.hasGFX940Insts()) 9118 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX940); 9119 if (NMCOp == (uint16_t)-1) 9120 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX90A); 9121 if (NMCOp == (uint16_t)-1) 9122 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX9); 9123 if (NMCOp != (uint16_t)-1) 9124 MCOp = NMCOp; 9125 } 9126 9127 // (uint16_t)-1 means that Opcode is a pseudo instruction that has 9128 // no encoding in the given subtarget generation. 9129 if (MCOp == (uint16_t)-1) 9130 return -1; 9131 9132 if (isAsmOnlyOpcode(MCOp)) 9133 return -1; 9134 9135 return MCOp; 9136 } 9137 9138 static 9139 TargetInstrInfo::RegSubRegPair getRegOrUndef(const MachineOperand &RegOpnd) { 9140 assert(RegOpnd.isReg()); 9141 return RegOpnd.isUndef() ? TargetInstrInfo::RegSubRegPair() : 9142 getRegSubRegPair(RegOpnd); 9143 } 9144 9145 TargetInstrInfo::RegSubRegPair 9146 llvm::getRegSequenceSubReg(MachineInstr &MI, unsigned SubReg) { 9147 assert(MI.isRegSequence()); 9148 for (unsigned I = 0, E = (MI.getNumOperands() - 1)/ 2; I < E; ++I) 9149 if (MI.getOperand(1 + 2 * I + 1).getImm() == SubReg) { 9150 auto &RegOp = MI.getOperand(1 + 2 * I); 9151 return getRegOrUndef(RegOp); 9152 } 9153 return TargetInstrInfo::RegSubRegPair(); 9154 } 9155 9156 // Try to find the definition of reg:subreg in subreg-manipulation pseudos 9157 // Following a subreg of reg:subreg isn't supported 9158 static bool followSubRegDef(MachineInstr &MI, 9159 TargetInstrInfo::RegSubRegPair &RSR) { 9160 if (!RSR.SubReg) 9161 return false; 9162 switch (MI.getOpcode()) { 9163 default: break; 9164 case AMDGPU::REG_SEQUENCE: 9165 RSR = getRegSequenceSubReg(MI, RSR.SubReg); 9166 return true; 9167 // EXTRACT_SUBREG ins't supported as this would follow a subreg of subreg 9168 case AMDGPU::INSERT_SUBREG: 9169 if (RSR.SubReg == (unsigned)MI.getOperand(3).getImm()) 9170 // inserted the subreg we're looking for 9171 RSR = getRegOrUndef(MI.getOperand(2)); 9172 else { // the subreg in the rest of the reg 9173 auto R1 = getRegOrUndef(MI.getOperand(1)); 9174 if (R1.SubReg) // subreg of subreg isn't supported 9175 return false; 9176 RSR.Reg = R1.Reg; 9177 } 9178 return true; 9179 } 9180 return false; 9181 } 9182 9183 MachineInstr *llvm::getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P, 9184 MachineRegisterInfo &MRI) { 9185 assert(MRI.isSSA()); 9186 if (!P.Reg.isVirtual()) 9187 return nullptr; 9188 9189 auto RSR = P; 9190 auto *DefInst = MRI.getVRegDef(RSR.Reg); 9191 while (auto *MI = DefInst) { 9192 DefInst = nullptr; 9193 switch (MI->getOpcode()) { 9194 case AMDGPU::COPY: 9195 case AMDGPU::V_MOV_B32_e32: { 9196 auto &Op1 = MI->getOperand(1); 9197 if (Op1.isReg() && Op1.getReg().isVirtual()) { 9198 if (Op1.isUndef()) 9199 return nullptr; 9200 RSR = getRegSubRegPair(Op1); 9201 DefInst = MRI.getVRegDef(RSR.Reg); 9202 } 9203 break; 9204 } 9205 default: 9206 if (followSubRegDef(*MI, RSR)) { 9207 if (!RSR.Reg) 9208 return nullptr; 9209 DefInst = MRI.getVRegDef(RSR.Reg); 9210 } 9211 } 9212 if (!DefInst) 9213 return MI; 9214 } 9215 return nullptr; 9216 } 9217 9218 bool llvm::execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI, 9219 Register VReg, 9220 const MachineInstr &DefMI, 9221 const MachineInstr &UseMI) { 9222 assert(MRI.isSSA() && "Must be run on SSA"); 9223 9224 auto *TRI = MRI.getTargetRegisterInfo(); 9225 auto *DefBB = DefMI.getParent(); 9226 9227 // Don't bother searching between blocks, although it is possible this block 9228 // doesn't modify exec. 9229 if (UseMI.getParent() != DefBB) 9230 return true; 9231 9232 const int MaxInstScan = 20; 9233 int NumInst = 0; 9234 9235 // Stop scan at the use. 9236 auto E = UseMI.getIterator(); 9237 for (auto I = std::next(DefMI.getIterator()); I != E; ++I) { 9238 if (I->isDebugInstr()) 9239 continue; 9240 9241 if (++NumInst > MaxInstScan) 9242 return true; 9243 9244 if (I->modifiesRegister(AMDGPU::EXEC, TRI)) 9245 return true; 9246 } 9247 9248 return false; 9249 } 9250 9251 bool llvm::execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI, 9252 Register VReg, 9253 const MachineInstr &DefMI) { 9254 assert(MRI.isSSA() && "Must be run on SSA"); 9255 9256 auto *TRI = MRI.getTargetRegisterInfo(); 9257 auto *DefBB = DefMI.getParent(); 9258 9259 const int MaxUseScan = 10; 9260 int NumUse = 0; 9261 9262 for (auto &Use : MRI.use_nodbg_operands(VReg)) { 9263 auto &UseInst = *Use.getParent(); 9264 // Don't bother searching between blocks, although it is possible this block 9265 // doesn't modify exec. 9266 if (UseInst.getParent() != DefBB || UseInst.isPHI()) 9267 return true; 9268 9269 if (++NumUse > MaxUseScan) 9270 return true; 9271 } 9272 9273 if (NumUse == 0) 9274 return false; 9275 9276 const int MaxInstScan = 20; 9277 int NumInst = 0; 9278 9279 // Stop scan when we have seen all the uses. 9280 for (auto I = std::next(DefMI.getIterator()); ; ++I) { 9281 assert(I != DefBB->end()); 9282 9283 if (I->isDebugInstr()) 9284 continue; 9285 9286 if (++NumInst > MaxInstScan) 9287 return true; 9288 9289 for (const MachineOperand &Op : I->operands()) { 9290 // We don't check reg masks here as they're used only on calls: 9291 // 1. EXEC is only considered const within one BB 9292 // 2. Call should be a terminator instruction if present in a BB 9293 9294 if (!Op.isReg()) 9295 continue; 9296 9297 Register Reg = Op.getReg(); 9298 if (Op.isUse()) { 9299 if (Reg == VReg && --NumUse == 0) 9300 return false; 9301 } else if (TRI->regsOverlap(Reg, AMDGPU::EXEC)) 9302 return true; 9303 } 9304 } 9305 } 9306 9307 MachineInstr *SIInstrInfo::createPHIDestinationCopy( 9308 MachineBasicBlock &MBB, MachineBasicBlock::iterator LastPHIIt, 9309 const DebugLoc &DL, Register Src, Register Dst) const { 9310 auto Cur = MBB.begin(); 9311 if (Cur != MBB.end()) 9312 do { 9313 if (!Cur->isPHI() && Cur->readsRegister(Dst)) 9314 return BuildMI(MBB, Cur, DL, get(TargetOpcode::COPY), Dst).addReg(Src); 9315 ++Cur; 9316 } while (Cur != MBB.end() && Cur != LastPHIIt); 9317 9318 return TargetInstrInfo::createPHIDestinationCopy(MBB, LastPHIIt, DL, Src, 9319 Dst); 9320 } 9321 9322 MachineInstr *SIInstrInfo::createPHISourceCopy( 9323 MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt, 9324 const DebugLoc &DL, Register Src, unsigned SrcSubReg, Register Dst) const { 9325 if (InsPt != MBB.end() && 9326 (InsPt->getOpcode() == AMDGPU::SI_IF || 9327 InsPt->getOpcode() == AMDGPU::SI_ELSE || 9328 InsPt->getOpcode() == AMDGPU::SI_IF_BREAK) && 9329 InsPt->definesRegister(Src)) { 9330 InsPt++; 9331 return BuildMI(MBB, InsPt, DL, 9332 get(ST.isWave32() ? AMDGPU::S_MOV_B32_term 9333 : AMDGPU::S_MOV_B64_term), 9334 Dst) 9335 .addReg(Src, 0, SrcSubReg) 9336 .addReg(AMDGPU::EXEC, RegState::Implicit); 9337 } 9338 return TargetInstrInfo::createPHISourceCopy(MBB, InsPt, DL, Src, SrcSubReg, 9339 Dst); 9340 } 9341 9342 bool llvm::SIInstrInfo::isWave32() const { return ST.isWave32(); } 9343 9344 MachineInstr *SIInstrInfo::foldMemoryOperandImpl( 9345 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, 9346 MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS, 9347 VirtRegMap *VRM) const { 9348 // This is a bit of a hack (copied from AArch64). Consider this instruction: 9349 // 9350 // %0:sreg_32 = COPY $m0 9351 // 9352 // We explicitly chose SReg_32 for the virtual register so such a copy might 9353 // be eliminated by RegisterCoalescer. However, that may not be possible, and 9354 // %0 may even spill. We can't spill $m0 normally (it would require copying to 9355 // a numbered SGPR anyway), and since it is in the SReg_32 register class, 9356 // TargetInstrInfo::foldMemoryOperand() is going to try. 9357 // A similar issue also exists with spilling and reloading $exec registers. 9358 // 9359 // To prevent that, constrain the %0 register class here. 9360 if (isFullCopyInstr(MI)) { 9361 Register DstReg = MI.getOperand(0).getReg(); 9362 Register SrcReg = MI.getOperand(1).getReg(); 9363 if ((DstReg.isVirtual() || SrcReg.isVirtual()) && 9364 (DstReg.isVirtual() != SrcReg.isVirtual())) { 9365 MachineRegisterInfo &MRI = MF.getRegInfo(); 9366 Register VirtReg = DstReg.isVirtual() ? DstReg : SrcReg; 9367 const TargetRegisterClass *RC = MRI.getRegClass(VirtReg); 9368 if (RC->hasSuperClassEq(&AMDGPU::SReg_32RegClass)) { 9369 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); 9370 return nullptr; 9371 } else if (RC->hasSuperClassEq(&AMDGPU::SReg_64RegClass)) { 9372 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_64_XEXECRegClass); 9373 return nullptr; 9374 } 9375 } 9376 } 9377 9378 return nullptr; 9379 } 9380 9381 unsigned SIInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, 9382 const MachineInstr &MI, 9383 unsigned *PredCost) const { 9384 if (MI.isBundle()) { 9385 MachineBasicBlock::const_instr_iterator I(MI.getIterator()); 9386 MachineBasicBlock::const_instr_iterator E(MI.getParent()->instr_end()); 9387 unsigned Lat = 0, Count = 0; 9388 for (++I; I != E && I->isBundledWithPred(); ++I) { 9389 ++Count; 9390 Lat = std::max(Lat, SchedModel.computeInstrLatency(&*I)); 9391 } 9392 return Lat + Count - 1; 9393 } 9394 9395 return SchedModel.computeInstrLatency(&MI); 9396 } 9397 9398 InstructionUniformity 9399 SIInstrInfo::getGenericInstructionUniformity(const MachineInstr &MI) const { 9400 unsigned opcode = MI.getOpcode(); 9401 if (auto *GI = dyn_cast<GIntrinsic>(&MI)) { 9402 auto IID = GI->getIntrinsicID(); 9403 if (AMDGPU::isIntrinsicSourceOfDivergence(IID)) 9404 return InstructionUniformity::NeverUniform; 9405 if (AMDGPU::isIntrinsicAlwaysUniform(IID)) 9406 return InstructionUniformity::AlwaysUniform; 9407 9408 switch (IID) { 9409 case Intrinsic::amdgcn_if: 9410 case Intrinsic::amdgcn_else: 9411 // FIXME: Uniform if second result 9412 break; 9413 } 9414 9415 return InstructionUniformity::Default; 9416 } 9417 9418 // Loads from the private and flat address spaces are divergent, because 9419 // threads can execute the load instruction with the same inputs and get 9420 // different results. 9421 // 9422 // All other loads are not divergent, because if threads issue loads with the 9423 // same arguments, they will always get the same result. 9424 if (opcode == AMDGPU::G_LOAD) { 9425 if (MI.memoperands_empty()) 9426 return InstructionUniformity::NeverUniform; // conservative assumption 9427 9428 if (llvm::any_of(MI.memoperands(), [](const MachineMemOperand *mmo) { 9429 return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS || 9430 mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS; 9431 })) { 9432 // At least one MMO in a non-global address space. 9433 return InstructionUniformity::NeverUniform; 9434 } 9435 return InstructionUniformity::Default; 9436 } 9437 9438 if (SIInstrInfo::isGenericAtomicRMWOpcode(opcode) || 9439 opcode == AMDGPU::G_ATOMIC_CMPXCHG || 9440 opcode == AMDGPU::G_ATOMIC_CMPXCHG_WITH_SUCCESS || 9441 AMDGPU::isGenericAtomic(opcode)) { 9442 return InstructionUniformity::NeverUniform; 9443 } 9444 return InstructionUniformity::Default; 9445 } 9446 9447 InstructionUniformity 9448 SIInstrInfo::getInstructionUniformity(const MachineInstr &MI) const { 9449 9450 if (isNeverUniform(MI)) 9451 return InstructionUniformity::NeverUniform; 9452 9453 unsigned opcode = MI.getOpcode(); 9454 if (opcode == AMDGPU::V_READLANE_B32 || 9455 opcode == AMDGPU::V_READFIRSTLANE_B32 || 9456 opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR) 9457 return InstructionUniformity::AlwaysUniform; 9458 9459 if (isCopyInstr(MI)) { 9460 const MachineOperand &srcOp = MI.getOperand(1); 9461 if (srcOp.isReg() && srcOp.getReg().isPhysical()) { 9462 const TargetRegisterClass *regClass = 9463 RI.getPhysRegBaseClass(srcOp.getReg()); 9464 return RI.isSGPRClass(regClass) ? InstructionUniformity::AlwaysUniform 9465 : InstructionUniformity::NeverUniform; 9466 } 9467 return InstructionUniformity::Default; 9468 } 9469 9470 // GMIR handling 9471 if (MI.isPreISelOpcode()) 9472 return SIInstrInfo::getGenericInstructionUniformity(MI); 9473 9474 // Atomics are divergent because they are executed sequentially: when an 9475 // atomic operation refers to the same address in each thread, then each 9476 // thread after the first sees the value written by the previous thread as 9477 // original value. 9478 9479 if (isAtomic(MI)) 9480 return InstructionUniformity::NeverUniform; 9481 9482 // Loads from the private and flat address spaces are divergent, because 9483 // threads can execute the load instruction with the same inputs and get 9484 // different results. 9485 if (isFLAT(MI) && MI.mayLoad()) { 9486 if (MI.memoperands_empty()) 9487 return InstructionUniformity::NeverUniform; // conservative assumption 9488 9489 if (llvm::any_of(MI.memoperands(), [](const MachineMemOperand *mmo) { 9490 return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS || 9491 mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS; 9492 })) { 9493 // At least one MMO in a non-global address space. 9494 return InstructionUniformity::NeverUniform; 9495 } 9496 9497 return InstructionUniformity::Default; 9498 } 9499 9500 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 9501 const AMDGPURegisterBankInfo *RBI = ST.getRegBankInfo(); 9502 9503 // FIXME: It's conceptually broken to report this for an instruction, and not 9504 // a specific def operand. For inline asm in particular, there could be mixed 9505 // uniform and divergent results. 9506 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) { 9507 const MachineOperand &SrcOp = MI.getOperand(I); 9508 if (!SrcOp.isReg()) 9509 continue; 9510 9511 Register Reg = SrcOp.getReg(); 9512 if (!Reg || !SrcOp.readsReg()) 9513 continue; 9514 9515 // If RegBank is null, this is unassigned or an unallocatable special 9516 // register, which are all scalars. 9517 const RegisterBank *RegBank = RBI->getRegBank(Reg, MRI, RI); 9518 if (RegBank && RegBank->getID() != AMDGPU::SGPRRegBankID) 9519 return InstructionUniformity::NeverUniform; 9520 } 9521 9522 // TODO: Uniformity check condtions above can be rearranged for more 9523 // redability 9524 9525 // TODO: amdgcn.{ballot, [if]cmp} should be AlwaysUniform, but they are 9526 // currently turned into no-op COPYs by SelectionDAG ISel and are 9527 // therefore no longer recognizable. 9528 9529 return InstructionUniformity::Default; 9530 } 9531 9532 unsigned SIInstrInfo::getDSShaderTypeValue(const MachineFunction &MF) { 9533 switch (MF.getFunction().getCallingConv()) { 9534 case CallingConv::AMDGPU_PS: 9535 return 1; 9536 case CallingConv::AMDGPU_VS: 9537 return 2; 9538 case CallingConv::AMDGPU_GS: 9539 return 3; 9540 case CallingConv::AMDGPU_HS: 9541 case CallingConv::AMDGPU_LS: 9542 case CallingConv::AMDGPU_ES: 9543 report_fatal_error("ds_ordered_count unsupported for this calling conv"); 9544 case CallingConv::AMDGPU_CS: 9545 case CallingConv::AMDGPU_KERNEL: 9546 case CallingConv::C: 9547 case CallingConv::Fast: 9548 default: 9549 // Assume other calling conventions are various compute callable functions 9550 return 0; 9551 } 9552 } 9553 9554 bool SIInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg, 9555 Register &SrcReg2, int64_t &CmpMask, 9556 int64_t &CmpValue) const { 9557 if (!MI.getOperand(0).isReg() || MI.getOperand(0).getSubReg()) 9558 return false; 9559 9560 switch (MI.getOpcode()) { 9561 default: 9562 break; 9563 case AMDGPU::S_CMP_EQ_U32: 9564 case AMDGPU::S_CMP_EQ_I32: 9565 case AMDGPU::S_CMP_LG_U32: 9566 case AMDGPU::S_CMP_LG_I32: 9567 case AMDGPU::S_CMP_LT_U32: 9568 case AMDGPU::S_CMP_LT_I32: 9569 case AMDGPU::S_CMP_GT_U32: 9570 case AMDGPU::S_CMP_GT_I32: 9571 case AMDGPU::S_CMP_LE_U32: 9572 case AMDGPU::S_CMP_LE_I32: 9573 case AMDGPU::S_CMP_GE_U32: 9574 case AMDGPU::S_CMP_GE_I32: 9575 case AMDGPU::S_CMP_EQ_U64: 9576 case AMDGPU::S_CMP_LG_U64: 9577 SrcReg = MI.getOperand(0).getReg(); 9578 if (MI.getOperand(1).isReg()) { 9579 if (MI.getOperand(1).getSubReg()) 9580 return false; 9581 SrcReg2 = MI.getOperand(1).getReg(); 9582 CmpValue = 0; 9583 } else if (MI.getOperand(1).isImm()) { 9584 SrcReg2 = Register(); 9585 CmpValue = MI.getOperand(1).getImm(); 9586 } else { 9587 return false; 9588 } 9589 CmpMask = ~0; 9590 return true; 9591 case AMDGPU::S_CMPK_EQ_U32: 9592 case AMDGPU::S_CMPK_EQ_I32: 9593 case AMDGPU::S_CMPK_LG_U32: 9594 case AMDGPU::S_CMPK_LG_I32: 9595 case AMDGPU::S_CMPK_LT_U32: 9596 case AMDGPU::S_CMPK_LT_I32: 9597 case AMDGPU::S_CMPK_GT_U32: 9598 case AMDGPU::S_CMPK_GT_I32: 9599 case AMDGPU::S_CMPK_LE_U32: 9600 case AMDGPU::S_CMPK_LE_I32: 9601 case AMDGPU::S_CMPK_GE_U32: 9602 case AMDGPU::S_CMPK_GE_I32: 9603 SrcReg = MI.getOperand(0).getReg(); 9604 SrcReg2 = Register(); 9605 CmpValue = MI.getOperand(1).getImm(); 9606 CmpMask = ~0; 9607 return true; 9608 } 9609 9610 return false; 9611 } 9612 9613 bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, 9614 Register SrcReg2, int64_t CmpMask, 9615 int64_t CmpValue, 9616 const MachineRegisterInfo *MRI) const { 9617 if (!SrcReg || SrcReg.isPhysical()) 9618 return false; 9619 9620 if (SrcReg2 && !getFoldableImm(SrcReg2, *MRI, CmpValue)) 9621 return false; 9622 9623 const auto optimizeCmpAnd = [&CmpInstr, SrcReg, CmpValue, MRI, 9624 this](int64_t ExpectedValue, unsigned SrcSize, 9625 bool IsReversible, bool IsSigned) -> bool { 9626 // s_cmp_eq_u32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n 9627 // s_cmp_eq_i32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n 9628 // s_cmp_ge_u32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n 9629 // s_cmp_ge_i32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n 9630 // s_cmp_eq_u64 (s_and_b64 $src, 1 << n), 1 << n => s_and_b64 $src, 1 << n 9631 // s_cmp_lg_u32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n 9632 // s_cmp_lg_i32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n 9633 // s_cmp_gt_u32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n 9634 // s_cmp_gt_i32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n 9635 // s_cmp_lg_u64 (s_and_b64 $src, 1 << n), 0 => s_and_b64 $src, 1 << n 9636 // 9637 // Signed ge/gt are not used for the sign bit. 9638 // 9639 // If result of the AND is unused except in the compare: 9640 // s_and_b(32|64) $src, 1 << n => s_bitcmp1_b(32|64) $src, n 9641 // 9642 // s_cmp_eq_u32 (s_and_b32 $src, 1 << n), 0 => s_bitcmp0_b32 $src, n 9643 // s_cmp_eq_i32 (s_and_b32 $src, 1 << n), 0 => s_bitcmp0_b32 $src, n 9644 // s_cmp_eq_u64 (s_and_b64 $src, 1 << n), 0 => s_bitcmp0_b64 $src, n 9645 // s_cmp_lg_u32 (s_and_b32 $src, 1 << n), 1 << n => s_bitcmp0_b32 $src, n 9646 // s_cmp_lg_i32 (s_and_b32 $src, 1 << n), 1 << n => s_bitcmp0_b32 $src, n 9647 // s_cmp_lg_u64 (s_and_b64 $src, 1 << n), 1 << n => s_bitcmp0_b64 $src, n 9648 9649 MachineInstr *Def = MRI->getUniqueVRegDef(SrcReg); 9650 if (!Def || Def->getParent() != CmpInstr.getParent()) 9651 return false; 9652 9653 if (Def->getOpcode() != AMDGPU::S_AND_B32 && 9654 Def->getOpcode() != AMDGPU::S_AND_B64) 9655 return false; 9656 9657 int64_t Mask; 9658 const auto isMask = [&Mask, SrcSize](const MachineOperand *MO) -> bool { 9659 if (MO->isImm()) 9660 Mask = MO->getImm(); 9661 else if (!getFoldableImm(MO, Mask)) 9662 return false; 9663 Mask &= maxUIntN(SrcSize); 9664 return isPowerOf2_64(Mask); 9665 }; 9666 9667 MachineOperand *SrcOp = &Def->getOperand(1); 9668 if (isMask(SrcOp)) 9669 SrcOp = &Def->getOperand(2); 9670 else if (isMask(&Def->getOperand(2))) 9671 SrcOp = &Def->getOperand(1); 9672 else 9673 return false; 9674 9675 unsigned BitNo = llvm::countr_zero((uint64_t)Mask); 9676 if (IsSigned && BitNo == SrcSize - 1) 9677 return false; 9678 9679 ExpectedValue <<= BitNo; 9680 9681 bool IsReversedCC = false; 9682 if (CmpValue != ExpectedValue) { 9683 if (!IsReversible) 9684 return false; 9685 IsReversedCC = CmpValue == (ExpectedValue ^ Mask); 9686 if (!IsReversedCC) 9687 return false; 9688 } 9689 9690 Register DefReg = Def->getOperand(0).getReg(); 9691 if (IsReversedCC && !MRI->hasOneNonDBGUse(DefReg)) 9692 return false; 9693 9694 for (auto I = std::next(Def->getIterator()), E = CmpInstr.getIterator(); 9695 I != E; ++I) { 9696 if (I->modifiesRegister(AMDGPU::SCC, &RI) || 9697 I->killsRegister(AMDGPU::SCC, &RI)) 9698 return false; 9699 } 9700 9701 MachineOperand *SccDef = Def->findRegisterDefOperand(AMDGPU::SCC); 9702 SccDef->setIsDead(false); 9703 CmpInstr.eraseFromParent(); 9704 9705 if (!MRI->use_nodbg_empty(DefReg)) { 9706 assert(!IsReversedCC); 9707 return true; 9708 } 9709 9710 // Replace AND with unused result with a S_BITCMP. 9711 MachineBasicBlock *MBB = Def->getParent(); 9712 9713 unsigned NewOpc = (SrcSize == 32) ? IsReversedCC ? AMDGPU::S_BITCMP0_B32 9714 : AMDGPU::S_BITCMP1_B32 9715 : IsReversedCC ? AMDGPU::S_BITCMP0_B64 9716 : AMDGPU::S_BITCMP1_B64; 9717 9718 BuildMI(*MBB, Def, Def->getDebugLoc(), get(NewOpc)) 9719 .add(*SrcOp) 9720 .addImm(BitNo); 9721 Def->eraseFromParent(); 9722 9723 return true; 9724 }; 9725 9726 switch (CmpInstr.getOpcode()) { 9727 default: 9728 break; 9729 case AMDGPU::S_CMP_EQ_U32: 9730 case AMDGPU::S_CMP_EQ_I32: 9731 case AMDGPU::S_CMPK_EQ_U32: 9732 case AMDGPU::S_CMPK_EQ_I32: 9733 return optimizeCmpAnd(1, 32, true, false); 9734 case AMDGPU::S_CMP_GE_U32: 9735 case AMDGPU::S_CMPK_GE_U32: 9736 return optimizeCmpAnd(1, 32, false, false); 9737 case AMDGPU::S_CMP_GE_I32: 9738 case AMDGPU::S_CMPK_GE_I32: 9739 return optimizeCmpAnd(1, 32, false, true); 9740 case AMDGPU::S_CMP_EQ_U64: 9741 return optimizeCmpAnd(1, 64, true, false); 9742 case AMDGPU::S_CMP_LG_U32: 9743 case AMDGPU::S_CMP_LG_I32: 9744 case AMDGPU::S_CMPK_LG_U32: 9745 case AMDGPU::S_CMPK_LG_I32: 9746 return optimizeCmpAnd(0, 32, true, false); 9747 case AMDGPU::S_CMP_GT_U32: 9748 case AMDGPU::S_CMPK_GT_U32: 9749 return optimizeCmpAnd(0, 32, false, false); 9750 case AMDGPU::S_CMP_GT_I32: 9751 case AMDGPU::S_CMPK_GT_I32: 9752 return optimizeCmpAnd(0, 32, false, true); 9753 case AMDGPU::S_CMP_LG_U64: 9754 return optimizeCmpAnd(0, 64, true, false); 9755 } 9756 9757 return false; 9758 } 9759 9760 void SIInstrInfo::enforceOperandRCAlignment(MachineInstr &MI, 9761 unsigned OpName) const { 9762 if (!ST.needsAlignedVGPRs()) 9763 return; 9764 9765 int OpNo = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpName); 9766 if (OpNo < 0) 9767 return; 9768 MachineOperand &Op = MI.getOperand(OpNo); 9769 if (getOpSize(MI, OpNo) > 4) 9770 return; 9771 9772 // Add implicit aligned super-reg to force alignment on the data operand. 9773 const DebugLoc &DL = MI.getDebugLoc(); 9774 MachineBasicBlock *BB = MI.getParent(); 9775 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); 9776 Register DataReg = Op.getReg(); 9777 bool IsAGPR = RI.isAGPR(MRI, DataReg); 9778 Register Undef = MRI.createVirtualRegister( 9779 IsAGPR ? &AMDGPU::AGPR_32RegClass : &AMDGPU::VGPR_32RegClass); 9780 BuildMI(*BB, MI, DL, get(AMDGPU::IMPLICIT_DEF), Undef); 9781 Register NewVR = 9782 MRI.createVirtualRegister(IsAGPR ? &AMDGPU::AReg_64_Align2RegClass 9783 : &AMDGPU::VReg_64_Align2RegClass); 9784 BuildMI(*BB, MI, DL, get(AMDGPU::REG_SEQUENCE), NewVR) 9785 .addReg(DataReg, 0, Op.getSubReg()) 9786 .addImm(AMDGPU::sub0) 9787 .addReg(Undef) 9788 .addImm(AMDGPU::sub1); 9789 Op.setReg(NewVR); 9790 Op.setSubReg(AMDGPU::sub0); 9791 MI.addOperand(MachineOperand::CreateReg(NewVR, false, true)); 9792 } 9793