1 //===- SIInstrInfo.cpp - SI Instruction Information ----------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// SI Implementation of TargetInstrInfo. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "SIInstrInfo.h" 15 #include "AMDGPU.h" 16 #include "AMDGPUInstrInfo.h" 17 #include "GCNHazardRecognizer.h" 18 #include "GCNSubtarget.h" 19 #include "SIMachineFunctionInfo.h" 20 #include "llvm/Analysis/ValueTracking.h" 21 #include "llvm/CodeGen/LiveIntervals.h" 22 #include "llvm/CodeGen/LiveVariables.h" 23 #include "llvm/CodeGen/MachineDominators.h" 24 #include "llvm/CodeGen/MachineFrameInfo.h" 25 #include "llvm/CodeGen/MachineScheduler.h" 26 #include "llvm/CodeGen/RegisterScavenging.h" 27 #include "llvm/CodeGen/ScheduleDAG.h" 28 #include "llvm/IR/DiagnosticInfo.h" 29 #include "llvm/IR/IntrinsicsAMDGPU.h" 30 #include "llvm/MC/MCContext.h" 31 #include "llvm/Support/CommandLine.h" 32 #include "llvm/Target/TargetMachine.h" 33 34 using namespace llvm; 35 36 #define DEBUG_TYPE "si-instr-info" 37 38 #define GET_INSTRINFO_CTOR_DTOR 39 #include "AMDGPUGenInstrInfo.inc" 40 41 namespace llvm { 42 namespace AMDGPU { 43 #define GET_D16ImageDimIntrinsics_IMPL 44 #define GET_ImageDimIntrinsicTable_IMPL 45 #define GET_RsrcIntrinsics_IMPL 46 #include "AMDGPUGenSearchableTables.inc" 47 } 48 } 49 50 51 // Must be at least 4 to be able to branch over minimum unconditional branch 52 // code. This is only for making it possible to write reasonably small tests for 53 // long branches. 54 static cl::opt<unsigned> 55 BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16), 56 cl::desc("Restrict range of branch instructions (DEBUG)")); 57 58 static cl::opt<bool> Fix16BitCopies( 59 "amdgpu-fix-16-bit-physreg-copies", 60 cl::desc("Fix copies between 32 and 16 bit registers by extending to 32 bit"), 61 cl::init(true), 62 cl::ReallyHidden); 63 64 SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST) 65 : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN), 66 RI(ST), ST(ST) { 67 SchedModel.init(&ST); 68 } 69 70 //===----------------------------------------------------------------------===// 71 // TargetInstrInfo callbacks 72 //===----------------------------------------------------------------------===// 73 74 static unsigned getNumOperandsNoGlue(SDNode *Node) { 75 unsigned N = Node->getNumOperands(); 76 while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue) 77 --N; 78 return N; 79 } 80 81 /// Returns true if both nodes have the same value for the given 82 /// operand \p Op, or if both nodes do not have this operand. 83 static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) { 84 unsigned Opc0 = N0->getMachineOpcode(); 85 unsigned Opc1 = N1->getMachineOpcode(); 86 87 int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName); 88 int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName); 89 90 if (Op0Idx == -1 && Op1Idx == -1) 91 return true; 92 93 94 if ((Op0Idx == -1 && Op1Idx != -1) || 95 (Op1Idx == -1 && Op0Idx != -1)) 96 return false; 97 98 // getNamedOperandIdx returns the index for the MachineInstr's operands, 99 // which includes the result as the first operand. We are indexing into the 100 // MachineSDNode's operands, so we need to skip the result operand to get 101 // the real index. 102 --Op0Idx; 103 --Op1Idx; 104 105 return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx); 106 } 107 108 bool SIInstrInfo::isReallyTriviallyReMaterializable( 109 const MachineInstr &MI) const { 110 if (isVOP1(MI) || isVOP2(MI) || isVOP3(MI) || isSDWA(MI) || isSALU(MI)) { 111 // Normally VALU use of exec would block the rematerialization, but that 112 // is OK in this case to have an implicit exec read as all VALU do. 113 // We really want all of the generic logic for this except for this. 114 115 // Another potential implicit use is mode register. The core logic of 116 // the RA will not attempt rematerialization if mode is set anywhere 117 // in the function, otherwise it is safe since mode is not changed. 118 119 // There is difference to generic method which does not allow 120 // rematerialization if there are virtual register uses. We allow this, 121 // therefore this method includes SOP instructions as well. 122 return !MI.hasImplicitDef() && 123 MI.getNumImplicitOperands() == MI.getDesc().implicit_uses().size() && 124 !MI.mayRaiseFPException(); 125 } 126 127 return false; 128 } 129 130 // Returns true if the scalar result of a VALU instruction depends on exec. 131 static bool resultDependsOnExec(const MachineInstr &MI) { 132 // Ignore comparisons which are only used masked with exec. 133 // This allows some hoisting/sinking of VALU comparisons. 134 if (MI.isCompare()) { 135 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 136 Register DstReg = MI.getOperand(0).getReg(); 137 if (!DstReg.isVirtual()) 138 return true; 139 for (MachineInstr &Use : MRI.use_nodbg_instructions(DstReg)) { 140 switch (Use.getOpcode()) { 141 case AMDGPU::S_AND_SAVEEXEC_B32: 142 case AMDGPU::S_AND_SAVEEXEC_B64: 143 break; 144 case AMDGPU::S_AND_B32: 145 case AMDGPU::S_AND_B64: 146 if (!Use.readsRegister(AMDGPU::EXEC)) 147 return true; 148 break; 149 default: 150 return true; 151 } 152 } 153 return false; 154 } 155 156 switch (MI.getOpcode()) { 157 default: 158 break; 159 case AMDGPU::V_READFIRSTLANE_B32: 160 return true; 161 } 162 163 return false; 164 } 165 166 bool SIInstrInfo::isIgnorableUse(const MachineOperand &MO) const { 167 // Any implicit use of exec by VALU is not a real register read. 168 return MO.getReg() == AMDGPU::EXEC && MO.isImplicit() && 169 isVALU(*MO.getParent()) && !resultDependsOnExec(*MO.getParent()); 170 } 171 172 bool SIInstrInfo::areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1, 173 int64_t &Offset0, 174 int64_t &Offset1) const { 175 if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode()) 176 return false; 177 178 unsigned Opc0 = Load0->getMachineOpcode(); 179 unsigned Opc1 = Load1->getMachineOpcode(); 180 181 // Make sure both are actually loads. 182 if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad()) 183 return false; 184 185 if (isDS(Opc0) && isDS(Opc1)) { 186 187 // FIXME: Handle this case: 188 if (getNumOperandsNoGlue(Load0) != getNumOperandsNoGlue(Load1)) 189 return false; 190 191 // Check base reg. 192 if (Load0->getOperand(0) != Load1->getOperand(0)) 193 return false; 194 195 // Skip read2 / write2 variants for simplicity. 196 // TODO: We should report true if the used offsets are adjacent (excluded 197 // st64 versions). 198 int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset); 199 int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset); 200 if (Offset0Idx == -1 || Offset1Idx == -1) 201 return false; 202 203 // XXX - be careful of dataless loads 204 // getNamedOperandIdx returns the index for MachineInstrs. Since they 205 // include the output in the operand list, but SDNodes don't, we need to 206 // subtract the index by one. 207 Offset0Idx -= get(Opc0).NumDefs; 208 Offset1Idx -= get(Opc1).NumDefs; 209 Offset0 = cast<ConstantSDNode>(Load0->getOperand(Offset0Idx))->getZExtValue(); 210 Offset1 = cast<ConstantSDNode>(Load1->getOperand(Offset1Idx))->getZExtValue(); 211 return true; 212 } 213 214 if (isSMRD(Opc0) && isSMRD(Opc1)) { 215 // Skip time and cache invalidation instructions. 216 if (!AMDGPU::hasNamedOperand(Opc0, AMDGPU::OpName::sbase) || 217 !AMDGPU::hasNamedOperand(Opc1, AMDGPU::OpName::sbase)) 218 return false; 219 220 unsigned NumOps = getNumOperandsNoGlue(Load0); 221 if (NumOps != getNumOperandsNoGlue(Load1)) 222 return false; 223 224 // Check base reg. 225 if (Load0->getOperand(0) != Load1->getOperand(0)) 226 return false; 227 228 // Match register offsets, if both register and immediate offsets present. 229 assert(NumOps == 4 || NumOps == 5); 230 if (NumOps == 5 && Load0->getOperand(1) != Load1->getOperand(1)) 231 return false; 232 233 const ConstantSDNode *Load0Offset = 234 dyn_cast<ConstantSDNode>(Load0->getOperand(NumOps - 3)); 235 const ConstantSDNode *Load1Offset = 236 dyn_cast<ConstantSDNode>(Load1->getOperand(NumOps - 3)); 237 238 if (!Load0Offset || !Load1Offset) 239 return false; 240 241 Offset0 = Load0Offset->getZExtValue(); 242 Offset1 = Load1Offset->getZExtValue(); 243 return true; 244 } 245 246 // MUBUF and MTBUF can access the same addresses. 247 if ((isMUBUF(Opc0) || isMTBUF(Opc0)) && (isMUBUF(Opc1) || isMTBUF(Opc1))) { 248 249 // MUBUF and MTBUF have vaddr at different indices. 250 if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) || 251 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) || 252 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc)) 253 return false; 254 255 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset); 256 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset); 257 258 if (OffIdx0 == -1 || OffIdx1 == -1) 259 return false; 260 261 // getNamedOperandIdx returns the index for MachineInstrs. Since they 262 // include the output in the operand list, but SDNodes don't, we need to 263 // subtract the index by one. 264 OffIdx0 -= get(Opc0).NumDefs; 265 OffIdx1 -= get(Opc1).NumDefs; 266 267 SDValue Off0 = Load0->getOperand(OffIdx0); 268 SDValue Off1 = Load1->getOperand(OffIdx1); 269 270 // The offset might be a FrameIndexSDNode. 271 if (!isa<ConstantSDNode>(Off0) || !isa<ConstantSDNode>(Off1)) 272 return false; 273 274 Offset0 = cast<ConstantSDNode>(Off0)->getZExtValue(); 275 Offset1 = cast<ConstantSDNode>(Off1)->getZExtValue(); 276 return true; 277 } 278 279 return false; 280 } 281 282 static bool isStride64(unsigned Opc) { 283 switch (Opc) { 284 case AMDGPU::DS_READ2ST64_B32: 285 case AMDGPU::DS_READ2ST64_B64: 286 case AMDGPU::DS_WRITE2ST64_B32: 287 case AMDGPU::DS_WRITE2ST64_B64: 288 return true; 289 default: 290 return false; 291 } 292 } 293 294 bool SIInstrInfo::getMemOperandsWithOffsetWidth( 295 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps, 296 int64_t &Offset, bool &OffsetIsScalable, unsigned &Width, 297 const TargetRegisterInfo *TRI) const { 298 if (!LdSt.mayLoadOrStore()) 299 return false; 300 301 unsigned Opc = LdSt.getOpcode(); 302 OffsetIsScalable = false; 303 const MachineOperand *BaseOp, *OffsetOp; 304 int DataOpIdx; 305 306 if (isDS(LdSt)) { 307 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr); 308 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset); 309 if (OffsetOp) { 310 // Normal, single offset LDS instruction. 311 if (!BaseOp) { 312 // DS_CONSUME/DS_APPEND use M0 for the base address. 313 // TODO: find the implicit use operand for M0 and use that as BaseOp? 314 return false; 315 } 316 BaseOps.push_back(BaseOp); 317 Offset = OffsetOp->getImm(); 318 // Get appropriate operand, and compute width accordingly. 319 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 320 if (DataOpIdx == -1) 321 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); 322 Width = getOpSize(LdSt, DataOpIdx); 323 } else { 324 // The 2 offset instructions use offset0 and offset1 instead. We can treat 325 // these as a load with a single offset if the 2 offsets are consecutive. 326 // We will use this for some partially aligned loads. 327 const MachineOperand *Offset0Op = 328 getNamedOperand(LdSt, AMDGPU::OpName::offset0); 329 const MachineOperand *Offset1Op = 330 getNamedOperand(LdSt, AMDGPU::OpName::offset1); 331 332 unsigned Offset0 = Offset0Op->getImm(); 333 unsigned Offset1 = Offset1Op->getImm(); 334 if (Offset0 + 1 != Offset1) 335 return false; 336 337 // Each of these offsets is in element sized units, so we need to convert 338 // to bytes of the individual reads. 339 340 unsigned EltSize; 341 if (LdSt.mayLoad()) 342 EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, 0)) / 16; 343 else { 344 assert(LdSt.mayStore()); 345 int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); 346 EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, Data0Idx)) / 8; 347 } 348 349 if (isStride64(Opc)) 350 EltSize *= 64; 351 352 BaseOps.push_back(BaseOp); 353 Offset = EltSize * Offset0; 354 // Get appropriate operand(s), and compute width accordingly. 355 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 356 if (DataOpIdx == -1) { 357 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); 358 Width = getOpSize(LdSt, DataOpIdx); 359 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1); 360 Width += getOpSize(LdSt, DataOpIdx); 361 } else { 362 Width = getOpSize(LdSt, DataOpIdx); 363 } 364 } 365 return true; 366 } 367 368 if (isMUBUF(LdSt) || isMTBUF(LdSt)) { 369 const MachineOperand *RSrc = getNamedOperand(LdSt, AMDGPU::OpName::srsrc); 370 if (!RSrc) // e.g. BUFFER_WBINVL1_VOL 371 return false; 372 BaseOps.push_back(RSrc); 373 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr); 374 if (BaseOp && !BaseOp->isFI()) 375 BaseOps.push_back(BaseOp); 376 const MachineOperand *OffsetImm = 377 getNamedOperand(LdSt, AMDGPU::OpName::offset); 378 Offset = OffsetImm->getImm(); 379 const MachineOperand *SOffset = 380 getNamedOperand(LdSt, AMDGPU::OpName::soffset); 381 if (SOffset) { 382 if (SOffset->isReg()) 383 BaseOps.push_back(SOffset); 384 else 385 Offset += SOffset->getImm(); 386 } 387 // Get appropriate operand, and compute width accordingly. 388 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 389 if (DataOpIdx == -1) 390 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); 391 if (DataOpIdx == -1) // LDS DMA 392 return false; 393 Width = getOpSize(LdSt, DataOpIdx); 394 return true; 395 } 396 397 if (isMIMG(LdSt)) { 398 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc); 399 BaseOps.push_back(&LdSt.getOperand(SRsrcIdx)); 400 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0); 401 if (VAddr0Idx >= 0) { 402 // GFX10 possible NSA encoding. 403 for (int I = VAddr0Idx; I < SRsrcIdx; ++I) 404 BaseOps.push_back(&LdSt.getOperand(I)); 405 } else { 406 BaseOps.push_back(getNamedOperand(LdSt, AMDGPU::OpName::vaddr)); 407 } 408 Offset = 0; 409 // Get appropriate operand, and compute width accordingly. 410 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); 411 Width = getOpSize(LdSt, DataOpIdx); 412 return true; 413 } 414 415 if (isSMRD(LdSt)) { 416 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::sbase); 417 if (!BaseOp) // e.g. S_MEMTIME 418 return false; 419 BaseOps.push_back(BaseOp); 420 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset); 421 Offset = OffsetOp ? OffsetOp->getImm() : 0; 422 // Get appropriate operand, and compute width accordingly. 423 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::sdst); 424 Width = getOpSize(LdSt, DataOpIdx); 425 return true; 426 } 427 428 if (isFLAT(LdSt)) { 429 // Instructions have either vaddr or saddr or both or none. 430 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr); 431 if (BaseOp) 432 BaseOps.push_back(BaseOp); 433 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr); 434 if (BaseOp) 435 BaseOps.push_back(BaseOp); 436 Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm(); 437 // Get appropriate operand, and compute width accordingly. 438 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 439 if (DataOpIdx == -1) 440 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); 441 if (DataOpIdx == -1) // LDS DMA 442 return false; 443 Width = getOpSize(LdSt, DataOpIdx); 444 return true; 445 } 446 447 return false; 448 } 449 450 static bool memOpsHaveSameBasePtr(const MachineInstr &MI1, 451 ArrayRef<const MachineOperand *> BaseOps1, 452 const MachineInstr &MI2, 453 ArrayRef<const MachineOperand *> BaseOps2) { 454 // Only examine the first "base" operand of each instruction, on the 455 // assumption that it represents the real base address of the memory access. 456 // Other operands are typically offsets or indices from this base address. 457 if (BaseOps1.front()->isIdenticalTo(*BaseOps2.front())) 458 return true; 459 460 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand()) 461 return false; 462 463 auto MO1 = *MI1.memoperands_begin(); 464 auto MO2 = *MI2.memoperands_begin(); 465 if (MO1->getAddrSpace() != MO2->getAddrSpace()) 466 return false; 467 468 auto Base1 = MO1->getValue(); 469 auto Base2 = MO2->getValue(); 470 if (!Base1 || !Base2) 471 return false; 472 Base1 = getUnderlyingObject(Base1); 473 Base2 = getUnderlyingObject(Base2); 474 475 if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2)) 476 return false; 477 478 return Base1 == Base2; 479 } 480 481 bool SIInstrInfo::shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1, 482 ArrayRef<const MachineOperand *> BaseOps2, 483 unsigned NumLoads, 484 unsigned NumBytes) const { 485 // If the mem ops (to be clustered) do not have the same base ptr, then they 486 // should not be clustered 487 if (!BaseOps1.empty() && !BaseOps2.empty()) { 488 const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent(); 489 const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent(); 490 if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOps1, SecondLdSt, BaseOps2)) 491 return false; 492 } else if (!BaseOps1.empty() || !BaseOps2.empty()) { 493 // If only one base op is empty, they do not have the same base ptr 494 return false; 495 } 496 497 // In order to avoid register pressure, on an average, the number of DWORDS 498 // loaded together by all clustered mem ops should not exceed 8. This is an 499 // empirical value based on certain observations and performance related 500 // experiments. 501 // The good thing about this heuristic is - it avoids clustering of too many 502 // sub-word loads, and also avoids clustering of wide loads. Below is the 503 // brief summary of how the heuristic behaves for various `LoadSize`. 504 // (1) 1 <= LoadSize <= 4: cluster at max 8 mem ops 505 // (2) 5 <= LoadSize <= 8: cluster at max 4 mem ops 506 // (3) 9 <= LoadSize <= 12: cluster at max 2 mem ops 507 // (4) 13 <= LoadSize <= 16: cluster at max 2 mem ops 508 // (5) LoadSize >= 17: do not cluster 509 const unsigned LoadSize = NumBytes / NumLoads; 510 const unsigned NumDWORDs = ((LoadSize + 3) / 4) * NumLoads; 511 return NumDWORDs <= 8; 512 } 513 514 // FIXME: This behaves strangely. If, for example, you have 32 load + stores, 515 // the first 16 loads will be interleaved with the stores, and the next 16 will 516 // be clustered as expected. It should really split into 2 16 store batches. 517 // 518 // Loads are clustered until this returns false, rather than trying to schedule 519 // groups of stores. This also means we have to deal with saying different 520 // address space loads should be clustered, and ones which might cause bank 521 // conflicts. 522 // 523 // This might be deprecated so it might not be worth that much effort to fix. 524 bool SIInstrInfo::shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1, 525 int64_t Offset0, int64_t Offset1, 526 unsigned NumLoads) const { 527 assert(Offset1 > Offset0 && 528 "Second offset should be larger than first offset!"); 529 // If we have less than 16 loads in a row, and the offsets are within 64 530 // bytes, then schedule together. 531 532 // A cacheline is 64 bytes (for global memory). 533 return (NumLoads <= 16 && (Offset1 - Offset0) < 64); 534 } 535 536 static void reportIllegalCopy(const SIInstrInfo *TII, MachineBasicBlock &MBB, 537 MachineBasicBlock::iterator MI, 538 const DebugLoc &DL, MCRegister DestReg, 539 MCRegister SrcReg, bool KillSrc, 540 const char *Msg = "illegal SGPR to VGPR copy") { 541 MachineFunction *MF = MBB.getParent(); 542 DiagnosticInfoUnsupported IllegalCopy(MF->getFunction(), Msg, DL, DS_Error); 543 LLVMContext &C = MF->getFunction().getContext(); 544 C.diagnose(IllegalCopy); 545 546 BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg) 547 .addReg(SrcReg, getKillRegState(KillSrc)); 548 } 549 550 /// Handle copying from SGPR to AGPR, or from AGPR to AGPR on GFX908. It is not 551 /// possible to have a direct copy in these cases on GFX908, so an intermediate 552 /// VGPR copy is required. 553 static void indirectCopyToAGPR(const SIInstrInfo &TII, 554 MachineBasicBlock &MBB, 555 MachineBasicBlock::iterator MI, 556 const DebugLoc &DL, MCRegister DestReg, 557 MCRegister SrcReg, bool KillSrc, 558 RegScavenger &RS, bool RegsOverlap, 559 Register ImpDefSuperReg = Register(), 560 Register ImpUseSuperReg = Register()) { 561 assert((TII.getSubtarget().hasMAIInsts() && 562 !TII.getSubtarget().hasGFX90AInsts()) && 563 "Expected GFX908 subtarget."); 564 565 assert((AMDGPU::SReg_32RegClass.contains(SrcReg) || 566 AMDGPU::AGPR_32RegClass.contains(SrcReg)) && 567 "Source register of the copy should be either an SGPR or an AGPR."); 568 569 assert(AMDGPU::AGPR_32RegClass.contains(DestReg) && 570 "Destination register of the copy should be an AGPR."); 571 572 const SIRegisterInfo &RI = TII.getRegisterInfo(); 573 574 // First try to find defining accvgpr_write to avoid temporary registers. 575 // In the case of copies of overlapping AGPRs, we conservatively do not 576 // reuse previous accvgpr_writes. Otherwise, we may incorrectly pick up 577 // an accvgpr_write used for this same copy due to implicit-defs 578 if (!RegsOverlap) { 579 for (auto Def = MI, E = MBB.begin(); Def != E; ) { 580 --Def; 581 if (!Def->definesRegister(SrcReg, &RI)) 582 continue; 583 if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) 584 break; 585 586 MachineOperand &DefOp = Def->getOperand(1); 587 assert(DefOp.isReg() || DefOp.isImm()); 588 589 if (DefOp.isReg()) { 590 bool SafeToPropagate = true; 591 // Check that register source operand is not clobbered before MI. 592 // Immediate operands are always safe to propagate. 593 for (auto I = Def; I != MI && SafeToPropagate; ++I) 594 if (I->modifiesRegister(DefOp.getReg(), &RI)) 595 SafeToPropagate = false; 596 597 if (!SafeToPropagate) 598 break; 599 600 DefOp.setIsKill(false); 601 } 602 603 MachineInstrBuilder Builder = 604 BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) 605 .add(DefOp); 606 if (ImpDefSuperReg) 607 Builder.addReg(ImpDefSuperReg, RegState::Define | RegState::Implicit); 608 609 if (ImpUseSuperReg) { 610 Builder.addReg(ImpUseSuperReg, 611 getKillRegState(KillSrc) | RegState::Implicit); 612 } 613 614 return; 615 } 616 } 617 618 RS.enterBasicBlock(MBB); 619 RS.forward(MI); 620 621 // Ideally we want to have three registers for a long reg_sequence copy 622 // to hide 2 waitstates between v_mov_b32 and accvgpr_write. 623 unsigned MaxVGPRs = RI.getRegPressureLimit(&AMDGPU::VGPR_32RegClass, 624 *MBB.getParent()); 625 626 // Registers in the sequence are allocated contiguously so we can just 627 // use register number to pick one of three round-robin temps. 628 unsigned RegNo = (DestReg - AMDGPU::AGPR0) % 3; 629 Register Tmp = 630 MBB.getParent()->getInfo<SIMachineFunctionInfo>()->getVGPRForAGPRCopy(); 631 assert(MBB.getParent()->getRegInfo().isReserved(Tmp) && 632 "VGPR used for an intermediate copy should have been reserved."); 633 634 // Only loop through if there are any free registers left, otherwise 635 // scavenger may report a fatal error without emergency spill slot 636 // or spill with the slot. 637 while (RegNo-- && RS.FindUnusedReg(&AMDGPU::VGPR_32RegClass)) { 638 Register Tmp2 = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0); 639 if (!Tmp2 || RI.getHWRegIndex(Tmp2) >= MaxVGPRs) 640 break; 641 Tmp = Tmp2; 642 RS.setRegUsed(Tmp); 643 } 644 645 // Insert copy to temporary VGPR. 646 unsigned TmpCopyOp = AMDGPU::V_MOV_B32_e32; 647 if (AMDGPU::AGPR_32RegClass.contains(SrcReg)) { 648 TmpCopyOp = AMDGPU::V_ACCVGPR_READ_B32_e64; 649 } else { 650 assert(AMDGPU::SReg_32RegClass.contains(SrcReg)); 651 } 652 653 MachineInstrBuilder UseBuilder = BuildMI(MBB, MI, DL, TII.get(TmpCopyOp), Tmp) 654 .addReg(SrcReg, getKillRegState(KillSrc)); 655 if (ImpUseSuperReg) { 656 UseBuilder.addReg(ImpUseSuperReg, 657 getKillRegState(KillSrc) | RegState::Implicit); 658 } 659 660 MachineInstrBuilder DefBuilder 661 = BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) 662 .addReg(Tmp, RegState::Kill); 663 664 if (ImpDefSuperReg) 665 DefBuilder.addReg(ImpDefSuperReg, RegState::Define | RegState::Implicit); 666 } 667 668 static void expandSGPRCopy(const SIInstrInfo &TII, MachineBasicBlock &MBB, 669 MachineBasicBlock::iterator MI, const DebugLoc &DL, 670 MCRegister DestReg, MCRegister SrcReg, bool KillSrc, 671 const TargetRegisterClass *RC, bool Forward) { 672 const SIRegisterInfo &RI = TII.getRegisterInfo(); 673 ArrayRef<int16_t> BaseIndices = RI.getRegSplitParts(RC, 4); 674 MachineBasicBlock::iterator I = MI; 675 MachineInstr *FirstMI = nullptr, *LastMI = nullptr; 676 677 for (unsigned Idx = 0; Idx < BaseIndices.size(); ++Idx) { 678 int16_t SubIdx = BaseIndices[Idx]; 679 Register Reg = RI.getSubReg(DestReg, SubIdx); 680 unsigned Opcode = AMDGPU::S_MOV_B32; 681 682 // Is SGPR aligned? If so try to combine with next. 683 Register Src = RI.getSubReg(SrcReg, SubIdx); 684 bool AlignedDest = ((Reg - AMDGPU::SGPR0) % 2) == 0; 685 bool AlignedSrc = ((Src - AMDGPU::SGPR0) % 2) == 0; 686 if (AlignedDest && AlignedSrc && (Idx + 1 < BaseIndices.size())) { 687 // Can use SGPR64 copy 688 unsigned Channel = RI.getChannelFromSubReg(SubIdx); 689 SubIdx = RI.getSubRegFromChannel(Channel, 2); 690 Opcode = AMDGPU::S_MOV_B64; 691 Idx++; 692 } 693 694 LastMI = BuildMI(MBB, I, DL, TII.get(Opcode), RI.getSubReg(DestReg, SubIdx)) 695 .addReg(RI.getSubReg(SrcReg, SubIdx)) 696 .addReg(SrcReg, RegState::Implicit); 697 698 if (!FirstMI) 699 FirstMI = LastMI; 700 701 if (!Forward) 702 I--; 703 } 704 705 assert(FirstMI && LastMI); 706 if (!Forward) 707 std::swap(FirstMI, LastMI); 708 709 FirstMI->addOperand( 710 MachineOperand::CreateReg(DestReg, true /*IsDef*/, true /*IsImp*/)); 711 712 if (KillSrc) 713 LastMI->addRegisterKilled(SrcReg, &RI); 714 } 715 716 void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB, 717 MachineBasicBlock::iterator MI, 718 const DebugLoc &DL, MCRegister DestReg, 719 MCRegister SrcReg, bool KillSrc) const { 720 const TargetRegisterClass *RC = RI.getPhysRegBaseClass(DestReg); 721 722 // FIXME: This is hack to resolve copies between 16 bit and 32 bit 723 // registers until all patterns are fixed. 724 if (Fix16BitCopies && 725 ((RI.getRegSizeInBits(*RC) == 16) ^ 726 (RI.getRegSizeInBits(*RI.getPhysRegBaseClass(SrcReg)) == 16))) { 727 MCRegister &RegToFix = (RI.getRegSizeInBits(*RC) == 16) ? DestReg : SrcReg; 728 MCRegister Super = RI.get32BitRegister(RegToFix); 729 assert(RI.getSubReg(Super, AMDGPU::lo16) == RegToFix); 730 RegToFix = Super; 731 732 if (DestReg == SrcReg) { 733 // Insert empty bundle since ExpandPostRA expects an instruction here. 734 BuildMI(MBB, MI, DL, get(AMDGPU::BUNDLE)); 735 return; 736 } 737 738 RC = RI.getPhysRegBaseClass(DestReg); 739 } 740 741 if (RC == &AMDGPU::VGPR_32RegClass) { 742 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) || 743 AMDGPU::SReg_32RegClass.contains(SrcReg) || 744 AMDGPU::AGPR_32RegClass.contains(SrcReg)); 745 unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ? 746 AMDGPU::V_ACCVGPR_READ_B32_e64 : AMDGPU::V_MOV_B32_e32; 747 BuildMI(MBB, MI, DL, get(Opc), DestReg) 748 .addReg(SrcReg, getKillRegState(KillSrc)); 749 return; 750 } 751 752 if (RC == &AMDGPU::SReg_32_XM0RegClass || 753 RC == &AMDGPU::SReg_32RegClass) { 754 if (SrcReg == AMDGPU::SCC) { 755 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg) 756 .addImm(1) 757 .addImm(0); 758 return; 759 } 760 761 if (DestReg == AMDGPU::VCC_LO) { 762 if (AMDGPU::SReg_32RegClass.contains(SrcReg)) { 763 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), AMDGPU::VCC_LO) 764 .addReg(SrcReg, getKillRegState(KillSrc)); 765 } else { 766 // FIXME: Hack until VReg_1 removed. 767 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg)); 768 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32)) 769 .addImm(0) 770 .addReg(SrcReg, getKillRegState(KillSrc)); 771 } 772 773 return; 774 } 775 776 if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) { 777 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc); 778 return; 779 } 780 781 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg) 782 .addReg(SrcReg, getKillRegState(KillSrc)); 783 return; 784 } 785 786 if (RC == &AMDGPU::SReg_64RegClass) { 787 if (SrcReg == AMDGPU::SCC) { 788 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B64), DestReg) 789 .addImm(1) 790 .addImm(0); 791 return; 792 } 793 794 if (DestReg == AMDGPU::VCC) { 795 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) { 796 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC) 797 .addReg(SrcReg, getKillRegState(KillSrc)); 798 } else { 799 // FIXME: Hack until VReg_1 removed. 800 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg)); 801 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32)) 802 .addImm(0) 803 .addReg(SrcReg, getKillRegState(KillSrc)); 804 } 805 806 return; 807 } 808 809 if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) { 810 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc); 811 return; 812 } 813 814 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg) 815 .addReg(SrcReg, getKillRegState(KillSrc)); 816 return; 817 } 818 819 if (DestReg == AMDGPU::SCC) { 820 // Copying 64-bit or 32-bit sources to SCC barely makes sense, 821 // but SelectionDAG emits such copies for i1 sources. 822 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) { 823 // This copy can only be produced by patterns 824 // with explicit SCC, which are known to be enabled 825 // only for subtargets with S_CMP_LG_U64 present. 826 assert(ST.hasScalarCompareEq64()); 827 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U64)) 828 .addReg(SrcReg, getKillRegState(KillSrc)) 829 .addImm(0); 830 } else { 831 assert(AMDGPU::SReg_32RegClass.contains(SrcReg)); 832 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32)) 833 .addReg(SrcReg, getKillRegState(KillSrc)) 834 .addImm(0); 835 } 836 837 return; 838 } 839 840 if (RC == &AMDGPU::AGPR_32RegClass) { 841 if (AMDGPU::VGPR_32RegClass.contains(SrcReg) || 842 (ST.hasGFX90AInsts() && AMDGPU::SReg_32RegClass.contains(SrcReg))) { 843 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) 844 .addReg(SrcReg, getKillRegState(KillSrc)); 845 return; 846 } 847 848 if (AMDGPU::AGPR_32RegClass.contains(SrcReg) && ST.hasGFX90AInsts()) { 849 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_MOV_B32), DestReg) 850 .addReg(SrcReg, getKillRegState(KillSrc)); 851 return; 852 } 853 854 // FIXME: Pass should maintain scavenger to avoid scan through the block on 855 // every AGPR spill. 856 RegScavenger RS; 857 const bool Overlap = RI.regsOverlap(SrcReg, DestReg); 858 indirectCopyToAGPR(*this, MBB, MI, DL, DestReg, SrcReg, KillSrc, RS, Overlap); 859 return; 860 } 861 862 const unsigned Size = RI.getRegSizeInBits(*RC); 863 if (Size == 16) { 864 assert(AMDGPU::VGPR_LO16RegClass.contains(SrcReg) || 865 AMDGPU::VGPR_HI16RegClass.contains(SrcReg) || 866 AMDGPU::SReg_LO16RegClass.contains(SrcReg) || 867 AMDGPU::AGPR_LO16RegClass.contains(SrcReg)); 868 869 bool IsSGPRDst = AMDGPU::SReg_LO16RegClass.contains(DestReg); 870 bool IsSGPRSrc = AMDGPU::SReg_LO16RegClass.contains(SrcReg); 871 bool IsAGPRDst = AMDGPU::AGPR_LO16RegClass.contains(DestReg); 872 bool IsAGPRSrc = AMDGPU::AGPR_LO16RegClass.contains(SrcReg); 873 bool DstLow = AMDGPU::VGPR_LO16RegClass.contains(DestReg) || 874 AMDGPU::SReg_LO16RegClass.contains(DestReg) || 875 AMDGPU::AGPR_LO16RegClass.contains(DestReg); 876 bool SrcLow = AMDGPU::VGPR_LO16RegClass.contains(SrcReg) || 877 AMDGPU::SReg_LO16RegClass.contains(SrcReg) || 878 AMDGPU::AGPR_LO16RegClass.contains(SrcReg); 879 MCRegister NewDestReg = RI.get32BitRegister(DestReg); 880 MCRegister NewSrcReg = RI.get32BitRegister(SrcReg); 881 882 if (IsSGPRDst) { 883 if (!IsSGPRSrc) { 884 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc); 885 return; 886 } 887 888 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), NewDestReg) 889 .addReg(NewSrcReg, getKillRegState(KillSrc)); 890 return; 891 } 892 893 if (IsAGPRDst || IsAGPRSrc) { 894 if (!DstLow || !SrcLow) { 895 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc, 896 "Cannot use hi16 subreg with an AGPR!"); 897 } 898 899 copyPhysReg(MBB, MI, DL, NewDestReg, NewSrcReg, KillSrc); 900 return; 901 } 902 903 if (IsSGPRSrc && !ST.hasSDWAScalar()) { 904 if (!DstLow || !SrcLow) { 905 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc, 906 "Cannot use hi16 subreg on VI!"); 907 } 908 909 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), NewDestReg) 910 .addReg(NewSrcReg, getKillRegState(KillSrc)); 911 return; 912 } 913 914 auto MIB = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_sdwa), NewDestReg) 915 .addImm(0) // src0_modifiers 916 .addReg(NewSrcReg) 917 .addImm(0) // clamp 918 .addImm(DstLow ? AMDGPU::SDWA::SdwaSel::WORD_0 919 : AMDGPU::SDWA::SdwaSel::WORD_1) 920 .addImm(AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE) 921 .addImm(SrcLow ? AMDGPU::SDWA::SdwaSel::WORD_0 922 : AMDGPU::SDWA::SdwaSel::WORD_1) 923 .addReg(NewDestReg, RegState::Implicit | RegState::Undef); 924 // First implicit operand is $exec. 925 MIB->tieOperands(0, MIB->getNumOperands() - 1); 926 return; 927 } 928 929 const TargetRegisterClass *SrcRC = RI.getPhysRegBaseClass(SrcReg); 930 if (RC == RI.getVGPR64Class() && (SrcRC == RC || RI.isSGPRClass(SrcRC))) { 931 if (ST.hasMovB64()) { 932 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_e32), DestReg) 933 .addReg(SrcReg, getKillRegState(KillSrc)); 934 return; 935 } 936 if (ST.hasPackedFP32Ops()) { 937 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DestReg) 938 .addImm(SISrcMods::OP_SEL_1) 939 .addReg(SrcReg) 940 .addImm(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1) 941 .addReg(SrcReg) 942 .addImm(0) // op_sel_lo 943 .addImm(0) // op_sel_hi 944 .addImm(0) // neg_lo 945 .addImm(0) // neg_hi 946 .addImm(0) // clamp 947 .addReg(SrcReg, getKillRegState(KillSrc) | RegState::Implicit); 948 return; 949 } 950 } 951 952 const bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg); 953 if (RI.isSGPRClass(RC)) { 954 if (!RI.isSGPRClass(SrcRC)) { 955 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc); 956 return; 957 } 958 const bool CanKillSuperReg = KillSrc && !RI.regsOverlap(SrcReg, DestReg); 959 expandSGPRCopy(*this, MBB, MI, DL, DestReg, SrcReg, CanKillSuperReg, RC, 960 Forward); 961 return; 962 } 963 964 unsigned EltSize = 4; 965 unsigned Opcode = AMDGPU::V_MOV_B32_e32; 966 if (RI.isAGPRClass(RC)) { 967 if (ST.hasGFX90AInsts() && RI.isAGPRClass(SrcRC)) 968 Opcode = AMDGPU::V_ACCVGPR_MOV_B32; 969 else if (RI.hasVGPRs(SrcRC) || 970 (ST.hasGFX90AInsts() && RI.isSGPRClass(SrcRC))) 971 Opcode = AMDGPU::V_ACCVGPR_WRITE_B32_e64; 972 else 973 Opcode = AMDGPU::INSTRUCTION_LIST_END; 974 } else if (RI.hasVGPRs(RC) && RI.isAGPRClass(SrcRC)) { 975 Opcode = AMDGPU::V_ACCVGPR_READ_B32_e64; 976 } else if ((Size % 64 == 0) && RI.hasVGPRs(RC) && 977 (RI.isProperlyAlignedRC(*RC) && 978 (SrcRC == RC || RI.isSGPRClass(SrcRC)))) { 979 // TODO: In 96-bit case, could do a 64-bit mov and then a 32-bit mov. 980 if (ST.hasMovB64()) { 981 Opcode = AMDGPU::V_MOV_B64_e32; 982 EltSize = 8; 983 } else if (ST.hasPackedFP32Ops()) { 984 Opcode = AMDGPU::V_PK_MOV_B32; 985 EltSize = 8; 986 } 987 } 988 989 // For the cases where we need an intermediate instruction/temporary register 990 // (destination is an AGPR), we need a scavenger. 991 // 992 // FIXME: The pass should maintain this for us so we don't have to re-scan the 993 // whole block for every handled copy. 994 std::unique_ptr<RegScavenger> RS; 995 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) 996 RS.reset(new RegScavenger()); 997 998 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize); 999 1000 // If there is an overlap, we can't kill the super-register on the last 1001 // instruction, since it will also kill the components made live by this def. 1002 const bool Overlap = RI.regsOverlap(SrcReg, DestReg); 1003 const bool CanKillSuperReg = KillSrc && !Overlap; 1004 1005 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) { 1006 unsigned SubIdx; 1007 if (Forward) 1008 SubIdx = SubIndices[Idx]; 1009 else 1010 SubIdx = SubIndices[SubIndices.size() - Idx - 1]; 1011 1012 bool IsFirstSubreg = Idx == 0; 1013 bool UseKill = CanKillSuperReg && Idx == SubIndices.size() - 1; 1014 1015 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) { 1016 Register ImpDefSuper = IsFirstSubreg ? Register(DestReg) : Register(); 1017 Register ImpUseSuper = SrcReg; 1018 indirectCopyToAGPR(*this, MBB, MI, DL, RI.getSubReg(DestReg, SubIdx), 1019 RI.getSubReg(SrcReg, SubIdx), UseKill, *RS, Overlap, 1020 ImpDefSuper, ImpUseSuper); 1021 } else if (Opcode == AMDGPU::V_PK_MOV_B32) { 1022 Register DstSubReg = RI.getSubReg(DestReg, SubIdx); 1023 Register SrcSubReg = RI.getSubReg(SrcReg, SubIdx); 1024 MachineInstrBuilder MIB = 1025 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DstSubReg) 1026 .addImm(SISrcMods::OP_SEL_1) 1027 .addReg(SrcSubReg) 1028 .addImm(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1) 1029 .addReg(SrcSubReg) 1030 .addImm(0) // op_sel_lo 1031 .addImm(0) // op_sel_hi 1032 .addImm(0) // neg_lo 1033 .addImm(0) // neg_hi 1034 .addImm(0) // clamp 1035 .addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit); 1036 if (IsFirstSubreg) 1037 MIB.addReg(DestReg, RegState::Define | RegState::Implicit); 1038 } else { 1039 MachineInstrBuilder Builder = 1040 BuildMI(MBB, MI, DL, get(Opcode), RI.getSubReg(DestReg, SubIdx)) 1041 .addReg(RI.getSubReg(SrcReg, SubIdx)); 1042 if (IsFirstSubreg) 1043 Builder.addReg(DestReg, RegState::Define | RegState::Implicit); 1044 1045 Builder.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit); 1046 } 1047 } 1048 } 1049 1050 int SIInstrInfo::commuteOpcode(unsigned Opcode) const { 1051 int NewOpc; 1052 1053 // Try to map original to commuted opcode 1054 NewOpc = AMDGPU::getCommuteRev(Opcode); 1055 if (NewOpc != -1) 1056 // Check if the commuted (REV) opcode exists on the target. 1057 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1; 1058 1059 // Try to map commuted to original opcode 1060 NewOpc = AMDGPU::getCommuteOrig(Opcode); 1061 if (NewOpc != -1) 1062 // Check if the original (non-REV) opcode exists on the target. 1063 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1; 1064 1065 return Opcode; 1066 } 1067 1068 void SIInstrInfo::materializeImmediate(MachineBasicBlock &MBB, 1069 MachineBasicBlock::iterator MI, 1070 const DebugLoc &DL, Register DestReg, 1071 int64_t Value) const { 1072 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 1073 const TargetRegisterClass *RegClass = MRI.getRegClass(DestReg); 1074 if (RegClass == &AMDGPU::SReg_32RegClass || 1075 RegClass == &AMDGPU::SGPR_32RegClass || 1076 RegClass == &AMDGPU::SReg_32_XM0RegClass || 1077 RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) { 1078 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg) 1079 .addImm(Value); 1080 return; 1081 } 1082 1083 if (RegClass == &AMDGPU::SReg_64RegClass || 1084 RegClass == &AMDGPU::SGPR_64RegClass || 1085 RegClass == &AMDGPU::SReg_64_XEXECRegClass) { 1086 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg) 1087 .addImm(Value); 1088 return; 1089 } 1090 1091 if (RegClass == &AMDGPU::VGPR_32RegClass) { 1092 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg) 1093 .addImm(Value); 1094 return; 1095 } 1096 if (RegClass->hasSuperClassEq(&AMDGPU::VReg_64RegClass)) { 1097 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg) 1098 .addImm(Value); 1099 return; 1100 } 1101 1102 unsigned EltSize = 4; 1103 unsigned Opcode = AMDGPU::V_MOV_B32_e32; 1104 if (RI.isSGPRClass(RegClass)) { 1105 if (RI.getRegSizeInBits(*RegClass) > 32) { 1106 Opcode = AMDGPU::S_MOV_B64; 1107 EltSize = 8; 1108 } else { 1109 Opcode = AMDGPU::S_MOV_B32; 1110 EltSize = 4; 1111 } 1112 } 1113 1114 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RegClass, EltSize); 1115 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) { 1116 int64_t IdxValue = Idx == 0 ? Value : 0; 1117 1118 MachineInstrBuilder Builder = BuildMI(MBB, MI, DL, 1119 get(Opcode), RI.getSubReg(DestReg, SubIndices[Idx])); 1120 Builder.addImm(IdxValue); 1121 } 1122 } 1123 1124 const TargetRegisterClass * 1125 SIInstrInfo::getPreferredSelectRegClass(unsigned Size) const { 1126 return &AMDGPU::VGPR_32RegClass; 1127 } 1128 1129 void SIInstrInfo::insertVectorSelect(MachineBasicBlock &MBB, 1130 MachineBasicBlock::iterator I, 1131 const DebugLoc &DL, Register DstReg, 1132 ArrayRef<MachineOperand> Cond, 1133 Register TrueReg, 1134 Register FalseReg) const { 1135 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 1136 const TargetRegisterClass *BoolXExecRC = 1137 RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 1138 assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass && 1139 "Not a VGPR32 reg"); 1140 1141 if (Cond.size() == 1) { 1142 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1143 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) 1144 .add(Cond[0]); 1145 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1146 .addImm(0) 1147 .addReg(FalseReg) 1148 .addImm(0) 1149 .addReg(TrueReg) 1150 .addReg(SReg); 1151 } else if (Cond.size() == 2) { 1152 assert(Cond[0].isImm() && "Cond[0] is not an immediate"); 1153 switch (Cond[0].getImm()) { 1154 case SIInstrInfo::SCC_TRUE: { 1155 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1156 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 1157 : AMDGPU::S_CSELECT_B64), SReg) 1158 .addImm(1) 1159 .addImm(0); 1160 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1161 .addImm(0) 1162 .addReg(FalseReg) 1163 .addImm(0) 1164 .addReg(TrueReg) 1165 .addReg(SReg); 1166 break; 1167 } 1168 case SIInstrInfo::SCC_FALSE: { 1169 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1170 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 1171 : AMDGPU::S_CSELECT_B64), SReg) 1172 .addImm(0) 1173 .addImm(1); 1174 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1175 .addImm(0) 1176 .addReg(FalseReg) 1177 .addImm(0) 1178 .addReg(TrueReg) 1179 .addReg(SReg); 1180 break; 1181 } 1182 case SIInstrInfo::VCCNZ: { 1183 MachineOperand RegOp = Cond[1]; 1184 RegOp.setImplicit(false); 1185 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1186 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) 1187 .add(RegOp); 1188 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1189 .addImm(0) 1190 .addReg(FalseReg) 1191 .addImm(0) 1192 .addReg(TrueReg) 1193 .addReg(SReg); 1194 break; 1195 } 1196 case SIInstrInfo::VCCZ: { 1197 MachineOperand RegOp = Cond[1]; 1198 RegOp.setImplicit(false); 1199 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1200 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) 1201 .add(RegOp); 1202 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1203 .addImm(0) 1204 .addReg(TrueReg) 1205 .addImm(0) 1206 .addReg(FalseReg) 1207 .addReg(SReg); 1208 break; 1209 } 1210 case SIInstrInfo::EXECNZ: { 1211 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1212 Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC()); 1213 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 1214 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2) 1215 .addImm(0); 1216 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 1217 : AMDGPU::S_CSELECT_B64), SReg) 1218 .addImm(1) 1219 .addImm(0); 1220 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1221 .addImm(0) 1222 .addReg(FalseReg) 1223 .addImm(0) 1224 .addReg(TrueReg) 1225 .addReg(SReg); 1226 break; 1227 } 1228 case SIInstrInfo::EXECZ: { 1229 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1230 Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC()); 1231 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 1232 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2) 1233 .addImm(0); 1234 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 1235 : AMDGPU::S_CSELECT_B64), SReg) 1236 .addImm(0) 1237 .addImm(1); 1238 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1239 .addImm(0) 1240 .addReg(FalseReg) 1241 .addImm(0) 1242 .addReg(TrueReg) 1243 .addReg(SReg); 1244 llvm_unreachable("Unhandled branch predicate EXECZ"); 1245 break; 1246 } 1247 default: 1248 llvm_unreachable("invalid branch predicate"); 1249 } 1250 } else { 1251 llvm_unreachable("Can only handle Cond size 1 or 2"); 1252 } 1253 } 1254 1255 Register SIInstrInfo::insertEQ(MachineBasicBlock *MBB, 1256 MachineBasicBlock::iterator I, 1257 const DebugLoc &DL, 1258 Register SrcReg, int Value) const { 1259 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 1260 Register Reg = MRI.createVirtualRegister(RI.getBoolRC()); 1261 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg) 1262 .addImm(Value) 1263 .addReg(SrcReg); 1264 1265 return Reg; 1266 } 1267 1268 Register SIInstrInfo::insertNE(MachineBasicBlock *MBB, 1269 MachineBasicBlock::iterator I, 1270 const DebugLoc &DL, 1271 Register SrcReg, int Value) const { 1272 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 1273 Register Reg = MRI.createVirtualRegister(RI.getBoolRC()); 1274 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg) 1275 .addImm(Value) 1276 .addReg(SrcReg); 1277 1278 return Reg; 1279 } 1280 1281 unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const { 1282 1283 if (RI.isAGPRClass(DstRC)) 1284 return AMDGPU::COPY; 1285 if (RI.getRegSizeInBits(*DstRC) == 32) { 1286 return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; 1287 } else if (RI.getRegSizeInBits(*DstRC) == 64 && RI.isSGPRClass(DstRC)) { 1288 return AMDGPU::S_MOV_B64; 1289 } else if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC)) { 1290 return AMDGPU::V_MOV_B64_PSEUDO; 1291 } 1292 return AMDGPU::COPY; 1293 } 1294 1295 const MCInstrDesc & 1296 SIInstrInfo::getIndirectGPRIDXPseudo(unsigned VecSize, 1297 bool IsIndirectSrc) const { 1298 if (IsIndirectSrc) { 1299 if (VecSize <= 32) // 4 bytes 1300 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1); 1301 if (VecSize <= 64) // 8 bytes 1302 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2); 1303 if (VecSize <= 96) // 12 bytes 1304 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3); 1305 if (VecSize <= 128) // 16 bytes 1306 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4); 1307 if (VecSize <= 160) // 20 bytes 1308 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5); 1309 if (VecSize <= 256) // 32 bytes 1310 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8); 1311 if (VecSize <= 288) // 36 bytes 1312 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9); 1313 if (VecSize <= 320) // 40 bytes 1314 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10); 1315 if (VecSize <= 352) // 44 bytes 1316 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11); 1317 if (VecSize <= 384) // 48 bytes 1318 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12); 1319 if (VecSize <= 512) // 64 bytes 1320 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16); 1321 if (VecSize <= 1024) // 128 bytes 1322 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32); 1323 1324 llvm_unreachable("unsupported size for IndirectRegReadGPRIDX pseudos"); 1325 } 1326 1327 if (VecSize <= 32) // 4 bytes 1328 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1); 1329 if (VecSize <= 64) // 8 bytes 1330 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2); 1331 if (VecSize <= 96) // 12 bytes 1332 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3); 1333 if (VecSize <= 128) // 16 bytes 1334 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4); 1335 if (VecSize <= 160) // 20 bytes 1336 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5); 1337 if (VecSize <= 256) // 32 bytes 1338 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8); 1339 if (VecSize <= 288) // 36 bytes 1340 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9); 1341 if (VecSize <= 320) // 40 bytes 1342 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10); 1343 if (VecSize <= 352) // 44 bytes 1344 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11); 1345 if (VecSize <= 384) // 48 bytes 1346 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12); 1347 if (VecSize <= 512) // 64 bytes 1348 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16); 1349 if (VecSize <= 1024) // 128 bytes 1350 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32); 1351 1352 llvm_unreachable("unsupported size for IndirectRegWriteGPRIDX pseudos"); 1353 } 1354 1355 static unsigned getIndirectVGPRWriteMovRelPseudoOpc(unsigned VecSize) { 1356 if (VecSize <= 32) // 4 bytes 1357 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1; 1358 if (VecSize <= 64) // 8 bytes 1359 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2; 1360 if (VecSize <= 96) // 12 bytes 1361 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3; 1362 if (VecSize <= 128) // 16 bytes 1363 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4; 1364 if (VecSize <= 160) // 20 bytes 1365 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5; 1366 if (VecSize <= 256) // 32 bytes 1367 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8; 1368 if (VecSize <= 288) // 36 bytes 1369 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9; 1370 if (VecSize <= 320) // 40 bytes 1371 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10; 1372 if (VecSize <= 352) // 44 bytes 1373 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11; 1374 if (VecSize <= 384) // 48 bytes 1375 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12; 1376 if (VecSize <= 512) // 64 bytes 1377 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16; 1378 if (VecSize <= 1024) // 128 bytes 1379 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32; 1380 1381 llvm_unreachable("unsupported size for IndirectRegWrite pseudos"); 1382 } 1383 1384 static unsigned getIndirectSGPRWriteMovRelPseudo32(unsigned VecSize) { 1385 if (VecSize <= 32) // 4 bytes 1386 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1; 1387 if (VecSize <= 64) // 8 bytes 1388 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2; 1389 if (VecSize <= 96) // 12 bytes 1390 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3; 1391 if (VecSize <= 128) // 16 bytes 1392 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4; 1393 if (VecSize <= 160) // 20 bytes 1394 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5; 1395 if (VecSize <= 256) // 32 bytes 1396 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8; 1397 if (VecSize <= 512) // 64 bytes 1398 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16; 1399 if (VecSize <= 1024) // 128 bytes 1400 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32; 1401 1402 llvm_unreachable("unsupported size for IndirectRegWrite pseudos"); 1403 } 1404 1405 static unsigned getIndirectSGPRWriteMovRelPseudo64(unsigned VecSize) { 1406 if (VecSize <= 64) // 8 bytes 1407 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1; 1408 if (VecSize <= 128) // 16 bytes 1409 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2; 1410 if (VecSize <= 256) // 32 bytes 1411 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4; 1412 if (VecSize <= 512) // 64 bytes 1413 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8; 1414 if (VecSize <= 1024) // 128 bytes 1415 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16; 1416 1417 llvm_unreachable("unsupported size for IndirectRegWrite pseudos"); 1418 } 1419 1420 const MCInstrDesc & 1421 SIInstrInfo::getIndirectRegWriteMovRelPseudo(unsigned VecSize, unsigned EltSize, 1422 bool IsSGPR) const { 1423 if (IsSGPR) { 1424 switch (EltSize) { 1425 case 32: 1426 return get(getIndirectSGPRWriteMovRelPseudo32(VecSize)); 1427 case 64: 1428 return get(getIndirectSGPRWriteMovRelPseudo64(VecSize)); 1429 default: 1430 llvm_unreachable("invalid reg indexing elt size"); 1431 } 1432 } 1433 1434 assert(EltSize == 32 && "invalid reg indexing elt size"); 1435 return get(getIndirectVGPRWriteMovRelPseudoOpc(VecSize)); 1436 } 1437 1438 static unsigned getSGPRSpillSaveOpcode(unsigned Size) { 1439 switch (Size) { 1440 case 4: 1441 return AMDGPU::SI_SPILL_S32_SAVE; 1442 case 8: 1443 return AMDGPU::SI_SPILL_S64_SAVE; 1444 case 12: 1445 return AMDGPU::SI_SPILL_S96_SAVE; 1446 case 16: 1447 return AMDGPU::SI_SPILL_S128_SAVE; 1448 case 20: 1449 return AMDGPU::SI_SPILL_S160_SAVE; 1450 case 24: 1451 return AMDGPU::SI_SPILL_S192_SAVE; 1452 case 28: 1453 return AMDGPU::SI_SPILL_S224_SAVE; 1454 case 32: 1455 return AMDGPU::SI_SPILL_S256_SAVE; 1456 case 36: 1457 return AMDGPU::SI_SPILL_S288_SAVE; 1458 case 40: 1459 return AMDGPU::SI_SPILL_S320_SAVE; 1460 case 44: 1461 return AMDGPU::SI_SPILL_S352_SAVE; 1462 case 48: 1463 return AMDGPU::SI_SPILL_S384_SAVE; 1464 case 64: 1465 return AMDGPU::SI_SPILL_S512_SAVE; 1466 case 128: 1467 return AMDGPU::SI_SPILL_S1024_SAVE; 1468 default: 1469 llvm_unreachable("unknown register size"); 1470 } 1471 } 1472 1473 static unsigned getVGPRSpillSaveOpcode(unsigned Size) { 1474 switch (Size) { 1475 case 4: 1476 return AMDGPU::SI_SPILL_V32_SAVE; 1477 case 8: 1478 return AMDGPU::SI_SPILL_V64_SAVE; 1479 case 12: 1480 return AMDGPU::SI_SPILL_V96_SAVE; 1481 case 16: 1482 return AMDGPU::SI_SPILL_V128_SAVE; 1483 case 20: 1484 return AMDGPU::SI_SPILL_V160_SAVE; 1485 case 24: 1486 return AMDGPU::SI_SPILL_V192_SAVE; 1487 case 28: 1488 return AMDGPU::SI_SPILL_V224_SAVE; 1489 case 32: 1490 return AMDGPU::SI_SPILL_V256_SAVE; 1491 case 36: 1492 return AMDGPU::SI_SPILL_V288_SAVE; 1493 case 40: 1494 return AMDGPU::SI_SPILL_V320_SAVE; 1495 case 44: 1496 return AMDGPU::SI_SPILL_V352_SAVE; 1497 case 48: 1498 return AMDGPU::SI_SPILL_V384_SAVE; 1499 case 64: 1500 return AMDGPU::SI_SPILL_V512_SAVE; 1501 case 128: 1502 return AMDGPU::SI_SPILL_V1024_SAVE; 1503 default: 1504 llvm_unreachable("unknown register size"); 1505 } 1506 } 1507 1508 static unsigned getAGPRSpillSaveOpcode(unsigned Size) { 1509 switch (Size) { 1510 case 4: 1511 return AMDGPU::SI_SPILL_A32_SAVE; 1512 case 8: 1513 return AMDGPU::SI_SPILL_A64_SAVE; 1514 case 12: 1515 return AMDGPU::SI_SPILL_A96_SAVE; 1516 case 16: 1517 return AMDGPU::SI_SPILL_A128_SAVE; 1518 case 20: 1519 return AMDGPU::SI_SPILL_A160_SAVE; 1520 case 24: 1521 return AMDGPU::SI_SPILL_A192_SAVE; 1522 case 28: 1523 return AMDGPU::SI_SPILL_A224_SAVE; 1524 case 32: 1525 return AMDGPU::SI_SPILL_A256_SAVE; 1526 case 36: 1527 return AMDGPU::SI_SPILL_A288_SAVE; 1528 case 40: 1529 return AMDGPU::SI_SPILL_A320_SAVE; 1530 case 44: 1531 return AMDGPU::SI_SPILL_A352_SAVE; 1532 case 48: 1533 return AMDGPU::SI_SPILL_A384_SAVE; 1534 case 64: 1535 return AMDGPU::SI_SPILL_A512_SAVE; 1536 case 128: 1537 return AMDGPU::SI_SPILL_A1024_SAVE; 1538 default: 1539 llvm_unreachable("unknown register size"); 1540 } 1541 } 1542 1543 static unsigned getAVSpillSaveOpcode(unsigned Size) { 1544 switch (Size) { 1545 case 4: 1546 return AMDGPU::SI_SPILL_AV32_SAVE; 1547 case 8: 1548 return AMDGPU::SI_SPILL_AV64_SAVE; 1549 case 12: 1550 return AMDGPU::SI_SPILL_AV96_SAVE; 1551 case 16: 1552 return AMDGPU::SI_SPILL_AV128_SAVE; 1553 case 20: 1554 return AMDGPU::SI_SPILL_AV160_SAVE; 1555 case 24: 1556 return AMDGPU::SI_SPILL_AV192_SAVE; 1557 case 28: 1558 return AMDGPU::SI_SPILL_AV224_SAVE; 1559 case 32: 1560 return AMDGPU::SI_SPILL_AV256_SAVE; 1561 case 36: 1562 return AMDGPU::SI_SPILL_AV288_SAVE; 1563 case 40: 1564 return AMDGPU::SI_SPILL_AV320_SAVE; 1565 case 44: 1566 return AMDGPU::SI_SPILL_AV352_SAVE; 1567 case 48: 1568 return AMDGPU::SI_SPILL_AV384_SAVE; 1569 case 64: 1570 return AMDGPU::SI_SPILL_AV512_SAVE; 1571 case 128: 1572 return AMDGPU::SI_SPILL_AV1024_SAVE; 1573 default: 1574 llvm_unreachable("unknown register size"); 1575 } 1576 } 1577 1578 void SIInstrInfo::storeRegToStackSlot( 1579 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg, 1580 bool isKill, int FrameIndex, const TargetRegisterClass *RC, 1581 const TargetRegisterInfo *TRI, Register VReg) const { 1582 MachineFunction *MF = MBB.getParent(); 1583 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 1584 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 1585 const DebugLoc &DL = MBB.findDebugLoc(MI); 1586 1587 MachinePointerInfo PtrInfo 1588 = MachinePointerInfo::getFixedStack(*MF, FrameIndex); 1589 MachineMemOperand *MMO = MF->getMachineMemOperand( 1590 PtrInfo, MachineMemOperand::MOStore, FrameInfo.getObjectSize(FrameIndex), 1591 FrameInfo.getObjectAlign(FrameIndex)); 1592 unsigned SpillSize = TRI->getSpillSize(*RC); 1593 1594 MachineRegisterInfo &MRI = MF->getRegInfo(); 1595 if (RI.isSGPRClass(RC)) { 1596 MFI->setHasSpilledSGPRs(); 1597 assert(SrcReg != AMDGPU::M0 && "m0 should not be spilled"); 1598 assert(SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI && 1599 SrcReg != AMDGPU::EXEC && "exec should not be spilled"); 1600 1601 // We are only allowed to create one new instruction when spilling 1602 // registers, so we need to use pseudo instruction for spilling SGPRs. 1603 const MCInstrDesc &OpDesc = get(getSGPRSpillSaveOpcode(SpillSize)); 1604 1605 // The SGPR spill/restore instructions only work on number sgprs, so we need 1606 // to make sure we are using the correct register class. 1607 if (SrcReg.isVirtual() && SpillSize == 4) { 1608 MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); 1609 } 1610 1611 BuildMI(MBB, MI, DL, OpDesc) 1612 .addReg(SrcReg, getKillRegState(isKill)) // data 1613 .addFrameIndex(FrameIndex) // addr 1614 .addMemOperand(MMO) 1615 .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit); 1616 1617 if (RI.spillSGPRToVGPR()) 1618 FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill); 1619 return; 1620 } 1621 1622 unsigned Opcode = RI.isVectorSuperClass(RC) 1623 ? getAVSpillSaveOpcode(SpillSize) 1624 : RI.isAGPRClass(RC) 1625 ? getAGPRSpillSaveOpcode(SpillSize) 1626 : getVGPRSpillSaveOpcode(SpillSize); 1627 MFI->setHasSpilledVGPRs(); 1628 1629 BuildMI(MBB, MI, DL, get(Opcode)) 1630 .addReg(SrcReg, getKillRegState(isKill)) // data 1631 .addFrameIndex(FrameIndex) // addr 1632 .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset 1633 .addImm(0) // offset 1634 .addMemOperand(MMO); 1635 } 1636 1637 static unsigned getSGPRSpillRestoreOpcode(unsigned Size) { 1638 switch (Size) { 1639 case 4: 1640 return AMDGPU::SI_SPILL_S32_RESTORE; 1641 case 8: 1642 return AMDGPU::SI_SPILL_S64_RESTORE; 1643 case 12: 1644 return AMDGPU::SI_SPILL_S96_RESTORE; 1645 case 16: 1646 return AMDGPU::SI_SPILL_S128_RESTORE; 1647 case 20: 1648 return AMDGPU::SI_SPILL_S160_RESTORE; 1649 case 24: 1650 return AMDGPU::SI_SPILL_S192_RESTORE; 1651 case 28: 1652 return AMDGPU::SI_SPILL_S224_RESTORE; 1653 case 32: 1654 return AMDGPU::SI_SPILL_S256_RESTORE; 1655 case 36: 1656 return AMDGPU::SI_SPILL_S288_RESTORE; 1657 case 40: 1658 return AMDGPU::SI_SPILL_S320_RESTORE; 1659 case 44: 1660 return AMDGPU::SI_SPILL_S352_RESTORE; 1661 case 48: 1662 return AMDGPU::SI_SPILL_S384_RESTORE; 1663 case 64: 1664 return AMDGPU::SI_SPILL_S512_RESTORE; 1665 case 128: 1666 return AMDGPU::SI_SPILL_S1024_RESTORE; 1667 default: 1668 llvm_unreachable("unknown register size"); 1669 } 1670 } 1671 1672 static unsigned getVGPRSpillRestoreOpcode(unsigned Size) { 1673 switch (Size) { 1674 case 4: 1675 return AMDGPU::SI_SPILL_V32_RESTORE; 1676 case 8: 1677 return AMDGPU::SI_SPILL_V64_RESTORE; 1678 case 12: 1679 return AMDGPU::SI_SPILL_V96_RESTORE; 1680 case 16: 1681 return AMDGPU::SI_SPILL_V128_RESTORE; 1682 case 20: 1683 return AMDGPU::SI_SPILL_V160_RESTORE; 1684 case 24: 1685 return AMDGPU::SI_SPILL_V192_RESTORE; 1686 case 28: 1687 return AMDGPU::SI_SPILL_V224_RESTORE; 1688 case 32: 1689 return AMDGPU::SI_SPILL_V256_RESTORE; 1690 case 36: 1691 return AMDGPU::SI_SPILL_V288_RESTORE; 1692 case 40: 1693 return AMDGPU::SI_SPILL_V320_RESTORE; 1694 case 44: 1695 return AMDGPU::SI_SPILL_V352_RESTORE; 1696 case 48: 1697 return AMDGPU::SI_SPILL_V384_RESTORE; 1698 case 64: 1699 return AMDGPU::SI_SPILL_V512_RESTORE; 1700 case 128: 1701 return AMDGPU::SI_SPILL_V1024_RESTORE; 1702 default: 1703 llvm_unreachable("unknown register size"); 1704 } 1705 } 1706 1707 static unsigned getAGPRSpillRestoreOpcode(unsigned Size) { 1708 switch (Size) { 1709 case 4: 1710 return AMDGPU::SI_SPILL_A32_RESTORE; 1711 case 8: 1712 return AMDGPU::SI_SPILL_A64_RESTORE; 1713 case 12: 1714 return AMDGPU::SI_SPILL_A96_RESTORE; 1715 case 16: 1716 return AMDGPU::SI_SPILL_A128_RESTORE; 1717 case 20: 1718 return AMDGPU::SI_SPILL_A160_RESTORE; 1719 case 24: 1720 return AMDGPU::SI_SPILL_A192_RESTORE; 1721 case 28: 1722 return AMDGPU::SI_SPILL_A224_RESTORE; 1723 case 32: 1724 return AMDGPU::SI_SPILL_A256_RESTORE; 1725 case 36: 1726 return AMDGPU::SI_SPILL_A288_RESTORE; 1727 case 40: 1728 return AMDGPU::SI_SPILL_A320_RESTORE; 1729 case 44: 1730 return AMDGPU::SI_SPILL_A352_RESTORE; 1731 case 48: 1732 return AMDGPU::SI_SPILL_A384_RESTORE; 1733 case 64: 1734 return AMDGPU::SI_SPILL_A512_RESTORE; 1735 case 128: 1736 return AMDGPU::SI_SPILL_A1024_RESTORE; 1737 default: 1738 llvm_unreachable("unknown register size"); 1739 } 1740 } 1741 1742 static unsigned getAVSpillRestoreOpcode(unsigned Size) { 1743 switch (Size) { 1744 case 4: 1745 return AMDGPU::SI_SPILL_AV32_RESTORE; 1746 case 8: 1747 return AMDGPU::SI_SPILL_AV64_RESTORE; 1748 case 12: 1749 return AMDGPU::SI_SPILL_AV96_RESTORE; 1750 case 16: 1751 return AMDGPU::SI_SPILL_AV128_RESTORE; 1752 case 20: 1753 return AMDGPU::SI_SPILL_AV160_RESTORE; 1754 case 24: 1755 return AMDGPU::SI_SPILL_AV192_RESTORE; 1756 case 28: 1757 return AMDGPU::SI_SPILL_AV224_RESTORE; 1758 case 32: 1759 return AMDGPU::SI_SPILL_AV256_RESTORE; 1760 case 36: 1761 return AMDGPU::SI_SPILL_AV288_RESTORE; 1762 case 40: 1763 return AMDGPU::SI_SPILL_AV320_RESTORE; 1764 case 44: 1765 return AMDGPU::SI_SPILL_AV352_RESTORE; 1766 case 48: 1767 return AMDGPU::SI_SPILL_AV384_RESTORE; 1768 case 64: 1769 return AMDGPU::SI_SPILL_AV512_RESTORE; 1770 case 128: 1771 return AMDGPU::SI_SPILL_AV1024_RESTORE; 1772 default: 1773 llvm_unreachable("unknown register size"); 1774 } 1775 } 1776 1777 void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, 1778 MachineBasicBlock::iterator MI, 1779 Register DestReg, int FrameIndex, 1780 const TargetRegisterClass *RC, 1781 const TargetRegisterInfo *TRI, 1782 Register VReg) const { 1783 MachineFunction *MF = MBB.getParent(); 1784 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 1785 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 1786 const DebugLoc &DL = MBB.findDebugLoc(MI); 1787 unsigned SpillSize = TRI->getSpillSize(*RC); 1788 1789 MachinePointerInfo PtrInfo 1790 = MachinePointerInfo::getFixedStack(*MF, FrameIndex); 1791 1792 MachineMemOperand *MMO = MF->getMachineMemOperand( 1793 PtrInfo, MachineMemOperand::MOLoad, FrameInfo.getObjectSize(FrameIndex), 1794 FrameInfo.getObjectAlign(FrameIndex)); 1795 1796 if (RI.isSGPRClass(RC)) { 1797 MFI->setHasSpilledSGPRs(); 1798 assert(DestReg != AMDGPU::M0 && "m0 should not be reloaded into"); 1799 assert(DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI && 1800 DestReg != AMDGPU::EXEC && "exec should not be spilled"); 1801 1802 // FIXME: Maybe this should not include a memoperand because it will be 1803 // lowered to non-memory instructions. 1804 const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(SpillSize)); 1805 if (DestReg.isVirtual() && SpillSize == 4) { 1806 MachineRegisterInfo &MRI = MF->getRegInfo(); 1807 MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); 1808 } 1809 1810 if (RI.spillSGPRToVGPR()) 1811 FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill); 1812 BuildMI(MBB, MI, DL, OpDesc, DestReg) 1813 .addFrameIndex(FrameIndex) // addr 1814 .addMemOperand(MMO) 1815 .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit); 1816 1817 return; 1818 } 1819 1820 unsigned Opcode = RI.isVectorSuperClass(RC) 1821 ? getAVSpillRestoreOpcode(SpillSize) 1822 : RI.isAGPRClass(RC) 1823 ? getAGPRSpillRestoreOpcode(SpillSize) 1824 : getVGPRSpillRestoreOpcode(SpillSize); 1825 BuildMI(MBB, MI, DL, get(Opcode), DestReg) 1826 .addFrameIndex(FrameIndex) // vaddr 1827 .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset 1828 .addImm(0) // offset 1829 .addMemOperand(MMO); 1830 } 1831 1832 void SIInstrInfo::insertNoop(MachineBasicBlock &MBB, 1833 MachineBasicBlock::iterator MI) const { 1834 insertNoops(MBB, MI, 1); 1835 } 1836 1837 void SIInstrInfo::insertNoops(MachineBasicBlock &MBB, 1838 MachineBasicBlock::iterator MI, 1839 unsigned Quantity) const { 1840 DebugLoc DL = MBB.findDebugLoc(MI); 1841 while (Quantity > 0) { 1842 unsigned Arg = std::min(Quantity, 8u); 1843 Quantity -= Arg; 1844 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP)).addImm(Arg - 1); 1845 } 1846 } 1847 1848 void SIInstrInfo::insertReturn(MachineBasicBlock &MBB) const { 1849 auto MF = MBB.getParent(); 1850 SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>(); 1851 1852 assert(Info->isEntryFunction()); 1853 1854 if (MBB.succ_empty()) { 1855 bool HasNoTerminator = MBB.getFirstTerminator() == MBB.end(); 1856 if (HasNoTerminator) { 1857 if (Info->returnsVoid()) { 1858 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::S_ENDPGM)).addImm(0); 1859 } else { 1860 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::SI_RETURN_TO_EPILOG)); 1861 } 1862 } 1863 } 1864 } 1865 1866 unsigned SIInstrInfo::getNumWaitStates(const MachineInstr &MI) { 1867 switch (MI.getOpcode()) { 1868 default: 1869 if (MI.isMetaInstruction()) 1870 return 0; 1871 return 1; // FIXME: Do wait states equal cycles? 1872 1873 case AMDGPU::S_NOP: 1874 return MI.getOperand(0).getImm() + 1; 1875 // SI_RETURN_TO_EPILOG is a fallthrough to code outside of the function. The 1876 // hazard, even if one exist, won't really be visible. Should we handle it? 1877 } 1878 } 1879 1880 bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const { 1881 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 1882 MachineBasicBlock &MBB = *MI.getParent(); 1883 DebugLoc DL = MBB.findDebugLoc(MI); 1884 switch (MI.getOpcode()) { 1885 default: return TargetInstrInfo::expandPostRAPseudo(MI); 1886 case AMDGPU::S_MOV_B64_term: 1887 // This is only a terminator to get the correct spill code placement during 1888 // register allocation. 1889 MI.setDesc(get(AMDGPU::S_MOV_B64)); 1890 break; 1891 1892 case AMDGPU::S_MOV_B32_term: 1893 // This is only a terminator to get the correct spill code placement during 1894 // register allocation. 1895 MI.setDesc(get(AMDGPU::S_MOV_B32)); 1896 break; 1897 1898 case AMDGPU::S_XOR_B64_term: 1899 // This is only a terminator to get the correct spill code placement during 1900 // register allocation. 1901 MI.setDesc(get(AMDGPU::S_XOR_B64)); 1902 break; 1903 1904 case AMDGPU::S_XOR_B32_term: 1905 // This is only a terminator to get the correct spill code placement during 1906 // register allocation. 1907 MI.setDesc(get(AMDGPU::S_XOR_B32)); 1908 break; 1909 case AMDGPU::S_OR_B64_term: 1910 // This is only a terminator to get the correct spill code placement during 1911 // register allocation. 1912 MI.setDesc(get(AMDGPU::S_OR_B64)); 1913 break; 1914 case AMDGPU::S_OR_B32_term: 1915 // This is only a terminator to get the correct spill code placement during 1916 // register allocation. 1917 MI.setDesc(get(AMDGPU::S_OR_B32)); 1918 break; 1919 1920 case AMDGPU::S_ANDN2_B64_term: 1921 // This is only a terminator to get the correct spill code placement during 1922 // register allocation. 1923 MI.setDesc(get(AMDGPU::S_ANDN2_B64)); 1924 break; 1925 1926 case AMDGPU::S_ANDN2_B32_term: 1927 // This is only a terminator to get the correct spill code placement during 1928 // register allocation. 1929 MI.setDesc(get(AMDGPU::S_ANDN2_B32)); 1930 break; 1931 1932 case AMDGPU::S_AND_B64_term: 1933 // This is only a terminator to get the correct spill code placement during 1934 // register allocation. 1935 MI.setDesc(get(AMDGPU::S_AND_B64)); 1936 break; 1937 1938 case AMDGPU::S_AND_B32_term: 1939 // This is only a terminator to get the correct spill code placement during 1940 // register allocation. 1941 MI.setDesc(get(AMDGPU::S_AND_B32)); 1942 break; 1943 1944 case AMDGPU::V_MOV_B64_PSEUDO: { 1945 Register Dst = MI.getOperand(0).getReg(); 1946 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0); 1947 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1); 1948 1949 const MachineOperand &SrcOp = MI.getOperand(1); 1950 // FIXME: Will this work for 64-bit floating point immediates? 1951 assert(!SrcOp.isFPImm()); 1952 if (ST.hasMovB64()) { 1953 MI.setDesc(get(AMDGPU::V_MOV_B64_e32)); 1954 if (SrcOp.isReg() || isInlineConstant(MI, 1) || 1955 isUInt<32>(SrcOp.getImm())) 1956 break; 1957 } 1958 if (SrcOp.isImm()) { 1959 APInt Imm(64, SrcOp.getImm()); 1960 APInt Lo(32, Imm.getLoBits(32).getZExtValue()); 1961 APInt Hi(32, Imm.getHiBits(32).getZExtValue()); 1962 if (ST.hasPackedFP32Ops() && Lo == Hi && isInlineConstant(Lo)) { 1963 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst) 1964 .addImm(SISrcMods::OP_SEL_1) 1965 .addImm(Lo.getSExtValue()) 1966 .addImm(SISrcMods::OP_SEL_1) 1967 .addImm(Lo.getSExtValue()) 1968 .addImm(0) // op_sel_lo 1969 .addImm(0) // op_sel_hi 1970 .addImm(0) // neg_lo 1971 .addImm(0) // neg_hi 1972 .addImm(0); // clamp 1973 } else { 1974 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo) 1975 .addImm(Lo.getSExtValue()) 1976 .addReg(Dst, RegState::Implicit | RegState::Define); 1977 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi) 1978 .addImm(Hi.getSExtValue()) 1979 .addReg(Dst, RegState::Implicit | RegState::Define); 1980 } 1981 } else { 1982 assert(SrcOp.isReg()); 1983 if (ST.hasPackedFP32Ops() && 1984 !RI.isAGPR(MBB.getParent()->getRegInfo(), SrcOp.getReg())) { 1985 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst) 1986 .addImm(SISrcMods::OP_SEL_1) // src0_mod 1987 .addReg(SrcOp.getReg()) 1988 .addImm(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1) // src1_mod 1989 .addReg(SrcOp.getReg()) 1990 .addImm(0) // op_sel_lo 1991 .addImm(0) // op_sel_hi 1992 .addImm(0) // neg_lo 1993 .addImm(0) // neg_hi 1994 .addImm(0); // clamp 1995 } else { 1996 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo) 1997 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0)) 1998 .addReg(Dst, RegState::Implicit | RegState::Define); 1999 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi) 2000 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1)) 2001 .addReg(Dst, RegState::Implicit | RegState::Define); 2002 } 2003 } 2004 MI.eraseFromParent(); 2005 break; 2006 } 2007 case AMDGPU::V_MOV_B64_DPP_PSEUDO: { 2008 expandMovDPP64(MI); 2009 break; 2010 } 2011 case AMDGPU::S_MOV_B64_IMM_PSEUDO: { 2012 const MachineOperand &SrcOp = MI.getOperand(1); 2013 assert(!SrcOp.isFPImm()); 2014 APInt Imm(64, SrcOp.getImm()); 2015 if (Imm.isIntN(32) || isInlineConstant(Imm)) { 2016 MI.setDesc(get(AMDGPU::S_MOV_B64)); 2017 break; 2018 } 2019 2020 Register Dst = MI.getOperand(0).getReg(); 2021 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0); 2022 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1); 2023 2024 APInt Lo(32, Imm.getLoBits(32).getZExtValue()); 2025 APInt Hi(32, Imm.getHiBits(32).getZExtValue()); 2026 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstLo) 2027 .addImm(Lo.getSExtValue()) 2028 .addReg(Dst, RegState::Implicit | RegState::Define); 2029 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstHi) 2030 .addImm(Hi.getSExtValue()) 2031 .addReg(Dst, RegState::Implicit | RegState::Define); 2032 MI.eraseFromParent(); 2033 break; 2034 } 2035 case AMDGPU::V_SET_INACTIVE_B32: { 2036 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64; 2037 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 2038 // FIXME: We may possibly optimize the COPY once we find ways to make LLVM 2039 // optimizations (mainly Register Coalescer) aware of WWM register liveness. 2040 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg()) 2041 .add(MI.getOperand(1)); 2042 auto FirstNot = BuildMI(MBB, MI, DL, get(NotOpc), Exec).addReg(Exec); 2043 FirstNot->addRegisterDead(AMDGPU::SCC, TRI); // SCC is overwritten 2044 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg()) 2045 .add(MI.getOperand(2)); 2046 BuildMI(MBB, MI, DL, get(NotOpc), Exec) 2047 .addReg(Exec); 2048 MI.eraseFromParent(); 2049 break; 2050 } 2051 case AMDGPU::V_SET_INACTIVE_B64: { 2052 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64; 2053 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 2054 MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), 2055 MI.getOperand(0).getReg()) 2056 .add(MI.getOperand(1)); 2057 expandPostRAPseudo(*Copy); 2058 auto FirstNot = BuildMI(MBB, MI, DL, get(NotOpc), Exec).addReg(Exec); 2059 FirstNot->addRegisterDead(AMDGPU::SCC, TRI); // SCC is overwritten 2060 Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), 2061 MI.getOperand(0).getReg()) 2062 .add(MI.getOperand(2)); 2063 expandPostRAPseudo(*Copy); 2064 BuildMI(MBB, MI, DL, get(NotOpc), Exec) 2065 .addReg(Exec); 2066 MI.eraseFromParent(); 2067 break; 2068 } 2069 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1: 2070 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2: 2071 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3: 2072 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4: 2073 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5: 2074 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8: 2075 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9: 2076 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10: 2077 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11: 2078 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12: 2079 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16: 2080 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32: 2081 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1: 2082 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2: 2083 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3: 2084 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4: 2085 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5: 2086 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8: 2087 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16: 2088 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32: 2089 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1: 2090 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2: 2091 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4: 2092 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8: 2093 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16: { 2094 const TargetRegisterClass *EltRC = getOpRegClass(MI, 2); 2095 2096 unsigned Opc; 2097 if (RI.hasVGPRs(EltRC)) { 2098 Opc = AMDGPU::V_MOVRELD_B32_e32; 2099 } else { 2100 Opc = RI.getRegSizeInBits(*EltRC) == 64 ? AMDGPU::S_MOVRELD_B64 2101 : AMDGPU::S_MOVRELD_B32; 2102 } 2103 2104 const MCInstrDesc &OpDesc = get(Opc); 2105 Register VecReg = MI.getOperand(0).getReg(); 2106 bool IsUndef = MI.getOperand(1).isUndef(); 2107 unsigned SubReg = MI.getOperand(3).getImm(); 2108 assert(VecReg == MI.getOperand(1).getReg()); 2109 2110 MachineInstrBuilder MIB = 2111 BuildMI(MBB, MI, DL, OpDesc) 2112 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef) 2113 .add(MI.getOperand(2)) 2114 .addReg(VecReg, RegState::ImplicitDefine) 2115 .addReg(VecReg, RegState::Implicit | (IsUndef ? RegState::Undef : 0)); 2116 2117 const int ImpDefIdx = 2118 OpDesc.getNumOperands() + OpDesc.implicit_uses().size(); 2119 const int ImpUseIdx = ImpDefIdx + 1; 2120 MIB->tieOperands(ImpDefIdx, ImpUseIdx); 2121 MI.eraseFromParent(); 2122 break; 2123 } 2124 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1: 2125 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2: 2126 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3: 2127 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4: 2128 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5: 2129 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8: 2130 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9: 2131 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10: 2132 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11: 2133 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12: 2134 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16: 2135 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32: { 2136 assert(ST.useVGPRIndexMode()); 2137 Register VecReg = MI.getOperand(0).getReg(); 2138 bool IsUndef = MI.getOperand(1).isUndef(); 2139 Register Idx = MI.getOperand(3).getReg(); 2140 Register SubReg = MI.getOperand(4).getImm(); 2141 2142 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON)) 2143 .addReg(Idx) 2144 .addImm(AMDGPU::VGPRIndexMode::DST_ENABLE); 2145 SetOn->getOperand(3).setIsUndef(); 2146 2147 const MCInstrDesc &OpDesc = get(AMDGPU::V_MOV_B32_indirect_write); 2148 MachineInstrBuilder MIB = 2149 BuildMI(MBB, MI, DL, OpDesc) 2150 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef) 2151 .add(MI.getOperand(2)) 2152 .addReg(VecReg, RegState::ImplicitDefine) 2153 .addReg(VecReg, 2154 RegState::Implicit | (IsUndef ? RegState::Undef : 0)); 2155 2156 const int ImpDefIdx = 2157 OpDesc.getNumOperands() + OpDesc.implicit_uses().size(); 2158 const int ImpUseIdx = ImpDefIdx + 1; 2159 MIB->tieOperands(ImpDefIdx, ImpUseIdx); 2160 2161 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF)); 2162 2163 finalizeBundle(MBB, SetOn->getIterator(), std::next(SetOff->getIterator())); 2164 2165 MI.eraseFromParent(); 2166 break; 2167 } 2168 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1: 2169 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2: 2170 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3: 2171 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4: 2172 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5: 2173 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8: 2174 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9: 2175 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10: 2176 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11: 2177 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12: 2178 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16: 2179 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32: { 2180 assert(ST.useVGPRIndexMode()); 2181 Register Dst = MI.getOperand(0).getReg(); 2182 Register VecReg = MI.getOperand(1).getReg(); 2183 bool IsUndef = MI.getOperand(1).isUndef(); 2184 Register Idx = MI.getOperand(2).getReg(); 2185 Register SubReg = MI.getOperand(3).getImm(); 2186 2187 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON)) 2188 .addReg(Idx) 2189 .addImm(AMDGPU::VGPRIndexMode::SRC0_ENABLE); 2190 SetOn->getOperand(3).setIsUndef(); 2191 2192 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_indirect_read)) 2193 .addDef(Dst) 2194 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef) 2195 .addReg(VecReg, RegState::Implicit | (IsUndef ? RegState::Undef : 0)); 2196 2197 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF)); 2198 2199 finalizeBundle(MBB, SetOn->getIterator(), std::next(SetOff->getIterator())); 2200 2201 MI.eraseFromParent(); 2202 break; 2203 } 2204 case AMDGPU::SI_PC_ADD_REL_OFFSET: { 2205 MachineFunction &MF = *MBB.getParent(); 2206 Register Reg = MI.getOperand(0).getReg(); 2207 Register RegLo = RI.getSubReg(Reg, AMDGPU::sub0); 2208 Register RegHi = RI.getSubReg(Reg, AMDGPU::sub1); 2209 2210 // Create a bundle so these instructions won't be re-ordered by the 2211 // post-RA scheduler. 2212 MIBundleBuilder Bundler(MBB, MI); 2213 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg)); 2214 2215 // Add 32-bit offset from this instruction to the start of the 2216 // constant data. 2217 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo) 2218 .addReg(RegLo) 2219 .add(MI.getOperand(1))); 2220 2221 MachineInstrBuilder MIB = BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi) 2222 .addReg(RegHi); 2223 MIB.add(MI.getOperand(2)); 2224 2225 Bundler.append(MIB); 2226 finalizeBundle(MBB, Bundler.begin()); 2227 2228 MI.eraseFromParent(); 2229 break; 2230 } 2231 case AMDGPU::ENTER_STRICT_WWM: { 2232 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when 2233 // Whole Wave Mode is entered. 2234 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 2235 : AMDGPU::S_OR_SAVEEXEC_B64)); 2236 break; 2237 } 2238 case AMDGPU::ENTER_STRICT_WQM: { 2239 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when 2240 // STRICT_WQM is entered. 2241 const unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 2242 const unsigned WQMOp = ST.isWave32() ? AMDGPU::S_WQM_B32 : AMDGPU::S_WQM_B64; 2243 const unsigned MovOp = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; 2244 BuildMI(MBB, MI, DL, get(MovOp), MI.getOperand(0).getReg()).addReg(Exec); 2245 BuildMI(MBB, MI, DL, get(WQMOp), Exec).addReg(Exec); 2246 2247 MI.eraseFromParent(); 2248 break; 2249 } 2250 case AMDGPU::EXIT_STRICT_WWM: 2251 case AMDGPU::EXIT_STRICT_WQM: { 2252 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when 2253 // WWM/STICT_WQM is exited. 2254 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64)); 2255 break; 2256 } 2257 case AMDGPU::ENTER_PSEUDO_WM: 2258 case AMDGPU::EXIT_PSEUDO_WM: { 2259 // These do nothing. 2260 MI.eraseFromParent(); 2261 break; 2262 } 2263 case AMDGPU::SI_RETURN: { 2264 const MachineFunction *MF = MBB.getParent(); 2265 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); 2266 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 2267 // Hiding the return address use with SI_RETURN may lead to extra kills in 2268 // the function and missing live-ins. We are fine in practice because callee 2269 // saved register handling ensures the register value is restored before 2270 // RET, but we need the undef flag here to appease the MachineVerifier 2271 // liveness checks. 2272 MachineInstrBuilder MIB = 2273 BuildMI(MBB, MI, DL, get(AMDGPU::S_SETPC_B64_return)) 2274 .addReg(TRI->getReturnAddressReg(*MF), RegState::Undef); 2275 2276 MIB.copyImplicitOps(MI); 2277 MI.eraseFromParent(); 2278 break; 2279 } 2280 } 2281 return true; 2282 } 2283 2284 std::pair<MachineInstr*, MachineInstr*> 2285 SIInstrInfo::expandMovDPP64(MachineInstr &MI) const { 2286 assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO); 2287 2288 if (ST.hasMovB64() && 2289 AMDGPU::isLegal64BitDPPControl( 2290 getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl)->getImm())) { 2291 MI.setDesc(get(AMDGPU::V_MOV_B64_dpp)); 2292 return std::pair(&MI, nullptr); 2293 } 2294 2295 MachineBasicBlock &MBB = *MI.getParent(); 2296 DebugLoc DL = MBB.findDebugLoc(MI); 2297 MachineFunction *MF = MBB.getParent(); 2298 MachineRegisterInfo &MRI = MF->getRegInfo(); 2299 Register Dst = MI.getOperand(0).getReg(); 2300 unsigned Part = 0; 2301 MachineInstr *Split[2]; 2302 2303 for (auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) { 2304 auto MovDPP = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_dpp)); 2305 if (Dst.isPhysical()) { 2306 MovDPP.addDef(RI.getSubReg(Dst, Sub)); 2307 } else { 2308 assert(MRI.isSSA()); 2309 auto Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 2310 MovDPP.addDef(Tmp); 2311 } 2312 2313 for (unsigned I = 1; I <= 2; ++I) { // old and src operands. 2314 const MachineOperand &SrcOp = MI.getOperand(I); 2315 assert(!SrcOp.isFPImm()); 2316 if (SrcOp.isImm()) { 2317 APInt Imm(64, SrcOp.getImm()); 2318 Imm.ashrInPlace(Part * 32); 2319 MovDPP.addImm(Imm.getLoBits(32).getZExtValue()); 2320 } else { 2321 assert(SrcOp.isReg()); 2322 Register Src = SrcOp.getReg(); 2323 if (Src.isPhysical()) 2324 MovDPP.addReg(RI.getSubReg(Src, Sub)); 2325 else 2326 MovDPP.addReg(Src, SrcOp.isUndef() ? RegState::Undef : 0, Sub); 2327 } 2328 } 2329 2330 for (const MachineOperand &MO : llvm::drop_begin(MI.explicit_operands(), 3)) 2331 MovDPP.addImm(MO.getImm()); 2332 2333 Split[Part] = MovDPP; 2334 ++Part; 2335 } 2336 2337 if (Dst.isVirtual()) 2338 BuildMI(MBB, MI, DL, get(AMDGPU::REG_SEQUENCE), Dst) 2339 .addReg(Split[0]->getOperand(0).getReg()) 2340 .addImm(AMDGPU::sub0) 2341 .addReg(Split[1]->getOperand(0).getReg()) 2342 .addImm(AMDGPU::sub1); 2343 2344 MI.eraseFromParent(); 2345 return std::pair(Split[0], Split[1]); 2346 } 2347 2348 bool SIInstrInfo::swapSourceModifiers(MachineInstr &MI, 2349 MachineOperand &Src0, 2350 unsigned Src0OpName, 2351 MachineOperand &Src1, 2352 unsigned Src1OpName) const { 2353 MachineOperand *Src0Mods = getNamedOperand(MI, Src0OpName); 2354 if (!Src0Mods) 2355 return false; 2356 2357 MachineOperand *Src1Mods = getNamedOperand(MI, Src1OpName); 2358 assert(Src1Mods && 2359 "All commutable instructions have both src0 and src1 modifiers"); 2360 2361 int Src0ModsVal = Src0Mods->getImm(); 2362 int Src1ModsVal = Src1Mods->getImm(); 2363 2364 Src1Mods->setImm(Src0ModsVal); 2365 Src0Mods->setImm(Src1ModsVal); 2366 return true; 2367 } 2368 2369 static MachineInstr *swapRegAndNonRegOperand(MachineInstr &MI, 2370 MachineOperand &RegOp, 2371 MachineOperand &NonRegOp) { 2372 Register Reg = RegOp.getReg(); 2373 unsigned SubReg = RegOp.getSubReg(); 2374 bool IsKill = RegOp.isKill(); 2375 bool IsDead = RegOp.isDead(); 2376 bool IsUndef = RegOp.isUndef(); 2377 bool IsDebug = RegOp.isDebug(); 2378 2379 if (NonRegOp.isImm()) 2380 RegOp.ChangeToImmediate(NonRegOp.getImm()); 2381 else if (NonRegOp.isFI()) 2382 RegOp.ChangeToFrameIndex(NonRegOp.getIndex()); 2383 else if (NonRegOp.isGlobal()) { 2384 RegOp.ChangeToGA(NonRegOp.getGlobal(), NonRegOp.getOffset(), 2385 NonRegOp.getTargetFlags()); 2386 } else 2387 return nullptr; 2388 2389 // Make sure we don't reinterpret a subreg index in the target flags. 2390 RegOp.setTargetFlags(NonRegOp.getTargetFlags()); 2391 2392 NonRegOp.ChangeToRegister(Reg, false, false, IsKill, IsDead, IsUndef, IsDebug); 2393 NonRegOp.setSubReg(SubReg); 2394 2395 return &MI; 2396 } 2397 2398 MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI, 2399 unsigned Src0Idx, 2400 unsigned Src1Idx) const { 2401 assert(!NewMI && "this should never be used"); 2402 2403 unsigned Opc = MI.getOpcode(); 2404 int CommutedOpcode = commuteOpcode(Opc); 2405 if (CommutedOpcode == -1) 2406 return nullptr; 2407 2408 assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) == 2409 static_cast<int>(Src0Idx) && 2410 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) == 2411 static_cast<int>(Src1Idx) && 2412 "inconsistency with findCommutedOpIndices"); 2413 2414 MachineOperand &Src0 = MI.getOperand(Src0Idx); 2415 MachineOperand &Src1 = MI.getOperand(Src1Idx); 2416 2417 MachineInstr *CommutedMI = nullptr; 2418 if (Src0.isReg() && Src1.isReg()) { 2419 if (isOperandLegal(MI, Src1Idx, &Src0)) { 2420 // Be sure to copy the source modifiers to the right place. 2421 CommutedMI 2422 = TargetInstrInfo::commuteInstructionImpl(MI, NewMI, Src0Idx, Src1Idx); 2423 } 2424 2425 } else if (Src0.isReg() && !Src1.isReg()) { 2426 // src0 should always be able to support any operand type, so no need to 2427 // check operand legality. 2428 CommutedMI = swapRegAndNonRegOperand(MI, Src0, Src1); 2429 } else if (!Src0.isReg() && Src1.isReg()) { 2430 if (isOperandLegal(MI, Src1Idx, &Src0)) 2431 CommutedMI = swapRegAndNonRegOperand(MI, Src1, Src0); 2432 } else { 2433 // FIXME: Found two non registers to commute. This does happen. 2434 return nullptr; 2435 } 2436 2437 if (CommutedMI) { 2438 swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers, 2439 Src1, AMDGPU::OpName::src1_modifiers); 2440 2441 CommutedMI->setDesc(get(CommutedOpcode)); 2442 } 2443 2444 return CommutedMI; 2445 } 2446 2447 // This needs to be implemented because the source modifiers may be inserted 2448 // between the true commutable operands, and the base 2449 // TargetInstrInfo::commuteInstruction uses it. 2450 bool SIInstrInfo::findCommutedOpIndices(const MachineInstr &MI, 2451 unsigned &SrcOpIdx0, 2452 unsigned &SrcOpIdx1) const { 2453 return findCommutedOpIndices(MI.getDesc(), SrcOpIdx0, SrcOpIdx1); 2454 } 2455 2456 bool SIInstrInfo::findCommutedOpIndices(const MCInstrDesc &Desc, 2457 unsigned &SrcOpIdx0, 2458 unsigned &SrcOpIdx1) const { 2459 if (!Desc.isCommutable()) 2460 return false; 2461 2462 unsigned Opc = Desc.getOpcode(); 2463 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); 2464 if (Src0Idx == -1) 2465 return false; 2466 2467 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); 2468 if (Src1Idx == -1) 2469 return false; 2470 2471 return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx); 2472 } 2473 2474 bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp, 2475 int64_t BrOffset) const { 2476 // BranchRelaxation should never have to check s_setpc_b64 because its dest 2477 // block is unanalyzable. 2478 assert(BranchOp != AMDGPU::S_SETPC_B64); 2479 2480 // Convert to dwords. 2481 BrOffset /= 4; 2482 2483 // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is 2484 // from the next instruction. 2485 BrOffset -= 1; 2486 2487 return isIntN(BranchOffsetBits, BrOffset); 2488 } 2489 2490 MachineBasicBlock *SIInstrInfo::getBranchDestBlock( 2491 const MachineInstr &MI) const { 2492 if (MI.getOpcode() == AMDGPU::S_SETPC_B64) { 2493 // This would be a difficult analysis to perform, but can always be legal so 2494 // there's no need to analyze it. 2495 return nullptr; 2496 } 2497 2498 return MI.getOperand(0).getMBB(); 2499 } 2500 2501 bool SIInstrInfo::hasDivergentBranch(const MachineBasicBlock *MBB) const { 2502 for (const MachineInstr &MI : MBB->terminators()) { 2503 if (MI.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO || 2504 MI.getOpcode() == AMDGPU::SI_IF || MI.getOpcode() == AMDGPU::SI_ELSE || 2505 MI.getOpcode() == AMDGPU::SI_LOOP) 2506 return true; 2507 } 2508 return false; 2509 } 2510 2511 void SIInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB, 2512 MachineBasicBlock &DestBB, 2513 MachineBasicBlock &RestoreBB, 2514 const DebugLoc &DL, int64_t BrOffset, 2515 RegScavenger *RS) const { 2516 assert(RS && "RegScavenger required for long branching"); 2517 assert(MBB.empty() && 2518 "new block should be inserted for expanding unconditional branch"); 2519 assert(MBB.pred_size() == 1); 2520 assert(RestoreBB.empty() && 2521 "restore block should be inserted for restoring clobbered registers"); 2522 2523 MachineFunction *MF = MBB.getParent(); 2524 MachineRegisterInfo &MRI = MF->getRegInfo(); 2525 2526 // FIXME: Virtual register workaround for RegScavenger not working with empty 2527 // blocks. 2528 Register PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); 2529 2530 auto I = MBB.end(); 2531 2532 // We need to compute the offset relative to the instruction immediately after 2533 // s_getpc_b64. Insert pc arithmetic code before last terminator. 2534 MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg); 2535 2536 auto &MCCtx = MF->getContext(); 2537 MCSymbol *PostGetPCLabel = 2538 MCCtx.createTempSymbol("post_getpc", /*AlwaysAddSuffix=*/true); 2539 GetPC->setPostInstrSymbol(*MF, PostGetPCLabel); 2540 2541 MCSymbol *OffsetLo = 2542 MCCtx.createTempSymbol("offset_lo", /*AlwaysAddSuffix=*/true); 2543 MCSymbol *OffsetHi = 2544 MCCtx.createTempSymbol("offset_hi", /*AlwaysAddSuffix=*/true); 2545 BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32)) 2546 .addReg(PCReg, RegState::Define, AMDGPU::sub0) 2547 .addReg(PCReg, 0, AMDGPU::sub0) 2548 .addSym(OffsetLo, MO_FAR_BRANCH_OFFSET); 2549 BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32)) 2550 .addReg(PCReg, RegState::Define, AMDGPU::sub1) 2551 .addReg(PCReg, 0, AMDGPU::sub1) 2552 .addSym(OffsetHi, MO_FAR_BRANCH_OFFSET); 2553 2554 // Insert the indirect branch after the other terminator. 2555 BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64)) 2556 .addReg(PCReg); 2557 2558 // FIXME: If spilling is necessary, this will fail because this scavenger has 2559 // no emergency stack slots. It is non-trivial to spill in this situation, 2560 // because the restore code needs to be specially placed after the 2561 // jump. BranchRelaxation then needs to be made aware of the newly inserted 2562 // block. 2563 // 2564 // If a spill is needed for the pc register pair, we need to insert a spill 2565 // restore block right before the destination block, and insert a short branch 2566 // into the old destination block's fallthrough predecessor. 2567 // e.g.: 2568 // 2569 // s_cbranch_scc0 skip_long_branch: 2570 // 2571 // long_branch_bb: 2572 // spill s[8:9] 2573 // s_getpc_b64 s[8:9] 2574 // s_add_u32 s8, s8, restore_bb 2575 // s_addc_u32 s9, s9, 0 2576 // s_setpc_b64 s[8:9] 2577 // 2578 // skip_long_branch: 2579 // foo; 2580 // 2581 // ..... 2582 // 2583 // dest_bb_fallthrough_predecessor: 2584 // bar; 2585 // s_branch dest_bb 2586 // 2587 // restore_bb: 2588 // restore s[8:9] 2589 // fallthrough dest_bb 2590 /// 2591 // dest_bb: 2592 // buzz; 2593 2594 RS->enterBasicBlockEnd(MBB); 2595 Register Scav = RS->scavengeRegisterBackwards( 2596 AMDGPU::SReg_64RegClass, MachineBasicBlock::iterator(GetPC), 2597 /* RestoreAfter */ false, 0, /* AllowSpill */ false); 2598 if (Scav) { 2599 RS->setRegUsed(Scav); 2600 MRI.replaceRegWith(PCReg, Scav); 2601 MRI.clearVirtRegs(); 2602 } else { 2603 // As SGPR needs VGPR to be spilled, we reuse the slot of temporary VGPR for 2604 // SGPR spill. 2605 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); 2606 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 2607 TRI->spillEmergencySGPR(GetPC, RestoreBB, AMDGPU::SGPR0_SGPR1, RS); 2608 MRI.replaceRegWith(PCReg, AMDGPU::SGPR0_SGPR1); 2609 MRI.clearVirtRegs(); 2610 } 2611 2612 MCSymbol *DestLabel = Scav ? DestBB.getSymbol() : RestoreBB.getSymbol(); 2613 // Now, the distance could be defined. 2614 auto *Offset = MCBinaryExpr::createSub( 2615 MCSymbolRefExpr::create(DestLabel, MCCtx), 2616 MCSymbolRefExpr::create(PostGetPCLabel, MCCtx), MCCtx); 2617 // Add offset assignments. 2618 auto *Mask = MCConstantExpr::create(0xFFFFFFFFULL, MCCtx); 2619 OffsetLo->setVariableValue(MCBinaryExpr::createAnd(Offset, Mask, MCCtx)); 2620 auto *ShAmt = MCConstantExpr::create(32, MCCtx); 2621 OffsetHi->setVariableValue(MCBinaryExpr::createAShr(Offset, ShAmt, MCCtx)); 2622 } 2623 2624 unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) { 2625 switch (Cond) { 2626 case SIInstrInfo::SCC_TRUE: 2627 return AMDGPU::S_CBRANCH_SCC1; 2628 case SIInstrInfo::SCC_FALSE: 2629 return AMDGPU::S_CBRANCH_SCC0; 2630 case SIInstrInfo::VCCNZ: 2631 return AMDGPU::S_CBRANCH_VCCNZ; 2632 case SIInstrInfo::VCCZ: 2633 return AMDGPU::S_CBRANCH_VCCZ; 2634 case SIInstrInfo::EXECNZ: 2635 return AMDGPU::S_CBRANCH_EXECNZ; 2636 case SIInstrInfo::EXECZ: 2637 return AMDGPU::S_CBRANCH_EXECZ; 2638 default: 2639 llvm_unreachable("invalid branch predicate"); 2640 } 2641 } 2642 2643 SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) { 2644 switch (Opcode) { 2645 case AMDGPU::S_CBRANCH_SCC0: 2646 return SCC_FALSE; 2647 case AMDGPU::S_CBRANCH_SCC1: 2648 return SCC_TRUE; 2649 case AMDGPU::S_CBRANCH_VCCNZ: 2650 return VCCNZ; 2651 case AMDGPU::S_CBRANCH_VCCZ: 2652 return VCCZ; 2653 case AMDGPU::S_CBRANCH_EXECNZ: 2654 return EXECNZ; 2655 case AMDGPU::S_CBRANCH_EXECZ: 2656 return EXECZ; 2657 default: 2658 return INVALID_BR; 2659 } 2660 } 2661 2662 bool SIInstrInfo::analyzeBranchImpl(MachineBasicBlock &MBB, 2663 MachineBasicBlock::iterator I, 2664 MachineBasicBlock *&TBB, 2665 MachineBasicBlock *&FBB, 2666 SmallVectorImpl<MachineOperand> &Cond, 2667 bool AllowModify) const { 2668 if (I->getOpcode() == AMDGPU::S_BRANCH) { 2669 // Unconditional Branch 2670 TBB = I->getOperand(0).getMBB(); 2671 return false; 2672 } 2673 2674 MachineBasicBlock *CondBB = nullptr; 2675 2676 if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { 2677 CondBB = I->getOperand(1).getMBB(); 2678 Cond.push_back(I->getOperand(0)); 2679 } else { 2680 BranchPredicate Pred = getBranchPredicate(I->getOpcode()); 2681 if (Pred == INVALID_BR) 2682 return true; 2683 2684 CondBB = I->getOperand(0).getMBB(); 2685 Cond.push_back(MachineOperand::CreateImm(Pred)); 2686 Cond.push_back(I->getOperand(1)); // Save the branch register. 2687 } 2688 ++I; 2689 2690 if (I == MBB.end()) { 2691 // Conditional branch followed by fall-through. 2692 TBB = CondBB; 2693 return false; 2694 } 2695 2696 if (I->getOpcode() == AMDGPU::S_BRANCH) { 2697 TBB = CondBB; 2698 FBB = I->getOperand(0).getMBB(); 2699 return false; 2700 } 2701 2702 return true; 2703 } 2704 2705 bool SIInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, 2706 MachineBasicBlock *&FBB, 2707 SmallVectorImpl<MachineOperand> &Cond, 2708 bool AllowModify) const { 2709 MachineBasicBlock::iterator I = MBB.getFirstTerminator(); 2710 auto E = MBB.end(); 2711 if (I == E) 2712 return false; 2713 2714 // Skip over the instructions that are artificially terminators for special 2715 // exec management. 2716 while (I != E && !I->isBranch() && !I->isReturn()) { 2717 switch (I->getOpcode()) { 2718 case AMDGPU::S_MOV_B64_term: 2719 case AMDGPU::S_XOR_B64_term: 2720 case AMDGPU::S_OR_B64_term: 2721 case AMDGPU::S_ANDN2_B64_term: 2722 case AMDGPU::S_AND_B64_term: 2723 case AMDGPU::S_MOV_B32_term: 2724 case AMDGPU::S_XOR_B32_term: 2725 case AMDGPU::S_OR_B32_term: 2726 case AMDGPU::S_ANDN2_B32_term: 2727 case AMDGPU::S_AND_B32_term: 2728 break; 2729 case AMDGPU::SI_IF: 2730 case AMDGPU::SI_ELSE: 2731 case AMDGPU::SI_KILL_I1_TERMINATOR: 2732 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR: 2733 // FIXME: It's messy that these need to be considered here at all. 2734 return true; 2735 default: 2736 llvm_unreachable("unexpected non-branch terminator inst"); 2737 } 2738 2739 ++I; 2740 } 2741 2742 if (I == E) 2743 return false; 2744 2745 return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify); 2746 } 2747 2748 unsigned SIInstrInfo::removeBranch(MachineBasicBlock &MBB, 2749 int *BytesRemoved) const { 2750 unsigned Count = 0; 2751 unsigned RemovedSize = 0; 2752 for (MachineInstr &MI : llvm::make_early_inc_range(MBB.terminators())) { 2753 // Skip over artificial terminators when removing instructions. 2754 if (MI.isBranch() || MI.isReturn()) { 2755 RemovedSize += getInstSizeInBytes(MI); 2756 MI.eraseFromParent(); 2757 ++Count; 2758 } 2759 } 2760 2761 if (BytesRemoved) 2762 *BytesRemoved = RemovedSize; 2763 2764 return Count; 2765 } 2766 2767 // Copy the flags onto the implicit condition register operand. 2768 static void preserveCondRegFlags(MachineOperand &CondReg, 2769 const MachineOperand &OrigCond) { 2770 CondReg.setIsUndef(OrigCond.isUndef()); 2771 CondReg.setIsKill(OrigCond.isKill()); 2772 } 2773 2774 unsigned SIInstrInfo::insertBranch(MachineBasicBlock &MBB, 2775 MachineBasicBlock *TBB, 2776 MachineBasicBlock *FBB, 2777 ArrayRef<MachineOperand> Cond, 2778 const DebugLoc &DL, 2779 int *BytesAdded) const { 2780 if (!FBB && Cond.empty()) { 2781 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH)) 2782 .addMBB(TBB); 2783 if (BytesAdded) 2784 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4; 2785 return 1; 2786 } 2787 2788 if(Cond.size() == 1 && Cond[0].isReg()) { 2789 BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO)) 2790 .add(Cond[0]) 2791 .addMBB(TBB); 2792 return 1; 2793 } 2794 2795 assert(TBB && Cond[0].isImm()); 2796 2797 unsigned Opcode 2798 = getBranchOpcode(static_cast<BranchPredicate>(Cond[0].getImm())); 2799 2800 if (!FBB) { 2801 Cond[1].isUndef(); 2802 MachineInstr *CondBr = 2803 BuildMI(&MBB, DL, get(Opcode)) 2804 .addMBB(TBB); 2805 2806 // Copy the flags onto the implicit condition register operand. 2807 preserveCondRegFlags(CondBr->getOperand(1), Cond[1]); 2808 fixImplicitOperands(*CondBr); 2809 2810 if (BytesAdded) 2811 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4; 2812 return 1; 2813 } 2814 2815 assert(TBB && FBB); 2816 2817 MachineInstr *CondBr = 2818 BuildMI(&MBB, DL, get(Opcode)) 2819 .addMBB(TBB); 2820 fixImplicitOperands(*CondBr); 2821 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH)) 2822 .addMBB(FBB); 2823 2824 MachineOperand &CondReg = CondBr->getOperand(1); 2825 CondReg.setIsUndef(Cond[1].isUndef()); 2826 CondReg.setIsKill(Cond[1].isKill()); 2827 2828 if (BytesAdded) 2829 *BytesAdded = ST.hasOffset3fBug() ? 16 : 8; 2830 2831 return 2; 2832 } 2833 2834 bool SIInstrInfo::reverseBranchCondition( 2835 SmallVectorImpl<MachineOperand> &Cond) const { 2836 if (Cond.size() != 2) { 2837 return true; 2838 } 2839 2840 if (Cond[0].isImm()) { 2841 Cond[0].setImm(-Cond[0].getImm()); 2842 return false; 2843 } 2844 2845 return true; 2846 } 2847 2848 bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB, 2849 ArrayRef<MachineOperand> Cond, 2850 Register DstReg, Register TrueReg, 2851 Register FalseReg, int &CondCycles, 2852 int &TrueCycles, int &FalseCycles) const { 2853 switch (Cond[0].getImm()) { 2854 case VCCNZ: 2855 case VCCZ: { 2856 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 2857 const TargetRegisterClass *RC = MRI.getRegClass(TrueReg); 2858 if (MRI.getRegClass(FalseReg) != RC) 2859 return false; 2860 2861 int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32; 2862 CondCycles = TrueCycles = FalseCycles = NumInsts; // ??? 2863 2864 // Limit to equal cost for branch vs. N v_cndmask_b32s. 2865 return RI.hasVGPRs(RC) && NumInsts <= 6; 2866 } 2867 case SCC_TRUE: 2868 case SCC_FALSE: { 2869 // FIXME: We could insert for VGPRs if we could replace the original compare 2870 // with a vector one. 2871 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 2872 const TargetRegisterClass *RC = MRI.getRegClass(TrueReg); 2873 if (MRI.getRegClass(FalseReg) != RC) 2874 return false; 2875 2876 int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32; 2877 2878 // Multiples of 8 can do s_cselect_b64 2879 if (NumInsts % 2 == 0) 2880 NumInsts /= 2; 2881 2882 CondCycles = TrueCycles = FalseCycles = NumInsts; // ??? 2883 return RI.isSGPRClass(RC); 2884 } 2885 default: 2886 return false; 2887 } 2888 } 2889 2890 void SIInstrInfo::insertSelect(MachineBasicBlock &MBB, 2891 MachineBasicBlock::iterator I, const DebugLoc &DL, 2892 Register DstReg, ArrayRef<MachineOperand> Cond, 2893 Register TrueReg, Register FalseReg) const { 2894 BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm()); 2895 if (Pred == VCCZ || Pred == SCC_FALSE) { 2896 Pred = static_cast<BranchPredicate>(-Pred); 2897 std::swap(TrueReg, FalseReg); 2898 } 2899 2900 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 2901 const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg); 2902 unsigned DstSize = RI.getRegSizeInBits(*DstRC); 2903 2904 if (DstSize == 32) { 2905 MachineInstr *Select; 2906 if (Pred == SCC_TRUE) { 2907 Select = BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B32), DstReg) 2908 .addReg(TrueReg) 2909 .addReg(FalseReg); 2910 } else { 2911 // Instruction's operands are backwards from what is expected. 2912 Select = BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e32), DstReg) 2913 .addReg(FalseReg) 2914 .addReg(TrueReg); 2915 } 2916 2917 preserveCondRegFlags(Select->getOperand(3), Cond[1]); 2918 return; 2919 } 2920 2921 if (DstSize == 64 && Pred == SCC_TRUE) { 2922 MachineInstr *Select = 2923 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg) 2924 .addReg(TrueReg) 2925 .addReg(FalseReg); 2926 2927 preserveCondRegFlags(Select->getOperand(3), Cond[1]); 2928 return; 2929 } 2930 2931 static const int16_t Sub0_15[] = { 2932 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, 2933 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7, 2934 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11, 2935 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15, 2936 }; 2937 2938 static const int16_t Sub0_15_64[] = { 2939 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3, 2940 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7, 2941 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11, 2942 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15, 2943 }; 2944 2945 unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32; 2946 const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass; 2947 const int16_t *SubIndices = Sub0_15; 2948 int NElts = DstSize / 32; 2949 2950 // 64-bit select is only available for SALU. 2951 // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit. 2952 if (Pred == SCC_TRUE) { 2953 if (NElts % 2) { 2954 SelOp = AMDGPU::S_CSELECT_B32; 2955 EltRC = &AMDGPU::SGPR_32RegClass; 2956 } else { 2957 SelOp = AMDGPU::S_CSELECT_B64; 2958 EltRC = &AMDGPU::SGPR_64RegClass; 2959 SubIndices = Sub0_15_64; 2960 NElts /= 2; 2961 } 2962 } 2963 2964 MachineInstrBuilder MIB = BuildMI( 2965 MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg); 2966 2967 I = MIB->getIterator(); 2968 2969 SmallVector<Register, 8> Regs; 2970 for (int Idx = 0; Idx != NElts; ++Idx) { 2971 Register DstElt = MRI.createVirtualRegister(EltRC); 2972 Regs.push_back(DstElt); 2973 2974 unsigned SubIdx = SubIndices[Idx]; 2975 2976 MachineInstr *Select; 2977 if (SelOp == AMDGPU::V_CNDMASK_B32_e32) { 2978 Select = 2979 BuildMI(MBB, I, DL, get(SelOp), DstElt) 2980 .addReg(FalseReg, 0, SubIdx) 2981 .addReg(TrueReg, 0, SubIdx); 2982 } else { 2983 Select = 2984 BuildMI(MBB, I, DL, get(SelOp), DstElt) 2985 .addReg(TrueReg, 0, SubIdx) 2986 .addReg(FalseReg, 0, SubIdx); 2987 } 2988 2989 preserveCondRegFlags(Select->getOperand(3), Cond[1]); 2990 fixImplicitOperands(*Select); 2991 2992 MIB.addReg(DstElt) 2993 .addImm(SubIdx); 2994 } 2995 } 2996 2997 bool SIInstrInfo::isFoldableCopy(const MachineInstr &MI) { 2998 switch (MI.getOpcode()) { 2999 case AMDGPU::V_MOV_B32_e32: 3000 case AMDGPU::V_MOV_B32_e64: 3001 case AMDGPU::V_MOV_B64_PSEUDO: 3002 case AMDGPU::V_MOV_B64_e32: 3003 case AMDGPU::V_MOV_B64_e64: 3004 case AMDGPU::S_MOV_B32: 3005 case AMDGPU::S_MOV_B64: 3006 case AMDGPU::COPY: 3007 case AMDGPU::V_ACCVGPR_WRITE_B32_e64: 3008 case AMDGPU::V_ACCVGPR_READ_B32_e64: 3009 case AMDGPU::V_ACCVGPR_MOV_B32: 3010 return true; 3011 default: 3012 return false; 3013 } 3014 } 3015 3016 static constexpr unsigned ModifierOpNames[] = { 3017 AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src1_modifiers, 3018 AMDGPU::OpName::src2_modifiers, AMDGPU::OpName::clamp, 3019 AMDGPU::OpName::omod, AMDGPU::OpName::op_sel}; 3020 3021 void SIInstrInfo::removeModOperands(MachineInstr &MI) const { 3022 unsigned Opc = MI.getOpcode(); 3023 for (unsigned Name : reverse(ModifierOpNames)) { 3024 int Idx = AMDGPU::getNamedOperandIdx(Opc, Name); 3025 if (Idx >= 0) 3026 MI.removeOperand(Idx); 3027 } 3028 } 3029 3030 bool SIInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, 3031 Register Reg, MachineRegisterInfo *MRI) const { 3032 if (!MRI->hasOneNonDBGUse(Reg)) 3033 return false; 3034 3035 switch (DefMI.getOpcode()) { 3036 default: 3037 return false; 3038 case AMDGPU::S_MOV_B64: 3039 // TODO: We could fold 64-bit immediates, but this get complicated 3040 // when there are sub-registers. 3041 return false; 3042 3043 case AMDGPU::V_MOV_B32_e32: 3044 case AMDGPU::S_MOV_B32: 3045 case AMDGPU::V_ACCVGPR_WRITE_B32_e64: 3046 break; 3047 } 3048 3049 const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0); 3050 assert(ImmOp); 3051 // FIXME: We could handle FrameIndex values here. 3052 if (!ImmOp->isImm()) 3053 return false; 3054 3055 unsigned Opc = UseMI.getOpcode(); 3056 if (Opc == AMDGPU::COPY) { 3057 Register DstReg = UseMI.getOperand(0).getReg(); 3058 bool Is16Bit = getOpSize(UseMI, 0) == 2; 3059 bool isVGPRCopy = RI.isVGPR(*MRI, DstReg); 3060 unsigned NewOpc = isVGPRCopy ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32; 3061 APInt Imm(32, ImmOp->getImm()); 3062 3063 if (UseMI.getOperand(1).getSubReg() == AMDGPU::hi16) 3064 Imm = Imm.ashr(16); 3065 3066 if (RI.isAGPR(*MRI, DstReg)) { 3067 if (!isInlineConstant(Imm)) 3068 return false; 3069 NewOpc = AMDGPU::V_ACCVGPR_WRITE_B32_e64; 3070 } 3071 3072 if (Is16Bit) { 3073 if (isVGPRCopy) 3074 return false; // Do not clobber vgpr_hi16 3075 3076 if (DstReg.isVirtual() && UseMI.getOperand(0).getSubReg() != AMDGPU::lo16) 3077 return false; 3078 3079 UseMI.getOperand(0).setSubReg(0); 3080 if (DstReg.isPhysical()) { 3081 DstReg = RI.get32BitRegister(DstReg); 3082 UseMI.getOperand(0).setReg(DstReg); 3083 } 3084 assert(UseMI.getOperand(1).getReg().isVirtual()); 3085 } 3086 3087 UseMI.setDesc(get(NewOpc)); 3088 UseMI.getOperand(1).ChangeToImmediate(Imm.getSExtValue()); 3089 UseMI.addImplicitDefUseOperands(*UseMI.getParent()->getParent()); 3090 return true; 3091 } 3092 3093 if (Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 || 3094 Opc == AMDGPU::V_MAD_F16_e64 || Opc == AMDGPU::V_MAC_F16_e64 || 3095 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 || 3096 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64 || 3097 Opc == AMDGPU::V_FMAC_F16_t16_e64) { 3098 // Don't fold if we are using source or output modifiers. The new VOP2 3099 // instructions don't have them. 3100 if (hasAnyModifiersSet(UseMI)) 3101 return false; 3102 3103 // If this is a free constant, there's no reason to do this. 3104 // TODO: We could fold this here instead of letting SIFoldOperands do it 3105 // later. 3106 MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0); 3107 3108 // Any src operand can be used for the legality check. 3109 if (isInlineConstant(UseMI, *Src0, *ImmOp)) 3110 return false; 3111 3112 bool IsF32 = Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 || 3113 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64; 3114 bool IsFMA = 3115 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 || 3116 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64 || 3117 Opc == AMDGPU::V_FMAC_F16_t16_e64; 3118 MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1); 3119 MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2); 3120 3121 // Multiplied part is the constant: Use v_madmk_{f16, f32}. 3122 // We should only expect these to be on src0 due to canonicalization. 3123 if (Src0->isReg() && Src0->getReg() == Reg) { 3124 if (!Src1->isReg() || RI.isSGPRClass(MRI->getRegClass(Src1->getReg()))) 3125 return false; 3126 3127 if (!Src2->isReg() || RI.isSGPRClass(MRI->getRegClass(Src2->getReg()))) 3128 return false; 3129 3130 unsigned NewOpc = 3131 IsFMA ? (IsF32 ? AMDGPU::V_FMAMK_F32 3132 : ST.hasTrue16BitInsts() ? AMDGPU::V_FMAMK_F16_t16 3133 : AMDGPU::V_FMAMK_F16) 3134 : (IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16); 3135 if (pseudoToMCOpcode(NewOpc) == -1) 3136 return false; 3137 3138 // We need to swap operands 0 and 1 since madmk constant is at operand 1. 3139 3140 const int64_t Imm = ImmOp->getImm(); 3141 3142 // FIXME: This would be a lot easier if we could return a new instruction 3143 // instead of having to modify in place. 3144 3145 Register Src1Reg = Src1->getReg(); 3146 unsigned Src1SubReg = Src1->getSubReg(); 3147 Src0->setReg(Src1Reg); 3148 Src0->setSubReg(Src1SubReg); 3149 Src0->setIsKill(Src1->isKill()); 3150 3151 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 || 3152 Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 || 3153 Opc == AMDGPU::V_FMAC_F16_e64) 3154 UseMI.untieRegOperand( 3155 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)); 3156 3157 Src1->ChangeToImmediate(Imm); 3158 3159 removeModOperands(UseMI); 3160 UseMI.setDesc(get(NewOpc)); 3161 3162 bool DeleteDef = MRI->use_nodbg_empty(Reg); 3163 if (DeleteDef) 3164 DefMI.eraseFromParent(); 3165 3166 return true; 3167 } 3168 3169 // Added part is the constant: Use v_madak_{f16, f32}. 3170 if (Src2->isReg() && Src2->getReg() == Reg) { 3171 // Not allowed to use constant bus for another operand. 3172 // We can however allow an inline immediate as src0. 3173 bool Src0Inlined = false; 3174 if (Src0->isReg()) { 3175 // Try to inline constant if possible. 3176 // If the Def moves immediate and the use is single 3177 // We are saving VGPR here. 3178 MachineInstr *Def = MRI->getUniqueVRegDef(Src0->getReg()); 3179 if (Def && Def->isMoveImmediate() && 3180 isInlineConstant(Def->getOperand(1)) && 3181 MRI->hasOneUse(Src0->getReg())) { 3182 Src0->ChangeToImmediate(Def->getOperand(1).getImm()); 3183 Src0Inlined = true; 3184 } else if ((Src0->getReg().isPhysical() && 3185 (ST.getConstantBusLimit(Opc) <= 1 && 3186 RI.isSGPRClass(RI.getPhysRegBaseClass(Src0->getReg())))) || 3187 (Src0->getReg().isVirtual() && 3188 (ST.getConstantBusLimit(Opc) <= 1 && 3189 RI.isSGPRClass(MRI->getRegClass(Src0->getReg()))))) 3190 return false; 3191 // VGPR is okay as Src0 - fallthrough 3192 } 3193 3194 if (Src1->isReg() && !Src0Inlined ) { 3195 // We have one slot for inlinable constant so far - try to fill it 3196 MachineInstr *Def = MRI->getUniqueVRegDef(Src1->getReg()); 3197 if (Def && Def->isMoveImmediate() && 3198 isInlineConstant(Def->getOperand(1)) && 3199 MRI->hasOneUse(Src1->getReg()) && 3200 commuteInstruction(UseMI)) { 3201 Src0->ChangeToImmediate(Def->getOperand(1).getImm()); 3202 } else if ((Src1->getReg().isPhysical() && 3203 RI.isSGPRClass(RI.getPhysRegBaseClass(Src1->getReg()))) || 3204 (Src1->getReg().isVirtual() && 3205 RI.isSGPRClass(MRI->getRegClass(Src1->getReg())))) 3206 return false; 3207 // VGPR is okay as Src1 - fallthrough 3208 } 3209 3210 unsigned NewOpc = 3211 IsFMA ? (IsF32 ? AMDGPU::V_FMAAK_F32 3212 : ST.hasTrue16BitInsts() ? AMDGPU::V_FMAAK_F16_t16 3213 : AMDGPU::V_FMAAK_F16) 3214 : (IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16); 3215 if (pseudoToMCOpcode(NewOpc) == -1) 3216 return false; 3217 3218 const int64_t Imm = ImmOp->getImm(); 3219 3220 // FIXME: This would be a lot easier if we could return a new instruction 3221 // instead of having to modify in place. 3222 3223 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 || 3224 Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 || 3225 Opc == AMDGPU::V_FMAC_F16_e64) 3226 UseMI.untieRegOperand( 3227 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)); 3228 3229 // ChangingToImmediate adds Src2 back to the instruction. 3230 Src2->ChangeToImmediate(Imm); 3231 3232 // These come before src2. 3233 removeModOperands(UseMI); 3234 UseMI.setDesc(get(NewOpc)); 3235 // It might happen that UseMI was commuted 3236 // and we now have SGPR as SRC1. If so 2 inlined 3237 // constant and SGPR are illegal. 3238 legalizeOperands(UseMI); 3239 3240 bool DeleteDef = MRI->use_nodbg_empty(Reg); 3241 if (DeleteDef) 3242 DefMI.eraseFromParent(); 3243 3244 return true; 3245 } 3246 } 3247 3248 return false; 3249 } 3250 3251 static bool 3252 memOpsHaveSameBaseOperands(ArrayRef<const MachineOperand *> BaseOps1, 3253 ArrayRef<const MachineOperand *> BaseOps2) { 3254 if (BaseOps1.size() != BaseOps2.size()) 3255 return false; 3256 for (size_t I = 0, E = BaseOps1.size(); I < E; ++I) { 3257 if (!BaseOps1[I]->isIdenticalTo(*BaseOps2[I])) 3258 return false; 3259 } 3260 return true; 3261 } 3262 3263 static bool offsetsDoNotOverlap(int WidthA, int OffsetA, 3264 int WidthB, int OffsetB) { 3265 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB; 3266 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA; 3267 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB; 3268 return LowOffset + LowWidth <= HighOffset; 3269 } 3270 3271 bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa, 3272 const MachineInstr &MIb) const { 3273 SmallVector<const MachineOperand *, 4> BaseOps0, BaseOps1; 3274 int64_t Offset0, Offset1; 3275 unsigned Dummy0, Dummy1; 3276 bool Offset0IsScalable, Offset1IsScalable; 3277 if (!getMemOperandsWithOffsetWidth(MIa, BaseOps0, Offset0, Offset0IsScalable, 3278 Dummy0, &RI) || 3279 !getMemOperandsWithOffsetWidth(MIb, BaseOps1, Offset1, Offset1IsScalable, 3280 Dummy1, &RI)) 3281 return false; 3282 3283 if (!memOpsHaveSameBaseOperands(BaseOps0, BaseOps1)) 3284 return false; 3285 3286 if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) { 3287 // FIXME: Handle ds_read2 / ds_write2. 3288 return false; 3289 } 3290 unsigned Width0 = MIa.memoperands().front()->getSize(); 3291 unsigned Width1 = MIb.memoperands().front()->getSize(); 3292 return offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1); 3293 } 3294 3295 bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, 3296 const MachineInstr &MIb) const { 3297 assert(MIa.mayLoadOrStore() && 3298 "MIa must load from or modify a memory location"); 3299 assert(MIb.mayLoadOrStore() && 3300 "MIb must load from or modify a memory location"); 3301 3302 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects()) 3303 return false; 3304 3305 // XXX - Can we relax this between address spaces? 3306 if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef()) 3307 return false; 3308 3309 // TODO: Should we check the address space from the MachineMemOperand? That 3310 // would allow us to distinguish objects we know don't alias based on the 3311 // underlying address space, even if it was lowered to a different one, 3312 // e.g. private accesses lowered to use MUBUF instructions on a scratch 3313 // buffer. 3314 if (isDS(MIa)) { 3315 if (isDS(MIb)) 3316 return checkInstOffsetsDoNotOverlap(MIa, MIb); 3317 3318 return !isFLAT(MIb) || isSegmentSpecificFLAT(MIb); 3319 } 3320 3321 if (isMUBUF(MIa) || isMTBUF(MIa)) { 3322 if (isMUBUF(MIb) || isMTBUF(MIb)) 3323 return checkInstOffsetsDoNotOverlap(MIa, MIb); 3324 3325 return !isFLAT(MIb) && !isSMRD(MIb); 3326 } 3327 3328 if (isSMRD(MIa)) { 3329 if (isSMRD(MIb)) 3330 return checkInstOffsetsDoNotOverlap(MIa, MIb); 3331 3332 return !isFLAT(MIb) && !isMUBUF(MIb) && !isMTBUF(MIb); 3333 } 3334 3335 if (isFLAT(MIa)) { 3336 if (isFLAT(MIb)) 3337 return checkInstOffsetsDoNotOverlap(MIa, MIb); 3338 3339 return false; 3340 } 3341 3342 return false; 3343 } 3344 3345 static bool getFoldableImm(Register Reg, const MachineRegisterInfo &MRI, 3346 int64_t &Imm, MachineInstr **DefMI = nullptr) { 3347 if (Reg.isPhysical()) 3348 return false; 3349 auto *Def = MRI.getUniqueVRegDef(Reg); 3350 if (Def && SIInstrInfo::isFoldableCopy(*Def) && Def->getOperand(1).isImm()) { 3351 Imm = Def->getOperand(1).getImm(); 3352 if (DefMI) 3353 *DefMI = Def; 3354 return true; 3355 } 3356 return false; 3357 } 3358 3359 static bool getFoldableImm(const MachineOperand *MO, int64_t &Imm, 3360 MachineInstr **DefMI = nullptr) { 3361 if (!MO->isReg()) 3362 return false; 3363 const MachineFunction *MF = MO->getParent()->getParent()->getParent(); 3364 const MachineRegisterInfo &MRI = MF->getRegInfo(); 3365 return getFoldableImm(MO->getReg(), MRI, Imm, DefMI); 3366 } 3367 3368 static void updateLiveVariables(LiveVariables *LV, MachineInstr &MI, 3369 MachineInstr &NewMI) { 3370 if (LV) { 3371 unsigned NumOps = MI.getNumOperands(); 3372 for (unsigned I = 1; I < NumOps; ++I) { 3373 MachineOperand &Op = MI.getOperand(I); 3374 if (Op.isReg() && Op.isKill()) 3375 LV->replaceKillInstruction(Op.getReg(), MI, NewMI); 3376 } 3377 } 3378 } 3379 3380 MachineInstr *SIInstrInfo::convertToThreeAddress(MachineInstr &MI, 3381 LiveVariables *LV, 3382 LiveIntervals *LIS) const { 3383 MachineBasicBlock &MBB = *MI.getParent(); 3384 unsigned Opc = MI.getOpcode(); 3385 3386 // Handle MFMA. 3387 int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(Opc); 3388 if (NewMFMAOpc != -1) { 3389 MachineInstrBuilder MIB = 3390 BuildMI(MBB, MI, MI.getDebugLoc(), get(NewMFMAOpc)); 3391 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) 3392 MIB.add(MI.getOperand(I)); 3393 updateLiveVariables(LV, MI, *MIB); 3394 if (LIS) 3395 LIS->ReplaceMachineInstrInMaps(MI, *MIB); 3396 return MIB; 3397 } 3398 3399 if (SIInstrInfo::isWMMA(MI)) { 3400 unsigned NewOpc = AMDGPU::mapWMMA2AddrTo3AddrOpcode(MI.getOpcode()); 3401 MachineInstrBuilder MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3402 .setMIFlags(MI.getFlags()); 3403 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) 3404 MIB->addOperand(MI.getOperand(I)); 3405 3406 updateLiveVariables(LV, MI, *MIB); 3407 if (LIS) 3408 LIS->ReplaceMachineInstrInMaps(MI, *MIB); 3409 3410 return MIB; 3411 } 3412 3413 assert(Opc != AMDGPU::V_FMAC_F16_t16_e32 && 3414 "V_FMAC_F16_t16_e32 is not supported and not expected to be present " 3415 "pre-RA"); 3416 3417 // Handle MAC/FMAC. 3418 bool IsF16 = Opc == AMDGPU::V_MAC_F16_e32 || Opc == AMDGPU::V_MAC_F16_e64 || 3419 Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64 || 3420 Opc == AMDGPU::V_FMAC_F16_t16_e64; 3421 bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64 || 3422 Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 || 3423 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64 || 3424 Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64 || 3425 Opc == AMDGPU::V_FMAC_F16_t16_e64 || 3426 Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64; 3427 bool IsF64 = Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64; 3428 bool IsLegacy = Opc == AMDGPU::V_MAC_LEGACY_F32_e32 || 3429 Opc == AMDGPU::V_MAC_LEGACY_F32_e64 || 3430 Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 || 3431 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64; 3432 bool Src0Literal = false; 3433 3434 switch (Opc) { 3435 default: 3436 return nullptr; 3437 case AMDGPU::V_MAC_F16_e64: 3438 case AMDGPU::V_FMAC_F16_e64: 3439 case AMDGPU::V_FMAC_F16_t16_e64: 3440 case AMDGPU::V_MAC_F32_e64: 3441 case AMDGPU::V_MAC_LEGACY_F32_e64: 3442 case AMDGPU::V_FMAC_F32_e64: 3443 case AMDGPU::V_FMAC_LEGACY_F32_e64: 3444 case AMDGPU::V_FMAC_F64_e64: 3445 break; 3446 case AMDGPU::V_MAC_F16_e32: 3447 case AMDGPU::V_FMAC_F16_e32: 3448 case AMDGPU::V_MAC_F32_e32: 3449 case AMDGPU::V_MAC_LEGACY_F32_e32: 3450 case AMDGPU::V_FMAC_F32_e32: 3451 case AMDGPU::V_FMAC_LEGACY_F32_e32: 3452 case AMDGPU::V_FMAC_F64_e32: { 3453 int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), 3454 AMDGPU::OpName::src0); 3455 const MachineOperand *Src0 = &MI.getOperand(Src0Idx); 3456 if (!Src0->isReg() && !Src0->isImm()) 3457 return nullptr; 3458 3459 if (Src0->isImm() && !isInlineConstant(MI, Src0Idx, *Src0)) 3460 Src0Literal = true; 3461 3462 break; 3463 } 3464 } 3465 3466 MachineInstrBuilder MIB; 3467 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); 3468 const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0); 3469 const MachineOperand *Src0Mods = 3470 getNamedOperand(MI, AMDGPU::OpName::src0_modifiers); 3471 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); 3472 const MachineOperand *Src1Mods = 3473 getNamedOperand(MI, AMDGPU::OpName::src1_modifiers); 3474 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); 3475 const MachineOperand *Src2Mods = 3476 getNamedOperand(MI, AMDGPU::OpName::src2_modifiers); 3477 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp); 3478 const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod); 3479 const MachineOperand *OpSel = getNamedOperand(MI, AMDGPU::OpName::op_sel); 3480 3481 if (!Src0Mods && !Src1Mods && !Src2Mods && !Clamp && !Omod && !IsF64 && 3482 !IsLegacy && 3483 // If we have an SGPR input, we will violate the constant bus restriction. 3484 (ST.getConstantBusLimit(Opc) > 1 || !Src0->isReg() || 3485 !RI.isSGPRReg(MBB.getParent()->getRegInfo(), Src0->getReg()))) { 3486 MachineInstr *DefMI; 3487 const auto killDef = [&]() -> void { 3488 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 3489 // The only user is the instruction which will be killed. 3490 Register DefReg = DefMI->getOperand(0).getReg(); 3491 if (!MRI.hasOneNonDBGUse(DefReg)) 3492 return; 3493 // We cannot just remove the DefMI here, calling pass will crash. 3494 DefMI->setDesc(get(AMDGPU::IMPLICIT_DEF)); 3495 for (unsigned I = DefMI->getNumOperands() - 1; I != 0; --I) 3496 DefMI->removeOperand(I); 3497 if (LV) 3498 LV->getVarInfo(DefReg).AliveBlocks.clear(); 3499 }; 3500 3501 int64_t Imm; 3502 if (!Src0Literal && getFoldableImm(Src2, Imm, &DefMI)) { 3503 unsigned NewOpc = 3504 IsFMA ? (IsF16 ? (ST.hasTrue16BitInsts() ? AMDGPU::V_FMAAK_F16_t16 3505 : AMDGPU::V_FMAAK_F16) 3506 : AMDGPU::V_FMAAK_F32) 3507 : (IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32); 3508 if (pseudoToMCOpcode(NewOpc) != -1) { 3509 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3510 .add(*Dst) 3511 .add(*Src0) 3512 .add(*Src1) 3513 .addImm(Imm); 3514 updateLiveVariables(LV, MI, *MIB); 3515 if (LIS) 3516 LIS->ReplaceMachineInstrInMaps(MI, *MIB); 3517 killDef(); 3518 return MIB; 3519 } 3520 } 3521 unsigned NewOpc = 3522 IsFMA ? (IsF16 ? (ST.hasTrue16BitInsts() ? AMDGPU::V_FMAMK_F16_t16 3523 : AMDGPU::V_FMAMK_F16) 3524 : AMDGPU::V_FMAMK_F32) 3525 : (IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32); 3526 if (!Src0Literal && getFoldableImm(Src1, Imm, &DefMI)) { 3527 if (pseudoToMCOpcode(NewOpc) != -1) { 3528 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3529 .add(*Dst) 3530 .add(*Src0) 3531 .addImm(Imm) 3532 .add(*Src2); 3533 updateLiveVariables(LV, MI, *MIB); 3534 if (LIS) 3535 LIS->ReplaceMachineInstrInMaps(MI, *MIB); 3536 killDef(); 3537 return MIB; 3538 } 3539 } 3540 if (Src0Literal || getFoldableImm(Src0, Imm, &DefMI)) { 3541 if (Src0Literal) { 3542 Imm = Src0->getImm(); 3543 DefMI = nullptr; 3544 } 3545 if (pseudoToMCOpcode(NewOpc) != -1 && 3546 isOperandLegal( 3547 MI, AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::src0), 3548 Src1)) { 3549 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3550 .add(*Dst) 3551 .add(*Src1) 3552 .addImm(Imm) 3553 .add(*Src2); 3554 updateLiveVariables(LV, MI, *MIB); 3555 if (LIS) 3556 LIS->ReplaceMachineInstrInMaps(MI, *MIB); 3557 if (DefMI) 3558 killDef(); 3559 return MIB; 3560 } 3561 } 3562 } 3563 3564 // VOP2 mac/fmac with a literal operand cannot be converted to VOP3 mad/fma 3565 // if VOP3 does not allow a literal operand. 3566 if (Src0Literal && !ST.hasVOP3Literal()) 3567 return nullptr; 3568 3569 unsigned NewOpc = IsFMA ? IsF16 ? AMDGPU::V_FMA_F16_gfx9_e64 3570 : IsF64 ? AMDGPU::V_FMA_F64_e64 3571 : IsLegacy 3572 ? AMDGPU::V_FMA_LEGACY_F32_e64 3573 : AMDGPU::V_FMA_F32_e64 3574 : IsF16 ? AMDGPU::V_MAD_F16_e64 3575 : IsLegacy ? AMDGPU::V_MAD_LEGACY_F32_e64 3576 : AMDGPU::V_MAD_F32_e64; 3577 if (pseudoToMCOpcode(NewOpc) == -1) 3578 return nullptr; 3579 3580 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3581 .add(*Dst) 3582 .addImm(Src0Mods ? Src0Mods->getImm() : 0) 3583 .add(*Src0) 3584 .addImm(Src1Mods ? Src1Mods->getImm() : 0) 3585 .add(*Src1) 3586 .addImm(Src2Mods ? Src2Mods->getImm() : 0) 3587 .add(*Src2) 3588 .addImm(Clamp ? Clamp->getImm() : 0) 3589 .addImm(Omod ? Omod->getImm() : 0); 3590 if (AMDGPU::hasNamedOperand(NewOpc, AMDGPU::OpName::op_sel)) 3591 MIB.addImm(OpSel ? OpSel->getImm() : 0); 3592 updateLiveVariables(LV, MI, *MIB); 3593 if (LIS) 3594 LIS->ReplaceMachineInstrInMaps(MI, *MIB); 3595 return MIB; 3596 } 3597 3598 // It's not generally safe to move VALU instructions across these since it will 3599 // start using the register as a base index rather than directly. 3600 // XXX - Why isn't hasSideEffects sufficient for these? 3601 static bool changesVGPRIndexingMode(const MachineInstr &MI) { 3602 switch (MI.getOpcode()) { 3603 case AMDGPU::S_SET_GPR_IDX_ON: 3604 case AMDGPU::S_SET_GPR_IDX_MODE: 3605 case AMDGPU::S_SET_GPR_IDX_OFF: 3606 return true; 3607 default: 3608 return false; 3609 } 3610 } 3611 3612 bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI, 3613 const MachineBasicBlock *MBB, 3614 const MachineFunction &MF) const { 3615 // Skipping the check for SP writes in the base implementation. The reason it 3616 // was added was apparently due to compile time concerns. 3617 // 3618 // TODO: Do we really want this barrier? It triggers unnecessary hazard nops 3619 // but is probably avoidable. 3620 3621 // Copied from base implementation. 3622 // Terminators and labels can't be scheduled around. 3623 if (MI.isTerminator() || MI.isPosition()) 3624 return true; 3625 3626 // INLINEASM_BR can jump to another block 3627 if (MI.getOpcode() == TargetOpcode::INLINEASM_BR) 3628 return true; 3629 3630 if (MI.getOpcode() == AMDGPU::SCHED_BARRIER && MI.getOperand(0).getImm() == 0) 3631 return true; 3632 3633 // Target-independent instructions do not have an implicit-use of EXEC, even 3634 // when they operate on VGPRs. Treating EXEC modifications as scheduling 3635 // boundaries prevents incorrect movements of such instructions. 3636 return MI.modifiesRegister(AMDGPU::EXEC, &RI) || 3637 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 || 3638 MI.getOpcode() == AMDGPU::S_SETREG_B32 || 3639 MI.getOpcode() == AMDGPU::S_SETPRIO || 3640 changesVGPRIndexingMode(MI); 3641 } 3642 3643 bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const { 3644 return Opcode == AMDGPU::DS_ORDERED_COUNT || 3645 Opcode == AMDGPU::DS_GWS_INIT || 3646 Opcode == AMDGPU::DS_GWS_SEMA_V || 3647 Opcode == AMDGPU::DS_GWS_SEMA_BR || 3648 Opcode == AMDGPU::DS_GWS_SEMA_P || 3649 Opcode == AMDGPU::DS_GWS_SEMA_RELEASE_ALL || 3650 Opcode == AMDGPU::DS_GWS_BARRIER; 3651 } 3652 3653 bool SIInstrInfo::modifiesModeRegister(const MachineInstr &MI) { 3654 // Skip the full operand and register alias search modifiesRegister 3655 // does. There's only a handful of instructions that touch this, it's only an 3656 // implicit def, and doesn't alias any other registers. 3657 return is_contained(MI.getDesc().implicit_defs(), AMDGPU::MODE); 3658 } 3659 3660 bool SIInstrInfo::hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const { 3661 unsigned Opcode = MI.getOpcode(); 3662 3663 if (MI.mayStore() && isSMRD(MI)) 3664 return true; // scalar store or atomic 3665 3666 // This will terminate the function when other lanes may need to continue. 3667 if (MI.isReturn()) 3668 return true; 3669 3670 // These instructions cause shader I/O that may cause hardware lockups 3671 // when executed with an empty EXEC mask. 3672 // 3673 // Note: exp with VM = DONE = 0 is automatically skipped by hardware when 3674 // EXEC = 0, but checking for that case here seems not worth it 3675 // given the typical code patterns. 3676 if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT || 3677 isEXP(Opcode) || 3678 Opcode == AMDGPU::DS_ORDERED_COUNT || Opcode == AMDGPU::S_TRAP || 3679 Opcode == AMDGPU::DS_GWS_INIT || Opcode == AMDGPU::DS_GWS_BARRIER) 3680 return true; 3681 3682 if (MI.isCall() || MI.isInlineAsm()) 3683 return true; // conservative assumption 3684 3685 // A mode change is a scalar operation that influences vector instructions. 3686 if (modifiesModeRegister(MI)) 3687 return true; 3688 3689 // These are like SALU instructions in terms of effects, so it's questionable 3690 // whether we should return true for those. 3691 // 3692 // However, executing them with EXEC = 0 causes them to operate on undefined 3693 // data, which we avoid by returning true here. 3694 if (Opcode == AMDGPU::V_READFIRSTLANE_B32 || 3695 Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32) 3696 return true; 3697 3698 return false; 3699 } 3700 3701 bool SIInstrInfo::mayReadEXEC(const MachineRegisterInfo &MRI, 3702 const MachineInstr &MI) const { 3703 if (MI.isMetaInstruction()) 3704 return false; 3705 3706 // This won't read exec if this is an SGPR->SGPR copy. 3707 if (MI.isCopyLike()) { 3708 if (!RI.isSGPRReg(MRI, MI.getOperand(0).getReg())) 3709 return true; 3710 3711 // Make sure this isn't copying exec as a normal operand 3712 return MI.readsRegister(AMDGPU::EXEC, &RI); 3713 } 3714 3715 // Make a conservative assumption about the callee. 3716 if (MI.isCall()) 3717 return true; 3718 3719 // Be conservative with any unhandled generic opcodes. 3720 if (!isTargetSpecificOpcode(MI.getOpcode())) 3721 return true; 3722 3723 return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI); 3724 } 3725 3726 bool SIInstrInfo::isInlineConstant(const APInt &Imm) const { 3727 switch (Imm.getBitWidth()) { 3728 case 1: // This likely will be a condition code mask. 3729 return true; 3730 3731 case 32: 3732 return AMDGPU::isInlinableLiteral32(Imm.getSExtValue(), 3733 ST.hasInv2PiInlineImm()); 3734 case 64: 3735 return AMDGPU::isInlinableLiteral64(Imm.getSExtValue(), 3736 ST.hasInv2PiInlineImm()); 3737 case 16: 3738 return ST.has16BitInsts() && 3739 AMDGPU::isInlinableLiteral16(Imm.getSExtValue(), 3740 ST.hasInv2PiInlineImm()); 3741 default: 3742 llvm_unreachable("invalid bitwidth"); 3743 } 3744 } 3745 3746 bool SIInstrInfo::isInlineConstant(const MachineOperand &MO, 3747 uint8_t OperandType) const { 3748 assert(!MO.isReg() && "isInlineConstant called on register operand!"); 3749 if (!MO.isImm() || 3750 OperandType < AMDGPU::OPERAND_SRC_FIRST || 3751 OperandType > AMDGPU::OPERAND_SRC_LAST) 3752 return false; 3753 3754 // MachineOperand provides no way to tell the true operand size, since it only 3755 // records a 64-bit value. We need to know the size to determine if a 32-bit 3756 // floating point immediate bit pattern is legal for an integer immediate. It 3757 // would be for any 32-bit integer operand, but would not be for a 64-bit one. 3758 3759 int64_t Imm = MO.getImm(); 3760 switch (OperandType) { 3761 case AMDGPU::OPERAND_REG_IMM_INT32: 3762 case AMDGPU::OPERAND_REG_IMM_FP32: 3763 case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED: 3764 case AMDGPU::OPERAND_REG_INLINE_C_INT32: 3765 case AMDGPU::OPERAND_REG_INLINE_C_FP32: 3766 case AMDGPU::OPERAND_REG_IMM_V2FP32: 3767 case AMDGPU::OPERAND_REG_INLINE_C_V2FP32: 3768 case AMDGPU::OPERAND_REG_IMM_V2INT32: 3769 case AMDGPU::OPERAND_REG_INLINE_C_V2INT32: 3770 case AMDGPU::OPERAND_REG_INLINE_AC_INT32: 3771 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: { 3772 int32_t Trunc = static_cast<int32_t>(Imm); 3773 return AMDGPU::isInlinableLiteral32(Trunc, ST.hasInv2PiInlineImm()); 3774 } 3775 case AMDGPU::OPERAND_REG_IMM_INT64: 3776 case AMDGPU::OPERAND_REG_IMM_FP64: 3777 case AMDGPU::OPERAND_REG_INLINE_C_INT64: 3778 case AMDGPU::OPERAND_REG_INLINE_C_FP64: 3779 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: 3780 return AMDGPU::isInlinableLiteral64(MO.getImm(), 3781 ST.hasInv2PiInlineImm()); 3782 case AMDGPU::OPERAND_REG_IMM_INT16: 3783 case AMDGPU::OPERAND_REG_INLINE_C_INT16: 3784 case AMDGPU::OPERAND_REG_INLINE_AC_INT16: 3785 // We would expect inline immediates to not be concerned with an integer/fp 3786 // distinction. However, in the case of 16-bit integer operations, the 3787 // "floating point" values appear to not work. It seems read the low 16-bits 3788 // of 32-bit immediates, which happens to always work for the integer 3789 // values. 3790 // 3791 // See llvm bugzilla 46302. 3792 // 3793 // TODO: Theoretically we could use op-sel to use the high bits of the 3794 // 32-bit FP values. 3795 return AMDGPU::isInlinableIntLiteral(Imm); 3796 case AMDGPU::OPERAND_REG_IMM_V2INT16: 3797 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: 3798 case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16: 3799 // This suffers the same problem as the scalar 16-bit cases. 3800 return AMDGPU::isInlinableIntLiteralV216(Imm); 3801 case AMDGPU::OPERAND_REG_IMM_FP16: 3802 case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED: 3803 case AMDGPU::OPERAND_REG_INLINE_C_FP16: 3804 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: { 3805 if (isInt<16>(Imm) || isUInt<16>(Imm)) { 3806 // A few special case instructions have 16-bit operands on subtargets 3807 // where 16-bit instructions are not legal. 3808 // TODO: Do the 32-bit immediates work? We shouldn't really need to handle 3809 // constants in these cases 3810 int16_t Trunc = static_cast<int16_t>(Imm); 3811 return ST.has16BitInsts() && 3812 AMDGPU::isInlinableLiteral16(Trunc, ST.hasInv2PiInlineImm()); 3813 } 3814 3815 return false; 3816 } 3817 case AMDGPU::OPERAND_REG_IMM_V2FP16: 3818 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: 3819 case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: { 3820 uint32_t Trunc = static_cast<uint32_t>(Imm); 3821 return AMDGPU::isInlinableLiteralV216(Trunc, ST.hasInv2PiInlineImm()); 3822 } 3823 case AMDGPU::OPERAND_KIMM32: 3824 case AMDGPU::OPERAND_KIMM16: 3825 return false; 3826 default: 3827 llvm_unreachable("invalid bitwidth"); 3828 } 3829 } 3830 3831 static bool compareMachineOp(const MachineOperand &Op0, 3832 const MachineOperand &Op1) { 3833 if (Op0.getType() != Op1.getType()) 3834 return false; 3835 3836 switch (Op0.getType()) { 3837 case MachineOperand::MO_Register: 3838 return Op0.getReg() == Op1.getReg(); 3839 case MachineOperand::MO_Immediate: 3840 return Op0.getImm() == Op1.getImm(); 3841 default: 3842 llvm_unreachable("Didn't expect to be comparing these operand types"); 3843 } 3844 } 3845 3846 bool SIInstrInfo::isImmOperandLegal(const MachineInstr &MI, unsigned OpNo, 3847 const MachineOperand &MO) const { 3848 const MCInstrDesc &InstDesc = MI.getDesc(); 3849 const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo]; 3850 3851 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal()); 3852 3853 if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE) 3854 return true; 3855 3856 if (OpInfo.RegClass < 0) 3857 return false; 3858 3859 if (MO.isImm() && isInlineConstant(MO, OpInfo)) { 3860 if (isMAI(MI) && ST.hasMFMAInlineLiteralBug() && 3861 OpNo ==(unsigned)AMDGPU::getNamedOperandIdx(MI.getOpcode(), 3862 AMDGPU::OpName::src2)) 3863 return false; 3864 return RI.opCanUseInlineConstant(OpInfo.OperandType); 3865 } 3866 3867 if (!RI.opCanUseLiteralConstant(OpInfo.OperandType)) 3868 return false; 3869 3870 if (!isVOP3(MI) || !AMDGPU::isSISrcOperand(InstDesc, OpNo)) 3871 return true; 3872 3873 return ST.hasVOP3Literal(); 3874 } 3875 3876 bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const { 3877 // GFX90A does not have V_MUL_LEGACY_F32_e32. 3878 if (Opcode == AMDGPU::V_MUL_LEGACY_F32_e64 && ST.hasGFX90AInsts()) 3879 return false; 3880 3881 int Op32 = AMDGPU::getVOPe32(Opcode); 3882 if (Op32 == -1) 3883 return false; 3884 3885 return pseudoToMCOpcode(Op32) != -1; 3886 } 3887 3888 bool SIInstrInfo::hasModifiers(unsigned Opcode) const { 3889 // The src0_modifier operand is present on all instructions 3890 // that have modifiers. 3891 3892 return AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src0_modifiers); 3893 } 3894 3895 bool SIInstrInfo::hasModifiersSet(const MachineInstr &MI, 3896 unsigned OpName) const { 3897 const MachineOperand *Mods = getNamedOperand(MI, OpName); 3898 return Mods && Mods->getImm(); 3899 } 3900 3901 bool SIInstrInfo::hasAnyModifiersSet(const MachineInstr &MI) const { 3902 return any_of(ModifierOpNames, 3903 [&](unsigned Name) { return hasModifiersSet(MI, Name); }); 3904 } 3905 3906 bool SIInstrInfo::canShrink(const MachineInstr &MI, 3907 const MachineRegisterInfo &MRI) const { 3908 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); 3909 // Can't shrink instruction with three operands. 3910 if (Src2) { 3911 switch (MI.getOpcode()) { 3912 default: return false; 3913 3914 case AMDGPU::V_ADDC_U32_e64: 3915 case AMDGPU::V_SUBB_U32_e64: 3916 case AMDGPU::V_SUBBREV_U32_e64: { 3917 const MachineOperand *Src1 3918 = getNamedOperand(MI, AMDGPU::OpName::src1); 3919 if (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg())) 3920 return false; 3921 // Additional verification is needed for sdst/src2. 3922 return true; 3923 } 3924 case AMDGPU::V_MAC_F16_e64: 3925 case AMDGPU::V_MAC_F32_e64: 3926 case AMDGPU::V_MAC_LEGACY_F32_e64: 3927 case AMDGPU::V_FMAC_F16_e64: 3928 case AMDGPU::V_FMAC_F16_t16_e64: 3929 case AMDGPU::V_FMAC_F32_e64: 3930 case AMDGPU::V_FMAC_F64_e64: 3931 case AMDGPU::V_FMAC_LEGACY_F32_e64: 3932 if (!Src2->isReg() || !RI.isVGPR(MRI, Src2->getReg()) || 3933 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers)) 3934 return false; 3935 break; 3936 3937 case AMDGPU::V_CNDMASK_B32_e64: 3938 break; 3939 } 3940 } 3941 3942 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); 3943 if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()) || 3944 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers))) 3945 return false; 3946 3947 // We don't need to check src0, all input types are legal, so just make sure 3948 // src0 isn't using any modifiers. 3949 if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers)) 3950 return false; 3951 3952 // Can it be shrunk to a valid 32 bit opcode? 3953 if (!hasVALU32BitEncoding(MI.getOpcode())) 3954 return false; 3955 3956 // Check output modifiers 3957 return !hasModifiersSet(MI, AMDGPU::OpName::omod) && 3958 !hasModifiersSet(MI, AMDGPU::OpName::clamp); 3959 } 3960 3961 // Set VCC operand with all flags from \p Orig, except for setting it as 3962 // implicit. 3963 static void copyFlagsToImplicitVCC(MachineInstr &MI, 3964 const MachineOperand &Orig) { 3965 3966 for (MachineOperand &Use : MI.implicit_operands()) { 3967 if (Use.isUse() && 3968 (Use.getReg() == AMDGPU::VCC || Use.getReg() == AMDGPU::VCC_LO)) { 3969 Use.setIsUndef(Orig.isUndef()); 3970 Use.setIsKill(Orig.isKill()); 3971 return; 3972 } 3973 } 3974 } 3975 3976 MachineInstr *SIInstrInfo::buildShrunkInst(MachineInstr &MI, 3977 unsigned Op32) const { 3978 MachineBasicBlock *MBB = MI.getParent(); 3979 MachineInstrBuilder Inst32 = 3980 BuildMI(*MBB, MI, MI.getDebugLoc(), get(Op32)) 3981 .setMIFlags(MI.getFlags()); 3982 3983 // Add the dst operand if the 32-bit encoding also has an explicit $vdst. 3984 // For VOPC instructions, this is replaced by an implicit def of vcc. 3985 if (AMDGPU::hasNamedOperand(Op32, AMDGPU::OpName::vdst)) { 3986 // dst 3987 Inst32.add(MI.getOperand(0)); 3988 } else if (AMDGPU::hasNamedOperand(Op32, AMDGPU::OpName::sdst)) { 3989 // VOPCX instructions won't be writing to an explicit dst, so this should 3990 // not fail for these instructions. 3991 assert(((MI.getOperand(0).getReg() == AMDGPU::VCC) || 3992 (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) && 3993 "Unexpected case"); 3994 } 3995 3996 Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0)); 3997 3998 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); 3999 if (Src1) 4000 Inst32.add(*Src1); 4001 4002 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); 4003 4004 if (Src2) { 4005 int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2); 4006 if (Op32Src2Idx != -1) { 4007 Inst32.add(*Src2); 4008 } else { 4009 // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is 4010 // replaced with an implicit read of vcc or vcc_lo. The implicit read 4011 // of vcc was already added during the initial BuildMI, but we 4012 // 1) may need to change vcc to vcc_lo to preserve the original register 4013 // 2) have to preserve the original flags. 4014 fixImplicitOperands(*Inst32); 4015 copyFlagsToImplicitVCC(*Inst32, *Src2); 4016 } 4017 } 4018 4019 return Inst32; 4020 } 4021 4022 bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo &MRI, 4023 const MachineOperand &MO, 4024 const MCOperandInfo &OpInfo) const { 4025 // Literal constants use the constant bus. 4026 if (!MO.isReg()) 4027 return !isInlineConstant(MO, OpInfo); 4028 4029 if (!MO.isUse()) 4030 return false; 4031 4032 if (MO.getReg().isVirtual()) 4033 return RI.isSGPRClass(MRI.getRegClass(MO.getReg())); 4034 4035 // Null is free 4036 if (MO.getReg() == AMDGPU::SGPR_NULL || MO.getReg() == AMDGPU::SGPR_NULL64) 4037 return false; 4038 4039 // SGPRs use the constant bus 4040 if (MO.isImplicit()) { 4041 return MO.getReg() == AMDGPU::M0 || 4042 MO.getReg() == AMDGPU::VCC || 4043 MO.getReg() == AMDGPU::VCC_LO; 4044 } else { 4045 return AMDGPU::SReg_32RegClass.contains(MO.getReg()) || 4046 AMDGPU::SReg_64RegClass.contains(MO.getReg()); 4047 } 4048 } 4049 4050 static Register findImplicitSGPRRead(const MachineInstr &MI) { 4051 for (const MachineOperand &MO : MI.implicit_operands()) { 4052 // We only care about reads. 4053 if (MO.isDef()) 4054 continue; 4055 4056 switch (MO.getReg()) { 4057 case AMDGPU::VCC: 4058 case AMDGPU::VCC_LO: 4059 case AMDGPU::VCC_HI: 4060 case AMDGPU::M0: 4061 case AMDGPU::FLAT_SCR: 4062 return MO.getReg(); 4063 4064 default: 4065 break; 4066 } 4067 } 4068 4069 return Register(); 4070 } 4071 4072 static bool shouldReadExec(const MachineInstr &MI) { 4073 if (SIInstrInfo::isVALU(MI)) { 4074 switch (MI.getOpcode()) { 4075 case AMDGPU::V_READLANE_B32: 4076 case AMDGPU::V_WRITELANE_B32: 4077 return false; 4078 } 4079 4080 return true; 4081 } 4082 4083 if (MI.isPreISelOpcode() || 4084 SIInstrInfo::isGenericOpcode(MI.getOpcode()) || 4085 SIInstrInfo::isSALU(MI) || 4086 SIInstrInfo::isSMRD(MI)) 4087 return false; 4088 4089 return true; 4090 } 4091 4092 static bool isSubRegOf(const SIRegisterInfo &TRI, 4093 const MachineOperand &SuperVec, 4094 const MachineOperand &SubReg) { 4095 if (SubReg.getReg().isPhysical()) 4096 return TRI.isSubRegister(SuperVec.getReg(), SubReg.getReg()); 4097 4098 return SubReg.getSubReg() != AMDGPU::NoSubRegister && 4099 SubReg.getReg() == SuperVec.getReg(); 4100 } 4101 4102 bool SIInstrInfo::verifyInstruction(const MachineInstr &MI, 4103 StringRef &ErrInfo) const { 4104 uint16_t Opcode = MI.getOpcode(); 4105 if (SIInstrInfo::isGenericOpcode(MI.getOpcode())) 4106 return true; 4107 4108 const MachineFunction *MF = MI.getParent()->getParent(); 4109 const MachineRegisterInfo &MRI = MF->getRegInfo(); 4110 4111 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0); 4112 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1); 4113 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2); 4114 int Src3Idx = -1; 4115 if (Src0Idx == -1) { 4116 // VOPD V_DUAL_* instructions use different operand names. 4117 Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0X); 4118 Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vsrc1X); 4119 Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0Y); 4120 Src3Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vsrc1Y); 4121 } 4122 4123 // Make sure the number of operands is correct. 4124 const MCInstrDesc &Desc = get(Opcode); 4125 if (!Desc.isVariadic() && 4126 Desc.getNumOperands() != MI.getNumExplicitOperands()) { 4127 ErrInfo = "Instruction has wrong number of operands."; 4128 return false; 4129 } 4130 4131 if (MI.isInlineAsm()) { 4132 // Verify register classes for inlineasm constraints. 4133 for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands(); 4134 I != E; ++I) { 4135 const TargetRegisterClass *RC = MI.getRegClassConstraint(I, this, &RI); 4136 if (!RC) 4137 continue; 4138 4139 const MachineOperand &Op = MI.getOperand(I); 4140 if (!Op.isReg()) 4141 continue; 4142 4143 Register Reg = Op.getReg(); 4144 if (!Reg.isVirtual() && !RC->contains(Reg)) { 4145 ErrInfo = "inlineasm operand has incorrect register class."; 4146 return false; 4147 } 4148 } 4149 4150 return true; 4151 } 4152 4153 if (isMIMG(MI) && MI.memoperands_empty() && MI.mayLoadOrStore()) { 4154 ErrInfo = "missing memory operand from MIMG instruction."; 4155 return false; 4156 } 4157 4158 // Make sure the register classes are correct. 4159 for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) { 4160 const MachineOperand &MO = MI.getOperand(i); 4161 if (MO.isFPImm()) { 4162 ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast " 4163 "all fp values to integers."; 4164 return false; 4165 } 4166 4167 int RegClass = Desc.operands()[i].RegClass; 4168 4169 switch (Desc.operands()[i].OperandType) { 4170 case MCOI::OPERAND_REGISTER: 4171 if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) { 4172 ErrInfo = "Illegal immediate value for operand."; 4173 return false; 4174 } 4175 break; 4176 case AMDGPU::OPERAND_REG_IMM_INT32: 4177 case AMDGPU::OPERAND_REG_IMM_FP32: 4178 case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED: 4179 case AMDGPU::OPERAND_REG_IMM_V2FP32: 4180 break; 4181 case AMDGPU::OPERAND_REG_INLINE_C_INT32: 4182 case AMDGPU::OPERAND_REG_INLINE_C_FP32: 4183 case AMDGPU::OPERAND_REG_INLINE_C_INT64: 4184 case AMDGPU::OPERAND_REG_INLINE_C_FP64: 4185 case AMDGPU::OPERAND_REG_INLINE_C_INT16: 4186 case AMDGPU::OPERAND_REG_INLINE_C_FP16: 4187 case AMDGPU::OPERAND_REG_INLINE_AC_INT32: 4188 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: 4189 case AMDGPU::OPERAND_REG_INLINE_AC_INT16: 4190 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: 4191 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: { 4192 if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, i))) { 4193 ErrInfo = "Illegal immediate value for operand."; 4194 return false; 4195 } 4196 break; 4197 } 4198 case MCOI::OPERAND_IMMEDIATE: 4199 case AMDGPU::OPERAND_KIMM32: 4200 // Check if this operand is an immediate. 4201 // FrameIndex operands will be replaced by immediates, so they are 4202 // allowed. 4203 if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) { 4204 ErrInfo = "Expected immediate, but got non-immediate"; 4205 return false; 4206 } 4207 [[fallthrough]]; 4208 default: 4209 continue; 4210 } 4211 4212 if (!MO.isReg()) 4213 continue; 4214 Register Reg = MO.getReg(); 4215 if (!Reg) 4216 continue; 4217 4218 // FIXME: Ideally we would have separate instruction definitions with the 4219 // aligned register constraint. 4220 // FIXME: We do not verify inline asm operands, but custom inline asm 4221 // verification is broken anyway 4222 if (ST.needsAlignedVGPRs()) { 4223 const TargetRegisterClass *RC = RI.getRegClassForReg(MRI, Reg); 4224 if (RI.hasVectorRegisters(RC) && MO.getSubReg()) { 4225 const TargetRegisterClass *SubRC = 4226 RI.getSubRegisterClass(RC, MO.getSubReg()); 4227 RC = RI.getCompatibleSubRegClass(RC, SubRC, MO.getSubReg()); 4228 if (RC) 4229 RC = SubRC; 4230 } 4231 4232 // Check that this is the aligned version of the class. 4233 if (!RC || !RI.isProperlyAlignedRC(*RC)) { 4234 ErrInfo = "Subtarget requires even aligned vector registers"; 4235 return false; 4236 } 4237 } 4238 4239 if (RegClass != -1) { 4240 if (Reg.isVirtual()) 4241 continue; 4242 4243 const TargetRegisterClass *RC = RI.getRegClass(RegClass); 4244 if (!RC->contains(Reg)) { 4245 ErrInfo = "Operand has incorrect register class."; 4246 return false; 4247 } 4248 } 4249 } 4250 4251 // Verify SDWA 4252 if (isSDWA(MI)) { 4253 if (!ST.hasSDWA()) { 4254 ErrInfo = "SDWA is not supported on this target"; 4255 return false; 4256 } 4257 4258 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst); 4259 4260 for (int OpIdx : {DstIdx, Src0Idx, Src1Idx, Src2Idx}) { 4261 if (OpIdx == -1) 4262 continue; 4263 const MachineOperand &MO = MI.getOperand(OpIdx); 4264 4265 if (!ST.hasSDWAScalar()) { 4266 // Only VGPRS on VI 4267 if (!MO.isReg() || !RI.hasVGPRs(RI.getRegClassForReg(MRI, MO.getReg()))) { 4268 ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI"; 4269 return false; 4270 } 4271 } else { 4272 // No immediates on GFX9 4273 if (!MO.isReg()) { 4274 ErrInfo = 4275 "Only reg allowed as operands in SDWA instructions on GFX9+"; 4276 return false; 4277 } 4278 } 4279 } 4280 4281 if (!ST.hasSDWAOmod()) { 4282 // No omod allowed on VI 4283 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod); 4284 if (OMod != nullptr && 4285 (!OMod->isImm() || OMod->getImm() != 0)) { 4286 ErrInfo = "OMod not allowed in SDWA instructions on VI"; 4287 return false; 4288 } 4289 } 4290 4291 uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode); 4292 if (isVOPC(BasicOpcode)) { 4293 if (!ST.hasSDWASdst() && DstIdx != -1) { 4294 // Only vcc allowed as dst on VI for VOPC 4295 const MachineOperand &Dst = MI.getOperand(DstIdx); 4296 if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) { 4297 ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI"; 4298 return false; 4299 } 4300 } else if (!ST.hasSDWAOutModsVOPC()) { 4301 // No clamp allowed on GFX9 for VOPC 4302 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp); 4303 if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) { 4304 ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI"; 4305 return false; 4306 } 4307 4308 // No omod allowed on GFX9 for VOPC 4309 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod); 4310 if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) { 4311 ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI"; 4312 return false; 4313 } 4314 } 4315 } 4316 4317 const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused); 4318 if (DstUnused && DstUnused->isImm() && 4319 DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) { 4320 const MachineOperand &Dst = MI.getOperand(DstIdx); 4321 if (!Dst.isReg() || !Dst.isTied()) { 4322 ErrInfo = "Dst register should have tied register"; 4323 return false; 4324 } 4325 4326 const MachineOperand &TiedMO = 4327 MI.getOperand(MI.findTiedOperandIdx(DstIdx)); 4328 if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) { 4329 ErrInfo = 4330 "Dst register should be tied to implicit use of preserved register"; 4331 return false; 4332 } else if (TiedMO.getReg().isPhysical() && 4333 Dst.getReg() != TiedMO.getReg()) { 4334 ErrInfo = "Dst register should use same physical register as preserved"; 4335 return false; 4336 } 4337 } 4338 } 4339 4340 // Verify MIMG 4341 if (isMIMG(MI.getOpcode()) && !MI.mayStore()) { 4342 // Ensure that the return type used is large enough for all the options 4343 // being used TFE/LWE require an extra result register. 4344 const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask); 4345 if (DMask) { 4346 uint64_t DMaskImm = DMask->getImm(); 4347 uint32_t RegCount = 4348 isGather4(MI.getOpcode()) ? 4 : llvm::popcount(DMaskImm); 4349 const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe); 4350 const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe); 4351 const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16); 4352 4353 // Adjust for packed 16 bit values 4354 if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem()) 4355 RegCount >>= 1; 4356 4357 // Adjust if using LWE or TFE 4358 if ((LWE && LWE->getImm()) || (TFE && TFE->getImm())) 4359 RegCount += 1; 4360 4361 const uint32_t DstIdx = 4362 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata); 4363 const MachineOperand &Dst = MI.getOperand(DstIdx); 4364 if (Dst.isReg()) { 4365 const TargetRegisterClass *DstRC = getOpRegClass(MI, DstIdx); 4366 uint32_t DstSize = RI.getRegSizeInBits(*DstRC) / 32; 4367 if (RegCount > DstSize) { 4368 ErrInfo = "MIMG instruction returns too many registers for dst " 4369 "register class"; 4370 return false; 4371 } 4372 } 4373 } 4374 } 4375 4376 // Verify VOP*. Ignore multiple sgpr operands on writelane. 4377 if (isVALU(MI) && Desc.getOpcode() != AMDGPU::V_WRITELANE_B32) { 4378 unsigned ConstantBusCount = 0; 4379 bool UsesLiteral = false; 4380 const MachineOperand *LiteralVal = nullptr; 4381 4382 int ImmIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm); 4383 if (ImmIdx != -1) { 4384 ++ConstantBusCount; 4385 UsesLiteral = true; 4386 LiteralVal = &MI.getOperand(ImmIdx); 4387 } 4388 4389 SmallVector<Register, 2> SGPRsUsed; 4390 Register SGPRUsed; 4391 4392 // Only look at the true operands. Only a real operand can use the constant 4393 // bus, and we don't want to check pseudo-operands like the source modifier 4394 // flags. 4395 for (int OpIdx : {Src0Idx, Src1Idx, Src2Idx, Src3Idx}) { 4396 if (OpIdx == -1) 4397 continue; 4398 const MachineOperand &MO = MI.getOperand(OpIdx); 4399 if (usesConstantBus(MRI, MO, MI.getDesc().operands()[OpIdx])) { 4400 if (MO.isReg()) { 4401 SGPRUsed = MO.getReg(); 4402 if (!llvm::is_contained(SGPRsUsed, SGPRUsed)) { 4403 ++ConstantBusCount; 4404 SGPRsUsed.push_back(SGPRUsed); 4405 } 4406 } else { 4407 if (!UsesLiteral) { 4408 ++ConstantBusCount; 4409 UsesLiteral = true; 4410 LiteralVal = &MO; 4411 } else if (!MO.isIdenticalTo(*LiteralVal)) { 4412 assert(isVOP2(MI) || isVOP3(MI)); 4413 ErrInfo = "VOP2/VOP3 instruction uses more than one literal"; 4414 return false; 4415 } 4416 } 4417 } 4418 } 4419 4420 SGPRUsed = findImplicitSGPRRead(MI); 4421 if (SGPRUsed) { 4422 // Implicit uses may safely overlap true operands 4423 if (llvm::all_of(SGPRsUsed, [this, SGPRUsed](unsigned SGPR) { 4424 return !RI.regsOverlap(SGPRUsed, SGPR); 4425 })) { 4426 ++ConstantBusCount; 4427 SGPRsUsed.push_back(SGPRUsed); 4428 } 4429 } 4430 4431 // v_writelane_b32 is an exception from constant bus restriction: 4432 // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const 4433 if (ConstantBusCount > ST.getConstantBusLimit(Opcode) && 4434 Opcode != AMDGPU::V_WRITELANE_B32) { 4435 ErrInfo = "VOP* instruction violates constant bus restriction"; 4436 return false; 4437 } 4438 4439 if (isVOP3(MI) && UsesLiteral && !ST.hasVOP3Literal()) { 4440 ErrInfo = "VOP3 instruction uses literal"; 4441 return false; 4442 } 4443 } 4444 4445 // Special case for writelane - this can break the multiple constant bus rule, 4446 // but still can't use more than one SGPR register 4447 if (Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) { 4448 unsigned SGPRCount = 0; 4449 Register SGPRUsed; 4450 4451 for (int OpIdx : {Src0Idx, Src1Idx}) { 4452 if (OpIdx == -1) 4453 break; 4454 4455 const MachineOperand &MO = MI.getOperand(OpIdx); 4456 4457 if (usesConstantBus(MRI, MO, MI.getDesc().operands()[OpIdx])) { 4458 if (MO.isReg() && MO.getReg() != AMDGPU::M0) { 4459 if (MO.getReg() != SGPRUsed) 4460 ++SGPRCount; 4461 SGPRUsed = MO.getReg(); 4462 } 4463 } 4464 if (SGPRCount > ST.getConstantBusLimit(Opcode)) { 4465 ErrInfo = "WRITELANE instruction violates constant bus restriction"; 4466 return false; 4467 } 4468 } 4469 } 4470 4471 // Verify misc. restrictions on specific instructions. 4472 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32_e64 || 4473 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64_e64) { 4474 const MachineOperand &Src0 = MI.getOperand(Src0Idx); 4475 const MachineOperand &Src1 = MI.getOperand(Src1Idx); 4476 const MachineOperand &Src2 = MI.getOperand(Src2Idx); 4477 if (Src0.isReg() && Src1.isReg() && Src2.isReg()) { 4478 if (!compareMachineOp(Src0, Src1) && 4479 !compareMachineOp(Src0, Src2)) { 4480 ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2"; 4481 return false; 4482 } 4483 } 4484 if ((getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm() & 4485 SISrcMods::ABS) || 4486 (getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm() & 4487 SISrcMods::ABS) || 4488 (getNamedOperand(MI, AMDGPU::OpName::src2_modifiers)->getImm() & 4489 SISrcMods::ABS)) { 4490 ErrInfo = "ABS not allowed in VOP3B instructions"; 4491 return false; 4492 } 4493 } 4494 4495 if (isSOP2(MI) || isSOPC(MI)) { 4496 const MachineOperand &Src0 = MI.getOperand(Src0Idx); 4497 const MachineOperand &Src1 = MI.getOperand(Src1Idx); 4498 4499 if (!Src0.isReg() && !Src1.isReg() && 4500 !isInlineConstant(Src0, Desc.operands()[Src0Idx]) && 4501 !isInlineConstant(Src1, Desc.operands()[Src1Idx]) && 4502 !Src0.isIdenticalTo(Src1)) { 4503 ErrInfo = "SOP2/SOPC instruction requires too many immediate constants"; 4504 return false; 4505 } 4506 } 4507 4508 if (isSOPK(MI)) { 4509 auto Op = getNamedOperand(MI, AMDGPU::OpName::simm16); 4510 if (Desc.isBranch()) { 4511 if (!Op->isMBB()) { 4512 ErrInfo = "invalid branch target for SOPK instruction"; 4513 return false; 4514 } 4515 } else { 4516 uint64_t Imm = Op->getImm(); 4517 if (sopkIsZext(MI)) { 4518 if (!isUInt<16>(Imm)) { 4519 ErrInfo = "invalid immediate for SOPK instruction"; 4520 return false; 4521 } 4522 } else { 4523 if (!isInt<16>(Imm)) { 4524 ErrInfo = "invalid immediate for SOPK instruction"; 4525 return false; 4526 } 4527 } 4528 } 4529 } 4530 4531 if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 || 4532 Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 || 4533 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 || 4534 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) { 4535 const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 || 4536 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64; 4537 4538 const unsigned StaticNumOps = 4539 Desc.getNumOperands() + Desc.implicit_uses().size(); 4540 const unsigned NumImplicitOps = IsDst ? 2 : 1; 4541 4542 // Allow additional implicit operands. This allows a fixup done by the post 4543 // RA scheduler where the main implicit operand is killed and implicit-defs 4544 // are added for sub-registers that remain live after this instruction. 4545 if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) { 4546 ErrInfo = "missing implicit register operands"; 4547 return false; 4548 } 4549 4550 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); 4551 if (IsDst) { 4552 if (!Dst->isUse()) { 4553 ErrInfo = "v_movreld_b32 vdst should be a use operand"; 4554 return false; 4555 } 4556 4557 unsigned UseOpIdx; 4558 if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) || 4559 UseOpIdx != StaticNumOps + 1) { 4560 ErrInfo = "movrel implicit operands should be tied"; 4561 return false; 4562 } 4563 } 4564 4565 const MachineOperand &Src0 = MI.getOperand(Src0Idx); 4566 const MachineOperand &ImpUse 4567 = MI.getOperand(StaticNumOps + NumImplicitOps - 1); 4568 if (!ImpUse.isReg() || !ImpUse.isUse() || 4569 !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) { 4570 ErrInfo = "src0 should be subreg of implicit vector use"; 4571 return false; 4572 } 4573 } 4574 4575 // Make sure we aren't losing exec uses in the td files. This mostly requires 4576 // being careful when using let Uses to try to add other use registers. 4577 if (shouldReadExec(MI)) { 4578 if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) { 4579 ErrInfo = "VALU instruction does not implicitly read exec mask"; 4580 return false; 4581 } 4582 } 4583 4584 if (isSMRD(MI)) { 4585 if (MI.mayStore() && 4586 ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS) { 4587 // The register offset form of scalar stores may only use m0 as the 4588 // soffset register. 4589 const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soffset); 4590 if (Soff && Soff->getReg() != AMDGPU::M0) { 4591 ErrInfo = "scalar stores must use m0 as offset register"; 4592 return false; 4593 } 4594 } 4595 } 4596 4597 if (isFLAT(MI) && !ST.hasFlatInstOffsets()) { 4598 const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset); 4599 if (Offset->getImm() != 0) { 4600 ErrInfo = "subtarget does not support offsets in flat instructions"; 4601 return false; 4602 } 4603 } 4604 4605 if (isMIMG(MI)) { 4606 const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim); 4607 if (DimOp) { 4608 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode, 4609 AMDGPU::OpName::vaddr0); 4610 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc); 4611 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opcode); 4612 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = 4613 AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode); 4614 const AMDGPU::MIMGDimInfo *Dim = 4615 AMDGPU::getMIMGDimInfoByEncoding(DimOp->getImm()); 4616 4617 if (!Dim) { 4618 ErrInfo = "dim is out of range"; 4619 return false; 4620 } 4621 4622 bool IsA16 = false; 4623 if (ST.hasR128A16()) { 4624 const MachineOperand *R128A16 = getNamedOperand(MI, AMDGPU::OpName::r128); 4625 IsA16 = R128A16->getImm() != 0; 4626 } else if (ST.hasA16()) { 4627 const MachineOperand *A16 = getNamedOperand(MI, AMDGPU::OpName::a16); 4628 IsA16 = A16->getImm() != 0; 4629 } 4630 4631 bool IsNSA = SRsrcIdx - VAddr0Idx > 1; 4632 4633 unsigned AddrWords = 4634 AMDGPU::getAddrSizeMIMGOp(BaseOpcode, Dim, IsA16, ST.hasG16()); 4635 4636 unsigned VAddrWords; 4637 if (IsNSA) { 4638 VAddrWords = SRsrcIdx - VAddr0Idx; 4639 } else { 4640 const TargetRegisterClass *RC = getOpRegClass(MI, VAddr0Idx); 4641 VAddrWords = MRI.getTargetRegisterInfo()->getRegSizeInBits(*RC) / 32; 4642 if (AddrWords > 12) 4643 AddrWords = 16; 4644 } 4645 4646 if (VAddrWords != AddrWords) { 4647 LLVM_DEBUG(dbgs() << "bad vaddr size, expected " << AddrWords 4648 << " but got " << VAddrWords << "\n"); 4649 ErrInfo = "bad vaddr size"; 4650 return false; 4651 } 4652 } 4653 } 4654 4655 const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl); 4656 if (DppCt) { 4657 using namespace AMDGPU::DPP; 4658 4659 unsigned DC = DppCt->getImm(); 4660 if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 || 4661 DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST || 4662 (DC >= DppCtrl::DPP_UNUSED4_FIRST && DC <= DppCtrl::DPP_UNUSED4_LAST) || 4663 (DC >= DppCtrl::DPP_UNUSED5_FIRST && DC <= DppCtrl::DPP_UNUSED5_LAST) || 4664 (DC >= DppCtrl::DPP_UNUSED6_FIRST && DC <= DppCtrl::DPP_UNUSED6_LAST) || 4665 (DC >= DppCtrl::DPP_UNUSED7_FIRST && DC <= DppCtrl::DPP_UNUSED7_LAST) || 4666 (DC >= DppCtrl::DPP_UNUSED8_FIRST && DC <= DppCtrl::DPP_UNUSED8_LAST)) { 4667 ErrInfo = "Invalid dpp_ctrl value"; 4668 return false; 4669 } 4670 if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 && 4671 ST.getGeneration() >= AMDGPUSubtarget::GFX10) { 4672 ErrInfo = "Invalid dpp_ctrl value: " 4673 "wavefront shifts are not supported on GFX10+"; 4674 return false; 4675 } 4676 if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 && 4677 ST.getGeneration() >= AMDGPUSubtarget::GFX10) { 4678 ErrInfo = "Invalid dpp_ctrl value: " 4679 "broadcasts are not supported on GFX10+"; 4680 return false; 4681 } 4682 if (DC >= DppCtrl::ROW_SHARE_FIRST && DC <= DppCtrl::ROW_XMASK_LAST && 4683 ST.getGeneration() < AMDGPUSubtarget::GFX10) { 4684 if (DC >= DppCtrl::ROW_NEWBCAST_FIRST && 4685 DC <= DppCtrl::ROW_NEWBCAST_LAST && 4686 !ST.hasGFX90AInsts()) { 4687 ErrInfo = "Invalid dpp_ctrl value: " 4688 "row_newbroadcast/row_share is not supported before " 4689 "GFX90A/GFX10"; 4690 return false; 4691 } else if (DC > DppCtrl::ROW_NEWBCAST_LAST || !ST.hasGFX90AInsts()) { 4692 ErrInfo = "Invalid dpp_ctrl value: " 4693 "row_share and row_xmask are not supported before GFX10"; 4694 return false; 4695 } 4696 } 4697 4698 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst); 4699 4700 if (Opcode != AMDGPU::V_MOV_B64_DPP_PSEUDO && 4701 ((DstIdx >= 0 && 4702 (Desc.operands()[DstIdx].RegClass == AMDGPU::VReg_64RegClassID || 4703 Desc.operands()[DstIdx].RegClass == 4704 AMDGPU::VReg_64_Align2RegClassID)) || 4705 ((Src0Idx >= 0 && 4706 (Desc.operands()[Src0Idx].RegClass == AMDGPU::VReg_64RegClassID || 4707 Desc.operands()[Src0Idx].RegClass == 4708 AMDGPU::VReg_64_Align2RegClassID)))) && 4709 !AMDGPU::isLegal64BitDPPControl(DC)) { 4710 ErrInfo = "Invalid dpp_ctrl value: " 4711 "64 bit dpp only support row_newbcast"; 4712 return false; 4713 } 4714 } 4715 4716 if ((MI.mayStore() || MI.mayLoad()) && !isVGPRSpill(MI)) { 4717 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); 4718 uint16_t DataNameIdx = isDS(Opcode) ? AMDGPU::OpName::data0 4719 : AMDGPU::OpName::vdata; 4720 const MachineOperand *Data = getNamedOperand(MI, DataNameIdx); 4721 const MachineOperand *Data2 = getNamedOperand(MI, AMDGPU::OpName::data1); 4722 if (Data && !Data->isReg()) 4723 Data = nullptr; 4724 4725 if (ST.hasGFX90AInsts()) { 4726 if (Dst && Data && 4727 (RI.isAGPR(MRI, Dst->getReg()) != RI.isAGPR(MRI, Data->getReg()))) { 4728 ErrInfo = "Invalid register class: " 4729 "vdata and vdst should be both VGPR or AGPR"; 4730 return false; 4731 } 4732 if (Data && Data2 && 4733 (RI.isAGPR(MRI, Data->getReg()) != RI.isAGPR(MRI, Data2->getReg()))) { 4734 ErrInfo = "Invalid register class: " 4735 "both data operands should be VGPR or AGPR"; 4736 return false; 4737 } 4738 } else { 4739 if ((Dst && RI.isAGPR(MRI, Dst->getReg())) || 4740 (Data && RI.isAGPR(MRI, Data->getReg())) || 4741 (Data2 && RI.isAGPR(MRI, Data2->getReg()))) { 4742 ErrInfo = "Invalid register class: " 4743 "agpr loads and stores not supported on this GPU"; 4744 return false; 4745 } 4746 } 4747 } 4748 4749 if (ST.needsAlignedVGPRs()) { 4750 const auto isAlignedReg = [&MI, &MRI, this](unsigned OpName) -> bool { 4751 const MachineOperand *Op = getNamedOperand(MI, OpName); 4752 if (!Op) 4753 return true; 4754 Register Reg = Op->getReg(); 4755 if (Reg.isPhysical()) 4756 return !(RI.getHWRegIndex(Reg) & 1); 4757 const TargetRegisterClass &RC = *MRI.getRegClass(Reg); 4758 return RI.getRegSizeInBits(RC) > 32 && RI.isProperlyAlignedRC(RC) && 4759 !(RI.getChannelFromSubReg(Op->getSubReg()) & 1); 4760 }; 4761 4762 if (MI.getOpcode() == AMDGPU::DS_GWS_INIT || 4763 MI.getOpcode() == AMDGPU::DS_GWS_SEMA_BR || 4764 MI.getOpcode() == AMDGPU::DS_GWS_BARRIER) { 4765 4766 if (!isAlignedReg(AMDGPU::OpName::data0)) { 4767 ErrInfo = "Subtarget requires even aligned vector registers " 4768 "for DS_GWS instructions"; 4769 return false; 4770 } 4771 } 4772 4773 if (isMIMG(MI)) { 4774 if (!isAlignedReg(AMDGPU::OpName::vaddr)) { 4775 ErrInfo = "Subtarget requires even aligned vector registers " 4776 "for vaddr operand of image instructions"; 4777 return false; 4778 } 4779 } 4780 } 4781 4782 if (MI.getOpcode() == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && 4783 !ST.hasGFX90AInsts()) { 4784 const MachineOperand *Src = getNamedOperand(MI, AMDGPU::OpName::src0); 4785 if (Src->isReg() && RI.isSGPRReg(MRI, Src->getReg())) { 4786 ErrInfo = "Invalid register class: " 4787 "v_accvgpr_write with an SGPR is not supported on this GPU"; 4788 return false; 4789 } 4790 } 4791 4792 if (Desc.getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS) { 4793 const MachineOperand &SrcOp = MI.getOperand(1); 4794 if (!SrcOp.isReg() || SrcOp.getReg().isVirtual()) { 4795 ErrInfo = "pseudo expects only physical SGPRs"; 4796 return false; 4797 } 4798 } 4799 4800 return true; 4801 } 4802 4803 unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const { 4804 switch (MI.getOpcode()) { 4805 default: return AMDGPU::INSTRUCTION_LIST_END; 4806 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE; 4807 case AMDGPU::COPY: return AMDGPU::COPY; 4808 case AMDGPU::PHI: return AMDGPU::PHI; 4809 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG; 4810 case AMDGPU::WQM: return AMDGPU::WQM; 4811 case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM; 4812 case AMDGPU::STRICT_WWM: return AMDGPU::STRICT_WWM; 4813 case AMDGPU::STRICT_WQM: return AMDGPU::STRICT_WQM; 4814 case AMDGPU::S_MOV_B32: { 4815 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 4816 return MI.getOperand(1).isReg() || 4817 RI.isAGPR(MRI, MI.getOperand(0).getReg()) ? 4818 AMDGPU::COPY : AMDGPU::V_MOV_B32_e32; 4819 } 4820 case AMDGPU::S_ADD_I32: 4821 return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32; 4822 case AMDGPU::S_ADDC_U32: 4823 return AMDGPU::V_ADDC_U32_e32; 4824 case AMDGPU::S_SUB_I32: 4825 return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_CO_U32_e32; 4826 // FIXME: These are not consistently handled, and selected when the carry is 4827 // used. 4828 case AMDGPU::S_ADD_U32: 4829 return AMDGPU::V_ADD_CO_U32_e32; 4830 case AMDGPU::S_SUB_U32: 4831 return AMDGPU::V_SUB_CO_U32_e32; 4832 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32; 4833 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32_e64; 4834 case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32_e64; 4835 case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32_e64; 4836 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64; 4837 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64; 4838 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64; 4839 case AMDGPU::S_XNOR_B32: 4840 return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END; 4841 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64; 4842 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64; 4843 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64; 4844 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64; 4845 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32; 4846 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64_e64; 4847 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32; 4848 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64_e64; 4849 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32; 4850 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64_e64; 4851 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32_e64; 4852 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32_e64; 4853 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32_e64; 4854 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32_e64; 4855 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64; 4856 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32; 4857 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32; 4858 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32; 4859 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e64; 4860 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e64; 4861 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e64; 4862 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e64; 4863 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e64; 4864 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e64; 4865 case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e64; 4866 case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e64; 4867 case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e64; 4868 case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e64; 4869 case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e64; 4870 case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e64; 4871 case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e64; 4872 case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e64; 4873 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64; 4874 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32; 4875 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32; 4876 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64; 4877 case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ; 4878 case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ; 4879 } 4880 llvm_unreachable( 4881 "Unexpected scalar opcode without corresponding vector one!"); 4882 } 4883 4884 static const TargetRegisterClass * 4885 adjustAllocatableRegClass(const GCNSubtarget &ST, const SIRegisterInfo &RI, 4886 const MachineRegisterInfo &MRI, 4887 const MCInstrDesc &TID, unsigned RCID, 4888 bool IsAllocatable) { 4889 if ((IsAllocatable || !ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) && 4890 (((TID.mayLoad() || TID.mayStore()) && 4891 !(TID.TSFlags & SIInstrFlags::VGPRSpill)) || 4892 (TID.TSFlags & (SIInstrFlags::DS | SIInstrFlags::MIMG)))) { 4893 switch (RCID) { 4894 case AMDGPU::AV_32RegClassID: 4895 RCID = AMDGPU::VGPR_32RegClassID; 4896 break; 4897 case AMDGPU::AV_64RegClassID: 4898 RCID = AMDGPU::VReg_64RegClassID; 4899 break; 4900 case AMDGPU::AV_96RegClassID: 4901 RCID = AMDGPU::VReg_96RegClassID; 4902 break; 4903 case AMDGPU::AV_128RegClassID: 4904 RCID = AMDGPU::VReg_128RegClassID; 4905 break; 4906 case AMDGPU::AV_160RegClassID: 4907 RCID = AMDGPU::VReg_160RegClassID; 4908 break; 4909 case AMDGPU::AV_512RegClassID: 4910 RCID = AMDGPU::VReg_512RegClassID; 4911 break; 4912 default: 4913 break; 4914 } 4915 } 4916 4917 return RI.getProperlyAlignedRC(RI.getRegClass(RCID)); 4918 } 4919 4920 const TargetRegisterClass *SIInstrInfo::getRegClass(const MCInstrDesc &TID, 4921 unsigned OpNum, const TargetRegisterInfo *TRI, 4922 const MachineFunction &MF) 4923 const { 4924 if (OpNum >= TID.getNumOperands()) 4925 return nullptr; 4926 auto RegClass = TID.operands()[OpNum].RegClass; 4927 bool IsAllocatable = false; 4928 if (TID.TSFlags & (SIInstrFlags::DS | SIInstrFlags::FLAT)) { 4929 // vdst and vdata should be both VGPR or AGPR, same for the DS instructions 4930 // with two data operands. Request register class constrained to VGPR only 4931 // of both operands present as Machine Copy Propagation can not check this 4932 // constraint and possibly other passes too. 4933 // 4934 // The check is limited to FLAT and DS because atomics in non-flat encoding 4935 // have their vdst and vdata tied to be the same register. 4936 const int VDstIdx = AMDGPU::getNamedOperandIdx(TID.Opcode, 4937 AMDGPU::OpName::vdst); 4938 const int DataIdx = AMDGPU::getNamedOperandIdx(TID.Opcode, 4939 (TID.TSFlags & SIInstrFlags::DS) ? AMDGPU::OpName::data0 4940 : AMDGPU::OpName::vdata); 4941 if (DataIdx != -1) { 4942 IsAllocatable = VDstIdx != -1 || AMDGPU::hasNamedOperand( 4943 TID.Opcode, AMDGPU::OpName::data1); 4944 } 4945 } 4946 return adjustAllocatableRegClass(ST, RI, MF.getRegInfo(), TID, RegClass, 4947 IsAllocatable); 4948 } 4949 4950 const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI, 4951 unsigned OpNo) const { 4952 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 4953 const MCInstrDesc &Desc = get(MI.getOpcode()); 4954 if (MI.isVariadic() || OpNo >= Desc.getNumOperands() || 4955 Desc.operands()[OpNo].RegClass == -1) { 4956 Register Reg = MI.getOperand(OpNo).getReg(); 4957 4958 if (Reg.isVirtual()) 4959 return MRI.getRegClass(Reg); 4960 return RI.getPhysRegBaseClass(Reg); 4961 } 4962 4963 unsigned RCID = Desc.operands()[OpNo].RegClass; 4964 return adjustAllocatableRegClass(ST, RI, MRI, Desc, RCID, true); 4965 } 4966 4967 void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const { 4968 MachineBasicBlock::iterator I = MI; 4969 MachineBasicBlock *MBB = MI.getParent(); 4970 MachineOperand &MO = MI.getOperand(OpIdx); 4971 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 4972 unsigned RCID = get(MI.getOpcode()).operands()[OpIdx].RegClass; 4973 const TargetRegisterClass *RC = RI.getRegClass(RCID); 4974 unsigned Size = RI.getRegSizeInBits(*RC); 4975 unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO : AMDGPU::V_MOV_B32_e32; 4976 if (MO.isReg()) 4977 Opcode = AMDGPU::COPY; 4978 else if (RI.isSGPRClass(RC)) 4979 Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32; 4980 4981 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(RC); 4982 const TargetRegisterClass *VRC64 = RI.getVGPR64Class(); 4983 if (RI.getCommonSubClass(VRC64, VRC)) 4984 VRC = VRC64; 4985 else 4986 VRC = &AMDGPU::VGPR_32RegClass; 4987 4988 Register Reg = MRI.createVirtualRegister(VRC); 4989 DebugLoc DL = MBB->findDebugLoc(I); 4990 BuildMI(*MI.getParent(), I, DL, get(Opcode), Reg).add(MO); 4991 MO.ChangeToRegister(Reg, false); 4992 } 4993 4994 unsigned SIInstrInfo::buildExtractSubReg(MachineBasicBlock::iterator MI, 4995 MachineRegisterInfo &MRI, 4996 MachineOperand &SuperReg, 4997 const TargetRegisterClass *SuperRC, 4998 unsigned SubIdx, 4999 const TargetRegisterClass *SubRC) 5000 const { 5001 MachineBasicBlock *MBB = MI->getParent(); 5002 DebugLoc DL = MI->getDebugLoc(); 5003 Register SubReg = MRI.createVirtualRegister(SubRC); 5004 5005 if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) { 5006 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg) 5007 .addReg(SuperReg.getReg(), 0, SubIdx); 5008 return SubReg; 5009 } 5010 5011 // Just in case the super register is itself a sub-register, copy it to a new 5012 // value so we don't need to worry about merging its subreg index with the 5013 // SubIdx passed to this function. The register coalescer should be able to 5014 // eliminate this extra copy. 5015 Register NewSuperReg = MRI.createVirtualRegister(SuperRC); 5016 5017 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), NewSuperReg) 5018 .addReg(SuperReg.getReg(), 0, SuperReg.getSubReg()); 5019 5020 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg) 5021 .addReg(NewSuperReg, 0, SubIdx); 5022 5023 return SubReg; 5024 } 5025 5026 MachineOperand SIInstrInfo::buildExtractSubRegOrImm( 5027 MachineBasicBlock::iterator MII, 5028 MachineRegisterInfo &MRI, 5029 MachineOperand &Op, 5030 const TargetRegisterClass *SuperRC, 5031 unsigned SubIdx, 5032 const TargetRegisterClass *SubRC) const { 5033 if (Op.isImm()) { 5034 if (SubIdx == AMDGPU::sub0) 5035 return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm())); 5036 if (SubIdx == AMDGPU::sub1) 5037 return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm() >> 32)); 5038 5039 llvm_unreachable("Unhandled register index for immediate"); 5040 } 5041 5042 unsigned SubReg = buildExtractSubReg(MII, MRI, Op, SuperRC, 5043 SubIdx, SubRC); 5044 return MachineOperand::CreateReg(SubReg, false); 5045 } 5046 5047 // Change the order of operands from (0, 1, 2) to (0, 2, 1) 5048 void SIInstrInfo::swapOperands(MachineInstr &Inst) const { 5049 assert(Inst.getNumExplicitOperands() == 3); 5050 MachineOperand Op1 = Inst.getOperand(1); 5051 Inst.removeOperand(1); 5052 Inst.addOperand(Op1); 5053 } 5054 5055 bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo &MRI, 5056 const MCOperandInfo &OpInfo, 5057 const MachineOperand &MO) const { 5058 if (!MO.isReg()) 5059 return false; 5060 5061 Register Reg = MO.getReg(); 5062 5063 const TargetRegisterClass *DRC = RI.getRegClass(OpInfo.RegClass); 5064 if (Reg.isPhysical()) 5065 return DRC->contains(Reg); 5066 5067 const TargetRegisterClass *RC = MRI.getRegClass(Reg); 5068 5069 if (MO.getSubReg()) { 5070 const MachineFunction *MF = MO.getParent()->getParent()->getParent(); 5071 const TargetRegisterClass *SuperRC = RI.getLargestLegalSuperClass(RC, *MF); 5072 if (!SuperRC) 5073 return false; 5074 5075 DRC = RI.getMatchingSuperRegClass(SuperRC, DRC, MO.getSubReg()); 5076 if (!DRC) 5077 return false; 5078 } 5079 return RC->hasSuperClassEq(DRC); 5080 } 5081 5082 bool SIInstrInfo::isLegalVSrcOperand(const MachineRegisterInfo &MRI, 5083 const MCOperandInfo &OpInfo, 5084 const MachineOperand &MO) const { 5085 if (MO.isReg()) 5086 return isLegalRegOperand(MRI, OpInfo, MO); 5087 5088 // Handle non-register types that are treated like immediates. 5089 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal()); 5090 return true; 5091 } 5092 5093 bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx, 5094 const MachineOperand *MO) const { 5095 const MachineFunction &MF = *MI.getParent()->getParent(); 5096 const MachineRegisterInfo &MRI = MF.getRegInfo(); 5097 const MCInstrDesc &InstDesc = MI.getDesc(); 5098 const MCOperandInfo &OpInfo = InstDesc.operands()[OpIdx]; 5099 const TargetRegisterClass *DefinedRC = 5100 OpInfo.RegClass != -1 ? RI.getRegClass(OpInfo.RegClass) : nullptr; 5101 if (!MO) 5102 MO = &MI.getOperand(OpIdx); 5103 5104 int ConstantBusLimit = ST.getConstantBusLimit(MI.getOpcode()); 5105 int LiteralLimit = !isVOP3(MI) || ST.hasVOP3Literal() ? 1 : 0; 5106 if (isVALU(MI) && usesConstantBus(MRI, *MO, OpInfo)) { 5107 if (!MO->isReg() && !isInlineConstant(*MO, OpInfo) && !LiteralLimit--) 5108 return false; 5109 5110 SmallDenseSet<RegSubRegPair> SGPRsUsed; 5111 if (MO->isReg()) 5112 SGPRsUsed.insert(RegSubRegPair(MO->getReg(), MO->getSubReg())); 5113 5114 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 5115 if (i == OpIdx) 5116 continue; 5117 const MachineOperand &Op = MI.getOperand(i); 5118 if (Op.isReg()) { 5119 RegSubRegPair SGPR(Op.getReg(), Op.getSubReg()); 5120 if (!SGPRsUsed.count(SGPR) && 5121 // FIXME: This can access off the end of the operands() array. 5122 usesConstantBus(MRI, Op, InstDesc.operands().begin()[i])) { 5123 if (--ConstantBusLimit <= 0) 5124 return false; 5125 SGPRsUsed.insert(SGPR); 5126 } 5127 } else if (InstDesc.operands()[i].OperandType == AMDGPU::OPERAND_KIMM32 || 5128 (AMDGPU::isSISrcOperand(InstDesc, i) && 5129 !isInlineConstant(Op, InstDesc.operands()[i]))) { 5130 if (!LiteralLimit--) 5131 return false; 5132 if (--ConstantBusLimit <= 0) 5133 return false; 5134 } 5135 } 5136 } 5137 5138 if (MO->isReg()) { 5139 if (!DefinedRC) 5140 return OpInfo.OperandType == MCOI::OPERAND_UNKNOWN; 5141 if (!isLegalRegOperand(MRI, OpInfo, *MO)) 5142 return false; 5143 bool IsAGPR = RI.isAGPR(MRI, MO->getReg()); 5144 if (IsAGPR && !ST.hasMAIInsts()) 5145 return false; 5146 unsigned Opc = MI.getOpcode(); 5147 if (IsAGPR && 5148 (!ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) && 5149 (MI.mayLoad() || MI.mayStore() || isDS(Opc) || isMIMG(Opc))) 5150 return false; 5151 // Atomics should have both vdst and vdata either vgpr or agpr. 5152 const int VDstIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 5153 const int DataIdx = AMDGPU::getNamedOperandIdx(Opc, 5154 isDS(Opc) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata); 5155 if ((int)OpIdx == VDstIdx && DataIdx != -1 && 5156 MI.getOperand(DataIdx).isReg() && 5157 RI.isAGPR(MRI, MI.getOperand(DataIdx).getReg()) != IsAGPR) 5158 return false; 5159 if ((int)OpIdx == DataIdx) { 5160 if (VDstIdx != -1 && 5161 RI.isAGPR(MRI, MI.getOperand(VDstIdx).getReg()) != IsAGPR) 5162 return false; 5163 // DS instructions with 2 src operands also must have tied RC. 5164 const int Data1Idx = AMDGPU::getNamedOperandIdx(Opc, 5165 AMDGPU::OpName::data1); 5166 if (Data1Idx != -1 && MI.getOperand(Data1Idx).isReg() && 5167 RI.isAGPR(MRI, MI.getOperand(Data1Idx).getReg()) != IsAGPR) 5168 return false; 5169 } 5170 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && !ST.hasGFX90AInsts() && 5171 (int)OpIdx == AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) && 5172 RI.isSGPRReg(MRI, MO->getReg())) 5173 return false; 5174 return true; 5175 } 5176 5177 // Handle non-register types that are treated like immediates. 5178 assert(MO->isImm() || MO->isTargetIndex() || MO->isFI() || MO->isGlobal()); 5179 5180 if (!DefinedRC) { 5181 // This operand expects an immediate. 5182 return true; 5183 } 5184 5185 return isImmOperandLegal(MI, OpIdx, *MO); 5186 } 5187 5188 void SIInstrInfo::legalizeOperandsVOP2(MachineRegisterInfo &MRI, 5189 MachineInstr &MI) const { 5190 unsigned Opc = MI.getOpcode(); 5191 const MCInstrDesc &InstrDesc = get(Opc); 5192 5193 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); 5194 MachineOperand &Src0 = MI.getOperand(Src0Idx); 5195 5196 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); 5197 MachineOperand &Src1 = MI.getOperand(Src1Idx); 5198 5199 // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32 5200 // we need to only have one constant bus use before GFX10. 5201 bool HasImplicitSGPR = findImplicitSGPRRead(MI); 5202 if (HasImplicitSGPR && ST.getConstantBusLimit(Opc) <= 1 && Src0.isReg() && 5203 RI.isSGPRReg(MRI, Src0.getReg())) 5204 legalizeOpWithMove(MI, Src0Idx); 5205 5206 // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for 5207 // both the value to write (src0) and lane select (src1). Fix up non-SGPR 5208 // src0/src1 with V_READFIRSTLANE. 5209 if (Opc == AMDGPU::V_WRITELANE_B32) { 5210 const DebugLoc &DL = MI.getDebugLoc(); 5211 if (Src0.isReg() && RI.isVGPR(MRI, Src0.getReg())) { 5212 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 5213 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 5214 .add(Src0); 5215 Src0.ChangeToRegister(Reg, false); 5216 } 5217 if (Src1.isReg() && RI.isVGPR(MRI, Src1.getReg())) { 5218 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 5219 const DebugLoc &DL = MI.getDebugLoc(); 5220 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 5221 .add(Src1); 5222 Src1.ChangeToRegister(Reg, false); 5223 } 5224 return; 5225 } 5226 5227 // No VOP2 instructions support AGPRs. 5228 if (Src0.isReg() && RI.isAGPR(MRI, Src0.getReg())) 5229 legalizeOpWithMove(MI, Src0Idx); 5230 5231 if (Src1.isReg() && RI.isAGPR(MRI, Src1.getReg())) 5232 legalizeOpWithMove(MI, Src1Idx); 5233 5234 // VOP2 src0 instructions support all operand types, so we don't need to check 5235 // their legality. If src1 is already legal, we don't need to do anything. 5236 if (isLegalRegOperand(MRI, InstrDesc.operands()[Src1Idx], Src1)) 5237 return; 5238 5239 // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for 5240 // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane 5241 // select is uniform. 5242 if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() && 5243 RI.isVGPR(MRI, Src1.getReg())) { 5244 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 5245 const DebugLoc &DL = MI.getDebugLoc(); 5246 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 5247 .add(Src1); 5248 Src1.ChangeToRegister(Reg, false); 5249 return; 5250 } 5251 5252 // We do not use commuteInstruction here because it is too aggressive and will 5253 // commute if it is possible. We only want to commute here if it improves 5254 // legality. This can be called a fairly large number of times so don't waste 5255 // compile time pointlessly swapping and checking legality again. 5256 if (HasImplicitSGPR || !MI.isCommutable()) { 5257 legalizeOpWithMove(MI, Src1Idx); 5258 return; 5259 } 5260 5261 // If src0 can be used as src1, commuting will make the operands legal. 5262 // Otherwise we have to give up and insert a move. 5263 // 5264 // TODO: Other immediate-like operand kinds could be commuted if there was a 5265 // MachineOperand::ChangeTo* for them. 5266 if ((!Src1.isImm() && !Src1.isReg()) || 5267 !isLegalRegOperand(MRI, InstrDesc.operands()[Src1Idx], Src0)) { 5268 legalizeOpWithMove(MI, Src1Idx); 5269 return; 5270 } 5271 5272 int CommutedOpc = commuteOpcode(MI); 5273 if (CommutedOpc == -1) { 5274 legalizeOpWithMove(MI, Src1Idx); 5275 return; 5276 } 5277 5278 MI.setDesc(get(CommutedOpc)); 5279 5280 Register Src0Reg = Src0.getReg(); 5281 unsigned Src0SubReg = Src0.getSubReg(); 5282 bool Src0Kill = Src0.isKill(); 5283 5284 if (Src1.isImm()) 5285 Src0.ChangeToImmediate(Src1.getImm()); 5286 else if (Src1.isReg()) { 5287 Src0.ChangeToRegister(Src1.getReg(), false, false, Src1.isKill()); 5288 Src0.setSubReg(Src1.getSubReg()); 5289 } else 5290 llvm_unreachable("Should only have register or immediate operands"); 5291 5292 Src1.ChangeToRegister(Src0Reg, false, false, Src0Kill); 5293 Src1.setSubReg(Src0SubReg); 5294 fixImplicitOperands(MI); 5295 } 5296 5297 // Legalize VOP3 operands. All operand types are supported for any operand 5298 // but only one literal constant and only starting from GFX10. 5299 void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI, 5300 MachineInstr &MI) const { 5301 unsigned Opc = MI.getOpcode(); 5302 5303 int VOP3Idx[3] = { 5304 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0), 5305 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1), 5306 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2) 5307 }; 5308 5309 if (Opc == AMDGPU::V_PERMLANE16_B32_e64 || 5310 Opc == AMDGPU::V_PERMLANEX16_B32_e64) { 5311 // src1 and src2 must be scalar 5312 MachineOperand &Src1 = MI.getOperand(VOP3Idx[1]); 5313 MachineOperand &Src2 = MI.getOperand(VOP3Idx[2]); 5314 const DebugLoc &DL = MI.getDebugLoc(); 5315 if (Src1.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src1.getReg()))) { 5316 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 5317 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 5318 .add(Src1); 5319 Src1.ChangeToRegister(Reg, false); 5320 } 5321 if (Src2.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src2.getReg()))) { 5322 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 5323 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 5324 .add(Src2); 5325 Src2.ChangeToRegister(Reg, false); 5326 } 5327 } 5328 5329 // Find the one SGPR operand we are allowed to use. 5330 int ConstantBusLimit = ST.getConstantBusLimit(Opc); 5331 int LiteralLimit = ST.hasVOP3Literal() ? 1 : 0; 5332 SmallDenseSet<unsigned> SGPRsUsed; 5333 Register SGPRReg = findUsedSGPR(MI, VOP3Idx); 5334 if (SGPRReg) { 5335 SGPRsUsed.insert(SGPRReg); 5336 --ConstantBusLimit; 5337 } 5338 5339 for (int Idx : VOP3Idx) { 5340 if (Idx == -1) 5341 break; 5342 MachineOperand &MO = MI.getOperand(Idx); 5343 5344 if (!MO.isReg()) { 5345 if (isInlineConstant(MO, get(Opc).operands()[Idx])) 5346 continue; 5347 5348 if (LiteralLimit > 0 && ConstantBusLimit > 0) { 5349 --LiteralLimit; 5350 --ConstantBusLimit; 5351 continue; 5352 } 5353 5354 --LiteralLimit; 5355 --ConstantBusLimit; 5356 legalizeOpWithMove(MI, Idx); 5357 continue; 5358 } 5359 5360 if (RI.hasAGPRs(RI.getRegClassForReg(MRI, MO.getReg())) && 5361 !isOperandLegal(MI, Idx, &MO)) { 5362 legalizeOpWithMove(MI, Idx); 5363 continue; 5364 } 5365 5366 if (!RI.isSGPRClass(RI.getRegClassForReg(MRI, MO.getReg()))) 5367 continue; // VGPRs are legal 5368 5369 // We can use one SGPR in each VOP3 instruction prior to GFX10 5370 // and two starting from GFX10. 5371 if (SGPRsUsed.count(MO.getReg())) 5372 continue; 5373 if (ConstantBusLimit > 0) { 5374 SGPRsUsed.insert(MO.getReg()); 5375 --ConstantBusLimit; 5376 continue; 5377 } 5378 5379 // If we make it this far, then the operand is not legal and we must 5380 // legalize it. 5381 legalizeOpWithMove(MI, Idx); 5382 } 5383 } 5384 5385 Register SIInstrInfo::readlaneVGPRToSGPR(Register SrcReg, MachineInstr &UseMI, 5386 MachineRegisterInfo &MRI) const { 5387 const TargetRegisterClass *VRC = MRI.getRegClass(SrcReg); 5388 const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC); 5389 Register DstReg = MRI.createVirtualRegister(SRC); 5390 unsigned SubRegs = RI.getRegSizeInBits(*VRC) / 32; 5391 5392 if (RI.hasAGPRs(VRC)) { 5393 VRC = RI.getEquivalentVGPRClass(VRC); 5394 Register NewSrcReg = MRI.createVirtualRegister(VRC); 5395 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), 5396 get(TargetOpcode::COPY), NewSrcReg) 5397 .addReg(SrcReg); 5398 SrcReg = NewSrcReg; 5399 } 5400 5401 if (SubRegs == 1) { 5402 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), 5403 get(AMDGPU::V_READFIRSTLANE_B32), DstReg) 5404 .addReg(SrcReg); 5405 return DstReg; 5406 } 5407 5408 SmallVector<Register, 8> SRegs; 5409 for (unsigned i = 0; i < SubRegs; ++i) { 5410 Register SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 5411 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), 5412 get(AMDGPU::V_READFIRSTLANE_B32), SGPR) 5413 .addReg(SrcReg, 0, RI.getSubRegFromChannel(i)); 5414 SRegs.push_back(SGPR); 5415 } 5416 5417 MachineInstrBuilder MIB = 5418 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), 5419 get(AMDGPU::REG_SEQUENCE), DstReg); 5420 for (unsigned i = 0; i < SubRegs; ++i) { 5421 MIB.addReg(SRegs[i]); 5422 MIB.addImm(RI.getSubRegFromChannel(i)); 5423 } 5424 return DstReg; 5425 } 5426 5427 void SIInstrInfo::legalizeOperandsSMRD(MachineRegisterInfo &MRI, 5428 MachineInstr &MI) const { 5429 5430 // If the pointer is store in VGPRs, then we need to move them to 5431 // SGPRs using v_readfirstlane. This is safe because we only select 5432 // loads with uniform pointers to SMRD instruction so we know the 5433 // pointer value is uniform. 5434 MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase); 5435 if (SBase && !RI.isSGPRClass(MRI.getRegClass(SBase->getReg()))) { 5436 Register SGPR = readlaneVGPRToSGPR(SBase->getReg(), MI, MRI); 5437 SBase->setReg(SGPR); 5438 } 5439 MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soffset); 5440 if (SOff && !RI.isSGPRClass(MRI.getRegClass(SOff->getReg()))) { 5441 Register SGPR = readlaneVGPRToSGPR(SOff->getReg(), MI, MRI); 5442 SOff->setReg(SGPR); 5443 } 5444 } 5445 5446 bool SIInstrInfo::moveFlatAddrToVGPR(MachineInstr &Inst) const { 5447 unsigned Opc = Inst.getOpcode(); 5448 int OldSAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr); 5449 if (OldSAddrIdx < 0) 5450 return false; 5451 5452 assert(isSegmentSpecificFLAT(Inst)); 5453 5454 int NewOpc = AMDGPU::getGlobalVaddrOp(Opc); 5455 if (NewOpc < 0) 5456 NewOpc = AMDGPU::getFlatScratchInstSVfromSS(Opc); 5457 if (NewOpc < 0) 5458 return false; 5459 5460 MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo(); 5461 MachineOperand &SAddr = Inst.getOperand(OldSAddrIdx); 5462 if (RI.isSGPRReg(MRI, SAddr.getReg())) 5463 return false; 5464 5465 int NewVAddrIdx = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr); 5466 if (NewVAddrIdx < 0) 5467 return false; 5468 5469 int OldVAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr); 5470 5471 // Check vaddr, it shall be zero or absent. 5472 MachineInstr *VAddrDef = nullptr; 5473 if (OldVAddrIdx >= 0) { 5474 MachineOperand &VAddr = Inst.getOperand(OldVAddrIdx); 5475 VAddrDef = MRI.getUniqueVRegDef(VAddr.getReg()); 5476 if (!VAddrDef || VAddrDef->getOpcode() != AMDGPU::V_MOV_B32_e32 || 5477 !VAddrDef->getOperand(1).isImm() || 5478 VAddrDef->getOperand(1).getImm() != 0) 5479 return false; 5480 } 5481 5482 const MCInstrDesc &NewDesc = get(NewOpc); 5483 Inst.setDesc(NewDesc); 5484 5485 // Callers expect iterator to be valid after this call, so modify the 5486 // instruction in place. 5487 if (OldVAddrIdx == NewVAddrIdx) { 5488 MachineOperand &NewVAddr = Inst.getOperand(NewVAddrIdx); 5489 // Clear use list from the old vaddr holding a zero register. 5490 MRI.removeRegOperandFromUseList(&NewVAddr); 5491 MRI.moveOperands(&NewVAddr, &SAddr, 1); 5492 Inst.removeOperand(OldSAddrIdx); 5493 // Update the use list with the pointer we have just moved from vaddr to 5494 // saddr position. Otherwise new vaddr will be missing from the use list. 5495 MRI.removeRegOperandFromUseList(&NewVAddr); 5496 MRI.addRegOperandToUseList(&NewVAddr); 5497 } else { 5498 assert(OldSAddrIdx == NewVAddrIdx); 5499 5500 if (OldVAddrIdx >= 0) { 5501 int NewVDstIn = AMDGPU::getNamedOperandIdx(NewOpc, 5502 AMDGPU::OpName::vdst_in); 5503 5504 // removeOperand doesn't try to fixup tied operand indexes at it goes, so 5505 // it asserts. Untie the operands for now and retie them afterwards. 5506 if (NewVDstIn != -1) { 5507 int OldVDstIn = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst_in); 5508 Inst.untieRegOperand(OldVDstIn); 5509 } 5510 5511 Inst.removeOperand(OldVAddrIdx); 5512 5513 if (NewVDstIn != -1) { 5514 int NewVDst = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vdst); 5515 Inst.tieOperands(NewVDst, NewVDstIn); 5516 } 5517 } 5518 } 5519 5520 if (VAddrDef && MRI.use_nodbg_empty(VAddrDef->getOperand(0).getReg())) 5521 VAddrDef->eraseFromParent(); 5522 5523 return true; 5524 } 5525 5526 // FIXME: Remove this when SelectionDAG is obsoleted. 5527 void SIInstrInfo::legalizeOperandsFLAT(MachineRegisterInfo &MRI, 5528 MachineInstr &MI) const { 5529 if (!isSegmentSpecificFLAT(MI)) 5530 return; 5531 5532 // Fixup SGPR operands in VGPRs. We only select these when the DAG divergence 5533 // thinks they are uniform, so a readfirstlane should be valid. 5534 MachineOperand *SAddr = getNamedOperand(MI, AMDGPU::OpName::saddr); 5535 if (!SAddr || RI.isSGPRClass(MRI.getRegClass(SAddr->getReg()))) 5536 return; 5537 5538 if (moveFlatAddrToVGPR(MI)) 5539 return; 5540 5541 Register ToSGPR = readlaneVGPRToSGPR(SAddr->getReg(), MI, MRI); 5542 SAddr->setReg(ToSGPR); 5543 } 5544 5545 void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB, 5546 MachineBasicBlock::iterator I, 5547 const TargetRegisterClass *DstRC, 5548 MachineOperand &Op, 5549 MachineRegisterInfo &MRI, 5550 const DebugLoc &DL) const { 5551 Register OpReg = Op.getReg(); 5552 unsigned OpSubReg = Op.getSubReg(); 5553 5554 const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg( 5555 RI.getRegClassForReg(MRI, OpReg), OpSubReg); 5556 5557 // Check if operand is already the correct register class. 5558 if (DstRC == OpRC) 5559 return; 5560 5561 Register DstReg = MRI.createVirtualRegister(DstRC); 5562 auto Copy = BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).add(Op); 5563 5564 Op.setReg(DstReg); 5565 Op.setSubReg(0); 5566 5567 MachineInstr *Def = MRI.getVRegDef(OpReg); 5568 if (!Def) 5569 return; 5570 5571 // Try to eliminate the copy if it is copying an immediate value. 5572 if (Def->isMoveImmediate() && DstRC != &AMDGPU::VReg_1RegClass) 5573 FoldImmediate(*Copy, *Def, OpReg, &MRI); 5574 5575 bool ImpDef = Def->isImplicitDef(); 5576 while (!ImpDef && Def && Def->isCopy()) { 5577 if (Def->getOperand(1).getReg().isPhysical()) 5578 break; 5579 Def = MRI.getUniqueVRegDef(Def->getOperand(1).getReg()); 5580 ImpDef = Def && Def->isImplicitDef(); 5581 } 5582 if (!RI.isSGPRClass(DstRC) && !Copy->readsRegister(AMDGPU::EXEC, &RI) && 5583 !ImpDef) 5584 Copy.addReg(AMDGPU::EXEC, RegState::Implicit); 5585 } 5586 5587 // Emit the actual waterfall loop, executing the wrapped instruction for each 5588 // unique value of \p Rsrc across all lanes. In the best case we execute 1 5589 // iteration, in the worst case we execute 64 (once per lane). 5590 static void 5591 emitLoadSRsrcFromVGPRLoop(const SIInstrInfo &TII, MachineRegisterInfo &MRI, 5592 MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB, 5593 MachineBasicBlock &BodyBB, const DebugLoc &DL, 5594 MachineOperand &Rsrc) { 5595 MachineFunction &MF = *OrigBB.getParent(); 5596 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 5597 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 5598 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 5599 unsigned SaveExecOpc = 5600 ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64; 5601 unsigned XorTermOpc = 5602 ST.isWave32() ? AMDGPU::S_XOR_B32_term : AMDGPU::S_XOR_B64_term; 5603 unsigned AndOpc = 5604 ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64; 5605 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 5606 5607 MachineBasicBlock::iterator I = LoopBB.begin(); 5608 5609 SmallVector<Register, 8> ReadlanePieces; 5610 Register CondReg; 5611 5612 Register VRsrc = Rsrc.getReg(); 5613 unsigned VRsrcUndef = getUndefRegState(Rsrc.isUndef()); 5614 5615 unsigned RegSize = TRI->getRegSizeInBits(Rsrc.getReg(), MRI); 5616 unsigned NumSubRegs = RegSize / 32; 5617 assert(NumSubRegs % 2 == 0 && NumSubRegs <= 32 && "Unhandled register size"); 5618 5619 for (unsigned Idx = 0; Idx < NumSubRegs; Idx += 2) { 5620 5621 Register CurRegLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 5622 Register CurRegHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 5623 5624 // Read the next variant <- also loop target. 5625 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegLo) 5626 .addReg(VRsrc, VRsrcUndef, TRI->getSubRegFromChannel(Idx)); 5627 5628 // Read the next variant <- also loop target. 5629 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegHi) 5630 .addReg(VRsrc, VRsrcUndef, TRI->getSubRegFromChannel(Idx + 1)); 5631 5632 ReadlanePieces.push_back(CurRegLo); 5633 ReadlanePieces.push_back(CurRegHi); 5634 5635 // Comparison is to be done as 64-bit. 5636 Register CurReg = MRI.createVirtualRegister(&AMDGPU::SGPR_64RegClass); 5637 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), CurReg) 5638 .addReg(CurRegLo) 5639 .addImm(AMDGPU::sub0) 5640 .addReg(CurRegHi) 5641 .addImm(AMDGPU::sub1); 5642 5643 Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC); 5644 auto Cmp = 5645 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), NewCondReg) 5646 .addReg(CurReg); 5647 if (NumSubRegs <= 2) 5648 Cmp.addReg(VRsrc); 5649 else 5650 Cmp.addReg(VRsrc, VRsrcUndef, TRI->getSubRegFromChannel(Idx, 2)); 5651 5652 // Combine the comparison results with AND. 5653 if (!CondReg) // First. 5654 CondReg = NewCondReg; 5655 else { // If not the first, we create an AND. 5656 Register AndReg = MRI.createVirtualRegister(BoolXExecRC); 5657 BuildMI(LoopBB, I, DL, TII.get(AndOpc), AndReg) 5658 .addReg(CondReg) 5659 .addReg(NewCondReg); 5660 CondReg = AndReg; 5661 } 5662 } // End for loop. 5663 5664 auto SRsrcRC = TRI->getEquivalentSGPRClass(MRI.getRegClass(VRsrc)); 5665 Register SRsrc = MRI.createVirtualRegister(SRsrcRC); 5666 5667 // Build scalar Rsrc. 5668 auto Merge = BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SRsrc); 5669 unsigned Channel = 0; 5670 for (Register Piece : ReadlanePieces) { 5671 Merge.addReg(Piece) 5672 .addImm(TRI->getSubRegFromChannel(Channel++)); 5673 } 5674 5675 // Update Rsrc operand to use the SGPR Rsrc. 5676 Rsrc.setReg(SRsrc); 5677 Rsrc.setIsKill(); 5678 5679 Register SaveExec = MRI.createVirtualRegister(BoolXExecRC); 5680 MRI.setSimpleHint(SaveExec, CondReg); 5681 5682 // Update EXEC to matching lanes, saving original to SaveExec. 5683 BuildMI(LoopBB, I, DL, TII.get(SaveExecOpc), SaveExec) 5684 .addReg(CondReg, RegState::Kill); 5685 5686 // The original instruction is here; we insert the terminators after it. 5687 I = BodyBB.end(); 5688 5689 // Update EXEC, switch all done bits to 0 and all todo bits to 1. 5690 BuildMI(BodyBB, I, DL, TII.get(XorTermOpc), Exec) 5691 .addReg(Exec) 5692 .addReg(SaveExec); 5693 5694 BuildMI(BodyBB, I, DL, TII.get(AMDGPU::SI_WATERFALL_LOOP)).addMBB(&LoopBB); 5695 } 5696 5697 // Build a waterfall loop around \p MI, replacing the VGPR \p Rsrc register 5698 // with SGPRs by iterating over all unique values across all lanes. 5699 // Returns the loop basic block that now contains \p MI. 5700 static MachineBasicBlock * 5701 loadSRsrcFromVGPR(const SIInstrInfo &TII, MachineInstr &MI, 5702 MachineOperand &Rsrc, MachineDominatorTree *MDT, 5703 MachineBasicBlock::iterator Begin = nullptr, 5704 MachineBasicBlock::iterator End = nullptr) { 5705 MachineBasicBlock &MBB = *MI.getParent(); 5706 MachineFunction &MF = *MBB.getParent(); 5707 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 5708 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 5709 MachineRegisterInfo &MRI = MF.getRegInfo(); 5710 if (!Begin.isValid()) 5711 Begin = &MI; 5712 if (!End.isValid()) { 5713 End = &MI; 5714 ++End; 5715 } 5716 const DebugLoc &DL = MI.getDebugLoc(); 5717 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 5718 unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; 5719 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 5720 5721 Register SaveExec = MRI.createVirtualRegister(BoolXExecRC); 5722 5723 // Save the EXEC mask 5724 BuildMI(MBB, Begin, DL, TII.get(MovExecOpc), SaveExec).addReg(Exec); 5725 5726 // Killed uses in the instruction we are waterfalling around will be 5727 // incorrect due to the added control-flow. 5728 MachineBasicBlock::iterator AfterMI = MI; 5729 ++AfterMI; 5730 for (auto I = Begin; I != AfterMI; I++) { 5731 for (auto &MO : I->uses()) { 5732 if (MO.isReg() && MO.isUse()) { 5733 MRI.clearKillFlags(MO.getReg()); 5734 } 5735 } 5736 } 5737 5738 // To insert the loop we need to split the block. Move everything after this 5739 // point to a new block, and insert a new empty block between the two. 5740 MachineBasicBlock *LoopBB = MF.CreateMachineBasicBlock(); 5741 MachineBasicBlock *BodyBB = MF.CreateMachineBasicBlock(); 5742 MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock(); 5743 MachineFunction::iterator MBBI(MBB); 5744 ++MBBI; 5745 5746 MF.insert(MBBI, LoopBB); 5747 MF.insert(MBBI, BodyBB); 5748 MF.insert(MBBI, RemainderBB); 5749 5750 LoopBB->addSuccessor(BodyBB); 5751 BodyBB->addSuccessor(LoopBB); 5752 BodyBB->addSuccessor(RemainderBB); 5753 5754 // Move Begin to MI to the BodyBB, and the remainder of the block to 5755 // RemainderBB. 5756 RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB); 5757 RemainderBB->splice(RemainderBB->begin(), &MBB, End, MBB.end()); 5758 BodyBB->splice(BodyBB->begin(), &MBB, Begin, MBB.end()); 5759 5760 MBB.addSuccessor(LoopBB); 5761 5762 // Update dominators. We know that MBB immediately dominates LoopBB, that 5763 // LoopBB immediately dominates BodyBB, and BodyBB immediately dominates 5764 // RemainderBB. RemainderBB immediately dominates all of the successors 5765 // transferred to it from MBB that MBB used to properly dominate. 5766 if (MDT) { 5767 MDT->addNewBlock(LoopBB, &MBB); 5768 MDT->addNewBlock(BodyBB, LoopBB); 5769 MDT->addNewBlock(RemainderBB, BodyBB); 5770 for (auto &Succ : RemainderBB->successors()) { 5771 if (MDT->properlyDominates(&MBB, Succ)) { 5772 MDT->changeImmediateDominator(Succ, RemainderBB); 5773 } 5774 } 5775 } 5776 5777 emitLoadSRsrcFromVGPRLoop(TII, MRI, MBB, *LoopBB, *BodyBB, DL, Rsrc); 5778 5779 // Restore the EXEC mask 5780 MachineBasicBlock::iterator First = RemainderBB->begin(); 5781 BuildMI(*RemainderBB, First, DL, TII.get(MovExecOpc), Exec).addReg(SaveExec); 5782 return BodyBB; 5783 } 5784 5785 // Extract pointer from Rsrc and return a zero-value Rsrc replacement. 5786 static std::tuple<unsigned, unsigned> 5787 extractRsrcPtr(const SIInstrInfo &TII, MachineInstr &MI, MachineOperand &Rsrc) { 5788 MachineBasicBlock &MBB = *MI.getParent(); 5789 MachineFunction &MF = *MBB.getParent(); 5790 MachineRegisterInfo &MRI = MF.getRegInfo(); 5791 5792 // Extract the ptr from the resource descriptor. 5793 unsigned RsrcPtr = 5794 TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass, 5795 AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass); 5796 5797 // Create an empty resource descriptor 5798 Register Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); 5799 Register SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 5800 Register SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 5801 Register NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass); 5802 uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat(); 5803 5804 // Zero64 = 0 5805 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64) 5806 .addImm(0); 5807 5808 // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0} 5809 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo) 5810 .addImm(RsrcDataFormat & 0xFFFFFFFF); 5811 5812 // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32} 5813 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi) 5814 .addImm(RsrcDataFormat >> 32); 5815 5816 // NewSRsrc = {Zero64, SRsrcFormat} 5817 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc) 5818 .addReg(Zero64) 5819 .addImm(AMDGPU::sub0_sub1) 5820 .addReg(SRsrcFormatLo) 5821 .addImm(AMDGPU::sub2) 5822 .addReg(SRsrcFormatHi) 5823 .addImm(AMDGPU::sub3); 5824 5825 return std::tuple(RsrcPtr, NewSRsrc); 5826 } 5827 5828 MachineBasicBlock * 5829 SIInstrInfo::legalizeOperands(MachineInstr &MI, 5830 MachineDominatorTree *MDT) const { 5831 MachineFunction &MF = *MI.getParent()->getParent(); 5832 MachineRegisterInfo &MRI = MF.getRegInfo(); 5833 MachineBasicBlock *CreatedBB = nullptr; 5834 5835 // Legalize VOP2 5836 if (isVOP2(MI) || isVOPC(MI)) { 5837 legalizeOperandsVOP2(MRI, MI); 5838 return CreatedBB; 5839 } 5840 5841 // Legalize VOP3 5842 if (isVOP3(MI)) { 5843 legalizeOperandsVOP3(MRI, MI); 5844 return CreatedBB; 5845 } 5846 5847 // Legalize SMRD 5848 if (isSMRD(MI)) { 5849 legalizeOperandsSMRD(MRI, MI); 5850 return CreatedBB; 5851 } 5852 5853 // Legalize FLAT 5854 if (isFLAT(MI)) { 5855 legalizeOperandsFLAT(MRI, MI); 5856 return CreatedBB; 5857 } 5858 5859 // Legalize REG_SEQUENCE and PHI 5860 // The register class of the operands much be the same type as the register 5861 // class of the output. 5862 if (MI.getOpcode() == AMDGPU::PHI) { 5863 const TargetRegisterClass *RC = nullptr, *SRC = nullptr, *VRC = nullptr; 5864 for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) { 5865 if (!MI.getOperand(i).isReg() || !MI.getOperand(i).getReg().isVirtual()) 5866 continue; 5867 const TargetRegisterClass *OpRC = 5868 MRI.getRegClass(MI.getOperand(i).getReg()); 5869 if (RI.hasVectorRegisters(OpRC)) { 5870 VRC = OpRC; 5871 } else { 5872 SRC = OpRC; 5873 } 5874 } 5875 5876 // If any of the operands are VGPR registers, then they all most be 5877 // otherwise we will create illegal VGPR->SGPR copies when legalizing 5878 // them. 5879 if (VRC || !RI.isSGPRClass(getOpRegClass(MI, 0))) { 5880 if (!VRC) { 5881 assert(SRC); 5882 if (getOpRegClass(MI, 0) == &AMDGPU::VReg_1RegClass) { 5883 VRC = &AMDGPU::VReg_1RegClass; 5884 } else 5885 VRC = RI.isAGPRClass(getOpRegClass(MI, 0)) 5886 ? RI.getEquivalentAGPRClass(SRC) 5887 : RI.getEquivalentVGPRClass(SRC); 5888 } else { 5889 VRC = RI.isAGPRClass(getOpRegClass(MI, 0)) 5890 ? RI.getEquivalentAGPRClass(VRC) 5891 : RI.getEquivalentVGPRClass(VRC); 5892 } 5893 RC = VRC; 5894 } else { 5895 RC = SRC; 5896 } 5897 5898 // Update all the operands so they have the same type. 5899 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { 5900 MachineOperand &Op = MI.getOperand(I); 5901 if (!Op.isReg() || !Op.getReg().isVirtual()) 5902 continue; 5903 5904 // MI is a PHI instruction. 5905 MachineBasicBlock *InsertBB = MI.getOperand(I + 1).getMBB(); 5906 MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator(); 5907 5908 // Avoid creating no-op copies with the same src and dst reg class. These 5909 // confuse some of the machine passes. 5910 legalizeGenericOperand(*InsertBB, Insert, RC, Op, MRI, MI.getDebugLoc()); 5911 } 5912 } 5913 5914 // REG_SEQUENCE doesn't really require operand legalization, but if one has a 5915 // VGPR dest type and SGPR sources, insert copies so all operands are 5916 // VGPRs. This seems to help operand folding / the register coalescer. 5917 if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) { 5918 MachineBasicBlock *MBB = MI.getParent(); 5919 const TargetRegisterClass *DstRC = getOpRegClass(MI, 0); 5920 if (RI.hasVGPRs(DstRC)) { 5921 // Update all the operands so they are VGPR register classes. These may 5922 // not be the same register class because REG_SEQUENCE supports mixing 5923 // subregister index types e.g. sub0_sub1 + sub2 + sub3 5924 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { 5925 MachineOperand &Op = MI.getOperand(I); 5926 if (!Op.isReg() || !Op.getReg().isVirtual()) 5927 continue; 5928 5929 const TargetRegisterClass *OpRC = MRI.getRegClass(Op.getReg()); 5930 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(OpRC); 5931 if (VRC == OpRC) 5932 continue; 5933 5934 legalizeGenericOperand(*MBB, MI, VRC, Op, MRI, MI.getDebugLoc()); 5935 Op.setIsKill(); 5936 } 5937 } 5938 5939 return CreatedBB; 5940 } 5941 5942 // Legalize INSERT_SUBREG 5943 // src0 must have the same register class as dst 5944 if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) { 5945 Register Dst = MI.getOperand(0).getReg(); 5946 Register Src0 = MI.getOperand(1).getReg(); 5947 const TargetRegisterClass *DstRC = MRI.getRegClass(Dst); 5948 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0); 5949 if (DstRC != Src0RC) { 5950 MachineBasicBlock *MBB = MI.getParent(); 5951 MachineOperand &Op = MI.getOperand(1); 5952 legalizeGenericOperand(*MBB, MI, DstRC, Op, MRI, MI.getDebugLoc()); 5953 } 5954 return CreatedBB; 5955 } 5956 5957 // Legalize SI_INIT_M0 5958 if (MI.getOpcode() == AMDGPU::SI_INIT_M0) { 5959 MachineOperand &Src = MI.getOperand(0); 5960 if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg()))) 5961 Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI)); 5962 return CreatedBB; 5963 } 5964 5965 // Legalize MIMG and MUBUF/MTBUF for shaders. 5966 // 5967 // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via 5968 // scratch memory access. In both cases, the legalization never involves 5969 // conversion to the addr64 form. 5970 if (isMIMG(MI) || (AMDGPU::isGraphics(MF.getFunction().getCallingConv()) && 5971 (isMUBUF(MI) || isMTBUF(MI)))) { 5972 MachineOperand *SRsrc = getNamedOperand(MI, AMDGPU::OpName::srsrc); 5973 if (SRsrc && !RI.isSGPRClass(MRI.getRegClass(SRsrc->getReg()))) 5974 CreatedBB = loadSRsrcFromVGPR(*this, MI, *SRsrc, MDT); 5975 5976 MachineOperand *SSamp = getNamedOperand(MI, AMDGPU::OpName::ssamp); 5977 if (SSamp && !RI.isSGPRClass(MRI.getRegClass(SSamp->getReg()))) 5978 CreatedBB = loadSRsrcFromVGPR(*this, MI, *SSamp, MDT); 5979 5980 return CreatedBB; 5981 } 5982 5983 // Legalize SI_CALL 5984 if (MI.getOpcode() == AMDGPU::SI_CALL_ISEL) { 5985 MachineOperand *Dest = &MI.getOperand(0); 5986 if (!RI.isSGPRClass(MRI.getRegClass(Dest->getReg()))) { 5987 // Move everything between ADJCALLSTACKUP and ADJCALLSTACKDOWN and 5988 // following copies, we also need to move copies from and to physical 5989 // registers into the loop block. 5990 unsigned FrameSetupOpcode = getCallFrameSetupOpcode(); 5991 unsigned FrameDestroyOpcode = getCallFrameDestroyOpcode(); 5992 5993 // Also move the copies to physical registers into the loop block 5994 MachineBasicBlock &MBB = *MI.getParent(); 5995 MachineBasicBlock::iterator Start(&MI); 5996 while (Start->getOpcode() != FrameSetupOpcode) 5997 --Start; 5998 MachineBasicBlock::iterator End(&MI); 5999 while (End->getOpcode() != FrameDestroyOpcode) 6000 ++End; 6001 // Also include following copies of the return value 6002 ++End; 6003 while (End != MBB.end() && End->isCopy() && End->getOperand(1).isReg() && 6004 MI.definesRegister(End->getOperand(1).getReg())) 6005 ++End; 6006 CreatedBB = loadSRsrcFromVGPR(*this, MI, *Dest, MDT, Start, End); 6007 } 6008 } 6009 6010 // Legalize MUBUF* instructions. 6011 int RsrcIdx = 6012 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc); 6013 if (RsrcIdx != -1) { 6014 // We have an MUBUF instruction 6015 MachineOperand *Rsrc = &MI.getOperand(RsrcIdx); 6016 unsigned RsrcRC = get(MI.getOpcode()).operands()[RsrcIdx].RegClass; 6017 if (RI.getCommonSubClass(MRI.getRegClass(Rsrc->getReg()), 6018 RI.getRegClass(RsrcRC))) { 6019 // The operands are legal. 6020 // FIXME: We may need to legalize operands besides srsrc. 6021 return CreatedBB; 6022 } 6023 6024 // Legalize a VGPR Rsrc. 6025 // 6026 // If the instruction is _ADDR64, we can avoid a waterfall by extracting 6027 // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using 6028 // a zero-value SRsrc. 6029 // 6030 // If the instruction is _OFFSET (both idxen and offen disabled), and we 6031 // support ADDR64 instructions, we can convert to ADDR64 and do the same as 6032 // above. 6033 // 6034 // Otherwise we are on non-ADDR64 hardware, and/or we have 6035 // idxen/offen/bothen and we fall back to a waterfall loop. 6036 6037 MachineBasicBlock &MBB = *MI.getParent(); 6038 6039 MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr); 6040 if (VAddr && AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -1) { 6041 // This is already an ADDR64 instruction so we need to add the pointer 6042 // extracted from the resource descriptor to the current value of VAddr. 6043 Register NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6044 Register NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6045 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 6046 6047 const auto *BoolXExecRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 6048 Register CondReg0 = MRI.createVirtualRegister(BoolXExecRC); 6049 Register CondReg1 = MRI.createVirtualRegister(BoolXExecRC); 6050 6051 unsigned RsrcPtr, NewSRsrc; 6052 std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc); 6053 6054 // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0 6055 const DebugLoc &DL = MI.getDebugLoc(); 6056 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_CO_U32_e64), NewVAddrLo) 6057 .addDef(CondReg0) 6058 .addReg(RsrcPtr, 0, AMDGPU::sub0) 6059 .addReg(VAddr->getReg(), 0, AMDGPU::sub0) 6060 .addImm(0); 6061 6062 // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1 6063 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e64), NewVAddrHi) 6064 .addDef(CondReg1, RegState::Dead) 6065 .addReg(RsrcPtr, 0, AMDGPU::sub1) 6066 .addReg(VAddr->getReg(), 0, AMDGPU::sub1) 6067 .addReg(CondReg0, RegState::Kill) 6068 .addImm(0); 6069 6070 // NewVaddr = {NewVaddrHi, NewVaddrLo} 6071 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr) 6072 .addReg(NewVAddrLo) 6073 .addImm(AMDGPU::sub0) 6074 .addReg(NewVAddrHi) 6075 .addImm(AMDGPU::sub1); 6076 6077 VAddr->setReg(NewVAddr); 6078 Rsrc->setReg(NewSRsrc); 6079 } else if (!VAddr && ST.hasAddr64()) { 6080 // This instructions is the _OFFSET variant, so we need to convert it to 6081 // ADDR64. 6082 assert(ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS && 6083 "FIXME: Need to emit flat atomics here"); 6084 6085 unsigned RsrcPtr, NewSRsrc; 6086 std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc); 6087 6088 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 6089 MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata); 6090 MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset); 6091 MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset); 6092 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode()); 6093 6094 // Atomics with return have an additional tied operand and are 6095 // missing some of the special bits. 6096 MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in); 6097 MachineInstr *Addr64; 6098 6099 if (!VDataIn) { 6100 // Regular buffer load / store. 6101 MachineInstrBuilder MIB = 6102 BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode)) 6103 .add(*VData) 6104 .addReg(NewVAddr) 6105 .addReg(NewSRsrc) 6106 .add(*SOffset) 6107 .add(*Offset); 6108 6109 if (const MachineOperand *CPol = 6110 getNamedOperand(MI, AMDGPU::OpName::cpol)) { 6111 MIB.addImm(CPol->getImm()); 6112 } 6113 6114 if (const MachineOperand *TFE = 6115 getNamedOperand(MI, AMDGPU::OpName::tfe)) { 6116 MIB.addImm(TFE->getImm()); 6117 } 6118 6119 MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::swz)); 6120 6121 MIB.cloneMemRefs(MI); 6122 Addr64 = MIB; 6123 } else { 6124 // Atomics with return. 6125 Addr64 = BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode)) 6126 .add(*VData) 6127 .add(*VDataIn) 6128 .addReg(NewVAddr) 6129 .addReg(NewSRsrc) 6130 .add(*SOffset) 6131 .add(*Offset) 6132 .addImm(getNamedImmOperand(MI, AMDGPU::OpName::cpol)) 6133 .cloneMemRefs(MI); 6134 } 6135 6136 MI.removeFromParent(); 6137 6138 // NewVaddr = {NewVaddrHi, NewVaddrLo} 6139 BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE), 6140 NewVAddr) 6141 .addReg(RsrcPtr, 0, AMDGPU::sub0) 6142 .addImm(AMDGPU::sub0) 6143 .addReg(RsrcPtr, 0, AMDGPU::sub1) 6144 .addImm(AMDGPU::sub1); 6145 } else { 6146 // This is another variant; legalize Rsrc with waterfall loop from VGPRs 6147 // to SGPRs. 6148 CreatedBB = loadSRsrcFromVGPR(*this, MI, *Rsrc, MDT); 6149 return CreatedBB; 6150 } 6151 } 6152 return CreatedBB; 6153 } 6154 6155 MachineBasicBlock *SIInstrInfo::moveToVALU(MachineInstr &TopInst, 6156 MachineDominatorTree *MDT) const { 6157 SetVectorType Worklist; 6158 Worklist.insert(&TopInst); 6159 MachineBasicBlock *CreatedBB = nullptr; 6160 MachineBasicBlock *CreatedBBTmp = nullptr; 6161 6162 while (!Worklist.empty()) { 6163 MachineInstr &Inst = *Worklist.pop_back_val(); 6164 MachineBasicBlock *MBB = Inst.getParent(); 6165 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 6166 6167 unsigned Opcode = Inst.getOpcode(); 6168 unsigned NewOpcode = getVALUOp(Inst); 6169 6170 // Handle some special cases 6171 switch (Opcode) { 6172 default: 6173 break; 6174 case AMDGPU::S_ADD_U64_PSEUDO: 6175 case AMDGPU::S_SUB_U64_PSEUDO: 6176 splitScalar64BitAddSub(Worklist, Inst, MDT); 6177 Inst.eraseFromParent(); 6178 continue; 6179 case AMDGPU::S_ADD_I32: 6180 case AMDGPU::S_SUB_I32: { 6181 // FIXME: The u32 versions currently selected use the carry. 6182 bool Changed; 6183 std::tie(Changed, CreatedBBTmp) = moveScalarAddSub(Worklist, Inst, MDT); 6184 if (CreatedBBTmp && TopInst.getParent() == CreatedBBTmp) 6185 CreatedBB = CreatedBBTmp; 6186 if (Changed) 6187 continue; 6188 6189 // Default handling 6190 break; 6191 } 6192 case AMDGPU::S_AND_B64: 6193 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT); 6194 Inst.eraseFromParent(); 6195 continue; 6196 6197 case AMDGPU::S_OR_B64: 6198 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT); 6199 Inst.eraseFromParent(); 6200 continue; 6201 6202 case AMDGPU::S_XOR_B64: 6203 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT); 6204 Inst.eraseFromParent(); 6205 continue; 6206 6207 case AMDGPU::S_NAND_B64: 6208 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT); 6209 Inst.eraseFromParent(); 6210 continue; 6211 6212 case AMDGPU::S_NOR_B64: 6213 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT); 6214 Inst.eraseFromParent(); 6215 continue; 6216 6217 case AMDGPU::S_XNOR_B64: 6218 if (ST.hasDLInsts()) 6219 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT); 6220 else 6221 splitScalar64BitXnor(Worklist, Inst, MDT); 6222 Inst.eraseFromParent(); 6223 continue; 6224 6225 case AMDGPU::S_ANDN2_B64: 6226 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT); 6227 Inst.eraseFromParent(); 6228 continue; 6229 6230 case AMDGPU::S_ORN2_B64: 6231 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT); 6232 Inst.eraseFromParent(); 6233 continue; 6234 6235 case AMDGPU::S_BREV_B64: 6236 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_BREV_B32, true); 6237 Inst.eraseFromParent(); 6238 continue; 6239 6240 case AMDGPU::S_NOT_B64: 6241 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32); 6242 Inst.eraseFromParent(); 6243 continue; 6244 6245 case AMDGPU::S_BCNT1_I32_B64: 6246 splitScalar64BitBCNT(Worklist, Inst); 6247 Inst.eraseFromParent(); 6248 continue; 6249 6250 case AMDGPU::S_BFE_I64: 6251 splitScalar64BitBFE(Worklist, Inst); 6252 Inst.eraseFromParent(); 6253 continue; 6254 6255 case AMDGPU::S_LSHL_B32: 6256 if (ST.hasOnlyRevVALUShifts()) { 6257 NewOpcode = AMDGPU::V_LSHLREV_B32_e64; 6258 swapOperands(Inst); 6259 } 6260 break; 6261 case AMDGPU::S_ASHR_I32: 6262 if (ST.hasOnlyRevVALUShifts()) { 6263 NewOpcode = AMDGPU::V_ASHRREV_I32_e64; 6264 swapOperands(Inst); 6265 } 6266 break; 6267 case AMDGPU::S_LSHR_B32: 6268 if (ST.hasOnlyRevVALUShifts()) { 6269 NewOpcode = AMDGPU::V_LSHRREV_B32_e64; 6270 swapOperands(Inst); 6271 } 6272 break; 6273 case AMDGPU::S_LSHL_B64: 6274 if (ST.hasOnlyRevVALUShifts()) { 6275 NewOpcode = AMDGPU::V_LSHLREV_B64_e64; 6276 swapOperands(Inst); 6277 } 6278 break; 6279 case AMDGPU::S_ASHR_I64: 6280 if (ST.hasOnlyRevVALUShifts()) { 6281 NewOpcode = AMDGPU::V_ASHRREV_I64_e64; 6282 swapOperands(Inst); 6283 } 6284 break; 6285 case AMDGPU::S_LSHR_B64: 6286 if (ST.hasOnlyRevVALUShifts()) { 6287 NewOpcode = AMDGPU::V_LSHRREV_B64_e64; 6288 swapOperands(Inst); 6289 } 6290 break; 6291 6292 case AMDGPU::S_ABS_I32: 6293 lowerScalarAbs(Worklist, Inst); 6294 Inst.eraseFromParent(); 6295 continue; 6296 6297 case AMDGPU::S_CBRANCH_SCC0: 6298 case AMDGPU::S_CBRANCH_SCC1: { 6299 // Clear unused bits of vcc 6300 Register CondReg = Inst.getOperand(1).getReg(); 6301 bool IsSCC = CondReg == AMDGPU::SCC; 6302 Register VCC = RI.getVCC(); 6303 Register EXEC = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 6304 unsigned Opc = ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64; 6305 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(Opc), VCC) 6306 .addReg(EXEC) 6307 .addReg(IsSCC ? VCC : CondReg); 6308 Inst.removeOperand(1); 6309 } 6310 break; 6311 6312 case AMDGPU::S_BFE_U64: 6313 case AMDGPU::S_BFM_B64: 6314 llvm_unreachable("Moving this op to VALU not implemented"); 6315 6316 case AMDGPU::S_PACK_LL_B32_B16: 6317 case AMDGPU::S_PACK_LH_B32_B16: 6318 case AMDGPU::S_PACK_HL_B32_B16: 6319 case AMDGPU::S_PACK_HH_B32_B16: 6320 movePackToVALU(Worklist, MRI, Inst); 6321 Inst.eraseFromParent(); 6322 continue; 6323 6324 case AMDGPU::S_XNOR_B32: 6325 lowerScalarXnor(Worklist, Inst); 6326 Inst.eraseFromParent(); 6327 continue; 6328 6329 case AMDGPU::S_NAND_B32: 6330 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32); 6331 Inst.eraseFromParent(); 6332 continue; 6333 6334 case AMDGPU::S_NOR_B32: 6335 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32); 6336 Inst.eraseFromParent(); 6337 continue; 6338 6339 case AMDGPU::S_ANDN2_B32: 6340 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32); 6341 Inst.eraseFromParent(); 6342 continue; 6343 6344 case AMDGPU::S_ORN2_B32: 6345 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32); 6346 Inst.eraseFromParent(); 6347 continue; 6348 6349 // TODO: remove as soon as everything is ready 6350 // to replace VGPR to SGPR copy with V_READFIRSTLANEs. 6351 // S_ADD/SUB_CO_PSEUDO as well as S_UADDO/USUBO_PSEUDO 6352 // can only be selected from the uniform SDNode. 6353 case AMDGPU::S_ADD_CO_PSEUDO: 6354 case AMDGPU::S_SUB_CO_PSEUDO: { 6355 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO) 6356 ? AMDGPU::V_ADDC_U32_e64 6357 : AMDGPU::V_SUBB_U32_e64; 6358 const auto *CarryRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 6359 6360 Register CarryInReg = Inst.getOperand(4).getReg(); 6361 if (!MRI.constrainRegClass(CarryInReg, CarryRC)) { 6362 Register NewCarryReg = MRI.createVirtualRegister(CarryRC); 6363 BuildMI(*MBB, &Inst, Inst.getDebugLoc(), get(AMDGPU::COPY), NewCarryReg) 6364 .addReg(CarryInReg); 6365 } 6366 6367 Register CarryOutReg = Inst.getOperand(1).getReg(); 6368 6369 Register DestReg = MRI.createVirtualRegister(RI.getEquivalentVGPRClass( 6370 MRI.getRegClass(Inst.getOperand(0).getReg()))); 6371 MachineInstr *CarryOp = 6372 BuildMI(*MBB, &Inst, Inst.getDebugLoc(), get(Opc), DestReg) 6373 .addReg(CarryOutReg, RegState::Define) 6374 .add(Inst.getOperand(2)) 6375 .add(Inst.getOperand(3)) 6376 .addReg(CarryInReg) 6377 .addImm(0); 6378 CreatedBBTmp = legalizeOperands(*CarryOp); 6379 if (CreatedBBTmp && TopInst.getParent() == CreatedBBTmp) 6380 CreatedBB = CreatedBBTmp; 6381 MRI.replaceRegWith(Inst.getOperand(0).getReg(), DestReg); 6382 addUsersToMoveToVALUWorklist(DestReg, MRI, Worklist); 6383 Inst.eraseFromParent(); 6384 } 6385 continue; 6386 case AMDGPU::S_UADDO_PSEUDO: 6387 case AMDGPU::S_USUBO_PSEUDO: { 6388 const DebugLoc &DL = Inst.getDebugLoc(); 6389 MachineOperand &Dest0 = Inst.getOperand(0); 6390 MachineOperand &Dest1 = Inst.getOperand(1); 6391 MachineOperand &Src0 = Inst.getOperand(2); 6392 MachineOperand &Src1 = Inst.getOperand(3); 6393 6394 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_UADDO_PSEUDO) 6395 ? AMDGPU::V_ADD_CO_U32_e64 6396 : AMDGPU::V_SUB_CO_U32_e64; 6397 const TargetRegisterClass *NewRC = 6398 RI.getEquivalentVGPRClass(MRI.getRegClass(Dest0.getReg())); 6399 Register DestReg = MRI.createVirtualRegister(NewRC); 6400 MachineInstr *NewInstr = BuildMI(*MBB, &Inst, DL, get(Opc), DestReg) 6401 .addReg(Dest1.getReg(), RegState::Define) 6402 .add(Src0) 6403 .add(Src1) 6404 .addImm(0); // clamp bit 6405 6406 CreatedBBTmp = legalizeOperands(*NewInstr, MDT); 6407 if (CreatedBBTmp && TopInst.getParent() == CreatedBBTmp) 6408 CreatedBB = CreatedBBTmp; 6409 6410 MRI.replaceRegWith(Dest0.getReg(), DestReg); 6411 addUsersToMoveToVALUWorklist(NewInstr->getOperand(0).getReg(), MRI, 6412 Worklist); 6413 Inst.eraseFromParent(); 6414 } 6415 continue; 6416 6417 case AMDGPU::S_CSELECT_B32: 6418 case AMDGPU::S_CSELECT_B64: 6419 lowerSelect(Worklist, Inst, MDT); 6420 Inst.eraseFromParent(); 6421 continue; 6422 case AMDGPU::S_CMP_EQ_I32: 6423 case AMDGPU::S_CMP_LG_I32: 6424 case AMDGPU::S_CMP_GT_I32: 6425 case AMDGPU::S_CMP_GE_I32: 6426 case AMDGPU::S_CMP_LT_I32: 6427 case AMDGPU::S_CMP_LE_I32: 6428 case AMDGPU::S_CMP_EQ_U32: 6429 case AMDGPU::S_CMP_LG_U32: 6430 case AMDGPU::S_CMP_GT_U32: 6431 case AMDGPU::S_CMP_GE_U32: 6432 case AMDGPU::S_CMP_LT_U32: 6433 case AMDGPU::S_CMP_LE_U32: 6434 case AMDGPU::S_CMP_EQ_U64: 6435 case AMDGPU::S_CMP_LG_U64: { 6436 const MCInstrDesc &NewDesc = get(NewOpcode); 6437 Register CondReg = MRI.createVirtualRegister(RI.getWaveMaskRegClass()); 6438 MachineInstr *NewInstr = 6439 BuildMI(*MBB, Inst, Inst.getDebugLoc(), NewDesc, CondReg) 6440 .add(Inst.getOperand(0)) 6441 .add(Inst.getOperand(1)); 6442 legalizeOperands(*NewInstr, MDT); 6443 int SCCIdx = Inst.findRegisterDefOperandIdx(AMDGPU::SCC); 6444 MachineOperand SCCOp = Inst.getOperand(SCCIdx); 6445 addSCCDefUsersToVALUWorklist(SCCOp, Inst, Worklist, CondReg); 6446 Inst.eraseFromParent(); 6447 } 6448 continue; 6449 } 6450 6451 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) { 6452 // We cannot move this instruction to the VALU, so we should try to 6453 // legalize its operands instead. 6454 CreatedBBTmp = legalizeOperands(Inst, MDT); 6455 if (CreatedBBTmp && TopInst.getParent() == CreatedBBTmp) 6456 CreatedBB = CreatedBBTmp; 6457 continue; 6458 } 6459 6460 // Handle converting generic instructions like COPY-to-SGPR into 6461 // COPY-to-VGPR. 6462 if (NewOpcode == Opcode) { 6463 Register DstReg = Inst.getOperand(0).getReg(); 6464 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst); 6465 6466 if (Inst.isCopy() && Inst.getOperand(1).getReg().isVirtual() && 6467 NewDstRC == RI.getRegClassForReg(MRI, Inst.getOperand(1).getReg())) { 6468 // Instead of creating a copy where src and dst are the same register 6469 // class, we just replace all uses of dst with src. These kinds of 6470 // copies interfere with the heuristics MachineSink uses to decide 6471 // whether or not to split a critical edge. Since the pass assumes 6472 // that copies will end up as machine instructions and not be 6473 // eliminated. 6474 addUsersToMoveToVALUWorklist(DstReg, MRI, Worklist); 6475 MRI.replaceRegWith(DstReg, Inst.getOperand(1).getReg()); 6476 MRI.clearKillFlags(Inst.getOperand(1).getReg()); 6477 Inst.getOperand(0).setReg(DstReg); 6478 6479 // Make sure we don't leave around a dead VGPR->SGPR copy. Normally 6480 // these are deleted later, but at -O0 it would leave a suspicious 6481 // looking illegal copy of an undef register. 6482 for (unsigned I = Inst.getNumOperands() - 1; I != 0; --I) 6483 Inst.removeOperand(I); 6484 Inst.setDesc(get(AMDGPU::IMPLICIT_DEF)); 6485 continue; 6486 } 6487 6488 Register NewDstReg = MRI.createVirtualRegister(NewDstRC); 6489 MRI.replaceRegWith(DstReg, NewDstReg); 6490 legalizeOperands(Inst, MDT); 6491 addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist); 6492 continue; 6493 } 6494 6495 // Use the new VALU Opcode. 6496 auto NewInstr = BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(NewOpcode)) 6497 .setMIFlags(Inst.getFlags()); 6498 for (const MachineOperand &Op : Inst.explicit_operands()) 6499 NewInstr->addOperand(Op); 6500 6501 // Remove any references to SCC. Vector instructions can't read from it, and 6502 // We're just about to add the implicit use / defs of VCC, and we don't want 6503 // both. 6504 for (MachineOperand &Op : Inst.implicit_operands()) { 6505 if (Op.getReg() == AMDGPU::SCC) { 6506 // Only propagate through live-def of SCC. 6507 if (Op.isDef() && !Op.isDead()) 6508 addSCCDefUsersToVALUWorklist(Op, Inst, Worklist); 6509 if (Op.isUse()) 6510 addSCCDefsToVALUWorklist(NewInstr, Worklist); 6511 } 6512 } 6513 6514 Inst.eraseFromParent(); 6515 6516 Register NewDstReg; 6517 if (NewInstr->getOperand(0).isReg() && NewInstr->getOperand(0).isDef()) { 6518 Register DstReg = NewInstr->getOperand(0).getReg(); 6519 assert(DstReg.isVirtual()); 6520 6521 // Update the destination register class. 6522 const TargetRegisterClass *NewDstRC = 6523 getDestEquivalentVGPRClass(*NewInstr); 6524 assert(NewDstRC); 6525 6526 NewDstReg = MRI.createVirtualRegister(NewDstRC); 6527 MRI.replaceRegWith(DstReg, NewDstReg); 6528 } 6529 6530 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) { 6531 // We are converting these to a BFE, so we need to add the missing 6532 // operands for the size and offset. 6533 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16; 6534 NewInstr.addImm(0); 6535 NewInstr.addImm(Size); 6536 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) { 6537 // The VALU version adds the second operand to the result, so insert an 6538 // extra 0 operand. 6539 NewInstr.addImm(0); 6540 } 6541 6542 if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) { 6543 const MachineOperand &OffsetWidthOp = NewInstr->getOperand(2); 6544 // If we need to move this to VGPRs, we need to unpack the second operand 6545 // back into the 2 separate ones for bit offset and width. 6546 assert(OffsetWidthOp.isImm() && 6547 "Scalar BFE is only implemented for constant width and offset"); 6548 uint32_t Imm = OffsetWidthOp.getImm(); 6549 6550 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0]. 6551 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16]. 6552 NewInstr->removeOperand(2); 6553 NewInstr.addImm(Offset); 6554 NewInstr.addImm(BitWidth); 6555 } 6556 6557 fixImplicitOperands(*NewInstr); 6558 6559 // Legalize the operands 6560 CreatedBBTmp = legalizeOperands(*NewInstr, MDT); 6561 if (CreatedBBTmp && TopInst.getParent() == CreatedBBTmp) 6562 CreatedBB = CreatedBBTmp; 6563 6564 if (NewDstReg) 6565 addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist); 6566 } 6567 return CreatedBB; 6568 } 6569 6570 // Add/sub require special handling to deal with carry outs. 6571 std::pair<bool, MachineBasicBlock *> 6572 SIInstrInfo::moveScalarAddSub(SetVectorType &Worklist, MachineInstr &Inst, 6573 MachineDominatorTree *MDT) const { 6574 if (ST.hasAddNoCarry()) { 6575 // Assume there is no user of scc since we don't select this in that case. 6576 // Since scc isn't used, it doesn't really matter if the i32 or u32 variant 6577 // is used. 6578 6579 MachineBasicBlock &MBB = *Inst.getParent(); 6580 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6581 6582 Register OldDstReg = Inst.getOperand(0).getReg(); 6583 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6584 6585 unsigned Opc = Inst.getOpcode(); 6586 assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32); 6587 6588 unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ? 6589 AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64; 6590 6591 assert(Inst.getOperand(3).getReg() == AMDGPU::SCC); 6592 Inst.removeOperand(3); 6593 6594 Inst.setDesc(get(NewOpc)); 6595 Inst.addOperand(MachineOperand::CreateImm(0)); // clamp bit 6596 Inst.addImplicitDefUseOperands(*MBB.getParent()); 6597 MRI.replaceRegWith(OldDstReg, ResultReg); 6598 MachineBasicBlock *NewBB = legalizeOperands(Inst, MDT); 6599 6600 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 6601 return std::pair(true, NewBB); 6602 } 6603 6604 return std::pair(false, nullptr); 6605 } 6606 6607 void SIInstrInfo::lowerSelect(SetVectorType &Worklist, MachineInstr &Inst, 6608 MachineDominatorTree *MDT) const { 6609 6610 MachineBasicBlock &MBB = *Inst.getParent(); 6611 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6612 MachineBasicBlock::iterator MII = Inst; 6613 DebugLoc DL = Inst.getDebugLoc(); 6614 6615 MachineOperand &Dest = Inst.getOperand(0); 6616 MachineOperand &Src0 = Inst.getOperand(1); 6617 MachineOperand &Src1 = Inst.getOperand(2); 6618 MachineOperand &Cond = Inst.getOperand(3); 6619 6620 Register SCCSource = Cond.getReg(); 6621 bool IsSCC = (SCCSource == AMDGPU::SCC); 6622 6623 // If this is a trivial select where the condition is effectively not SCC 6624 // (SCCSource is a source of copy to SCC), then the select is semantically 6625 // equivalent to copying SCCSource. Hence, there is no need to create 6626 // V_CNDMASK, we can just use that and bail out. 6627 if (!IsSCC && Src0.isImm() && (Src0.getImm() == -1) && Src1.isImm() && 6628 (Src1.getImm() == 0)) { 6629 MRI.replaceRegWith(Dest.getReg(), SCCSource); 6630 return; 6631 } 6632 6633 const TargetRegisterClass *TC = 6634 RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 6635 6636 Register CopySCC = MRI.createVirtualRegister(TC); 6637 6638 if (IsSCC) { 6639 // Now look for the closest SCC def if it is a copy 6640 // replacing the SCCSource with the COPY source register 6641 bool CopyFound = false; 6642 for (MachineInstr &CandI : 6643 make_range(std::next(MachineBasicBlock::reverse_iterator(Inst)), 6644 Inst.getParent()->rend())) { 6645 if (CandI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != 6646 -1) { 6647 if (CandI.isCopy() && CandI.getOperand(0).getReg() == AMDGPU::SCC) { 6648 BuildMI(MBB, MII, DL, get(AMDGPU::COPY), CopySCC) 6649 .addReg(CandI.getOperand(1).getReg()); 6650 CopyFound = true; 6651 } 6652 break; 6653 } 6654 } 6655 if (!CopyFound) { 6656 // SCC def is not a copy 6657 // Insert a trivial select instead of creating a copy, because a copy from 6658 // SCC would semantically mean just copying a single bit, but we may need 6659 // the result to be a vector condition mask that needs preserving. 6660 unsigned Opcode = (ST.getWavefrontSize() == 64) ? AMDGPU::S_CSELECT_B64 6661 : AMDGPU::S_CSELECT_B32; 6662 auto NewSelect = 6663 BuildMI(MBB, MII, DL, get(Opcode), CopySCC).addImm(-1).addImm(0); 6664 NewSelect->getOperand(3).setIsUndef(Cond.isUndef()); 6665 } 6666 } 6667 6668 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6669 6670 auto UpdatedInst = 6671 BuildMI(MBB, MII, DL, get(AMDGPU::V_CNDMASK_B32_e64), ResultReg) 6672 .addImm(0) 6673 .add(Src1) // False 6674 .addImm(0) 6675 .add(Src0) // True 6676 .addReg(IsSCC ? CopySCC : SCCSource); 6677 6678 MRI.replaceRegWith(Dest.getReg(), ResultReg); 6679 legalizeOperands(*UpdatedInst, MDT); 6680 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 6681 } 6682 6683 void SIInstrInfo::lowerScalarAbs(SetVectorType &Worklist, 6684 MachineInstr &Inst) const { 6685 MachineBasicBlock &MBB = *Inst.getParent(); 6686 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6687 MachineBasicBlock::iterator MII = Inst; 6688 DebugLoc DL = Inst.getDebugLoc(); 6689 6690 MachineOperand &Dest = Inst.getOperand(0); 6691 MachineOperand &Src = Inst.getOperand(1); 6692 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6693 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6694 6695 unsigned SubOp = ST.hasAddNoCarry() ? 6696 AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_CO_U32_e32; 6697 6698 BuildMI(MBB, MII, DL, get(SubOp), TmpReg) 6699 .addImm(0) 6700 .addReg(Src.getReg()); 6701 6702 BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg) 6703 .addReg(Src.getReg()) 6704 .addReg(TmpReg); 6705 6706 MRI.replaceRegWith(Dest.getReg(), ResultReg); 6707 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 6708 } 6709 6710 void SIInstrInfo::lowerScalarXnor(SetVectorType &Worklist, 6711 MachineInstr &Inst) const { 6712 MachineBasicBlock &MBB = *Inst.getParent(); 6713 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6714 MachineBasicBlock::iterator MII = Inst; 6715 const DebugLoc &DL = Inst.getDebugLoc(); 6716 6717 MachineOperand &Dest = Inst.getOperand(0); 6718 MachineOperand &Src0 = Inst.getOperand(1); 6719 MachineOperand &Src1 = Inst.getOperand(2); 6720 6721 if (ST.hasDLInsts()) { 6722 Register NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6723 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL); 6724 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL); 6725 6726 BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest) 6727 .add(Src0) 6728 .add(Src1); 6729 6730 MRI.replaceRegWith(Dest.getReg(), NewDest); 6731 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist); 6732 } else { 6733 // Using the identity !(x ^ y) == (!x ^ y) == (x ^ !y), we can 6734 // invert either source and then perform the XOR. If either source is a 6735 // scalar register, then we can leave the inversion on the scalar unit to 6736 // achieve a better distribution of scalar and vector instructions. 6737 bool Src0IsSGPR = Src0.isReg() && 6738 RI.isSGPRClass(MRI.getRegClass(Src0.getReg())); 6739 bool Src1IsSGPR = Src1.isReg() && 6740 RI.isSGPRClass(MRI.getRegClass(Src1.getReg())); 6741 MachineInstr *Xor; 6742 Register Temp = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 6743 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 6744 6745 // Build a pair of scalar instructions and add them to the work list. 6746 // The next iteration over the work list will lower these to the vector 6747 // unit as necessary. 6748 if (Src0IsSGPR) { 6749 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src0); 6750 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest) 6751 .addReg(Temp) 6752 .add(Src1); 6753 } else if (Src1IsSGPR) { 6754 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src1); 6755 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest) 6756 .add(Src0) 6757 .addReg(Temp); 6758 } else { 6759 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), Temp) 6760 .add(Src0) 6761 .add(Src1); 6762 MachineInstr *Not = 6763 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest).addReg(Temp); 6764 Worklist.insert(Not); 6765 } 6766 6767 MRI.replaceRegWith(Dest.getReg(), NewDest); 6768 6769 Worklist.insert(Xor); 6770 6771 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist); 6772 } 6773 } 6774 6775 void SIInstrInfo::splitScalarNotBinop(SetVectorType &Worklist, 6776 MachineInstr &Inst, 6777 unsigned Opcode) const { 6778 MachineBasicBlock &MBB = *Inst.getParent(); 6779 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6780 MachineBasicBlock::iterator MII = Inst; 6781 const DebugLoc &DL = Inst.getDebugLoc(); 6782 6783 MachineOperand &Dest = Inst.getOperand(0); 6784 MachineOperand &Src0 = Inst.getOperand(1); 6785 MachineOperand &Src1 = Inst.getOperand(2); 6786 6787 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 6788 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 6789 6790 MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), Interm) 6791 .add(Src0) 6792 .add(Src1); 6793 6794 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest) 6795 .addReg(Interm); 6796 6797 Worklist.insert(&Op); 6798 Worklist.insert(&Not); 6799 6800 MRI.replaceRegWith(Dest.getReg(), NewDest); 6801 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist); 6802 } 6803 6804 void SIInstrInfo::splitScalarBinOpN2(SetVectorType& Worklist, 6805 MachineInstr &Inst, 6806 unsigned Opcode) const { 6807 MachineBasicBlock &MBB = *Inst.getParent(); 6808 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6809 MachineBasicBlock::iterator MII = Inst; 6810 const DebugLoc &DL = Inst.getDebugLoc(); 6811 6812 MachineOperand &Dest = Inst.getOperand(0); 6813 MachineOperand &Src0 = Inst.getOperand(1); 6814 MachineOperand &Src1 = Inst.getOperand(2); 6815 6816 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 6817 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 6818 6819 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Interm) 6820 .add(Src1); 6821 6822 MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), NewDest) 6823 .add(Src0) 6824 .addReg(Interm); 6825 6826 Worklist.insert(&Not); 6827 Worklist.insert(&Op); 6828 6829 MRI.replaceRegWith(Dest.getReg(), NewDest); 6830 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist); 6831 } 6832 6833 void SIInstrInfo::splitScalar64BitUnaryOp( 6834 SetVectorType &Worklist, MachineInstr &Inst, 6835 unsigned Opcode, bool Swap) const { 6836 MachineBasicBlock &MBB = *Inst.getParent(); 6837 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6838 6839 MachineOperand &Dest = Inst.getOperand(0); 6840 MachineOperand &Src0 = Inst.getOperand(1); 6841 DebugLoc DL = Inst.getDebugLoc(); 6842 6843 MachineBasicBlock::iterator MII = Inst; 6844 6845 const MCInstrDesc &InstDesc = get(Opcode); 6846 const TargetRegisterClass *Src0RC = Src0.isReg() ? 6847 MRI.getRegClass(Src0.getReg()) : 6848 &AMDGPU::SGPR_32RegClass; 6849 6850 const TargetRegisterClass *Src0SubRC = 6851 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0); 6852 6853 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 6854 AMDGPU::sub0, Src0SubRC); 6855 6856 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg()); 6857 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC); 6858 const TargetRegisterClass *NewDestSubRC = 6859 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0); 6860 6861 Register DestSub0 = MRI.createVirtualRegister(NewDestSubRC); 6862 MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0).add(SrcReg0Sub0); 6863 6864 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 6865 AMDGPU::sub1, Src0SubRC); 6866 6867 Register DestSub1 = MRI.createVirtualRegister(NewDestSubRC); 6868 MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1).add(SrcReg0Sub1); 6869 6870 if (Swap) 6871 std::swap(DestSub0, DestSub1); 6872 6873 Register FullDestReg = MRI.createVirtualRegister(NewDestRC); 6874 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg) 6875 .addReg(DestSub0) 6876 .addImm(AMDGPU::sub0) 6877 .addReg(DestSub1) 6878 .addImm(AMDGPU::sub1); 6879 6880 MRI.replaceRegWith(Dest.getReg(), FullDestReg); 6881 6882 Worklist.insert(&LoHalf); 6883 Worklist.insert(&HiHalf); 6884 6885 // We don't need to legalizeOperands here because for a single operand, src0 6886 // will support any kind of input. 6887 6888 // Move all users of this moved value. 6889 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist); 6890 } 6891 6892 void SIInstrInfo::splitScalar64BitAddSub(SetVectorType &Worklist, 6893 MachineInstr &Inst, 6894 MachineDominatorTree *MDT) const { 6895 bool IsAdd = (Inst.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO); 6896 6897 MachineBasicBlock &MBB = *Inst.getParent(); 6898 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6899 const auto *CarryRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 6900 6901 Register FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 6902 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6903 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6904 6905 Register CarryReg = MRI.createVirtualRegister(CarryRC); 6906 Register DeadCarryReg = MRI.createVirtualRegister(CarryRC); 6907 6908 MachineOperand &Dest = Inst.getOperand(0); 6909 MachineOperand &Src0 = Inst.getOperand(1); 6910 MachineOperand &Src1 = Inst.getOperand(2); 6911 const DebugLoc &DL = Inst.getDebugLoc(); 6912 MachineBasicBlock::iterator MII = Inst; 6913 6914 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0.getReg()); 6915 const TargetRegisterClass *Src1RC = MRI.getRegClass(Src1.getReg()); 6916 const TargetRegisterClass *Src0SubRC = 6917 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0); 6918 const TargetRegisterClass *Src1SubRC = 6919 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0); 6920 6921 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 6922 AMDGPU::sub0, Src0SubRC); 6923 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, 6924 AMDGPU::sub0, Src1SubRC); 6925 6926 6927 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 6928 AMDGPU::sub1, Src0SubRC); 6929 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, 6930 AMDGPU::sub1, Src1SubRC); 6931 6932 unsigned LoOpc = IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64; 6933 MachineInstr *LoHalf = 6934 BuildMI(MBB, MII, DL, get(LoOpc), DestSub0) 6935 .addReg(CarryReg, RegState::Define) 6936 .add(SrcReg0Sub0) 6937 .add(SrcReg1Sub0) 6938 .addImm(0); // clamp bit 6939 6940 unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64; 6941 MachineInstr *HiHalf = 6942 BuildMI(MBB, MII, DL, get(HiOpc), DestSub1) 6943 .addReg(DeadCarryReg, RegState::Define | RegState::Dead) 6944 .add(SrcReg0Sub1) 6945 .add(SrcReg1Sub1) 6946 .addReg(CarryReg, RegState::Kill) 6947 .addImm(0); // clamp bit 6948 6949 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg) 6950 .addReg(DestSub0) 6951 .addImm(AMDGPU::sub0) 6952 .addReg(DestSub1) 6953 .addImm(AMDGPU::sub1); 6954 6955 MRI.replaceRegWith(Dest.getReg(), FullDestReg); 6956 6957 // Try to legalize the operands in case we need to swap the order to keep it 6958 // valid. 6959 legalizeOperands(*LoHalf, MDT); 6960 legalizeOperands(*HiHalf, MDT); 6961 6962 // Move all users of this moved value. 6963 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist); 6964 } 6965 6966 void SIInstrInfo::splitScalar64BitBinaryOp(SetVectorType &Worklist, 6967 MachineInstr &Inst, unsigned Opcode, 6968 MachineDominatorTree *MDT) const { 6969 MachineBasicBlock &MBB = *Inst.getParent(); 6970 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6971 6972 MachineOperand &Dest = Inst.getOperand(0); 6973 MachineOperand &Src0 = Inst.getOperand(1); 6974 MachineOperand &Src1 = Inst.getOperand(2); 6975 DebugLoc DL = Inst.getDebugLoc(); 6976 6977 MachineBasicBlock::iterator MII = Inst; 6978 6979 const MCInstrDesc &InstDesc = get(Opcode); 6980 const TargetRegisterClass *Src0RC = Src0.isReg() ? 6981 MRI.getRegClass(Src0.getReg()) : 6982 &AMDGPU::SGPR_32RegClass; 6983 6984 const TargetRegisterClass *Src0SubRC = 6985 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0); 6986 const TargetRegisterClass *Src1RC = Src1.isReg() ? 6987 MRI.getRegClass(Src1.getReg()) : 6988 &AMDGPU::SGPR_32RegClass; 6989 6990 const TargetRegisterClass *Src1SubRC = 6991 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0); 6992 6993 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 6994 AMDGPU::sub0, Src0SubRC); 6995 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, 6996 AMDGPU::sub0, Src1SubRC); 6997 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 6998 AMDGPU::sub1, Src0SubRC); 6999 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, 7000 AMDGPU::sub1, Src1SubRC); 7001 7002 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg()); 7003 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC); 7004 const TargetRegisterClass *NewDestSubRC = 7005 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0); 7006 7007 Register DestSub0 = MRI.createVirtualRegister(NewDestSubRC); 7008 MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0) 7009 .add(SrcReg0Sub0) 7010 .add(SrcReg1Sub0); 7011 7012 Register DestSub1 = MRI.createVirtualRegister(NewDestSubRC); 7013 MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1) 7014 .add(SrcReg0Sub1) 7015 .add(SrcReg1Sub1); 7016 7017 Register FullDestReg = MRI.createVirtualRegister(NewDestRC); 7018 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg) 7019 .addReg(DestSub0) 7020 .addImm(AMDGPU::sub0) 7021 .addReg(DestSub1) 7022 .addImm(AMDGPU::sub1); 7023 7024 MRI.replaceRegWith(Dest.getReg(), FullDestReg); 7025 7026 Worklist.insert(&LoHalf); 7027 Worklist.insert(&HiHalf); 7028 7029 // Move all users of this moved value. 7030 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist); 7031 } 7032 7033 void SIInstrInfo::splitScalar64BitXnor(SetVectorType &Worklist, 7034 MachineInstr &Inst, 7035 MachineDominatorTree *MDT) const { 7036 MachineBasicBlock &MBB = *Inst.getParent(); 7037 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7038 7039 MachineOperand &Dest = Inst.getOperand(0); 7040 MachineOperand &Src0 = Inst.getOperand(1); 7041 MachineOperand &Src1 = Inst.getOperand(2); 7042 const DebugLoc &DL = Inst.getDebugLoc(); 7043 7044 MachineBasicBlock::iterator MII = Inst; 7045 7046 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg()); 7047 7048 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); 7049 7050 MachineOperand* Op0; 7051 MachineOperand* Op1; 7052 7053 if (Src0.isReg() && RI.isSGPRReg(MRI, Src0.getReg())) { 7054 Op0 = &Src0; 7055 Op1 = &Src1; 7056 } else { 7057 Op0 = &Src1; 7058 Op1 = &Src0; 7059 } 7060 7061 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B64), Interm) 7062 .add(*Op0); 7063 7064 Register NewDest = MRI.createVirtualRegister(DestRC); 7065 7066 MachineInstr &Xor = *BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B64), NewDest) 7067 .addReg(Interm) 7068 .add(*Op1); 7069 7070 MRI.replaceRegWith(Dest.getReg(), NewDest); 7071 7072 Worklist.insert(&Xor); 7073 } 7074 7075 void SIInstrInfo::splitScalar64BitBCNT( 7076 SetVectorType &Worklist, MachineInstr &Inst) const { 7077 MachineBasicBlock &MBB = *Inst.getParent(); 7078 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7079 7080 MachineBasicBlock::iterator MII = Inst; 7081 const DebugLoc &DL = Inst.getDebugLoc(); 7082 7083 MachineOperand &Dest = Inst.getOperand(0); 7084 MachineOperand &Src = Inst.getOperand(1); 7085 7086 const MCInstrDesc &InstDesc = get(AMDGPU::V_BCNT_U32_B32_e64); 7087 const TargetRegisterClass *SrcRC = Src.isReg() ? 7088 MRI.getRegClass(Src.getReg()) : 7089 &AMDGPU::SGPR_32RegClass; 7090 7091 Register MidReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7092 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7093 7094 const TargetRegisterClass *SrcSubRC = 7095 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0); 7096 7097 MachineOperand SrcRegSub0 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, 7098 AMDGPU::sub0, SrcSubRC); 7099 MachineOperand SrcRegSub1 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, 7100 AMDGPU::sub1, SrcSubRC); 7101 7102 BuildMI(MBB, MII, DL, InstDesc, MidReg).add(SrcRegSub0).addImm(0); 7103 7104 BuildMI(MBB, MII, DL, InstDesc, ResultReg).add(SrcRegSub1).addReg(MidReg); 7105 7106 MRI.replaceRegWith(Dest.getReg(), ResultReg); 7107 7108 // We don't need to legalize operands here. src0 for either instruction can be 7109 // an SGPR, and the second input is unused or determined here. 7110 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 7111 } 7112 7113 void SIInstrInfo::splitScalar64BitBFE(SetVectorType &Worklist, 7114 MachineInstr &Inst) const { 7115 MachineBasicBlock &MBB = *Inst.getParent(); 7116 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7117 MachineBasicBlock::iterator MII = Inst; 7118 const DebugLoc &DL = Inst.getDebugLoc(); 7119 7120 MachineOperand &Dest = Inst.getOperand(0); 7121 uint32_t Imm = Inst.getOperand(2).getImm(); 7122 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0]. 7123 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16]. 7124 7125 (void) Offset; 7126 7127 // Only sext_inreg cases handled. 7128 assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 && 7129 Offset == 0 && "Not implemented"); 7130 7131 if (BitWidth < 32) { 7132 Register MidRegLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7133 Register MidRegHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7134 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 7135 7136 BuildMI(MBB, MII, DL, get(AMDGPU::V_BFE_I32_e64), MidRegLo) 7137 .addReg(Inst.getOperand(1).getReg(), 0, AMDGPU::sub0) 7138 .addImm(0) 7139 .addImm(BitWidth); 7140 7141 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e32), MidRegHi) 7142 .addImm(31) 7143 .addReg(MidRegLo); 7144 7145 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg) 7146 .addReg(MidRegLo) 7147 .addImm(AMDGPU::sub0) 7148 .addReg(MidRegHi) 7149 .addImm(AMDGPU::sub1); 7150 7151 MRI.replaceRegWith(Dest.getReg(), ResultReg); 7152 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 7153 return; 7154 } 7155 7156 MachineOperand &Src = Inst.getOperand(1); 7157 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7158 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 7159 7160 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e64), TmpReg) 7161 .addImm(31) 7162 .addReg(Src.getReg(), 0, AMDGPU::sub0); 7163 7164 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg) 7165 .addReg(Src.getReg(), 0, AMDGPU::sub0) 7166 .addImm(AMDGPU::sub0) 7167 .addReg(TmpReg) 7168 .addImm(AMDGPU::sub1); 7169 7170 MRI.replaceRegWith(Dest.getReg(), ResultReg); 7171 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 7172 } 7173 7174 void SIInstrInfo::addUsersToMoveToVALUWorklist( 7175 Register DstReg, 7176 MachineRegisterInfo &MRI, 7177 SetVectorType &Worklist) const { 7178 for (MachineRegisterInfo::use_iterator I = MRI.use_begin(DstReg), 7179 E = MRI.use_end(); I != E;) { 7180 MachineInstr &UseMI = *I->getParent(); 7181 7182 unsigned OpNo = 0; 7183 7184 switch (UseMI.getOpcode()) { 7185 case AMDGPU::COPY: 7186 case AMDGPU::WQM: 7187 case AMDGPU::SOFT_WQM: 7188 case AMDGPU::STRICT_WWM: 7189 case AMDGPU::STRICT_WQM: 7190 case AMDGPU::REG_SEQUENCE: 7191 case AMDGPU::PHI: 7192 case AMDGPU::INSERT_SUBREG: 7193 break; 7194 default: 7195 OpNo = I.getOperandNo(); 7196 break; 7197 } 7198 7199 if (!RI.hasVectorRegisters(getOpRegClass(UseMI, OpNo))) { 7200 Worklist.insert(&UseMI); 7201 7202 do { 7203 ++I; 7204 } while (I != E && I->getParent() == &UseMI); 7205 } else { 7206 ++I; 7207 } 7208 } 7209 } 7210 7211 void SIInstrInfo::movePackToVALU(SetVectorType &Worklist, 7212 MachineRegisterInfo &MRI, 7213 MachineInstr &Inst) const { 7214 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7215 MachineBasicBlock *MBB = Inst.getParent(); 7216 MachineOperand &Src0 = Inst.getOperand(1); 7217 MachineOperand &Src1 = Inst.getOperand(2); 7218 const DebugLoc &DL = Inst.getDebugLoc(); 7219 7220 switch (Inst.getOpcode()) { 7221 case AMDGPU::S_PACK_LL_B32_B16: { 7222 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7223 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7224 7225 // FIXME: Can do a lot better if we know the high bits of src0 or src1 are 7226 // 0. 7227 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) 7228 .addImm(0xffff); 7229 7230 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_B32_e64), TmpReg) 7231 .addReg(ImmReg, RegState::Kill) 7232 .add(Src0); 7233 7234 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32_e64), ResultReg) 7235 .add(Src1) 7236 .addImm(16) 7237 .addReg(TmpReg, RegState::Kill); 7238 break; 7239 } 7240 case AMDGPU::S_PACK_LH_B32_B16: { 7241 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7242 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) 7243 .addImm(0xffff); 7244 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_BFI_B32_e64), ResultReg) 7245 .addReg(ImmReg, RegState::Kill) 7246 .add(Src0) 7247 .add(Src1); 7248 break; 7249 } 7250 case AMDGPU::S_PACK_HL_B32_B16: { 7251 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7252 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg) 7253 .addImm(16) 7254 .add(Src0); 7255 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32_e64), ResultReg) 7256 .add(Src1) 7257 .addImm(16) 7258 .addReg(TmpReg, RegState::Kill); 7259 break; 7260 } 7261 case AMDGPU::S_PACK_HH_B32_B16: { 7262 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7263 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 7264 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg) 7265 .addImm(16) 7266 .add(Src0); 7267 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) 7268 .addImm(0xffff0000); 7269 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_OR_B32_e64), ResultReg) 7270 .add(Src1) 7271 .addReg(ImmReg, RegState::Kill) 7272 .addReg(TmpReg, RegState::Kill); 7273 break; 7274 } 7275 default: 7276 llvm_unreachable("unhandled s_pack_* instruction"); 7277 } 7278 7279 MachineOperand &Dest = Inst.getOperand(0); 7280 MRI.replaceRegWith(Dest.getReg(), ResultReg); 7281 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 7282 } 7283 7284 void SIInstrInfo::addSCCDefUsersToVALUWorklist(MachineOperand &Op, 7285 MachineInstr &SCCDefInst, 7286 SetVectorType &Worklist, 7287 Register NewCond) const { 7288 7289 // Ensure that def inst defines SCC, which is still live. 7290 assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isDef() && 7291 !Op.isDead() && Op.getParent() == &SCCDefInst); 7292 SmallVector<MachineInstr *, 4> CopyToDelete; 7293 // This assumes that all the users of SCC are in the same block 7294 // as the SCC def. 7295 for (MachineInstr &MI : // Skip the def inst itself. 7296 make_range(std::next(MachineBasicBlock::iterator(SCCDefInst)), 7297 SCCDefInst.getParent()->end())) { 7298 // Check if SCC is used first. 7299 int SCCIdx = MI.findRegisterUseOperandIdx(AMDGPU::SCC, false, &RI); 7300 if (SCCIdx != -1) { 7301 if (MI.isCopy()) { 7302 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 7303 Register DestReg = MI.getOperand(0).getReg(); 7304 7305 MRI.replaceRegWith(DestReg, NewCond); 7306 CopyToDelete.push_back(&MI); 7307 } else { 7308 7309 if (NewCond.isValid()) 7310 MI.getOperand(SCCIdx).setReg(NewCond); 7311 7312 Worklist.insert(&MI); 7313 } 7314 } 7315 // Exit if we find another SCC def. 7316 if (MI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != -1) 7317 break; 7318 } 7319 for (auto &Copy : CopyToDelete) 7320 Copy->eraseFromParent(); 7321 } 7322 7323 // Instructions that use SCC may be converted to VALU instructions. When that 7324 // happens, the SCC register is changed to VCC_LO. The instruction that defines 7325 // SCC must be changed to an instruction that defines VCC. This function makes 7326 // sure that the instruction that defines SCC is added to the moveToVALU 7327 // worklist. 7328 void SIInstrInfo::addSCCDefsToVALUWorklist(MachineInstr *SCCUseInst, 7329 SetVectorType &Worklist) const { 7330 // Look for a preceding instruction that either defines VCC or SCC. If VCC 7331 // then there is nothing to do because the defining instruction has been 7332 // converted to a VALU already. If SCC then that instruction needs to be 7333 // converted to a VALU. 7334 for (MachineInstr &MI : 7335 make_range(std::next(MachineBasicBlock::reverse_iterator(SCCUseInst)), 7336 SCCUseInst->getParent()->rend())) { 7337 if (MI.modifiesRegister(AMDGPU::VCC, &RI)) 7338 break; 7339 if (MI.definesRegister(AMDGPU::SCC, &RI)) { 7340 Worklist.insert(&MI); 7341 break; 7342 } 7343 } 7344 } 7345 7346 const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass( 7347 const MachineInstr &Inst) const { 7348 const TargetRegisterClass *NewDstRC = getOpRegClass(Inst, 0); 7349 7350 switch (Inst.getOpcode()) { 7351 // For target instructions, getOpRegClass just returns the virtual register 7352 // class associated with the operand, so we need to find an equivalent VGPR 7353 // register class in order to move the instruction to the VALU. 7354 case AMDGPU::COPY: 7355 case AMDGPU::PHI: 7356 case AMDGPU::REG_SEQUENCE: 7357 case AMDGPU::INSERT_SUBREG: 7358 case AMDGPU::WQM: 7359 case AMDGPU::SOFT_WQM: 7360 case AMDGPU::STRICT_WWM: 7361 case AMDGPU::STRICT_WQM: { 7362 const TargetRegisterClass *SrcRC = getOpRegClass(Inst, 1); 7363 if (RI.isAGPRClass(SrcRC)) { 7364 if (RI.isAGPRClass(NewDstRC)) 7365 return nullptr; 7366 7367 switch (Inst.getOpcode()) { 7368 case AMDGPU::PHI: 7369 case AMDGPU::REG_SEQUENCE: 7370 case AMDGPU::INSERT_SUBREG: 7371 NewDstRC = RI.getEquivalentAGPRClass(NewDstRC); 7372 break; 7373 default: 7374 NewDstRC = RI.getEquivalentVGPRClass(NewDstRC); 7375 } 7376 7377 if (!NewDstRC) 7378 return nullptr; 7379 } else { 7380 if (RI.isVGPRClass(NewDstRC) || NewDstRC == &AMDGPU::VReg_1RegClass) 7381 return nullptr; 7382 7383 NewDstRC = RI.getEquivalentVGPRClass(NewDstRC); 7384 if (!NewDstRC) 7385 return nullptr; 7386 } 7387 7388 return NewDstRC; 7389 } 7390 default: 7391 return NewDstRC; 7392 } 7393 } 7394 7395 // Find the one SGPR operand we are allowed to use. 7396 Register SIInstrInfo::findUsedSGPR(const MachineInstr &MI, 7397 int OpIndices[3]) const { 7398 const MCInstrDesc &Desc = MI.getDesc(); 7399 7400 // Find the one SGPR operand we are allowed to use. 7401 // 7402 // First we need to consider the instruction's operand requirements before 7403 // legalizing. Some operands are required to be SGPRs, such as implicit uses 7404 // of VCC, but we are still bound by the constant bus requirement to only use 7405 // one. 7406 // 7407 // If the operand's class is an SGPR, we can never move it. 7408 7409 Register SGPRReg = findImplicitSGPRRead(MI); 7410 if (SGPRReg) 7411 return SGPRReg; 7412 7413 Register UsedSGPRs[3] = {Register()}; 7414 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 7415 7416 for (unsigned i = 0; i < 3; ++i) { 7417 int Idx = OpIndices[i]; 7418 if (Idx == -1) 7419 break; 7420 7421 const MachineOperand &MO = MI.getOperand(Idx); 7422 if (!MO.isReg()) 7423 continue; 7424 7425 // Is this operand statically required to be an SGPR based on the operand 7426 // constraints? 7427 const TargetRegisterClass *OpRC = 7428 RI.getRegClass(Desc.operands()[Idx].RegClass); 7429 bool IsRequiredSGPR = RI.isSGPRClass(OpRC); 7430 if (IsRequiredSGPR) 7431 return MO.getReg(); 7432 7433 // If this could be a VGPR or an SGPR, Check the dynamic register class. 7434 Register Reg = MO.getReg(); 7435 const TargetRegisterClass *RegRC = MRI.getRegClass(Reg); 7436 if (RI.isSGPRClass(RegRC)) 7437 UsedSGPRs[i] = Reg; 7438 } 7439 7440 // We don't have a required SGPR operand, so we have a bit more freedom in 7441 // selecting operands to move. 7442 7443 // Try to select the most used SGPR. If an SGPR is equal to one of the 7444 // others, we choose that. 7445 // 7446 // e.g. 7447 // V_FMA_F32 v0, s0, s0, s0 -> No moves 7448 // V_FMA_F32 v0, s0, s1, s0 -> Move s1 7449 7450 // TODO: If some of the operands are 64-bit SGPRs and some 32, we should 7451 // prefer those. 7452 7453 if (UsedSGPRs[0]) { 7454 if (UsedSGPRs[0] == UsedSGPRs[1] || UsedSGPRs[0] == UsedSGPRs[2]) 7455 SGPRReg = UsedSGPRs[0]; 7456 } 7457 7458 if (!SGPRReg && UsedSGPRs[1]) { 7459 if (UsedSGPRs[1] == UsedSGPRs[2]) 7460 SGPRReg = UsedSGPRs[1]; 7461 } 7462 7463 return SGPRReg; 7464 } 7465 7466 MachineOperand *SIInstrInfo::getNamedOperand(MachineInstr &MI, 7467 unsigned OperandName) const { 7468 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OperandName); 7469 if (Idx == -1) 7470 return nullptr; 7471 7472 return &MI.getOperand(Idx); 7473 } 7474 7475 uint64_t SIInstrInfo::getDefaultRsrcDataFormat() const { 7476 if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) { 7477 int64_t Format = ST.getGeneration() >= AMDGPUSubtarget::GFX11 7478 ? (int64_t)AMDGPU::UfmtGFX11::UFMT_32_FLOAT 7479 : (int64_t)AMDGPU::UfmtGFX10::UFMT_32_FLOAT; 7480 return (Format << 44) | 7481 (1ULL << 56) | // RESOURCE_LEVEL = 1 7482 (3ULL << 60); // OOB_SELECT = 3 7483 } 7484 7485 uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT; 7486 if (ST.isAmdHsaOS()) { 7487 // Set ATC = 1. GFX9 doesn't have this bit. 7488 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) 7489 RsrcDataFormat |= (1ULL << 56); 7490 7491 // Set MTYPE = 2 (MTYPE_UC = uncached). GFX9 doesn't have this. 7492 // BTW, it disables TC L2 and therefore decreases performance. 7493 if (ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS) 7494 RsrcDataFormat |= (2ULL << 59); 7495 } 7496 7497 return RsrcDataFormat; 7498 } 7499 7500 uint64_t SIInstrInfo::getScratchRsrcWords23() const { 7501 uint64_t Rsrc23 = getDefaultRsrcDataFormat() | 7502 AMDGPU::RSRC_TID_ENABLE | 7503 0xffffffff; // Size; 7504 7505 // GFX9 doesn't have ELEMENT_SIZE. 7506 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) { 7507 uint64_t EltSizeValue = Log2_32(ST.getMaxPrivateElementSize(true)) - 1; 7508 Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT; 7509 } 7510 7511 // IndexStride = 64 / 32. 7512 uint64_t IndexStride = ST.getWavefrontSize() == 64 ? 3 : 2; 7513 Rsrc23 |= IndexStride << AMDGPU::RSRC_INDEX_STRIDE_SHIFT; 7514 7515 // If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17]. 7516 // Clear them unless we want a huge stride. 7517 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS && 7518 ST.getGeneration() <= AMDGPUSubtarget::GFX9) 7519 Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT; 7520 7521 return Rsrc23; 7522 } 7523 7524 bool SIInstrInfo::isLowLatencyInstruction(const MachineInstr &MI) const { 7525 unsigned Opc = MI.getOpcode(); 7526 7527 return isSMRD(Opc); 7528 } 7529 7530 bool SIInstrInfo::isHighLatencyDef(int Opc) const { 7531 return get(Opc).mayLoad() && 7532 (isMUBUF(Opc) || isMTBUF(Opc) || isMIMG(Opc) || isFLAT(Opc)); 7533 } 7534 7535 unsigned SIInstrInfo::isStackAccess(const MachineInstr &MI, 7536 int &FrameIndex) const { 7537 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::vaddr); 7538 if (!Addr || !Addr->isFI()) 7539 return Register(); 7540 7541 assert(!MI.memoperands_empty() && 7542 (*MI.memoperands_begin())->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS); 7543 7544 FrameIndex = Addr->getIndex(); 7545 return getNamedOperand(MI, AMDGPU::OpName::vdata)->getReg(); 7546 } 7547 7548 unsigned SIInstrInfo::isSGPRStackAccess(const MachineInstr &MI, 7549 int &FrameIndex) const { 7550 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::addr); 7551 assert(Addr && Addr->isFI()); 7552 FrameIndex = Addr->getIndex(); 7553 return getNamedOperand(MI, AMDGPU::OpName::data)->getReg(); 7554 } 7555 7556 unsigned SIInstrInfo::isLoadFromStackSlot(const MachineInstr &MI, 7557 int &FrameIndex) const { 7558 if (!MI.mayLoad()) 7559 return Register(); 7560 7561 if (isMUBUF(MI) || isVGPRSpill(MI)) 7562 return isStackAccess(MI, FrameIndex); 7563 7564 if (isSGPRSpill(MI)) 7565 return isSGPRStackAccess(MI, FrameIndex); 7566 7567 return Register(); 7568 } 7569 7570 unsigned SIInstrInfo::isStoreToStackSlot(const MachineInstr &MI, 7571 int &FrameIndex) const { 7572 if (!MI.mayStore()) 7573 return Register(); 7574 7575 if (isMUBUF(MI) || isVGPRSpill(MI)) 7576 return isStackAccess(MI, FrameIndex); 7577 7578 if (isSGPRSpill(MI)) 7579 return isSGPRStackAccess(MI, FrameIndex); 7580 7581 return Register(); 7582 } 7583 7584 unsigned SIInstrInfo::getInstBundleSize(const MachineInstr &MI) const { 7585 unsigned Size = 0; 7586 MachineBasicBlock::const_instr_iterator I = MI.getIterator(); 7587 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end(); 7588 while (++I != E && I->isInsideBundle()) { 7589 assert(!I->isBundle() && "No nested bundle!"); 7590 Size += getInstSizeInBytes(*I); 7591 } 7592 7593 return Size; 7594 } 7595 7596 unsigned SIInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { 7597 unsigned Opc = MI.getOpcode(); 7598 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opc); 7599 unsigned DescSize = Desc.getSize(); 7600 7601 // If we have a definitive size, we can use it. Otherwise we need to inspect 7602 // the operands to know the size. 7603 if (isFixedSize(MI)) { 7604 unsigned Size = DescSize; 7605 7606 // If we hit the buggy offset, an extra nop will be inserted in MC so 7607 // estimate the worst case. 7608 if (MI.isBranch() && ST.hasOffset3fBug()) 7609 Size += 4; 7610 7611 return Size; 7612 } 7613 7614 // Instructions may have a 32-bit literal encoded after them. Check 7615 // operands that could ever be literals. 7616 if (isVALU(MI) || isSALU(MI)) { 7617 if (isDPP(MI)) 7618 return DescSize; 7619 bool HasLiteral = false; 7620 for (int I = 0, E = MI.getNumExplicitOperands(); I != E; ++I) { 7621 const MachineOperand &Op = MI.getOperand(I); 7622 const MCOperandInfo &OpInfo = Desc.operands()[I]; 7623 if (!Op.isReg() && !isInlineConstant(Op, OpInfo)) { 7624 HasLiteral = true; 7625 break; 7626 } 7627 } 7628 return HasLiteral ? DescSize + 4 : DescSize; 7629 } 7630 7631 // Check whether we have extra NSA words. 7632 if (isMIMG(MI)) { 7633 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0); 7634 if (VAddr0Idx < 0) 7635 return 8; 7636 7637 int RSrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc); 7638 return 8 + 4 * ((RSrcIdx - VAddr0Idx + 2) / 4); 7639 } 7640 7641 switch (Opc) { 7642 case TargetOpcode::BUNDLE: 7643 return getInstBundleSize(MI); 7644 case TargetOpcode::INLINEASM: 7645 case TargetOpcode::INLINEASM_BR: { 7646 const MachineFunction *MF = MI.getParent()->getParent(); 7647 const char *AsmStr = MI.getOperand(0).getSymbolName(); 7648 return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo(), &ST); 7649 } 7650 default: 7651 if (MI.isMetaInstruction()) 7652 return 0; 7653 return DescSize; 7654 } 7655 } 7656 7657 bool SIInstrInfo::mayAccessFlatAddressSpace(const MachineInstr &MI) const { 7658 if (!isFLAT(MI)) 7659 return false; 7660 7661 if (MI.memoperands_empty()) 7662 return true; 7663 7664 for (const MachineMemOperand *MMO : MI.memoperands()) { 7665 if (MMO->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS) 7666 return true; 7667 } 7668 return false; 7669 } 7670 7671 bool SIInstrInfo::isNonUniformBranchInstr(MachineInstr &Branch) const { 7672 return Branch.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO; 7673 } 7674 7675 void SIInstrInfo::convertNonUniformIfRegion(MachineBasicBlock *IfEntry, 7676 MachineBasicBlock *IfEnd) const { 7677 MachineBasicBlock::iterator TI = IfEntry->getFirstTerminator(); 7678 assert(TI != IfEntry->end()); 7679 7680 MachineInstr *Branch = &(*TI); 7681 MachineFunction *MF = IfEntry->getParent(); 7682 MachineRegisterInfo &MRI = IfEntry->getParent()->getRegInfo(); 7683 7684 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { 7685 Register DstReg = MRI.createVirtualRegister(RI.getBoolRC()); 7686 MachineInstr *SIIF = 7687 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_IF), DstReg) 7688 .add(Branch->getOperand(0)) 7689 .add(Branch->getOperand(1)); 7690 MachineInstr *SIEND = 7691 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_END_CF)) 7692 .addReg(DstReg); 7693 7694 IfEntry->erase(TI); 7695 IfEntry->insert(IfEntry->end(), SIIF); 7696 IfEnd->insert(IfEnd->getFirstNonPHI(), SIEND); 7697 } 7698 } 7699 7700 void SIInstrInfo::convertNonUniformLoopRegion( 7701 MachineBasicBlock *LoopEntry, MachineBasicBlock *LoopEnd) const { 7702 MachineBasicBlock::iterator TI = LoopEnd->getFirstTerminator(); 7703 // We expect 2 terminators, one conditional and one unconditional. 7704 assert(TI != LoopEnd->end()); 7705 7706 MachineInstr *Branch = &(*TI); 7707 MachineFunction *MF = LoopEnd->getParent(); 7708 MachineRegisterInfo &MRI = LoopEnd->getParent()->getRegInfo(); 7709 7710 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { 7711 7712 Register DstReg = MRI.createVirtualRegister(RI.getBoolRC()); 7713 Register BackEdgeReg = MRI.createVirtualRegister(RI.getBoolRC()); 7714 MachineInstrBuilder HeaderPHIBuilder = 7715 BuildMI(*(MF), Branch->getDebugLoc(), get(TargetOpcode::PHI), DstReg); 7716 for (MachineBasicBlock *PMBB : LoopEntry->predecessors()) { 7717 if (PMBB == LoopEnd) { 7718 HeaderPHIBuilder.addReg(BackEdgeReg); 7719 } else { 7720 Register ZeroReg = MRI.createVirtualRegister(RI.getBoolRC()); 7721 materializeImmediate(*PMBB, PMBB->getFirstTerminator(), DebugLoc(), 7722 ZeroReg, 0); 7723 HeaderPHIBuilder.addReg(ZeroReg); 7724 } 7725 HeaderPHIBuilder.addMBB(PMBB); 7726 } 7727 MachineInstr *HeaderPhi = HeaderPHIBuilder; 7728 MachineInstr *SIIFBREAK = BuildMI(*(MF), Branch->getDebugLoc(), 7729 get(AMDGPU::SI_IF_BREAK), BackEdgeReg) 7730 .addReg(DstReg) 7731 .add(Branch->getOperand(0)); 7732 MachineInstr *SILOOP = 7733 BuildMI(*(MF), Branch->getDebugLoc(), get(AMDGPU::SI_LOOP)) 7734 .addReg(BackEdgeReg) 7735 .addMBB(LoopEntry); 7736 7737 LoopEntry->insert(LoopEntry->begin(), HeaderPhi); 7738 LoopEnd->erase(TI); 7739 LoopEnd->insert(LoopEnd->end(), SIIFBREAK); 7740 LoopEnd->insert(LoopEnd->end(), SILOOP); 7741 } 7742 } 7743 7744 ArrayRef<std::pair<int, const char *>> 7745 SIInstrInfo::getSerializableTargetIndices() const { 7746 static const std::pair<int, const char *> TargetIndices[] = { 7747 {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"}, 7748 {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"}, 7749 {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"}, 7750 {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"}, 7751 {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}}; 7752 return ArrayRef(TargetIndices); 7753 } 7754 7755 /// This is used by the post-RA scheduler (SchedulePostRAList.cpp). The 7756 /// post-RA version of misched uses CreateTargetMIHazardRecognizer. 7757 ScheduleHazardRecognizer * 7758 SIInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II, 7759 const ScheduleDAG *DAG) const { 7760 return new GCNHazardRecognizer(DAG->MF); 7761 } 7762 7763 /// This is the hazard recognizer used at -O0 by the PostRAHazardRecognizer 7764 /// pass. 7765 ScheduleHazardRecognizer * 7766 SIInstrInfo::CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const { 7767 return new GCNHazardRecognizer(MF); 7768 } 7769 7770 // Called during: 7771 // - pre-RA scheduling and post-RA scheduling 7772 ScheduleHazardRecognizer * 7773 SIInstrInfo::CreateTargetMIHazardRecognizer(const InstrItineraryData *II, 7774 const ScheduleDAGMI *DAG) const { 7775 // Borrowed from Arm Target 7776 // We would like to restrict this hazard recognizer to only 7777 // post-RA scheduling; we can tell that we're post-RA because we don't 7778 // track VRegLiveness. 7779 if (!DAG->hasVRegLiveness()) 7780 return new GCNHazardRecognizer(DAG->MF); 7781 return TargetInstrInfo::CreateTargetMIHazardRecognizer(II, DAG); 7782 } 7783 7784 std::pair<unsigned, unsigned> 7785 SIInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const { 7786 return std::pair(TF & MO_MASK, TF & ~MO_MASK); 7787 } 7788 7789 ArrayRef<std::pair<unsigned, const char *>> 7790 SIInstrInfo::getSerializableDirectMachineOperandTargetFlags() const { 7791 static const std::pair<unsigned, const char *> TargetFlags[] = { 7792 { MO_GOTPCREL, "amdgpu-gotprel" }, 7793 { MO_GOTPCREL32_LO, "amdgpu-gotprel32-lo" }, 7794 { MO_GOTPCREL32_HI, "amdgpu-gotprel32-hi" }, 7795 { MO_REL32_LO, "amdgpu-rel32-lo" }, 7796 { MO_REL32_HI, "amdgpu-rel32-hi" }, 7797 { MO_ABS32_LO, "amdgpu-abs32-lo" }, 7798 { MO_ABS32_HI, "amdgpu-abs32-hi" }, 7799 }; 7800 7801 return ArrayRef(TargetFlags); 7802 } 7803 7804 ArrayRef<std::pair<MachineMemOperand::Flags, const char *>> 7805 SIInstrInfo::getSerializableMachineMemOperandTargetFlags() const { 7806 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] = 7807 { 7808 {MONoClobber, "amdgpu-noclobber"}, 7809 }; 7810 7811 return ArrayRef(TargetFlags); 7812 } 7813 7814 bool SIInstrInfo::isBasicBlockPrologue(const MachineInstr &MI) const { 7815 return !MI.isTerminator() && MI.getOpcode() != AMDGPU::COPY && 7816 MI.modifiesRegister(AMDGPU::EXEC, &RI); 7817 } 7818 7819 MachineInstrBuilder 7820 SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB, 7821 MachineBasicBlock::iterator I, 7822 const DebugLoc &DL, 7823 Register DestReg) const { 7824 if (ST.hasAddNoCarry()) 7825 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e64), DestReg); 7826 7827 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7828 Register UnusedCarry = MRI.createVirtualRegister(RI.getBoolRC()); 7829 MRI.setRegAllocationHint(UnusedCarry, 0, RI.getVCC()); 7830 7831 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg) 7832 .addReg(UnusedCarry, RegState::Define | RegState::Dead); 7833 } 7834 7835 MachineInstrBuilder SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB, 7836 MachineBasicBlock::iterator I, 7837 const DebugLoc &DL, 7838 Register DestReg, 7839 RegScavenger &RS) const { 7840 if (ST.hasAddNoCarry()) 7841 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e32), DestReg); 7842 7843 // If available, prefer to use vcc. 7844 Register UnusedCarry = !RS.isRegUsed(AMDGPU::VCC) 7845 ? Register(RI.getVCC()) 7846 : RS.scavengeRegister(RI.getBoolRC(), I, 0, false); 7847 7848 // TODO: Users need to deal with this. 7849 if (!UnusedCarry.isValid()) 7850 return MachineInstrBuilder(); 7851 7852 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg) 7853 .addReg(UnusedCarry, RegState::Define | RegState::Dead); 7854 } 7855 7856 bool SIInstrInfo::isKillTerminator(unsigned Opcode) { 7857 switch (Opcode) { 7858 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR: 7859 case AMDGPU::SI_KILL_I1_TERMINATOR: 7860 return true; 7861 default: 7862 return false; 7863 } 7864 } 7865 7866 const MCInstrDesc &SIInstrInfo::getKillTerminatorFromPseudo(unsigned Opcode) const { 7867 switch (Opcode) { 7868 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO: 7869 return get(AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR); 7870 case AMDGPU::SI_KILL_I1_PSEUDO: 7871 return get(AMDGPU::SI_KILL_I1_TERMINATOR); 7872 default: 7873 llvm_unreachable("invalid opcode, expected SI_KILL_*_PSEUDO"); 7874 } 7875 } 7876 7877 void SIInstrInfo::fixImplicitOperands(MachineInstr &MI) const { 7878 if (!ST.isWave32()) 7879 return; 7880 7881 for (auto &Op : MI.implicit_operands()) { 7882 if (Op.isReg() && Op.getReg() == AMDGPU::VCC) 7883 Op.setReg(AMDGPU::VCC_LO); 7884 } 7885 } 7886 7887 bool SIInstrInfo::isBufferSMRD(const MachineInstr &MI) const { 7888 if (!isSMRD(MI)) 7889 return false; 7890 7891 // Check that it is using a buffer resource. 7892 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::sbase); 7893 if (Idx == -1) // e.g. s_memtime 7894 return false; 7895 7896 const auto RCID = MI.getDesc().operands()[Idx].RegClass; 7897 return RI.getRegClass(RCID)->hasSubClassEq(&AMDGPU::SGPR_128RegClass); 7898 } 7899 7900 // Depending on the used address space and instructions, some immediate offsets 7901 // are allowed and some are not. 7902 // In general, flat instruction offsets can only be non-negative, global and 7903 // scratch instruction offsets can also be negative. 7904 // 7905 // There are several bugs related to these offsets: 7906 // On gfx10.1, flat instructions that go into the global address space cannot 7907 // use an offset. 7908 // 7909 // For scratch instructions, the address can be either an SGPR or a VGPR. 7910 // The following offsets can be used, depending on the architecture (x means 7911 // cannot be used): 7912 // +----------------------------+------+------+ 7913 // | Address-Mode | SGPR | VGPR | 7914 // +----------------------------+------+------+ 7915 // | gfx9 | | | 7916 // | negative, 4-aligned offset | x | ok | 7917 // | negative, unaligned offset | x | ok | 7918 // +----------------------------+------+------+ 7919 // | gfx10 | | | 7920 // | negative, 4-aligned offset | ok | ok | 7921 // | negative, unaligned offset | ok | x | 7922 // +----------------------------+------+------+ 7923 // | gfx10.3 | | | 7924 // | negative, 4-aligned offset | ok | ok | 7925 // | negative, unaligned offset | ok | ok | 7926 // +----------------------------+------+------+ 7927 // 7928 // This function ignores the addressing mode, so if an offset cannot be used in 7929 // one addressing mode, it is considered illegal. 7930 bool SIInstrInfo::isLegalFLATOffset(int64_t Offset, unsigned AddrSpace, 7931 uint64_t FlatVariant) const { 7932 // TODO: Should 0 be special cased? 7933 if (!ST.hasFlatInstOffsets()) 7934 return false; 7935 7936 if (ST.hasFlatSegmentOffsetBug() && FlatVariant == SIInstrFlags::FLAT && 7937 (AddrSpace == AMDGPUAS::FLAT_ADDRESS || 7938 AddrSpace == AMDGPUAS::GLOBAL_ADDRESS)) 7939 return false; 7940 7941 bool AllowNegative = FlatVariant != SIInstrFlags::FLAT; 7942 if (ST.hasNegativeScratchOffsetBug() && 7943 FlatVariant == SIInstrFlags::FlatScratch) 7944 AllowNegative = false; 7945 if (ST.hasNegativeUnalignedScratchOffsetBug() && 7946 FlatVariant == SIInstrFlags::FlatScratch && Offset < 0 && 7947 (Offset % 4) != 0) { 7948 return false; 7949 } 7950 7951 unsigned N = AMDGPU::getNumFlatOffsetBits(ST); 7952 return isIntN(N, Offset) && (AllowNegative || Offset >= 0); 7953 } 7954 7955 // See comment on SIInstrInfo::isLegalFLATOffset for what is legal and what not. 7956 std::pair<int64_t, int64_t> 7957 SIInstrInfo::splitFlatOffset(int64_t COffsetVal, unsigned AddrSpace, 7958 uint64_t FlatVariant) const { 7959 int64_t RemainderOffset = COffsetVal; 7960 int64_t ImmField = 0; 7961 bool AllowNegative = FlatVariant != SIInstrFlags::FLAT; 7962 if (ST.hasNegativeScratchOffsetBug() && 7963 FlatVariant == SIInstrFlags::FlatScratch) 7964 AllowNegative = false; 7965 7966 const unsigned NumBits = AMDGPU::getNumFlatOffsetBits(ST) - 1; 7967 if (AllowNegative) { 7968 // Use signed division by a power of two to truncate towards 0. 7969 int64_t D = 1LL << NumBits; 7970 RemainderOffset = (COffsetVal / D) * D; 7971 ImmField = COffsetVal - RemainderOffset; 7972 7973 if (ST.hasNegativeUnalignedScratchOffsetBug() && 7974 FlatVariant == SIInstrFlags::FlatScratch && ImmField < 0 && 7975 (ImmField % 4) != 0) { 7976 // Make ImmField a multiple of 4 7977 RemainderOffset += ImmField % 4; 7978 ImmField -= ImmField % 4; 7979 } 7980 } else if (COffsetVal >= 0) { 7981 ImmField = COffsetVal & maskTrailingOnes<uint64_t>(NumBits); 7982 RemainderOffset = COffsetVal - ImmField; 7983 } 7984 7985 assert(isLegalFLATOffset(ImmField, AddrSpace, FlatVariant)); 7986 assert(RemainderOffset + ImmField == COffsetVal); 7987 return {ImmField, RemainderOffset}; 7988 } 7989 7990 // This must be kept in sync with the SIEncodingFamily class in SIInstrInfo.td 7991 // and the columns of the getMCOpcodeGen table. 7992 enum SIEncodingFamily { 7993 SI = 0, 7994 VI = 1, 7995 SDWA = 2, 7996 SDWA9 = 3, 7997 GFX80 = 4, 7998 GFX9 = 5, 7999 GFX10 = 6, 8000 SDWA10 = 7, 8001 GFX90A = 8, 8002 GFX940 = 9, 8003 GFX11 = 10, 8004 }; 8005 8006 static SIEncodingFamily subtargetEncodingFamily(const GCNSubtarget &ST) { 8007 switch (ST.getGeneration()) { 8008 default: 8009 break; 8010 case AMDGPUSubtarget::SOUTHERN_ISLANDS: 8011 case AMDGPUSubtarget::SEA_ISLANDS: 8012 return SIEncodingFamily::SI; 8013 case AMDGPUSubtarget::VOLCANIC_ISLANDS: 8014 case AMDGPUSubtarget::GFX9: 8015 return SIEncodingFamily::VI; 8016 case AMDGPUSubtarget::GFX10: 8017 return SIEncodingFamily::GFX10; 8018 case AMDGPUSubtarget::GFX11: 8019 return SIEncodingFamily::GFX11; 8020 } 8021 llvm_unreachable("Unknown subtarget generation!"); 8022 } 8023 8024 bool SIInstrInfo::isAsmOnlyOpcode(int MCOp) const { 8025 switch(MCOp) { 8026 // These opcodes use indirect register addressing so 8027 // they need special handling by codegen (currently missing). 8028 // Therefore it is too risky to allow these opcodes 8029 // to be selected by dpp combiner or sdwa peepholer. 8030 case AMDGPU::V_MOVRELS_B32_dpp_gfx10: 8031 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10: 8032 case AMDGPU::V_MOVRELD_B32_dpp_gfx10: 8033 case AMDGPU::V_MOVRELD_B32_sdwa_gfx10: 8034 case AMDGPU::V_MOVRELSD_B32_dpp_gfx10: 8035 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10: 8036 case AMDGPU::V_MOVRELSD_2_B32_dpp_gfx10: 8037 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10: 8038 return true; 8039 default: 8040 return false; 8041 } 8042 } 8043 8044 int SIInstrInfo::pseudoToMCOpcode(int Opcode) const { 8045 SIEncodingFamily Gen = subtargetEncodingFamily(ST); 8046 8047 if ((get(Opcode).TSFlags & SIInstrFlags::renamedInGFX9) != 0 && 8048 ST.getGeneration() == AMDGPUSubtarget::GFX9) 8049 Gen = SIEncodingFamily::GFX9; 8050 8051 // Adjust the encoding family to GFX80 for D16 buffer instructions when the 8052 // subtarget has UnpackedD16VMem feature. 8053 // TODO: remove this when we discard GFX80 encoding. 8054 if (ST.hasUnpackedD16VMem() && (get(Opcode).TSFlags & SIInstrFlags::D16Buf)) 8055 Gen = SIEncodingFamily::GFX80; 8056 8057 if (get(Opcode).TSFlags & SIInstrFlags::SDWA) { 8058 switch (ST.getGeneration()) { 8059 default: 8060 Gen = SIEncodingFamily::SDWA; 8061 break; 8062 case AMDGPUSubtarget::GFX9: 8063 Gen = SIEncodingFamily::SDWA9; 8064 break; 8065 case AMDGPUSubtarget::GFX10: 8066 Gen = SIEncodingFamily::SDWA10; 8067 break; 8068 } 8069 } 8070 8071 if (isMAI(Opcode)) { 8072 int MFMAOp = AMDGPU::getMFMAEarlyClobberOp(Opcode); 8073 if (MFMAOp != -1) 8074 Opcode = MFMAOp; 8075 } 8076 8077 int MCOp = AMDGPU::getMCOpcode(Opcode, Gen); 8078 8079 // -1 means that Opcode is already a native instruction. 8080 if (MCOp == -1) 8081 return Opcode; 8082 8083 if (ST.hasGFX90AInsts()) { 8084 uint16_t NMCOp = (uint16_t)-1; 8085 if (ST.hasGFX940Insts()) 8086 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX940); 8087 if (NMCOp == (uint16_t)-1) 8088 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX90A); 8089 if (NMCOp == (uint16_t)-1) 8090 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX9); 8091 if (NMCOp != (uint16_t)-1) 8092 MCOp = NMCOp; 8093 } 8094 8095 // (uint16_t)-1 means that Opcode is a pseudo instruction that has 8096 // no encoding in the given subtarget generation. 8097 if (MCOp == (uint16_t)-1) 8098 return -1; 8099 8100 if (isAsmOnlyOpcode(MCOp)) 8101 return -1; 8102 8103 return MCOp; 8104 } 8105 8106 static 8107 TargetInstrInfo::RegSubRegPair getRegOrUndef(const MachineOperand &RegOpnd) { 8108 assert(RegOpnd.isReg()); 8109 return RegOpnd.isUndef() ? TargetInstrInfo::RegSubRegPair() : 8110 getRegSubRegPair(RegOpnd); 8111 } 8112 8113 TargetInstrInfo::RegSubRegPair 8114 llvm::getRegSequenceSubReg(MachineInstr &MI, unsigned SubReg) { 8115 assert(MI.isRegSequence()); 8116 for (unsigned I = 0, E = (MI.getNumOperands() - 1)/ 2; I < E; ++I) 8117 if (MI.getOperand(1 + 2 * I + 1).getImm() == SubReg) { 8118 auto &RegOp = MI.getOperand(1 + 2 * I); 8119 return getRegOrUndef(RegOp); 8120 } 8121 return TargetInstrInfo::RegSubRegPair(); 8122 } 8123 8124 // Try to find the definition of reg:subreg in subreg-manipulation pseudos 8125 // Following a subreg of reg:subreg isn't supported 8126 static bool followSubRegDef(MachineInstr &MI, 8127 TargetInstrInfo::RegSubRegPair &RSR) { 8128 if (!RSR.SubReg) 8129 return false; 8130 switch (MI.getOpcode()) { 8131 default: break; 8132 case AMDGPU::REG_SEQUENCE: 8133 RSR = getRegSequenceSubReg(MI, RSR.SubReg); 8134 return true; 8135 // EXTRACT_SUBREG ins't supported as this would follow a subreg of subreg 8136 case AMDGPU::INSERT_SUBREG: 8137 if (RSR.SubReg == (unsigned)MI.getOperand(3).getImm()) 8138 // inserted the subreg we're looking for 8139 RSR = getRegOrUndef(MI.getOperand(2)); 8140 else { // the subreg in the rest of the reg 8141 auto R1 = getRegOrUndef(MI.getOperand(1)); 8142 if (R1.SubReg) // subreg of subreg isn't supported 8143 return false; 8144 RSR.Reg = R1.Reg; 8145 } 8146 return true; 8147 } 8148 return false; 8149 } 8150 8151 MachineInstr *llvm::getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P, 8152 MachineRegisterInfo &MRI) { 8153 assert(MRI.isSSA()); 8154 if (!P.Reg.isVirtual()) 8155 return nullptr; 8156 8157 auto RSR = P; 8158 auto *DefInst = MRI.getVRegDef(RSR.Reg); 8159 while (auto *MI = DefInst) { 8160 DefInst = nullptr; 8161 switch (MI->getOpcode()) { 8162 case AMDGPU::COPY: 8163 case AMDGPU::V_MOV_B32_e32: { 8164 auto &Op1 = MI->getOperand(1); 8165 if (Op1.isReg() && Op1.getReg().isVirtual()) { 8166 if (Op1.isUndef()) 8167 return nullptr; 8168 RSR = getRegSubRegPair(Op1); 8169 DefInst = MRI.getVRegDef(RSR.Reg); 8170 } 8171 break; 8172 } 8173 default: 8174 if (followSubRegDef(*MI, RSR)) { 8175 if (!RSR.Reg) 8176 return nullptr; 8177 DefInst = MRI.getVRegDef(RSR.Reg); 8178 } 8179 } 8180 if (!DefInst) 8181 return MI; 8182 } 8183 return nullptr; 8184 } 8185 8186 bool llvm::execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI, 8187 Register VReg, 8188 const MachineInstr &DefMI, 8189 const MachineInstr &UseMI) { 8190 assert(MRI.isSSA() && "Must be run on SSA"); 8191 8192 auto *TRI = MRI.getTargetRegisterInfo(); 8193 auto *DefBB = DefMI.getParent(); 8194 8195 // Don't bother searching between blocks, although it is possible this block 8196 // doesn't modify exec. 8197 if (UseMI.getParent() != DefBB) 8198 return true; 8199 8200 const int MaxInstScan = 20; 8201 int NumInst = 0; 8202 8203 // Stop scan at the use. 8204 auto E = UseMI.getIterator(); 8205 for (auto I = std::next(DefMI.getIterator()); I != E; ++I) { 8206 if (I->isDebugInstr()) 8207 continue; 8208 8209 if (++NumInst > MaxInstScan) 8210 return true; 8211 8212 if (I->modifiesRegister(AMDGPU::EXEC, TRI)) 8213 return true; 8214 } 8215 8216 return false; 8217 } 8218 8219 bool llvm::execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI, 8220 Register VReg, 8221 const MachineInstr &DefMI) { 8222 assert(MRI.isSSA() && "Must be run on SSA"); 8223 8224 auto *TRI = MRI.getTargetRegisterInfo(); 8225 auto *DefBB = DefMI.getParent(); 8226 8227 const int MaxUseScan = 10; 8228 int NumUse = 0; 8229 8230 for (auto &Use : MRI.use_nodbg_operands(VReg)) { 8231 auto &UseInst = *Use.getParent(); 8232 // Don't bother searching between blocks, although it is possible this block 8233 // doesn't modify exec. 8234 if (UseInst.getParent() != DefBB || UseInst.isPHI()) 8235 return true; 8236 8237 if (++NumUse > MaxUseScan) 8238 return true; 8239 } 8240 8241 if (NumUse == 0) 8242 return false; 8243 8244 const int MaxInstScan = 20; 8245 int NumInst = 0; 8246 8247 // Stop scan when we have seen all the uses. 8248 for (auto I = std::next(DefMI.getIterator()); ; ++I) { 8249 assert(I != DefBB->end()); 8250 8251 if (I->isDebugInstr()) 8252 continue; 8253 8254 if (++NumInst > MaxInstScan) 8255 return true; 8256 8257 for (const MachineOperand &Op : I->operands()) { 8258 // We don't check reg masks here as they're used only on calls: 8259 // 1. EXEC is only considered const within one BB 8260 // 2. Call should be a terminator instruction if present in a BB 8261 8262 if (!Op.isReg()) 8263 continue; 8264 8265 Register Reg = Op.getReg(); 8266 if (Op.isUse()) { 8267 if (Reg == VReg && --NumUse == 0) 8268 return false; 8269 } else if (TRI->regsOverlap(Reg, AMDGPU::EXEC)) 8270 return true; 8271 } 8272 } 8273 } 8274 8275 MachineInstr *SIInstrInfo::createPHIDestinationCopy( 8276 MachineBasicBlock &MBB, MachineBasicBlock::iterator LastPHIIt, 8277 const DebugLoc &DL, Register Src, Register Dst) const { 8278 auto Cur = MBB.begin(); 8279 if (Cur != MBB.end()) 8280 do { 8281 if (!Cur->isPHI() && Cur->readsRegister(Dst)) 8282 return BuildMI(MBB, Cur, DL, get(TargetOpcode::COPY), Dst).addReg(Src); 8283 ++Cur; 8284 } while (Cur != MBB.end() && Cur != LastPHIIt); 8285 8286 return TargetInstrInfo::createPHIDestinationCopy(MBB, LastPHIIt, DL, Src, 8287 Dst); 8288 } 8289 8290 MachineInstr *SIInstrInfo::createPHISourceCopy( 8291 MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt, 8292 const DebugLoc &DL, Register Src, unsigned SrcSubReg, Register Dst) const { 8293 if (InsPt != MBB.end() && 8294 (InsPt->getOpcode() == AMDGPU::SI_IF || 8295 InsPt->getOpcode() == AMDGPU::SI_ELSE || 8296 InsPt->getOpcode() == AMDGPU::SI_IF_BREAK) && 8297 InsPt->definesRegister(Src)) { 8298 InsPt++; 8299 return BuildMI(MBB, InsPt, DL, 8300 get(ST.isWave32() ? AMDGPU::S_MOV_B32_term 8301 : AMDGPU::S_MOV_B64_term), 8302 Dst) 8303 .addReg(Src, 0, SrcSubReg) 8304 .addReg(AMDGPU::EXEC, RegState::Implicit); 8305 } 8306 return TargetInstrInfo::createPHISourceCopy(MBB, InsPt, DL, Src, SrcSubReg, 8307 Dst); 8308 } 8309 8310 bool llvm::SIInstrInfo::isWave32() const { return ST.isWave32(); } 8311 8312 MachineInstr *SIInstrInfo::foldMemoryOperandImpl( 8313 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, 8314 MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS, 8315 VirtRegMap *VRM) const { 8316 // This is a bit of a hack (copied from AArch64). Consider this instruction: 8317 // 8318 // %0:sreg_32 = COPY $m0 8319 // 8320 // We explicitly chose SReg_32 for the virtual register so such a copy might 8321 // be eliminated by RegisterCoalescer. However, that may not be possible, and 8322 // %0 may even spill. We can't spill $m0 normally (it would require copying to 8323 // a numbered SGPR anyway), and since it is in the SReg_32 register class, 8324 // TargetInstrInfo::foldMemoryOperand() is going to try. 8325 // A similar issue also exists with spilling and reloading $exec registers. 8326 // 8327 // To prevent that, constrain the %0 register class here. 8328 if (MI.isFullCopy()) { 8329 Register DstReg = MI.getOperand(0).getReg(); 8330 Register SrcReg = MI.getOperand(1).getReg(); 8331 if ((DstReg.isVirtual() || SrcReg.isVirtual()) && 8332 (DstReg.isVirtual() != SrcReg.isVirtual())) { 8333 MachineRegisterInfo &MRI = MF.getRegInfo(); 8334 Register VirtReg = DstReg.isVirtual() ? DstReg : SrcReg; 8335 const TargetRegisterClass *RC = MRI.getRegClass(VirtReg); 8336 if (RC->hasSuperClassEq(&AMDGPU::SReg_32RegClass)) { 8337 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); 8338 return nullptr; 8339 } else if (RC->hasSuperClassEq(&AMDGPU::SReg_64RegClass)) { 8340 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_64_XEXECRegClass); 8341 return nullptr; 8342 } 8343 } 8344 } 8345 8346 return nullptr; 8347 } 8348 8349 unsigned SIInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, 8350 const MachineInstr &MI, 8351 unsigned *PredCost) const { 8352 if (MI.isBundle()) { 8353 MachineBasicBlock::const_instr_iterator I(MI.getIterator()); 8354 MachineBasicBlock::const_instr_iterator E(MI.getParent()->instr_end()); 8355 unsigned Lat = 0, Count = 0; 8356 for (++I; I != E && I->isBundledWithPred(); ++I) { 8357 ++Count; 8358 Lat = std::max(Lat, SchedModel.computeInstrLatency(&*I)); 8359 } 8360 return Lat + Count - 1; 8361 } 8362 8363 return SchedModel.computeInstrLatency(&MI); 8364 } 8365 8366 InstructionUniformity 8367 SIInstrInfo::getGenericInstructionUniformity(const MachineInstr &MI) const { 8368 unsigned opcode = MI.getOpcode(); 8369 if (opcode == AMDGPU::G_INTRINSIC || 8370 opcode == AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS) { 8371 return AMDGPU::isIntrinsicSourceOfDivergence(MI.getIntrinsicID()) 8372 ? InstructionUniformity::NeverUniform 8373 : InstructionUniformity::AlwaysUniform; 8374 } 8375 8376 // Loads from the private and flat address spaces are divergent, because 8377 // threads can execute the load instruction with the same inputs and get 8378 // different results. 8379 // 8380 // All other loads are not divergent, because if threads issue loads with the 8381 // same arguments, they will always get the same result. 8382 if (opcode == AMDGPU::G_LOAD) { 8383 if (MI.memoperands_empty()) 8384 return InstructionUniformity::NeverUniform; // conservative assumption 8385 8386 if (llvm::any_of(MI.memoperands(), [](const MachineMemOperand *mmo) { 8387 return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS || 8388 mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS; 8389 })) { 8390 // At least one MMO in a non-global address space. 8391 return InstructionUniformity::NeverUniform; 8392 } 8393 return InstructionUniformity::Default; 8394 } 8395 8396 if (SIInstrInfo::isGenericAtomicRMWOpcode(opcode) || 8397 opcode == AMDGPU::G_ATOMIC_CMPXCHG || 8398 opcode == AMDGPU::G_ATOMIC_CMPXCHG_WITH_SUCCESS) { 8399 return InstructionUniformity::NeverUniform; 8400 } 8401 return InstructionUniformity::Default; 8402 } 8403 8404 InstructionUniformity 8405 SIInstrInfo::getInstructionUniformity(const MachineInstr &MI) const { 8406 // Atomics are divergent because they are executed sequentially: when an 8407 // atomic operation refers to the same address in each thread, then each 8408 // thread after the first sees the value written by the previous thread as 8409 // original value. 8410 8411 if (isAtomic(MI)) 8412 return InstructionUniformity::NeverUniform; 8413 8414 // Loads from the private and flat address spaces are divergent, because 8415 // threads can execute the load instruction with the same inputs and get 8416 // different results. 8417 if (isFLAT(MI) && MI.mayLoad()) { 8418 if (MI.memoperands_empty()) 8419 return InstructionUniformity::NeverUniform; // conservative assumption 8420 8421 if (llvm::any_of(MI.memoperands(), [](const MachineMemOperand *mmo) { 8422 return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS || 8423 mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS; 8424 })) { 8425 // At least one MMO in a non-global address space. 8426 return InstructionUniformity::NeverUniform; 8427 } 8428 8429 return InstructionUniformity::Default; 8430 } 8431 8432 unsigned opcode = MI.getOpcode(); 8433 if (opcode == AMDGPU::COPY) { 8434 const MachineOperand &srcOp = MI.getOperand(1); 8435 if (srcOp.isReg() && srcOp.getReg().isPhysical()) { 8436 const TargetRegisterClass *regClass = RI.getPhysRegBaseClass(srcOp.getReg()); 8437 return RI.isSGPRClass(regClass) ? InstructionUniformity::AlwaysUniform 8438 : InstructionUniformity::NeverUniform; 8439 } 8440 return InstructionUniformity::Default; 8441 } 8442 if (opcode == AMDGPU::INLINEASM || opcode == AMDGPU::INLINEASM_BR) { 8443 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 8444 for (auto &op : MI.operands()) { 8445 if (!op.isReg() || !op.isDef()) 8446 continue; 8447 auto *RC = MRI.getRegClass(op.getReg()); 8448 if (!RC || RI.isDivergentRegClass(RC)) 8449 return InstructionUniformity::NeverUniform; 8450 } 8451 return InstructionUniformity::AlwaysUniform; 8452 } 8453 if (opcode == AMDGPU::V_READLANE_B32 || opcode == AMDGPU::V_READFIRSTLANE_B32) 8454 return InstructionUniformity::AlwaysUniform; 8455 8456 if (opcode == AMDGPU::V_WRITELANE_B32) 8457 return InstructionUniformity::NeverUniform; 8458 8459 // GMIR handling 8460 if (SIInstrInfo::isGenericOpcode(opcode)) 8461 return SIInstrInfo::getGenericInstructionUniformity(MI); 8462 8463 // Handling $vpgr reads 8464 for (auto srcOp : MI.operands()) { 8465 if (srcOp.isReg() && srcOp.getReg().isPhysical()) { 8466 const TargetRegisterClass *regClass = RI.getPhysRegBaseClass(srcOp.getReg()); 8467 if (RI.isVGPRClass(regClass)) 8468 return InstructionUniformity::NeverUniform; 8469 } 8470 } 8471 8472 // TODO: Uniformity check condtions above can be rearranged for more 8473 // redability 8474 8475 // TODO: amdgcn.{ballot, [if]cmp} should be AlwaysUniform, but they are 8476 // currently turned into no-op COPYs by SelectionDAG ISel and are 8477 // therefore no longer recognizable. 8478 8479 return InstructionUniformity::Default; 8480 } 8481 8482 unsigned SIInstrInfo::getDSShaderTypeValue(const MachineFunction &MF) { 8483 switch (MF.getFunction().getCallingConv()) { 8484 case CallingConv::AMDGPU_PS: 8485 return 1; 8486 case CallingConv::AMDGPU_VS: 8487 return 2; 8488 case CallingConv::AMDGPU_GS: 8489 return 3; 8490 case CallingConv::AMDGPU_HS: 8491 case CallingConv::AMDGPU_LS: 8492 case CallingConv::AMDGPU_ES: 8493 report_fatal_error("ds_ordered_count unsupported for this calling conv"); 8494 case CallingConv::AMDGPU_CS: 8495 case CallingConv::AMDGPU_KERNEL: 8496 case CallingConv::C: 8497 case CallingConv::Fast: 8498 default: 8499 // Assume other calling conventions are various compute callable functions 8500 return 0; 8501 } 8502 } 8503 8504 bool SIInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg, 8505 Register &SrcReg2, int64_t &CmpMask, 8506 int64_t &CmpValue) const { 8507 if (!MI.getOperand(0).isReg() || MI.getOperand(0).getSubReg()) 8508 return false; 8509 8510 switch (MI.getOpcode()) { 8511 default: 8512 break; 8513 case AMDGPU::S_CMP_EQ_U32: 8514 case AMDGPU::S_CMP_EQ_I32: 8515 case AMDGPU::S_CMP_LG_U32: 8516 case AMDGPU::S_CMP_LG_I32: 8517 case AMDGPU::S_CMP_LT_U32: 8518 case AMDGPU::S_CMP_LT_I32: 8519 case AMDGPU::S_CMP_GT_U32: 8520 case AMDGPU::S_CMP_GT_I32: 8521 case AMDGPU::S_CMP_LE_U32: 8522 case AMDGPU::S_CMP_LE_I32: 8523 case AMDGPU::S_CMP_GE_U32: 8524 case AMDGPU::S_CMP_GE_I32: 8525 case AMDGPU::S_CMP_EQ_U64: 8526 case AMDGPU::S_CMP_LG_U64: 8527 SrcReg = MI.getOperand(0).getReg(); 8528 if (MI.getOperand(1).isReg()) { 8529 if (MI.getOperand(1).getSubReg()) 8530 return false; 8531 SrcReg2 = MI.getOperand(1).getReg(); 8532 CmpValue = 0; 8533 } else if (MI.getOperand(1).isImm()) { 8534 SrcReg2 = Register(); 8535 CmpValue = MI.getOperand(1).getImm(); 8536 } else { 8537 return false; 8538 } 8539 CmpMask = ~0; 8540 return true; 8541 case AMDGPU::S_CMPK_EQ_U32: 8542 case AMDGPU::S_CMPK_EQ_I32: 8543 case AMDGPU::S_CMPK_LG_U32: 8544 case AMDGPU::S_CMPK_LG_I32: 8545 case AMDGPU::S_CMPK_LT_U32: 8546 case AMDGPU::S_CMPK_LT_I32: 8547 case AMDGPU::S_CMPK_GT_U32: 8548 case AMDGPU::S_CMPK_GT_I32: 8549 case AMDGPU::S_CMPK_LE_U32: 8550 case AMDGPU::S_CMPK_LE_I32: 8551 case AMDGPU::S_CMPK_GE_U32: 8552 case AMDGPU::S_CMPK_GE_I32: 8553 SrcReg = MI.getOperand(0).getReg(); 8554 SrcReg2 = Register(); 8555 CmpValue = MI.getOperand(1).getImm(); 8556 CmpMask = ~0; 8557 return true; 8558 } 8559 8560 return false; 8561 } 8562 8563 bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, 8564 Register SrcReg2, int64_t CmpMask, 8565 int64_t CmpValue, 8566 const MachineRegisterInfo *MRI) const { 8567 if (!SrcReg || SrcReg.isPhysical()) 8568 return false; 8569 8570 if (SrcReg2 && !getFoldableImm(SrcReg2, *MRI, CmpValue)) 8571 return false; 8572 8573 const auto optimizeCmpAnd = [&CmpInstr, SrcReg, CmpValue, MRI, 8574 this](int64_t ExpectedValue, unsigned SrcSize, 8575 bool IsReversible, bool IsSigned) -> bool { 8576 // s_cmp_eq_u32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n 8577 // s_cmp_eq_i32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n 8578 // s_cmp_ge_u32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n 8579 // s_cmp_ge_i32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n 8580 // s_cmp_eq_u64 (s_and_b64 $src, 1 << n), 1 << n => s_and_b64 $src, 1 << n 8581 // s_cmp_lg_u32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n 8582 // s_cmp_lg_i32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n 8583 // s_cmp_gt_u32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n 8584 // s_cmp_gt_i32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n 8585 // s_cmp_lg_u64 (s_and_b64 $src, 1 << n), 0 => s_and_b64 $src, 1 << n 8586 // 8587 // Signed ge/gt are not used for the sign bit. 8588 // 8589 // If result of the AND is unused except in the compare: 8590 // s_and_b(32|64) $src, 1 << n => s_bitcmp1_b(32|64) $src, n 8591 // 8592 // s_cmp_eq_u32 (s_and_b32 $src, 1 << n), 0 => s_bitcmp0_b32 $src, n 8593 // s_cmp_eq_i32 (s_and_b32 $src, 1 << n), 0 => s_bitcmp0_b32 $src, n 8594 // s_cmp_eq_u64 (s_and_b64 $src, 1 << n), 0 => s_bitcmp0_b64 $src, n 8595 // s_cmp_lg_u32 (s_and_b32 $src, 1 << n), 1 << n => s_bitcmp0_b32 $src, n 8596 // s_cmp_lg_i32 (s_and_b32 $src, 1 << n), 1 << n => s_bitcmp0_b32 $src, n 8597 // s_cmp_lg_u64 (s_and_b64 $src, 1 << n), 1 << n => s_bitcmp0_b64 $src, n 8598 8599 MachineInstr *Def = MRI->getUniqueVRegDef(SrcReg); 8600 if (!Def || Def->getParent() != CmpInstr.getParent()) 8601 return false; 8602 8603 if (Def->getOpcode() != AMDGPU::S_AND_B32 && 8604 Def->getOpcode() != AMDGPU::S_AND_B64) 8605 return false; 8606 8607 int64_t Mask; 8608 const auto isMask = [&Mask, SrcSize](const MachineOperand *MO) -> bool { 8609 if (MO->isImm()) 8610 Mask = MO->getImm(); 8611 else if (!getFoldableImm(MO, Mask)) 8612 return false; 8613 Mask &= maxUIntN(SrcSize); 8614 return isPowerOf2_64(Mask); 8615 }; 8616 8617 MachineOperand *SrcOp = &Def->getOperand(1); 8618 if (isMask(SrcOp)) 8619 SrcOp = &Def->getOperand(2); 8620 else if (isMask(&Def->getOperand(2))) 8621 SrcOp = &Def->getOperand(1); 8622 else 8623 return false; 8624 8625 unsigned BitNo = countTrailingZeros((uint64_t)Mask); 8626 if (IsSigned && BitNo == SrcSize - 1) 8627 return false; 8628 8629 ExpectedValue <<= BitNo; 8630 8631 bool IsReversedCC = false; 8632 if (CmpValue != ExpectedValue) { 8633 if (!IsReversible) 8634 return false; 8635 IsReversedCC = CmpValue == (ExpectedValue ^ Mask); 8636 if (!IsReversedCC) 8637 return false; 8638 } 8639 8640 Register DefReg = Def->getOperand(0).getReg(); 8641 if (IsReversedCC && !MRI->hasOneNonDBGUse(DefReg)) 8642 return false; 8643 8644 for (auto I = std::next(Def->getIterator()), E = CmpInstr.getIterator(); 8645 I != E; ++I) { 8646 if (I->modifiesRegister(AMDGPU::SCC, &RI) || 8647 I->killsRegister(AMDGPU::SCC, &RI)) 8648 return false; 8649 } 8650 8651 MachineOperand *SccDef = Def->findRegisterDefOperand(AMDGPU::SCC); 8652 SccDef->setIsDead(false); 8653 CmpInstr.eraseFromParent(); 8654 8655 if (!MRI->use_nodbg_empty(DefReg)) { 8656 assert(!IsReversedCC); 8657 return true; 8658 } 8659 8660 // Replace AND with unused result with a S_BITCMP. 8661 MachineBasicBlock *MBB = Def->getParent(); 8662 8663 unsigned NewOpc = (SrcSize == 32) ? IsReversedCC ? AMDGPU::S_BITCMP0_B32 8664 : AMDGPU::S_BITCMP1_B32 8665 : IsReversedCC ? AMDGPU::S_BITCMP0_B64 8666 : AMDGPU::S_BITCMP1_B64; 8667 8668 BuildMI(*MBB, Def, Def->getDebugLoc(), get(NewOpc)) 8669 .add(*SrcOp) 8670 .addImm(BitNo); 8671 Def->eraseFromParent(); 8672 8673 return true; 8674 }; 8675 8676 switch (CmpInstr.getOpcode()) { 8677 default: 8678 break; 8679 case AMDGPU::S_CMP_EQ_U32: 8680 case AMDGPU::S_CMP_EQ_I32: 8681 case AMDGPU::S_CMPK_EQ_U32: 8682 case AMDGPU::S_CMPK_EQ_I32: 8683 return optimizeCmpAnd(1, 32, true, false); 8684 case AMDGPU::S_CMP_GE_U32: 8685 case AMDGPU::S_CMPK_GE_U32: 8686 return optimizeCmpAnd(1, 32, false, false); 8687 case AMDGPU::S_CMP_GE_I32: 8688 case AMDGPU::S_CMPK_GE_I32: 8689 return optimizeCmpAnd(1, 32, false, true); 8690 case AMDGPU::S_CMP_EQ_U64: 8691 return optimizeCmpAnd(1, 64, true, false); 8692 case AMDGPU::S_CMP_LG_U32: 8693 case AMDGPU::S_CMP_LG_I32: 8694 case AMDGPU::S_CMPK_LG_U32: 8695 case AMDGPU::S_CMPK_LG_I32: 8696 return optimizeCmpAnd(0, 32, true, false); 8697 case AMDGPU::S_CMP_GT_U32: 8698 case AMDGPU::S_CMPK_GT_U32: 8699 return optimizeCmpAnd(0, 32, false, false); 8700 case AMDGPU::S_CMP_GT_I32: 8701 case AMDGPU::S_CMPK_GT_I32: 8702 return optimizeCmpAnd(0, 32, false, true); 8703 case AMDGPU::S_CMP_LG_U64: 8704 return optimizeCmpAnd(0, 64, true, false); 8705 } 8706 8707 return false; 8708 } 8709 8710 void SIInstrInfo::enforceOperandRCAlignment(MachineInstr &MI, 8711 unsigned OpName) const { 8712 if (!ST.needsAlignedVGPRs()) 8713 return; 8714 8715 int OpNo = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpName); 8716 if (OpNo < 0) 8717 return; 8718 MachineOperand &Op = MI.getOperand(OpNo); 8719 if (getOpSize(MI, OpNo) > 4) 8720 return; 8721 8722 // Add implicit aligned super-reg to force alignment on the data operand. 8723 const DebugLoc &DL = MI.getDebugLoc(); 8724 MachineBasicBlock *BB = MI.getParent(); 8725 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); 8726 Register DataReg = Op.getReg(); 8727 bool IsAGPR = RI.isAGPR(MRI, DataReg); 8728 Register Undef = MRI.createVirtualRegister( 8729 IsAGPR ? &AMDGPU::AGPR_32RegClass : &AMDGPU::VGPR_32RegClass); 8730 BuildMI(*BB, MI, DL, get(AMDGPU::IMPLICIT_DEF), Undef); 8731 Register NewVR = 8732 MRI.createVirtualRegister(IsAGPR ? &AMDGPU::AReg_64_Align2RegClass 8733 : &AMDGPU::VReg_64_Align2RegClass); 8734 BuildMI(*BB, MI, DL, get(AMDGPU::REG_SEQUENCE), NewVR) 8735 .addReg(DataReg, 0, Op.getSubReg()) 8736 .addImm(AMDGPU::sub0) 8737 .addReg(Undef) 8738 .addImm(AMDGPU::sub1); 8739 Op.setReg(NewVR); 8740 Op.setSubReg(AMDGPU::sub0); 8741 MI.addOperand(MachineOperand::CreateReg(NewVR, false, true)); 8742 } 8743