1 //===- SIInstrInfo.cpp - SI Instruction Information ----------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// SI Implementation of TargetInstrInfo. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "SIInstrInfo.h" 15 #include "AMDGPU.h" 16 #include "AMDGPUInstrInfo.h" 17 #include "GCNHazardRecognizer.h" 18 #include "GCNSubtarget.h" 19 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 20 #include "SIMachineFunctionInfo.h" 21 #include "llvm/Analysis/ValueTracking.h" 22 #include "llvm/CodeGen/LiveVariables.h" 23 #include "llvm/CodeGen/MachineDominators.h" 24 #include "llvm/CodeGen/RegisterScavenging.h" 25 #include "llvm/CodeGen/ScheduleDAG.h" 26 #include "llvm/IR/DiagnosticInfo.h" 27 #include "llvm/IR/IntrinsicsAMDGPU.h" 28 #include "llvm/Support/CommandLine.h" 29 #include "llvm/Target/TargetMachine.h" 30 31 using namespace llvm; 32 33 #define DEBUG_TYPE "si-instr-info" 34 35 #define GET_INSTRINFO_CTOR_DTOR 36 #include "AMDGPUGenInstrInfo.inc" 37 38 namespace llvm { 39 40 class AAResults; 41 42 namespace AMDGPU { 43 #define GET_D16ImageDimIntrinsics_IMPL 44 #define GET_ImageDimIntrinsicTable_IMPL 45 #define GET_RsrcIntrinsics_IMPL 46 #include "AMDGPUGenSearchableTables.inc" 47 } 48 } 49 50 51 // Must be at least 4 to be able to branch over minimum unconditional branch 52 // code. This is only for making it possible to write reasonably small tests for 53 // long branches. 54 static cl::opt<unsigned> 55 BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16), 56 cl::desc("Restrict range of branch instructions (DEBUG)")); 57 58 static cl::opt<bool> Fix16BitCopies( 59 "amdgpu-fix-16-bit-physreg-copies", 60 cl::desc("Fix copies between 32 and 16 bit registers by extending to 32 bit"), 61 cl::init(true), 62 cl::ReallyHidden); 63 64 SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST) 65 : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN), 66 RI(ST), ST(ST) { 67 SchedModel.init(&ST); 68 } 69 70 //===----------------------------------------------------------------------===// 71 // TargetInstrInfo callbacks 72 //===----------------------------------------------------------------------===// 73 74 static unsigned getNumOperandsNoGlue(SDNode *Node) { 75 unsigned N = Node->getNumOperands(); 76 while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue) 77 --N; 78 return N; 79 } 80 81 /// Returns true if both nodes have the same value for the given 82 /// operand \p Op, or if both nodes do not have this operand. 83 static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) { 84 unsigned Opc0 = N0->getMachineOpcode(); 85 unsigned Opc1 = N1->getMachineOpcode(); 86 87 int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName); 88 int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName); 89 90 if (Op0Idx == -1 && Op1Idx == -1) 91 return true; 92 93 94 if ((Op0Idx == -1 && Op1Idx != -1) || 95 (Op1Idx == -1 && Op0Idx != -1)) 96 return false; 97 98 // getNamedOperandIdx returns the index for the MachineInstr's operands, 99 // which includes the result as the first operand. We are indexing into the 100 // MachineSDNode's operands, so we need to skip the result operand to get 101 // the real index. 102 --Op0Idx; 103 --Op1Idx; 104 105 return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx); 106 } 107 108 bool SIInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI, 109 AAResults *AA) const { 110 // TODO: The generic check fails for VALU instructions that should be 111 // rematerializable due to implicit reads of exec. We really want all of the 112 // generic logic for this except for this. 113 switch (MI.getOpcode()) { 114 case AMDGPU::V_MOV_B32_e32: 115 case AMDGPU::V_MOV_B32_e64: 116 case AMDGPU::V_MOV_B64_PSEUDO: 117 case AMDGPU::V_ACCVGPR_READ_B32_e64: 118 case AMDGPU::V_ACCVGPR_WRITE_B32_e64: 119 // No implicit operands. 120 return MI.getNumOperands() == MI.getDesc().getNumOperands(); 121 default: 122 return false; 123 } 124 } 125 126 bool SIInstrInfo::areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1, 127 int64_t &Offset0, 128 int64_t &Offset1) const { 129 if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode()) 130 return false; 131 132 unsigned Opc0 = Load0->getMachineOpcode(); 133 unsigned Opc1 = Load1->getMachineOpcode(); 134 135 // Make sure both are actually loads. 136 if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad()) 137 return false; 138 139 if (isDS(Opc0) && isDS(Opc1)) { 140 141 // FIXME: Handle this case: 142 if (getNumOperandsNoGlue(Load0) != getNumOperandsNoGlue(Load1)) 143 return false; 144 145 // Check base reg. 146 if (Load0->getOperand(0) != Load1->getOperand(0)) 147 return false; 148 149 // Skip read2 / write2 variants for simplicity. 150 // TODO: We should report true if the used offsets are adjacent (excluded 151 // st64 versions). 152 int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset); 153 int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset); 154 if (Offset0Idx == -1 || Offset1Idx == -1) 155 return false; 156 157 // XXX - be careful of datalesss loads 158 // getNamedOperandIdx returns the index for MachineInstrs. Since they 159 // include the output in the operand list, but SDNodes don't, we need to 160 // subtract the index by one. 161 Offset0Idx -= get(Opc0).NumDefs; 162 Offset1Idx -= get(Opc1).NumDefs; 163 Offset0 = cast<ConstantSDNode>(Load0->getOperand(Offset0Idx))->getZExtValue(); 164 Offset1 = cast<ConstantSDNode>(Load1->getOperand(Offset1Idx))->getZExtValue(); 165 return true; 166 } 167 168 if (isSMRD(Opc0) && isSMRD(Opc1)) { 169 // Skip time and cache invalidation instructions. 170 if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::sbase) == -1 || 171 AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::sbase) == -1) 172 return false; 173 174 assert(getNumOperandsNoGlue(Load0) == getNumOperandsNoGlue(Load1)); 175 176 // Check base reg. 177 if (Load0->getOperand(0) != Load1->getOperand(0)) 178 return false; 179 180 const ConstantSDNode *Load0Offset = 181 dyn_cast<ConstantSDNode>(Load0->getOperand(1)); 182 const ConstantSDNode *Load1Offset = 183 dyn_cast<ConstantSDNode>(Load1->getOperand(1)); 184 185 if (!Load0Offset || !Load1Offset) 186 return false; 187 188 Offset0 = Load0Offset->getZExtValue(); 189 Offset1 = Load1Offset->getZExtValue(); 190 return true; 191 } 192 193 // MUBUF and MTBUF can access the same addresses. 194 if ((isMUBUF(Opc0) || isMTBUF(Opc0)) && (isMUBUF(Opc1) || isMTBUF(Opc1))) { 195 196 // MUBUF and MTBUF have vaddr at different indices. 197 if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) || 198 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) || 199 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc)) 200 return false; 201 202 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset); 203 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset); 204 205 if (OffIdx0 == -1 || OffIdx1 == -1) 206 return false; 207 208 // getNamedOperandIdx returns the index for MachineInstrs. Since they 209 // include the output in the operand list, but SDNodes don't, we need to 210 // subtract the index by one. 211 OffIdx0 -= get(Opc0).NumDefs; 212 OffIdx1 -= get(Opc1).NumDefs; 213 214 SDValue Off0 = Load0->getOperand(OffIdx0); 215 SDValue Off1 = Load1->getOperand(OffIdx1); 216 217 // The offset might be a FrameIndexSDNode. 218 if (!isa<ConstantSDNode>(Off0) || !isa<ConstantSDNode>(Off1)) 219 return false; 220 221 Offset0 = cast<ConstantSDNode>(Off0)->getZExtValue(); 222 Offset1 = cast<ConstantSDNode>(Off1)->getZExtValue(); 223 return true; 224 } 225 226 return false; 227 } 228 229 static bool isStride64(unsigned Opc) { 230 switch (Opc) { 231 case AMDGPU::DS_READ2ST64_B32: 232 case AMDGPU::DS_READ2ST64_B64: 233 case AMDGPU::DS_WRITE2ST64_B32: 234 case AMDGPU::DS_WRITE2ST64_B64: 235 return true; 236 default: 237 return false; 238 } 239 } 240 241 bool SIInstrInfo::getMemOperandsWithOffsetWidth( 242 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps, 243 int64_t &Offset, bool &OffsetIsScalable, unsigned &Width, 244 const TargetRegisterInfo *TRI) const { 245 if (!LdSt.mayLoadOrStore()) 246 return false; 247 248 unsigned Opc = LdSt.getOpcode(); 249 OffsetIsScalable = false; 250 const MachineOperand *BaseOp, *OffsetOp; 251 int DataOpIdx; 252 253 if (isDS(LdSt)) { 254 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr); 255 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset); 256 if (OffsetOp) { 257 // Normal, single offset LDS instruction. 258 if (!BaseOp) { 259 // DS_CONSUME/DS_APPEND use M0 for the base address. 260 // TODO: find the implicit use operand for M0 and use that as BaseOp? 261 return false; 262 } 263 BaseOps.push_back(BaseOp); 264 Offset = OffsetOp->getImm(); 265 // Get appropriate operand, and compute width accordingly. 266 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 267 if (DataOpIdx == -1) 268 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); 269 Width = getOpSize(LdSt, DataOpIdx); 270 } else { 271 // The 2 offset instructions use offset0 and offset1 instead. We can treat 272 // these as a load with a single offset if the 2 offsets are consecutive. 273 // We will use this for some partially aligned loads. 274 const MachineOperand *Offset0Op = 275 getNamedOperand(LdSt, AMDGPU::OpName::offset0); 276 const MachineOperand *Offset1Op = 277 getNamedOperand(LdSt, AMDGPU::OpName::offset1); 278 279 unsigned Offset0 = Offset0Op->getImm(); 280 unsigned Offset1 = Offset1Op->getImm(); 281 if (Offset0 + 1 != Offset1) 282 return false; 283 284 // Each of these offsets is in element sized units, so we need to convert 285 // to bytes of the individual reads. 286 287 unsigned EltSize; 288 if (LdSt.mayLoad()) 289 EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, 0)) / 16; 290 else { 291 assert(LdSt.mayStore()); 292 int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); 293 EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, Data0Idx)) / 8; 294 } 295 296 if (isStride64(Opc)) 297 EltSize *= 64; 298 299 BaseOps.push_back(BaseOp); 300 Offset = EltSize * Offset0; 301 // Get appropriate operand(s), and compute width accordingly. 302 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 303 if (DataOpIdx == -1) { 304 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); 305 Width = getOpSize(LdSt, DataOpIdx); 306 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1); 307 Width += getOpSize(LdSt, DataOpIdx); 308 } else { 309 Width = getOpSize(LdSt, DataOpIdx); 310 } 311 } 312 return true; 313 } 314 315 if (isMUBUF(LdSt) || isMTBUF(LdSt)) { 316 const MachineOperand *SOffset = getNamedOperand(LdSt, AMDGPU::OpName::soffset); 317 if (SOffset && SOffset->isReg()) { 318 // We can only handle this if it's a stack access, as any other resource 319 // would require reporting multiple base registers. 320 const MachineOperand *AddrReg = getNamedOperand(LdSt, AMDGPU::OpName::vaddr); 321 if (AddrReg && !AddrReg->isFI()) 322 return false; 323 324 const MachineOperand *RSrc = getNamedOperand(LdSt, AMDGPU::OpName::srsrc); 325 const SIMachineFunctionInfo *MFI 326 = LdSt.getParent()->getParent()->getInfo<SIMachineFunctionInfo>(); 327 if (RSrc->getReg() != MFI->getScratchRSrcReg()) 328 return false; 329 330 const MachineOperand *OffsetImm = 331 getNamedOperand(LdSt, AMDGPU::OpName::offset); 332 BaseOps.push_back(RSrc); 333 BaseOps.push_back(SOffset); 334 Offset = OffsetImm->getImm(); 335 } else { 336 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::srsrc); 337 if (!BaseOp) // e.g. BUFFER_WBINVL1_VOL 338 return false; 339 BaseOps.push_back(BaseOp); 340 341 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr); 342 if (BaseOp) 343 BaseOps.push_back(BaseOp); 344 345 const MachineOperand *OffsetImm = 346 getNamedOperand(LdSt, AMDGPU::OpName::offset); 347 Offset = OffsetImm->getImm(); 348 if (SOffset) // soffset can be an inline immediate. 349 Offset += SOffset->getImm(); 350 } 351 // Get appropriate operand, and compute width accordingly. 352 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 353 if (DataOpIdx == -1) 354 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); 355 Width = getOpSize(LdSt, DataOpIdx); 356 return true; 357 } 358 359 if (isMIMG(LdSt)) { 360 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc); 361 BaseOps.push_back(&LdSt.getOperand(SRsrcIdx)); 362 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0); 363 if (VAddr0Idx >= 0) { 364 // GFX10 possible NSA encoding. 365 for (int I = VAddr0Idx; I < SRsrcIdx; ++I) 366 BaseOps.push_back(&LdSt.getOperand(I)); 367 } else { 368 BaseOps.push_back(getNamedOperand(LdSt, AMDGPU::OpName::vaddr)); 369 } 370 Offset = 0; 371 // Get appropriate operand, and compute width accordingly. 372 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); 373 Width = getOpSize(LdSt, DataOpIdx); 374 return true; 375 } 376 377 if (isSMRD(LdSt)) { 378 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::sbase); 379 if (!BaseOp) // e.g. S_MEMTIME 380 return false; 381 BaseOps.push_back(BaseOp); 382 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset); 383 Offset = OffsetOp ? OffsetOp->getImm() : 0; 384 // Get appropriate operand, and compute width accordingly. 385 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::sdst); 386 Width = getOpSize(LdSt, DataOpIdx); 387 return true; 388 } 389 390 if (isFLAT(LdSt)) { 391 // Instructions have either vaddr or saddr or both or none. 392 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr); 393 if (BaseOp) 394 BaseOps.push_back(BaseOp); 395 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr); 396 if (BaseOp) 397 BaseOps.push_back(BaseOp); 398 Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm(); 399 // Get appropriate operand, and compute width accordingly. 400 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 401 if (DataOpIdx == -1) 402 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); 403 Width = getOpSize(LdSt, DataOpIdx); 404 return true; 405 } 406 407 return false; 408 } 409 410 static bool memOpsHaveSameBasePtr(const MachineInstr &MI1, 411 ArrayRef<const MachineOperand *> BaseOps1, 412 const MachineInstr &MI2, 413 ArrayRef<const MachineOperand *> BaseOps2) { 414 // Only examine the first "base" operand of each instruction, on the 415 // assumption that it represents the real base address of the memory access. 416 // Other operands are typically offsets or indices from this base address. 417 if (BaseOps1.front()->isIdenticalTo(*BaseOps2.front())) 418 return true; 419 420 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand()) 421 return false; 422 423 auto MO1 = *MI1.memoperands_begin(); 424 auto MO2 = *MI2.memoperands_begin(); 425 if (MO1->getAddrSpace() != MO2->getAddrSpace()) 426 return false; 427 428 auto Base1 = MO1->getValue(); 429 auto Base2 = MO2->getValue(); 430 if (!Base1 || !Base2) 431 return false; 432 Base1 = getUnderlyingObject(Base1); 433 Base2 = getUnderlyingObject(Base2); 434 435 if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2)) 436 return false; 437 438 return Base1 == Base2; 439 } 440 441 bool SIInstrInfo::shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1, 442 ArrayRef<const MachineOperand *> BaseOps2, 443 unsigned NumLoads, 444 unsigned NumBytes) const { 445 // If the mem ops (to be clustered) do not have the same base ptr, then they 446 // should not be clustered 447 if (!BaseOps1.empty() && !BaseOps2.empty()) { 448 const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent(); 449 const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent(); 450 if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOps1, SecondLdSt, BaseOps2)) 451 return false; 452 } else if (!BaseOps1.empty() || !BaseOps2.empty()) { 453 // If only one base op is empty, they do not have the same base ptr 454 return false; 455 } 456 457 // In order to avoid regester pressure, on an average, the number of DWORDS 458 // loaded together by all clustered mem ops should not exceed 8. This is an 459 // empirical value based on certain observations and performance related 460 // experiments. 461 // The good thing about this heuristic is - it avoids clustering of too many 462 // sub-word loads, and also avoids clustering of wide loads. Below is the 463 // brief summary of how the heuristic behaves for various `LoadSize`. 464 // (1) 1 <= LoadSize <= 4: cluster at max 8 mem ops 465 // (2) 5 <= LoadSize <= 8: cluster at max 4 mem ops 466 // (3) 9 <= LoadSize <= 12: cluster at max 2 mem ops 467 // (4) 13 <= LoadSize <= 16: cluster at max 2 mem ops 468 // (5) LoadSize >= 17: do not cluster 469 const unsigned LoadSize = NumBytes / NumLoads; 470 const unsigned NumDWORDs = ((LoadSize + 3) / 4) * NumLoads; 471 return NumDWORDs <= 8; 472 } 473 474 // FIXME: This behaves strangely. If, for example, you have 32 load + stores, 475 // the first 16 loads will be interleaved with the stores, and the next 16 will 476 // be clustered as expected. It should really split into 2 16 store batches. 477 // 478 // Loads are clustered until this returns false, rather than trying to schedule 479 // groups of stores. This also means we have to deal with saying different 480 // address space loads should be clustered, and ones which might cause bank 481 // conflicts. 482 // 483 // This might be deprecated so it might not be worth that much effort to fix. 484 bool SIInstrInfo::shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1, 485 int64_t Offset0, int64_t Offset1, 486 unsigned NumLoads) const { 487 assert(Offset1 > Offset0 && 488 "Second offset should be larger than first offset!"); 489 // If we have less than 16 loads in a row, and the offsets are within 64 490 // bytes, then schedule together. 491 492 // A cacheline is 64 bytes (for global memory). 493 return (NumLoads <= 16 && (Offset1 - Offset0) < 64); 494 } 495 496 static void reportIllegalCopy(const SIInstrInfo *TII, MachineBasicBlock &MBB, 497 MachineBasicBlock::iterator MI, 498 const DebugLoc &DL, MCRegister DestReg, 499 MCRegister SrcReg, bool KillSrc, 500 const char *Msg = "illegal SGPR to VGPR copy") { 501 MachineFunction *MF = MBB.getParent(); 502 DiagnosticInfoUnsupported IllegalCopy(MF->getFunction(), Msg, DL, DS_Error); 503 LLVMContext &C = MF->getFunction().getContext(); 504 C.diagnose(IllegalCopy); 505 506 BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg) 507 .addReg(SrcReg, getKillRegState(KillSrc)); 508 } 509 510 /// Handle copying from SGPR to AGPR, or from AGPR to AGPR. It is not possible 511 /// to directly copy, so an intermediate VGPR needs to be used. 512 static void indirectCopyToAGPR(const SIInstrInfo &TII, 513 MachineBasicBlock &MBB, 514 MachineBasicBlock::iterator MI, 515 const DebugLoc &DL, MCRegister DestReg, 516 MCRegister SrcReg, bool KillSrc, 517 RegScavenger &RS, 518 Register ImpDefSuperReg = Register(), 519 Register ImpUseSuperReg = Register()) { 520 const SIRegisterInfo &RI = TII.getRegisterInfo(); 521 522 assert(AMDGPU::SReg_32RegClass.contains(SrcReg) || 523 AMDGPU::AGPR_32RegClass.contains(SrcReg)); 524 525 // First try to find defining accvgpr_write to avoid temporary registers. 526 for (auto Def = MI, E = MBB.begin(); Def != E; ) { 527 --Def; 528 if (!Def->definesRegister(SrcReg, &RI)) 529 continue; 530 if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) 531 break; 532 533 MachineOperand &DefOp = Def->getOperand(1); 534 assert(DefOp.isReg() || DefOp.isImm()); 535 536 if (DefOp.isReg()) { 537 // Check that register source operand if not clobbered before MI. 538 // Immediate operands are always safe to propagate. 539 bool SafeToPropagate = true; 540 for (auto I = Def; I != MI && SafeToPropagate; ++I) 541 if (I->modifiesRegister(DefOp.getReg(), &RI)) 542 SafeToPropagate = false; 543 544 if (!SafeToPropagate) 545 break; 546 547 DefOp.setIsKill(false); 548 } 549 550 MachineInstrBuilder Builder = 551 BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) 552 .add(DefOp); 553 if (ImpDefSuperReg) 554 Builder.addReg(ImpDefSuperReg, RegState::Define | RegState::Implicit); 555 556 if (ImpUseSuperReg) { 557 Builder.addReg(ImpUseSuperReg, 558 getKillRegState(KillSrc) | RegState::Implicit); 559 } 560 561 return; 562 } 563 564 RS.enterBasicBlock(MBB); 565 RS.forward(MI); 566 567 // Ideally we want to have three registers for a long reg_sequence copy 568 // to hide 2 waitstates between v_mov_b32 and accvgpr_write. 569 unsigned MaxVGPRs = RI.getRegPressureLimit(&AMDGPU::VGPR_32RegClass, 570 *MBB.getParent()); 571 572 // Registers in the sequence are allocated contiguously so we can just 573 // use register number to pick one of three round-robin temps. 574 unsigned RegNo = DestReg % 3; 575 Register Tmp = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0); 576 if (!Tmp) 577 report_fatal_error("Cannot scavenge VGPR to copy to AGPR"); 578 RS.setRegUsed(Tmp); 579 // Only loop through if there are any free registers left, otherwise 580 // scavenger may report a fatal error without emergency spill slot 581 // or spill with the slot. 582 while (RegNo-- && RS.FindUnusedReg(&AMDGPU::VGPR_32RegClass)) { 583 Register Tmp2 = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0); 584 if (!Tmp2 || RI.getHWRegIndex(Tmp2) >= MaxVGPRs) 585 break; 586 Tmp = Tmp2; 587 RS.setRegUsed(Tmp); 588 } 589 590 // Insert copy to temporary VGPR. 591 unsigned TmpCopyOp = AMDGPU::V_MOV_B32_e32; 592 if (AMDGPU::AGPR_32RegClass.contains(SrcReg)) { 593 TmpCopyOp = AMDGPU::V_ACCVGPR_READ_B32_e64; 594 } else { 595 assert(AMDGPU::SReg_32RegClass.contains(SrcReg)); 596 } 597 598 MachineInstrBuilder UseBuilder = BuildMI(MBB, MI, DL, TII.get(TmpCopyOp), Tmp) 599 .addReg(SrcReg, getKillRegState(KillSrc)); 600 if (ImpUseSuperReg) { 601 UseBuilder.addReg(ImpUseSuperReg, 602 getKillRegState(KillSrc) | RegState::Implicit); 603 } 604 605 MachineInstrBuilder DefBuilder 606 = BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) 607 .addReg(Tmp, RegState::Kill); 608 609 if (ImpDefSuperReg) 610 DefBuilder.addReg(ImpDefSuperReg, RegState::Define | RegState::Implicit); 611 } 612 613 static void expandSGPRCopy(const SIInstrInfo &TII, MachineBasicBlock &MBB, 614 MachineBasicBlock::iterator MI, const DebugLoc &DL, 615 MCRegister DestReg, MCRegister SrcReg, bool KillSrc, 616 const TargetRegisterClass *RC, bool Forward) { 617 const SIRegisterInfo &RI = TII.getRegisterInfo(); 618 ArrayRef<int16_t> BaseIndices = RI.getRegSplitParts(RC, 4); 619 MachineBasicBlock::iterator I = MI; 620 MachineInstr *FirstMI = nullptr, *LastMI = nullptr; 621 622 for (unsigned Idx = 0; Idx < BaseIndices.size(); ++Idx) { 623 int16_t SubIdx = BaseIndices[Idx]; 624 Register Reg = RI.getSubReg(DestReg, SubIdx); 625 unsigned Opcode = AMDGPU::S_MOV_B32; 626 627 // Is SGPR aligned? If so try to combine with next. 628 Register Src = RI.getSubReg(SrcReg, SubIdx); 629 bool AlignedDest = ((Reg - AMDGPU::SGPR0) % 2) == 0; 630 bool AlignedSrc = ((Src - AMDGPU::SGPR0) % 2) == 0; 631 if (AlignedDest && AlignedSrc && (Idx + 1 < BaseIndices.size())) { 632 // Can use SGPR64 copy 633 unsigned Channel = RI.getChannelFromSubReg(SubIdx); 634 SubIdx = RI.getSubRegFromChannel(Channel, 2); 635 Opcode = AMDGPU::S_MOV_B64; 636 Idx++; 637 } 638 639 LastMI = BuildMI(MBB, I, DL, TII.get(Opcode), RI.getSubReg(DestReg, SubIdx)) 640 .addReg(RI.getSubReg(SrcReg, SubIdx)) 641 .addReg(SrcReg, RegState::Implicit); 642 643 if (!FirstMI) 644 FirstMI = LastMI; 645 646 if (!Forward) 647 I--; 648 } 649 650 assert(FirstMI && LastMI); 651 if (!Forward) 652 std::swap(FirstMI, LastMI); 653 654 FirstMI->addOperand( 655 MachineOperand::CreateReg(DestReg, true /*IsDef*/, true /*IsImp*/)); 656 657 if (KillSrc) 658 LastMI->addRegisterKilled(SrcReg, &RI); 659 } 660 661 void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB, 662 MachineBasicBlock::iterator MI, 663 const DebugLoc &DL, MCRegister DestReg, 664 MCRegister SrcReg, bool KillSrc) const { 665 const TargetRegisterClass *RC = RI.getPhysRegClass(DestReg); 666 667 // FIXME: This is hack to resolve copies between 16 bit and 32 bit 668 // registers until all patterns are fixed. 669 if (Fix16BitCopies && 670 ((RI.getRegSizeInBits(*RC) == 16) ^ 671 (RI.getRegSizeInBits(*RI.getPhysRegClass(SrcReg)) == 16))) { 672 MCRegister &RegToFix = (RI.getRegSizeInBits(*RC) == 16) ? DestReg : SrcReg; 673 MCRegister Super = RI.get32BitRegister(RegToFix); 674 assert(RI.getSubReg(Super, AMDGPU::lo16) == RegToFix); 675 RegToFix = Super; 676 677 if (DestReg == SrcReg) { 678 // Insert empty bundle since ExpandPostRA expects an instruction here. 679 BuildMI(MBB, MI, DL, get(AMDGPU::BUNDLE)); 680 return; 681 } 682 683 RC = RI.getPhysRegClass(DestReg); 684 } 685 686 if (RC == &AMDGPU::VGPR_32RegClass) { 687 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) || 688 AMDGPU::SReg_32RegClass.contains(SrcReg) || 689 AMDGPU::AGPR_32RegClass.contains(SrcReg)); 690 unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ? 691 AMDGPU::V_ACCVGPR_READ_B32_e64 : AMDGPU::V_MOV_B32_e32; 692 BuildMI(MBB, MI, DL, get(Opc), DestReg) 693 .addReg(SrcReg, getKillRegState(KillSrc)); 694 return; 695 } 696 697 if (RC == &AMDGPU::SReg_32_XM0RegClass || 698 RC == &AMDGPU::SReg_32RegClass) { 699 if (SrcReg == AMDGPU::SCC) { 700 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg) 701 .addImm(1) 702 .addImm(0); 703 return; 704 } 705 706 if (DestReg == AMDGPU::VCC_LO) { 707 if (AMDGPU::SReg_32RegClass.contains(SrcReg)) { 708 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), AMDGPU::VCC_LO) 709 .addReg(SrcReg, getKillRegState(KillSrc)); 710 } else { 711 // FIXME: Hack until VReg_1 removed. 712 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg)); 713 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32)) 714 .addImm(0) 715 .addReg(SrcReg, getKillRegState(KillSrc)); 716 } 717 718 return; 719 } 720 721 if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) { 722 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc); 723 return; 724 } 725 726 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg) 727 .addReg(SrcReg, getKillRegState(KillSrc)); 728 return; 729 } 730 731 if (RC == &AMDGPU::SReg_64RegClass) { 732 if (SrcReg == AMDGPU::SCC) { 733 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B64), DestReg) 734 .addImm(1) 735 .addImm(0); 736 return; 737 } 738 739 if (DestReg == AMDGPU::VCC) { 740 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) { 741 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC) 742 .addReg(SrcReg, getKillRegState(KillSrc)); 743 } else { 744 // FIXME: Hack until VReg_1 removed. 745 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg)); 746 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32)) 747 .addImm(0) 748 .addReg(SrcReg, getKillRegState(KillSrc)); 749 } 750 751 return; 752 } 753 754 if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) { 755 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc); 756 return; 757 } 758 759 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg) 760 .addReg(SrcReg, getKillRegState(KillSrc)); 761 return; 762 } 763 764 if (DestReg == AMDGPU::SCC) { 765 // Copying 64-bit or 32-bit sources to SCC barely makes sense, 766 // but SelectionDAG emits such copies for i1 sources. 767 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) { 768 // This copy can only be produced by patterns 769 // with explicit SCC, which are known to be enabled 770 // only for subtargets with S_CMP_LG_U64 present. 771 assert(ST.hasScalarCompareEq64()); 772 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U64)) 773 .addReg(SrcReg, getKillRegState(KillSrc)) 774 .addImm(0); 775 } else { 776 assert(AMDGPU::SReg_32RegClass.contains(SrcReg)); 777 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32)) 778 .addReg(SrcReg, getKillRegState(KillSrc)) 779 .addImm(0); 780 } 781 782 return; 783 } 784 785 786 if (RC == &AMDGPU::AGPR_32RegClass) { 787 if (AMDGPU::VGPR_32RegClass.contains(SrcReg)) { 788 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) 789 .addReg(SrcReg, getKillRegState(KillSrc)); 790 return; 791 } 792 793 // FIXME: Pass should maintain scavenger to avoid scan through the block on 794 // every AGPR spill. 795 RegScavenger RS; 796 indirectCopyToAGPR(*this, MBB, MI, DL, DestReg, SrcReg, KillSrc, RS); 797 return; 798 } 799 800 if (RI.getRegSizeInBits(*RC) == 16) { 801 assert(AMDGPU::VGPR_LO16RegClass.contains(SrcReg) || 802 AMDGPU::VGPR_HI16RegClass.contains(SrcReg) || 803 AMDGPU::SReg_LO16RegClass.contains(SrcReg) || 804 AMDGPU::AGPR_LO16RegClass.contains(SrcReg)); 805 806 bool IsSGPRDst = AMDGPU::SReg_LO16RegClass.contains(DestReg); 807 bool IsSGPRSrc = AMDGPU::SReg_LO16RegClass.contains(SrcReg); 808 bool IsAGPRDst = AMDGPU::AGPR_LO16RegClass.contains(DestReg); 809 bool IsAGPRSrc = AMDGPU::AGPR_LO16RegClass.contains(SrcReg); 810 bool DstLow = AMDGPU::VGPR_LO16RegClass.contains(DestReg) || 811 AMDGPU::SReg_LO16RegClass.contains(DestReg) || 812 AMDGPU::AGPR_LO16RegClass.contains(DestReg); 813 bool SrcLow = AMDGPU::VGPR_LO16RegClass.contains(SrcReg) || 814 AMDGPU::SReg_LO16RegClass.contains(SrcReg) || 815 AMDGPU::AGPR_LO16RegClass.contains(SrcReg); 816 MCRegister NewDestReg = RI.get32BitRegister(DestReg); 817 MCRegister NewSrcReg = RI.get32BitRegister(SrcReg); 818 819 if (IsSGPRDst) { 820 if (!IsSGPRSrc) { 821 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc); 822 return; 823 } 824 825 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), NewDestReg) 826 .addReg(NewSrcReg, getKillRegState(KillSrc)); 827 return; 828 } 829 830 if (IsAGPRDst || IsAGPRSrc) { 831 if (!DstLow || !SrcLow) { 832 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc, 833 "Cannot use hi16 subreg with an AGPR!"); 834 } 835 836 copyPhysReg(MBB, MI, DL, NewDestReg, NewSrcReg, KillSrc); 837 return; 838 } 839 840 if (IsSGPRSrc && !ST.hasSDWAScalar()) { 841 if (!DstLow || !SrcLow) { 842 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc, 843 "Cannot use hi16 subreg on VI!"); 844 } 845 846 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), NewDestReg) 847 .addReg(NewSrcReg, getKillRegState(KillSrc)); 848 return; 849 } 850 851 auto MIB = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_sdwa), NewDestReg) 852 .addImm(0) // src0_modifiers 853 .addReg(NewSrcReg) 854 .addImm(0) // clamp 855 .addImm(DstLow ? AMDGPU::SDWA::SdwaSel::WORD_0 856 : AMDGPU::SDWA::SdwaSel::WORD_1) 857 .addImm(AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE) 858 .addImm(SrcLow ? AMDGPU::SDWA::SdwaSel::WORD_0 859 : AMDGPU::SDWA::SdwaSel::WORD_1) 860 .addReg(NewDestReg, RegState::Implicit | RegState::Undef); 861 // First implicit operand is $exec. 862 MIB->tieOperands(0, MIB->getNumOperands() - 1); 863 return; 864 } 865 866 const bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg); 867 if (RI.isSGPRClass(RC)) { 868 if (!RI.isSGPRClass(RI.getPhysRegClass(SrcReg))) { 869 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc); 870 return; 871 } 872 expandSGPRCopy(*this, MBB, MI, DL, DestReg, SrcReg, KillSrc, RC, Forward); 873 return; 874 } 875 876 unsigned Opcode = AMDGPU::V_MOV_B32_e32; 877 if (RI.hasAGPRs(RC)) { 878 Opcode = RI.hasVGPRs(RI.getPhysRegClass(SrcReg)) ? 879 AMDGPU::V_ACCVGPR_WRITE_B32_e64 : AMDGPU::INSTRUCTION_LIST_END; 880 } else if (RI.hasVGPRs(RC) && RI.hasAGPRs(RI.getPhysRegClass(SrcReg))) { 881 Opcode = AMDGPU::V_ACCVGPR_READ_B32_e64; 882 } 883 884 // For the cases where we need an intermediate instruction/temporary register 885 // (destination is an AGPR), we need a scavenger. 886 // 887 // FIXME: The pass should maintain this for us so we don't have to re-scan the 888 // whole block for every handled copy. 889 std::unique_ptr<RegScavenger> RS; 890 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) 891 RS.reset(new RegScavenger()); 892 893 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, 4); 894 895 // If there is an overlap, we can't kill the super-register on the last 896 // instruction, since it will also kill the components made live by this def. 897 const bool CanKillSuperReg = KillSrc && !RI.regsOverlap(SrcReg, DestReg); 898 899 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) { 900 unsigned SubIdx; 901 if (Forward) 902 SubIdx = SubIndices[Idx]; 903 else 904 SubIdx = SubIndices[SubIndices.size() - Idx - 1]; 905 906 bool UseKill = CanKillSuperReg && Idx == SubIndices.size() - 1; 907 908 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) { 909 Register ImpDefSuper = Idx == 0 ? Register(DestReg) : Register(); 910 Register ImpUseSuper = SrcReg; 911 indirectCopyToAGPR(*this, MBB, MI, DL, RI.getSubReg(DestReg, SubIdx), 912 RI.getSubReg(SrcReg, SubIdx), UseKill, *RS, 913 ImpDefSuper, ImpUseSuper); 914 } else { 915 MachineInstrBuilder Builder = 916 BuildMI(MBB, MI, DL, get(Opcode), RI.getSubReg(DestReg, SubIdx)) 917 .addReg(RI.getSubReg(SrcReg, SubIdx)); 918 if (Idx == 0) 919 Builder.addReg(DestReg, RegState::Define | RegState::Implicit); 920 921 Builder.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit); 922 } 923 } 924 } 925 926 int SIInstrInfo::commuteOpcode(unsigned Opcode) const { 927 int NewOpc; 928 929 // Try to map original to commuted opcode 930 NewOpc = AMDGPU::getCommuteRev(Opcode); 931 if (NewOpc != -1) 932 // Check if the commuted (REV) opcode exists on the target. 933 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1; 934 935 // Try to map commuted to original opcode 936 NewOpc = AMDGPU::getCommuteOrig(Opcode); 937 if (NewOpc != -1) 938 // Check if the original (non-REV) opcode exists on the target. 939 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1; 940 941 return Opcode; 942 } 943 944 void SIInstrInfo::materializeImmediate(MachineBasicBlock &MBB, 945 MachineBasicBlock::iterator MI, 946 const DebugLoc &DL, unsigned DestReg, 947 int64_t Value) const { 948 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 949 const TargetRegisterClass *RegClass = MRI.getRegClass(DestReg); 950 if (RegClass == &AMDGPU::SReg_32RegClass || 951 RegClass == &AMDGPU::SGPR_32RegClass || 952 RegClass == &AMDGPU::SReg_32_XM0RegClass || 953 RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) { 954 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg) 955 .addImm(Value); 956 return; 957 } 958 959 if (RegClass == &AMDGPU::SReg_64RegClass || 960 RegClass == &AMDGPU::SGPR_64RegClass || 961 RegClass == &AMDGPU::SReg_64_XEXECRegClass) { 962 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg) 963 .addImm(Value); 964 return; 965 } 966 967 if (RegClass == &AMDGPU::VGPR_32RegClass) { 968 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg) 969 .addImm(Value); 970 return; 971 } 972 if (RegClass == &AMDGPU::VReg_64RegClass) { 973 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg) 974 .addImm(Value); 975 return; 976 } 977 978 unsigned EltSize = 4; 979 unsigned Opcode = AMDGPU::V_MOV_B32_e32; 980 if (RI.isSGPRClass(RegClass)) { 981 if (RI.getRegSizeInBits(*RegClass) > 32) { 982 Opcode = AMDGPU::S_MOV_B64; 983 EltSize = 8; 984 } else { 985 Opcode = AMDGPU::S_MOV_B32; 986 EltSize = 4; 987 } 988 } 989 990 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RegClass, EltSize); 991 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) { 992 int64_t IdxValue = Idx == 0 ? Value : 0; 993 994 MachineInstrBuilder Builder = BuildMI(MBB, MI, DL, 995 get(Opcode), RI.getSubReg(DestReg, SubIndices[Idx])); 996 Builder.addImm(IdxValue); 997 } 998 } 999 1000 const TargetRegisterClass * 1001 SIInstrInfo::getPreferredSelectRegClass(unsigned Size) const { 1002 return &AMDGPU::VGPR_32RegClass; 1003 } 1004 1005 void SIInstrInfo::insertVectorSelect(MachineBasicBlock &MBB, 1006 MachineBasicBlock::iterator I, 1007 const DebugLoc &DL, Register DstReg, 1008 ArrayRef<MachineOperand> Cond, 1009 Register TrueReg, 1010 Register FalseReg) const { 1011 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 1012 const TargetRegisterClass *BoolXExecRC = 1013 RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 1014 assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass && 1015 "Not a VGPR32 reg"); 1016 1017 if (Cond.size() == 1) { 1018 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1019 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) 1020 .add(Cond[0]); 1021 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1022 .addImm(0) 1023 .addReg(FalseReg) 1024 .addImm(0) 1025 .addReg(TrueReg) 1026 .addReg(SReg); 1027 } else if (Cond.size() == 2) { 1028 assert(Cond[0].isImm() && "Cond[0] is not an immediate"); 1029 switch (Cond[0].getImm()) { 1030 case SIInstrInfo::SCC_TRUE: { 1031 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1032 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 1033 : AMDGPU::S_CSELECT_B64), SReg) 1034 .addImm(1) 1035 .addImm(0); 1036 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1037 .addImm(0) 1038 .addReg(FalseReg) 1039 .addImm(0) 1040 .addReg(TrueReg) 1041 .addReg(SReg); 1042 break; 1043 } 1044 case SIInstrInfo::SCC_FALSE: { 1045 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1046 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 1047 : AMDGPU::S_CSELECT_B64), SReg) 1048 .addImm(0) 1049 .addImm(1); 1050 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1051 .addImm(0) 1052 .addReg(FalseReg) 1053 .addImm(0) 1054 .addReg(TrueReg) 1055 .addReg(SReg); 1056 break; 1057 } 1058 case SIInstrInfo::VCCNZ: { 1059 MachineOperand RegOp = Cond[1]; 1060 RegOp.setImplicit(false); 1061 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1062 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) 1063 .add(RegOp); 1064 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1065 .addImm(0) 1066 .addReg(FalseReg) 1067 .addImm(0) 1068 .addReg(TrueReg) 1069 .addReg(SReg); 1070 break; 1071 } 1072 case SIInstrInfo::VCCZ: { 1073 MachineOperand RegOp = Cond[1]; 1074 RegOp.setImplicit(false); 1075 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1076 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) 1077 .add(RegOp); 1078 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1079 .addImm(0) 1080 .addReg(TrueReg) 1081 .addImm(0) 1082 .addReg(FalseReg) 1083 .addReg(SReg); 1084 break; 1085 } 1086 case SIInstrInfo::EXECNZ: { 1087 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1088 Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC()); 1089 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 1090 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2) 1091 .addImm(0); 1092 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 1093 : AMDGPU::S_CSELECT_B64), SReg) 1094 .addImm(1) 1095 .addImm(0); 1096 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1097 .addImm(0) 1098 .addReg(FalseReg) 1099 .addImm(0) 1100 .addReg(TrueReg) 1101 .addReg(SReg); 1102 break; 1103 } 1104 case SIInstrInfo::EXECZ: { 1105 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1106 Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC()); 1107 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 1108 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2) 1109 .addImm(0); 1110 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 1111 : AMDGPU::S_CSELECT_B64), SReg) 1112 .addImm(0) 1113 .addImm(1); 1114 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1115 .addImm(0) 1116 .addReg(FalseReg) 1117 .addImm(0) 1118 .addReg(TrueReg) 1119 .addReg(SReg); 1120 llvm_unreachable("Unhandled branch predicate EXECZ"); 1121 break; 1122 } 1123 default: 1124 llvm_unreachable("invalid branch predicate"); 1125 } 1126 } else { 1127 llvm_unreachable("Can only handle Cond size 1 or 2"); 1128 } 1129 } 1130 1131 Register SIInstrInfo::insertEQ(MachineBasicBlock *MBB, 1132 MachineBasicBlock::iterator I, 1133 const DebugLoc &DL, 1134 Register SrcReg, int Value) const { 1135 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 1136 Register Reg = MRI.createVirtualRegister(RI.getBoolRC()); 1137 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg) 1138 .addImm(Value) 1139 .addReg(SrcReg); 1140 1141 return Reg; 1142 } 1143 1144 Register SIInstrInfo::insertNE(MachineBasicBlock *MBB, 1145 MachineBasicBlock::iterator I, 1146 const DebugLoc &DL, 1147 Register SrcReg, int Value) const { 1148 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 1149 Register Reg = MRI.createVirtualRegister(RI.getBoolRC()); 1150 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg) 1151 .addImm(Value) 1152 .addReg(SrcReg); 1153 1154 return Reg; 1155 } 1156 1157 unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const { 1158 1159 if (RI.hasAGPRs(DstRC)) 1160 return AMDGPU::COPY; 1161 if (RI.getRegSizeInBits(*DstRC) == 32) { 1162 return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; 1163 } else if (RI.getRegSizeInBits(*DstRC) == 64 && RI.isSGPRClass(DstRC)) { 1164 return AMDGPU::S_MOV_B64; 1165 } else if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC)) { 1166 return AMDGPU::V_MOV_B64_PSEUDO; 1167 } 1168 return AMDGPU::COPY; 1169 } 1170 1171 const MCInstrDesc & 1172 SIInstrInfo::getIndirectGPRIDXPseudo(unsigned VecSize, 1173 bool IsIndirectSrc) const { 1174 if (IsIndirectSrc) { 1175 if (VecSize <= 32) // 4 bytes 1176 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1); 1177 if (VecSize <= 64) // 8 bytes 1178 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2); 1179 if (VecSize <= 96) // 12 bytes 1180 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3); 1181 if (VecSize <= 128) // 16 bytes 1182 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4); 1183 if (VecSize <= 160) // 20 bytes 1184 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5); 1185 if (VecSize <= 256) // 32 bytes 1186 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8); 1187 if (VecSize <= 512) // 64 bytes 1188 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16); 1189 if (VecSize <= 1024) // 128 bytes 1190 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32); 1191 1192 llvm_unreachable("unsupported size for IndirectRegReadGPRIDX pseudos"); 1193 } 1194 1195 if (VecSize <= 32) // 4 bytes 1196 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1); 1197 if (VecSize <= 64) // 8 bytes 1198 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2); 1199 if (VecSize <= 96) // 12 bytes 1200 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3); 1201 if (VecSize <= 128) // 16 bytes 1202 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4); 1203 if (VecSize <= 160) // 20 bytes 1204 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5); 1205 if (VecSize <= 256) // 32 bytes 1206 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8); 1207 if (VecSize <= 512) // 64 bytes 1208 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16); 1209 if (VecSize <= 1024) // 128 bytes 1210 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32); 1211 1212 llvm_unreachable("unsupported size for IndirectRegWriteGPRIDX pseudos"); 1213 } 1214 1215 static unsigned getIndirectVGPRWriteMovRelPseudoOpc(unsigned VecSize) { 1216 if (VecSize <= 32) // 4 bytes 1217 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1; 1218 if (VecSize <= 64) // 8 bytes 1219 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2; 1220 if (VecSize <= 96) // 12 bytes 1221 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3; 1222 if (VecSize <= 128) // 16 bytes 1223 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4; 1224 if (VecSize <= 160) // 20 bytes 1225 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5; 1226 if (VecSize <= 256) // 32 bytes 1227 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8; 1228 if (VecSize <= 512) // 64 bytes 1229 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16; 1230 if (VecSize <= 1024) // 128 bytes 1231 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32; 1232 1233 llvm_unreachable("unsupported size for IndirectRegWrite pseudos"); 1234 } 1235 1236 static unsigned getIndirectSGPRWriteMovRelPseudo32(unsigned VecSize) { 1237 if (VecSize <= 32) // 4 bytes 1238 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1; 1239 if (VecSize <= 64) // 8 bytes 1240 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2; 1241 if (VecSize <= 96) // 12 bytes 1242 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3; 1243 if (VecSize <= 128) // 16 bytes 1244 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4; 1245 if (VecSize <= 160) // 20 bytes 1246 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5; 1247 if (VecSize <= 256) // 32 bytes 1248 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8; 1249 if (VecSize <= 512) // 64 bytes 1250 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16; 1251 if (VecSize <= 1024) // 128 bytes 1252 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32; 1253 1254 llvm_unreachable("unsupported size for IndirectRegWrite pseudos"); 1255 } 1256 1257 static unsigned getIndirectSGPRWriteMovRelPseudo64(unsigned VecSize) { 1258 if (VecSize <= 64) // 8 bytes 1259 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1; 1260 if (VecSize <= 128) // 16 bytes 1261 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2; 1262 if (VecSize <= 256) // 32 bytes 1263 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4; 1264 if (VecSize <= 512) // 64 bytes 1265 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8; 1266 if (VecSize <= 1024) // 128 bytes 1267 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16; 1268 1269 llvm_unreachable("unsupported size for IndirectRegWrite pseudos"); 1270 } 1271 1272 const MCInstrDesc & 1273 SIInstrInfo::getIndirectRegWriteMovRelPseudo(unsigned VecSize, unsigned EltSize, 1274 bool IsSGPR) const { 1275 if (IsSGPR) { 1276 switch (EltSize) { 1277 case 32: 1278 return get(getIndirectSGPRWriteMovRelPseudo32(VecSize)); 1279 case 64: 1280 return get(getIndirectSGPRWriteMovRelPseudo64(VecSize)); 1281 default: 1282 llvm_unreachable("invalid reg indexing elt size"); 1283 } 1284 } 1285 1286 assert(EltSize == 32 && "invalid reg indexing elt size"); 1287 return get(getIndirectVGPRWriteMovRelPseudoOpc(VecSize)); 1288 } 1289 1290 static unsigned getSGPRSpillSaveOpcode(unsigned Size) { 1291 switch (Size) { 1292 case 4: 1293 return AMDGPU::SI_SPILL_S32_SAVE; 1294 case 8: 1295 return AMDGPU::SI_SPILL_S64_SAVE; 1296 case 12: 1297 return AMDGPU::SI_SPILL_S96_SAVE; 1298 case 16: 1299 return AMDGPU::SI_SPILL_S128_SAVE; 1300 case 20: 1301 return AMDGPU::SI_SPILL_S160_SAVE; 1302 case 24: 1303 return AMDGPU::SI_SPILL_S192_SAVE; 1304 case 32: 1305 return AMDGPU::SI_SPILL_S256_SAVE; 1306 case 64: 1307 return AMDGPU::SI_SPILL_S512_SAVE; 1308 case 128: 1309 return AMDGPU::SI_SPILL_S1024_SAVE; 1310 default: 1311 llvm_unreachable("unknown register size"); 1312 } 1313 } 1314 1315 static unsigned getVGPRSpillSaveOpcode(unsigned Size) { 1316 switch (Size) { 1317 case 4: 1318 return AMDGPU::SI_SPILL_V32_SAVE; 1319 case 8: 1320 return AMDGPU::SI_SPILL_V64_SAVE; 1321 case 12: 1322 return AMDGPU::SI_SPILL_V96_SAVE; 1323 case 16: 1324 return AMDGPU::SI_SPILL_V128_SAVE; 1325 case 20: 1326 return AMDGPU::SI_SPILL_V160_SAVE; 1327 case 24: 1328 return AMDGPU::SI_SPILL_V192_SAVE; 1329 case 32: 1330 return AMDGPU::SI_SPILL_V256_SAVE; 1331 case 64: 1332 return AMDGPU::SI_SPILL_V512_SAVE; 1333 case 128: 1334 return AMDGPU::SI_SPILL_V1024_SAVE; 1335 default: 1336 llvm_unreachable("unknown register size"); 1337 } 1338 } 1339 1340 static unsigned getAGPRSpillSaveOpcode(unsigned Size) { 1341 switch (Size) { 1342 case 4: 1343 return AMDGPU::SI_SPILL_A32_SAVE; 1344 case 8: 1345 return AMDGPU::SI_SPILL_A64_SAVE; 1346 case 12: 1347 return AMDGPU::SI_SPILL_A96_SAVE; 1348 case 16: 1349 return AMDGPU::SI_SPILL_A128_SAVE; 1350 case 20: 1351 return AMDGPU::SI_SPILL_A160_SAVE; 1352 case 24: 1353 return AMDGPU::SI_SPILL_A192_SAVE; 1354 case 32: 1355 return AMDGPU::SI_SPILL_A256_SAVE; 1356 case 64: 1357 return AMDGPU::SI_SPILL_A512_SAVE; 1358 case 128: 1359 return AMDGPU::SI_SPILL_A1024_SAVE; 1360 default: 1361 llvm_unreachable("unknown register size"); 1362 } 1363 } 1364 1365 void SIInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB, 1366 MachineBasicBlock::iterator MI, 1367 Register SrcReg, bool isKill, 1368 int FrameIndex, 1369 const TargetRegisterClass *RC, 1370 const TargetRegisterInfo *TRI) const { 1371 MachineFunction *MF = MBB.getParent(); 1372 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 1373 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 1374 const DebugLoc &DL = MBB.findDebugLoc(MI); 1375 1376 MachinePointerInfo PtrInfo 1377 = MachinePointerInfo::getFixedStack(*MF, FrameIndex); 1378 MachineMemOperand *MMO = MF->getMachineMemOperand( 1379 PtrInfo, MachineMemOperand::MOStore, FrameInfo.getObjectSize(FrameIndex), 1380 FrameInfo.getObjectAlign(FrameIndex)); 1381 unsigned SpillSize = TRI->getSpillSize(*RC); 1382 1383 if (RI.isSGPRClass(RC)) { 1384 MFI->setHasSpilledSGPRs(); 1385 assert(SrcReg != AMDGPU::M0 && "m0 should not be spilled"); 1386 assert(SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI && 1387 SrcReg != AMDGPU::EXEC && "exec should not be spilled"); 1388 1389 // We are only allowed to create one new instruction when spilling 1390 // registers, so we need to use pseudo instruction for spilling SGPRs. 1391 const MCInstrDesc &OpDesc = get(getSGPRSpillSaveOpcode(SpillSize)); 1392 1393 // The SGPR spill/restore instructions only work on number sgprs, so we need 1394 // to make sure we are using the correct register class. 1395 if (SrcReg.isVirtual() && SpillSize == 4) { 1396 MachineRegisterInfo &MRI = MF->getRegInfo(); 1397 MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); 1398 } 1399 1400 BuildMI(MBB, MI, DL, OpDesc) 1401 .addReg(SrcReg, getKillRegState(isKill)) // data 1402 .addFrameIndex(FrameIndex) // addr 1403 .addMemOperand(MMO) 1404 .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit); 1405 1406 if (RI.spillSGPRToVGPR()) 1407 FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill); 1408 return; 1409 } 1410 1411 unsigned Opcode = RI.hasAGPRs(RC) ? getAGPRSpillSaveOpcode(SpillSize) 1412 : getVGPRSpillSaveOpcode(SpillSize); 1413 MFI->setHasSpilledVGPRs(); 1414 1415 BuildMI(MBB, MI, DL, get(Opcode)) 1416 .addReg(SrcReg, getKillRegState(isKill)) // data 1417 .addFrameIndex(FrameIndex) // addr 1418 .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset 1419 .addImm(0) // offset 1420 .addMemOperand(MMO); 1421 } 1422 1423 static unsigned getSGPRSpillRestoreOpcode(unsigned Size) { 1424 switch (Size) { 1425 case 4: 1426 return AMDGPU::SI_SPILL_S32_RESTORE; 1427 case 8: 1428 return AMDGPU::SI_SPILL_S64_RESTORE; 1429 case 12: 1430 return AMDGPU::SI_SPILL_S96_RESTORE; 1431 case 16: 1432 return AMDGPU::SI_SPILL_S128_RESTORE; 1433 case 20: 1434 return AMDGPU::SI_SPILL_S160_RESTORE; 1435 case 24: 1436 return AMDGPU::SI_SPILL_S192_RESTORE; 1437 case 32: 1438 return AMDGPU::SI_SPILL_S256_RESTORE; 1439 case 64: 1440 return AMDGPU::SI_SPILL_S512_RESTORE; 1441 case 128: 1442 return AMDGPU::SI_SPILL_S1024_RESTORE; 1443 default: 1444 llvm_unreachable("unknown register size"); 1445 } 1446 } 1447 1448 static unsigned getVGPRSpillRestoreOpcode(unsigned Size) { 1449 switch (Size) { 1450 case 4: 1451 return AMDGPU::SI_SPILL_V32_RESTORE; 1452 case 8: 1453 return AMDGPU::SI_SPILL_V64_RESTORE; 1454 case 12: 1455 return AMDGPU::SI_SPILL_V96_RESTORE; 1456 case 16: 1457 return AMDGPU::SI_SPILL_V128_RESTORE; 1458 case 20: 1459 return AMDGPU::SI_SPILL_V160_RESTORE; 1460 case 24: 1461 return AMDGPU::SI_SPILL_V192_RESTORE; 1462 case 32: 1463 return AMDGPU::SI_SPILL_V256_RESTORE; 1464 case 64: 1465 return AMDGPU::SI_SPILL_V512_RESTORE; 1466 case 128: 1467 return AMDGPU::SI_SPILL_V1024_RESTORE; 1468 default: 1469 llvm_unreachable("unknown register size"); 1470 } 1471 } 1472 1473 static unsigned getAGPRSpillRestoreOpcode(unsigned Size) { 1474 switch (Size) { 1475 case 4: 1476 return AMDGPU::SI_SPILL_A32_RESTORE; 1477 case 8: 1478 return AMDGPU::SI_SPILL_A64_RESTORE; 1479 case 12: 1480 return AMDGPU::SI_SPILL_A96_RESTORE; 1481 case 16: 1482 return AMDGPU::SI_SPILL_A128_RESTORE; 1483 case 20: 1484 return AMDGPU::SI_SPILL_A160_RESTORE; 1485 case 24: 1486 return AMDGPU::SI_SPILL_A192_RESTORE; 1487 case 32: 1488 return AMDGPU::SI_SPILL_A256_RESTORE; 1489 case 64: 1490 return AMDGPU::SI_SPILL_A512_RESTORE; 1491 case 128: 1492 return AMDGPU::SI_SPILL_A1024_RESTORE; 1493 default: 1494 llvm_unreachable("unknown register size"); 1495 } 1496 } 1497 1498 void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, 1499 MachineBasicBlock::iterator MI, 1500 Register DestReg, int FrameIndex, 1501 const TargetRegisterClass *RC, 1502 const TargetRegisterInfo *TRI) const { 1503 MachineFunction *MF = MBB.getParent(); 1504 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 1505 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 1506 const DebugLoc &DL = MBB.findDebugLoc(MI); 1507 unsigned SpillSize = TRI->getSpillSize(*RC); 1508 1509 MachinePointerInfo PtrInfo 1510 = MachinePointerInfo::getFixedStack(*MF, FrameIndex); 1511 1512 MachineMemOperand *MMO = MF->getMachineMemOperand( 1513 PtrInfo, MachineMemOperand::MOLoad, FrameInfo.getObjectSize(FrameIndex), 1514 FrameInfo.getObjectAlign(FrameIndex)); 1515 1516 if (RI.isSGPRClass(RC)) { 1517 MFI->setHasSpilledSGPRs(); 1518 assert(DestReg != AMDGPU::M0 && "m0 should not be reloaded into"); 1519 assert(DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI && 1520 DestReg != AMDGPU::EXEC && "exec should not be spilled"); 1521 1522 // FIXME: Maybe this should not include a memoperand because it will be 1523 // lowered to non-memory instructions. 1524 const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(SpillSize)); 1525 if (DestReg.isVirtual() && SpillSize == 4) { 1526 MachineRegisterInfo &MRI = MF->getRegInfo(); 1527 MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); 1528 } 1529 1530 if (RI.spillSGPRToVGPR()) 1531 FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill); 1532 BuildMI(MBB, MI, DL, OpDesc, DestReg) 1533 .addFrameIndex(FrameIndex) // addr 1534 .addMemOperand(MMO) 1535 .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit); 1536 1537 return; 1538 } 1539 1540 unsigned Opcode = RI.hasAGPRs(RC) ? getAGPRSpillRestoreOpcode(SpillSize) 1541 : getVGPRSpillRestoreOpcode(SpillSize); 1542 BuildMI(MBB, MI, DL, get(Opcode), DestReg) 1543 .addFrameIndex(FrameIndex) // vaddr 1544 .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset 1545 .addImm(0) // offset 1546 .addMemOperand(MMO); 1547 } 1548 1549 void SIInstrInfo::insertNoop(MachineBasicBlock &MBB, 1550 MachineBasicBlock::iterator MI) const { 1551 insertNoops(MBB, MI, 1); 1552 } 1553 1554 void SIInstrInfo::insertNoops(MachineBasicBlock &MBB, 1555 MachineBasicBlock::iterator MI, 1556 unsigned Quantity) const { 1557 DebugLoc DL = MBB.findDebugLoc(MI); 1558 while (Quantity > 0) { 1559 unsigned Arg = std::min(Quantity, 8u); 1560 Quantity -= Arg; 1561 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP)).addImm(Arg - 1); 1562 } 1563 } 1564 1565 void SIInstrInfo::insertReturn(MachineBasicBlock &MBB) const { 1566 auto MF = MBB.getParent(); 1567 SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>(); 1568 1569 assert(Info->isEntryFunction()); 1570 1571 if (MBB.succ_empty()) { 1572 bool HasNoTerminator = MBB.getFirstTerminator() == MBB.end(); 1573 if (HasNoTerminator) { 1574 if (Info->returnsVoid()) { 1575 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::S_ENDPGM)).addImm(0); 1576 } else { 1577 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::SI_RETURN_TO_EPILOG)); 1578 } 1579 } 1580 } 1581 } 1582 1583 unsigned SIInstrInfo::getNumWaitStates(const MachineInstr &MI) { 1584 switch (MI.getOpcode()) { 1585 default: return 1; // FIXME: Do wait states equal cycles? 1586 1587 case AMDGPU::S_NOP: 1588 return MI.getOperand(0).getImm() + 1; 1589 } 1590 } 1591 1592 bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const { 1593 MachineBasicBlock &MBB = *MI.getParent(); 1594 DebugLoc DL = MBB.findDebugLoc(MI); 1595 switch (MI.getOpcode()) { 1596 default: return TargetInstrInfo::expandPostRAPseudo(MI); 1597 case AMDGPU::S_MOV_B64_term: 1598 // This is only a terminator to get the correct spill code placement during 1599 // register allocation. 1600 MI.setDesc(get(AMDGPU::S_MOV_B64)); 1601 break; 1602 1603 case AMDGPU::S_MOV_B32_term: 1604 // This is only a terminator to get the correct spill code placement during 1605 // register allocation. 1606 MI.setDesc(get(AMDGPU::S_MOV_B32)); 1607 break; 1608 1609 case AMDGPU::S_XOR_B64_term: 1610 // This is only a terminator to get the correct spill code placement during 1611 // register allocation. 1612 MI.setDesc(get(AMDGPU::S_XOR_B64)); 1613 break; 1614 1615 case AMDGPU::S_XOR_B32_term: 1616 // This is only a terminator to get the correct spill code placement during 1617 // register allocation. 1618 MI.setDesc(get(AMDGPU::S_XOR_B32)); 1619 break; 1620 case AMDGPU::S_OR_B64_term: 1621 // This is only a terminator to get the correct spill code placement during 1622 // register allocation. 1623 MI.setDesc(get(AMDGPU::S_OR_B64)); 1624 break; 1625 case AMDGPU::S_OR_B32_term: 1626 // This is only a terminator to get the correct spill code placement during 1627 // register allocation. 1628 MI.setDesc(get(AMDGPU::S_OR_B32)); 1629 break; 1630 1631 case AMDGPU::S_ANDN2_B64_term: 1632 // This is only a terminator to get the correct spill code placement during 1633 // register allocation. 1634 MI.setDesc(get(AMDGPU::S_ANDN2_B64)); 1635 break; 1636 1637 case AMDGPU::S_ANDN2_B32_term: 1638 // This is only a terminator to get the correct spill code placement during 1639 // register allocation. 1640 MI.setDesc(get(AMDGPU::S_ANDN2_B32)); 1641 break; 1642 1643 case AMDGPU::V_MOV_B64_PSEUDO: { 1644 Register Dst = MI.getOperand(0).getReg(); 1645 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0); 1646 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1); 1647 1648 const MachineOperand &SrcOp = MI.getOperand(1); 1649 // FIXME: Will this work for 64-bit floating point immediates? 1650 assert(!SrcOp.isFPImm()); 1651 if (SrcOp.isImm()) { 1652 APInt Imm(64, SrcOp.getImm()); 1653 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo) 1654 .addImm(Imm.getLoBits(32).getZExtValue()) 1655 .addReg(Dst, RegState::Implicit | RegState::Define); 1656 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi) 1657 .addImm(Imm.getHiBits(32).getZExtValue()) 1658 .addReg(Dst, RegState::Implicit | RegState::Define); 1659 } else { 1660 assert(SrcOp.isReg()); 1661 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo) 1662 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0)) 1663 .addReg(Dst, RegState::Implicit | RegState::Define); 1664 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi) 1665 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1)) 1666 .addReg(Dst, RegState::Implicit | RegState::Define); 1667 } 1668 MI.eraseFromParent(); 1669 break; 1670 } 1671 case AMDGPU::V_MOV_B64_DPP_PSEUDO: { 1672 expandMovDPP64(MI); 1673 break; 1674 } 1675 case AMDGPU::V_SET_INACTIVE_B32: { 1676 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64; 1677 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 1678 BuildMI(MBB, MI, DL, get(NotOpc), Exec) 1679 .addReg(Exec); 1680 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg()) 1681 .add(MI.getOperand(2)); 1682 BuildMI(MBB, MI, DL, get(NotOpc), Exec) 1683 .addReg(Exec); 1684 MI.eraseFromParent(); 1685 break; 1686 } 1687 case AMDGPU::V_SET_INACTIVE_B64: { 1688 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64; 1689 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 1690 BuildMI(MBB, MI, DL, get(NotOpc), Exec) 1691 .addReg(Exec); 1692 MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), 1693 MI.getOperand(0).getReg()) 1694 .add(MI.getOperand(2)); 1695 expandPostRAPseudo(*Copy); 1696 BuildMI(MBB, MI, DL, get(NotOpc), Exec) 1697 .addReg(Exec); 1698 MI.eraseFromParent(); 1699 break; 1700 } 1701 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1: 1702 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2: 1703 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3: 1704 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4: 1705 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5: 1706 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8: 1707 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16: 1708 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32: 1709 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1: 1710 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2: 1711 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3: 1712 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4: 1713 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5: 1714 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8: 1715 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16: 1716 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32: 1717 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1: 1718 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2: 1719 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4: 1720 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8: 1721 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16: { 1722 const TargetRegisterClass *EltRC = getOpRegClass(MI, 2); 1723 1724 unsigned Opc; 1725 if (RI.hasVGPRs(EltRC)) { 1726 Opc = AMDGPU::V_MOVRELD_B32_e32; 1727 } else { 1728 Opc = RI.getRegSizeInBits(*EltRC) == 64 ? AMDGPU::S_MOVRELD_B64 1729 : AMDGPU::S_MOVRELD_B32; 1730 } 1731 1732 const MCInstrDesc &OpDesc = get(Opc); 1733 Register VecReg = MI.getOperand(0).getReg(); 1734 bool IsUndef = MI.getOperand(1).isUndef(); 1735 unsigned SubReg = MI.getOperand(3).getImm(); 1736 assert(VecReg == MI.getOperand(1).getReg()); 1737 1738 MachineInstrBuilder MIB = 1739 BuildMI(MBB, MI, DL, OpDesc) 1740 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef) 1741 .add(MI.getOperand(2)) 1742 .addReg(VecReg, RegState::ImplicitDefine) 1743 .addReg(VecReg, RegState::Implicit | (IsUndef ? RegState::Undef : 0)); 1744 1745 const int ImpDefIdx = 1746 OpDesc.getNumOperands() + OpDesc.getNumImplicitUses(); 1747 const int ImpUseIdx = ImpDefIdx + 1; 1748 MIB->tieOperands(ImpDefIdx, ImpUseIdx); 1749 MI.eraseFromParent(); 1750 break; 1751 } 1752 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1: 1753 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2: 1754 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3: 1755 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4: 1756 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5: 1757 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8: 1758 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16: 1759 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32: { 1760 assert(ST.useVGPRIndexMode()); 1761 Register VecReg = MI.getOperand(0).getReg(); 1762 bool IsUndef = MI.getOperand(1).isUndef(); 1763 Register Idx = MI.getOperand(3).getReg(); 1764 Register SubReg = MI.getOperand(4).getImm(); 1765 1766 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON)) 1767 .addReg(Idx) 1768 .addImm(AMDGPU::VGPRIndexMode::DST_ENABLE); 1769 SetOn->getOperand(3).setIsUndef(); 1770 1771 const MCInstrDesc &OpDesc = get(AMDGPU::V_MOV_B32_indirect); 1772 MachineInstrBuilder MIB = 1773 BuildMI(MBB, MI, DL, OpDesc) 1774 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef) 1775 .add(MI.getOperand(2)) 1776 .addReg(VecReg, RegState::ImplicitDefine) 1777 .addReg(VecReg, 1778 RegState::Implicit | (IsUndef ? RegState::Undef : 0)); 1779 1780 const int ImpDefIdx = OpDesc.getNumOperands() + OpDesc.getNumImplicitUses(); 1781 const int ImpUseIdx = ImpDefIdx + 1; 1782 MIB->tieOperands(ImpDefIdx, ImpUseIdx); 1783 1784 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF)); 1785 1786 finalizeBundle(MBB, SetOn->getIterator(), std::next(SetOff->getIterator())); 1787 1788 MI.eraseFromParent(); 1789 break; 1790 } 1791 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1: 1792 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2: 1793 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3: 1794 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4: 1795 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5: 1796 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8: 1797 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16: 1798 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32: { 1799 assert(ST.useVGPRIndexMode()); 1800 Register Dst = MI.getOperand(0).getReg(); 1801 Register VecReg = MI.getOperand(1).getReg(); 1802 bool IsUndef = MI.getOperand(1).isUndef(); 1803 Register Idx = MI.getOperand(2).getReg(); 1804 Register SubReg = MI.getOperand(3).getImm(); 1805 1806 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON)) 1807 .addReg(Idx) 1808 .addImm(AMDGPU::VGPRIndexMode::SRC0_ENABLE); 1809 SetOn->getOperand(3).setIsUndef(); 1810 1811 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32)) 1812 .addDef(Dst) 1813 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef) 1814 .addReg(VecReg, RegState::Implicit | (IsUndef ? RegState::Undef : 0)) 1815 .addReg(AMDGPU::M0, RegState::Implicit); 1816 1817 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF)); 1818 1819 finalizeBundle(MBB, SetOn->getIterator(), std::next(SetOff->getIterator())); 1820 1821 MI.eraseFromParent(); 1822 break; 1823 } 1824 case AMDGPU::SI_PC_ADD_REL_OFFSET: { 1825 MachineFunction &MF = *MBB.getParent(); 1826 Register Reg = MI.getOperand(0).getReg(); 1827 Register RegLo = RI.getSubReg(Reg, AMDGPU::sub0); 1828 Register RegHi = RI.getSubReg(Reg, AMDGPU::sub1); 1829 1830 // Create a bundle so these instructions won't be re-ordered by the 1831 // post-RA scheduler. 1832 MIBundleBuilder Bundler(MBB, MI); 1833 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg)); 1834 1835 // Add 32-bit offset from this instruction to the start of the 1836 // constant data. 1837 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo) 1838 .addReg(RegLo) 1839 .add(MI.getOperand(1))); 1840 1841 MachineInstrBuilder MIB = BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi) 1842 .addReg(RegHi); 1843 MIB.add(MI.getOperand(2)); 1844 1845 Bundler.append(MIB); 1846 finalizeBundle(MBB, Bundler.begin()); 1847 1848 MI.eraseFromParent(); 1849 break; 1850 } 1851 case AMDGPU::ENTER_WWM: { 1852 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when 1853 // WWM is entered. 1854 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 1855 : AMDGPU::S_OR_SAVEEXEC_B64)); 1856 break; 1857 } 1858 case AMDGPU::EXIT_WWM: { 1859 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when 1860 // WWM is exited. 1861 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64)); 1862 break; 1863 } 1864 } 1865 return true; 1866 } 1867 1868 std::pair<MachineInstr*, MachineInstr*> 1869 SIInstrInfo::expandMovDPP64(MachineInstr &MI) const { 1870 assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO); 1871 1872 MachineBasicBlock &MBB = *MI.getParent(); 1873 DebugLoc DL = MBB.findDebugLoc(MI); 1874 MachineFunction *MF = MBB.getParent(); 1875 MachineRegisterInfo &MRI = MF->getRegInfo(); 1876 Register Dst = MI.getOperand(0).getReg(); 1877 unsigned Part = 0; 1878 MachineInstr *Split[2]; 1879 1880 1881 for (auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) { 1882 auto MovDPP = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_dpp)); 1883 if (Dst.isPhysical()) { 1884 MovDPP.addDef(RI.getSubReg(Dst, Sub)); 1885 } else { 1886 assert(MRI.isSSA()); 1887 auto Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 1888 MovDPP.addDef(Tmp); 1889 } 1890 1891 for (unsigned I = 1; I <= 2; ++I) { // old and src operands. 1892 const MachineOperand &SrcOp = MI.getOperand(I); 1893 assert(!SrcOp.isFPImm()); 1894 if (SrcOp.isImm()) { 1895 APInt Imm(64, SrcOp.getImm()); 1896 Imm.ashrInPlace(Part * 32); 1897 MovDPP.addImm(Imm.getLoBits(32).getZExtValue()); 1898 } else { 1899 assert(SrcOp.isReg()); 1900 Register Src = SrcOp.getReg(); 1901 if (Src.isPhysical()) 1902 MovDPP.addReg(RI.getSubReg(Src, Sub)); 1903 else 1904 MovDPP.addReg(Src, SrcOp.isUndef() ? RegState::Undef : 0, Sub); 1905 } 1906 } 1907 1908 for (unsigned I = 3; I < MI.getNumExplicitOperands(); ++I) 1909 MovDPP.addImm(MI.getOperand(I).getImm()); 1910 1911 Split[Part] = MovDPP; 1912 ++Part; 1913 } 1914 1915 if (Dst.isVirtual()) 1916 BuildMI(MBB, MI, DL, get(AMDGPU::REG_SEQUENCE), Dst) 1917 .addReg(Split[0]->getOperand(0).getReg()) 1918 .addImm(AMDGPU::sub0) 1919 .addReg(Split[1]->getOperand(0).getReg()) 1920 .addImm(AMDGPU::sub1); 1921 1922 MI.eraseFromParent(); 1923 return std::make_pair(Split[0], Split[1]); 1924 } 1925 1926 bool SIInstrInfo::swapSourceModifiers(MachineInstr &MI, 1927 MachineOperand &Src0, 1928 unsigned Src0OpName, 1929 MachineOperand &Src1, 1930 unsigned Src1OpName) const { 1931 MachineOperand *Src0Mods = getNamedOperand(MI, Src0OpName); 1932 if (!Src0Mods) 1933 return false; 1934 1935 MachineOperand *Src1Mods = getNamedOperand(MI, Src1OpName); 1936 assert(Src1Mods && 1937 "All commutable instructions have both src0 and src1 modifiers"); 1938 1939 int Src0ModsVal = Src0Mods->getImm(); 1940 int Src1ModsVal = Src1Mods->getImm(); 1941 1942 Src1Mods->setImm(Src0ModsVal); 1943 Src0Mods->setImm(Src1ModsVal); 1944 return true; 1945 } 1946 1947 static MachineInstr *swapRegAndNonRegOperand(MachineInstr &MI, 1948 MachineOperand &RegOp, 1949 MachineOperand &NonRegOp) { 1950 Register Reg = RegOp.getReg(); 1951 unsigned SubReg = RegOp.getSubReg(); 1952 bool IsKill = RegOp.isKill(); 1953 bool IsDead = RegOp.isDead(); 1954 bool IsUndef = RegOp.isUndef(); 1955 bool IsDebug = RegOp.isDebug(); 1956 1957 if (NonRegOp.isImm()) 1958 RegOp.ChangeToImmediate(NonRegOp.getImm()); 1959 else if (NonRegOp.isFI()) 1960 RegOp.ChangeToFrameIndex(NonRegOp.getIndex()); 1961 else if (NonRegOp.isGlobal()) { 1962 RegOp.ChangeToGA(NonRegOp.getGlobal(), NonRegOp.getOffset(), 1963 NonRegOp.getTargetFlags()); 1964 } else 1965 return nullptr; 1966 1967 // Make sure we don't reinterpret a subreg index in the target flags. 1968 RegOp.setTargetFlags(NonRegOp.getTargetFlags()); 1969 1970 NonRegOp.ChangeToRegister(Reg, false, false, IsKill, IsDead, IsUndef, IsDebug); 1971 NonRegOp.setSubReg(SubReg); 1972 1973 return &MI; 1974 } 1975 1976 MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI, 1977 unsigned Src0Idx, 1978 unsigned Src1Idx) const { 1979 assert(!NewMI && "this should never be used"); 1980 1981 unsigned Opc = MI.getOpcode(); 1982 int CommutedOpcode = commuteOpcode(Opc); 1983 if (CommutedOpcode == -1) 1984 return nullptr; 1985 1986 assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) == 1987 static_cast<int>(Src0Idx) && 1988 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) == 1989 static_cast<int>(Src1Idx) && 1990 "inconsistency with findCommutedOpIndices"); 1991 1992 MachineOperand &Src0 = MI.getOperand(Src0Idx); 1993 MachineOperand &Src1 = MI.getOperand(Src1Idx); 1994 1995 MachineInstr *CommutedMI = nullptr; 1996 if (Src0.isReg() && Src1.isReg()) { 1997 if (isOperandLegal(MI, Src1Idx, &Src0)) { 1998 // Be sure to copy the source modifiers to the right place. 1999 CommutedMI 2000 = TargetInstrInfo::commuteInstructionImpl(MI, NewMI, Src0Idx, Src1Idx); 2001 } 2002 2003 } else if (Src0.isReg() && !Src1.isReg()) { 2004 // src0 should always be able to support any operand type, so no need to 2005 // check operand legality. 2006 CommutedMI = swapRegAndNonRegOperand(MI, Src0, Src1); 2007 } else if (!Src0.isReg() && Src1.isReg()) { 2008 if (isOperandLegal(MI, Src1Idx, &Src0)) 2009 CommutedMI = swapRegAndNonRegOperand(MI, Src1, Src0); 2010 } else { 2011 // FIXME: Found two non registers to commute. This does happen. 2012 return nullptr; 2013 } 2014 2015 if (CommutedMI) { 2016 swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers, 2017 Src1, AMDGPU::OpName::src1_modifiers); 2018 2019 CommutedMI->setDesc(get(CommutedOpcode)); 2020 } 2021 2022 return CommutedMI; 2023 } 2024 2025 // This needs to be implemented because the source modifiers may be inserted 2026 // between the true commutable operands, and the base 2027 // TargetInstrInfo::commuteInstruction uses it. 2028 bool SIInstrInfo::findCommutedOpIndices(const MachineInstr &MI, 2029 unsigned &SrcOpIdx0, 2030 unsigned &SrcOpIdx1) const { 2031 return findCommutedOpIndices(MI.getDesc(), SrcOpIdx0, SrcOpIdx1); 2032 } 2033 2034 bool SIInstrInfo::findCommutedOpIndices(MCInstrDesc Desc, unsigned &SrcOpIdx0, 2035 unsigned &SrcOpIdx1) const { 2036 if (!Desc.isCommutable()) 2037 return false; 2038 2039 unsigned Opc = Desc.getOpcode(); 2040 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); 2041 if (Src0Idx == -1) 2042 return false; 2043 2044 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); 2045 if (Src1Idx == -1) 2046 return false; 2047 2048 return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx); 2049 } 2050 2051 bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp, 2052 int64_t BrOffset) const { 2053 // BranchRelaxation should never have to check s_setpc_b64 because its dest 2054 // block is unanalyzable. 2055 assert(BranchOp != AMDGPU::S_SETPC_B64); 2056 2057 // Convert to dwords. 2058 BrOffset /= 4; 2059 2060 // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is 2061 // from the next instruction. 2062 BrOffset -= 1; 2063 2064 return isIntN(BranchOffsetBits, BrOffset); 2065 } 2066 2067 MachineBasicBlock *SIInstrInfo::getBranchDestBlock( 2068 const MachineInstr &MI) const { 2069 if (MI.getOpcode() == AMDGPU::S_SETPC_B64) { 2070 // This would be a difficult analysis to perform, but can always be legal so 2071 // there's no need to analyze it. 2072 return nullptr; 2073 } 2074 2075 return MI.getOperand(0).getMBB(); 2076 } 2077 2078 unsigned SIInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB, 2079 MachineBasicBlock &DestBB, 2080 const DebugLoc &DL, 2081 int64_t BrOffset, 2082 RegScavenger *RS) const { 2083 assert(RS && "RegScavenger required for long branching"); 2084 assert(MBB.empty() && 2085 "new block should be inserted for expanding unconditional branch"); 2086 assert(MBB.pred_size() == 1); 2087 2088 MachineFunction *MF = MBB.getParent(); 2089 MachineRegisterInfo &MRI = MF->getRegInfo(); 2090 2091 // FIXME: Virtual register workaround for RegScavenger not working with empty 2092 // blocks. 2093 Register PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); 2094 2095 auto I = MBB.end(); 2096 2097 // We need to compute the offset relative to the instruction immediately after 2098 // s_getpc_b64. Insert pc arithmetic code before last terminator. 2099 MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg); 2100 2101 // TODO: Handle > 32-bit block address. 2102 if (BrOffset >= 0) { 2103 BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32)) 2104 .addReg(PCReg, RegState::Define, AMDGPU::sub0) 2105 .addReg(PCReg, 0, AMDGPU::sub0) 2106 .addMBB(&DestBB, MO_LONG_BRANCH_FORWARD); 2107 BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32)) 2108 .addReg(PCReg, RegState::Define, AMDGPU::sub1) 2109 .addReg(PCReg, 0, AMDGPU::sub1) 2110 .addImm(0); 2111 } else { 2112 // Backwards branch. 2113 BuildMI(MBB, I, DL, get(AMDGPU::S_SUB_U32)) 2114 .addReg(PCReg, RegState::Define, AMDGPU::sub0) 2115 .addReg(PCReg, 0, AMDGPU::sub0) 2116 .addMBB(&DestBB, MO_LONG_BRANCH_BACKWARD); 2117 BuildMI(MBB, I, DL, get(AMDGPU::S_SUBB_U32)) 2118 .addReg(PCReg, RegState::Define, AMDGPU::sub1) 2119 .addReg(PCReg, 0, AMDGPU::sub1) 2120 .addImm(0); 2121 } 2122 2123 // Insert the indirect branch after the other terminator. 2124 BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64)) 2125 .addReg(PCReg); 2126 2127 // FIXME: If spilling is necessary, this will fail because this scavenger has 2128 // no emergency stack slots. It is non-trivial to spill in this situation, 2129 // because the restore code needs to be specially placed after the 2130 // jump. BranchRelaxation then needs to be made aware of the newly inserted 2131 // block. 2132 // 2133 // If a spill is needed for the pc register pair, we need to insert a spill 2134 // restore block right before the destination block, and insert a short branch 2135 // into the old destination block's fallthrough predecessor. 2136 // e.g.: 2137 // 2138 // s_cbranch_scc0 skip_long_branch: 2139 // 2140 // long_branch_bb: 2141 // spill s[8:9] 2142 // s_getpc_b64 s[8:9] 2143 // s_add_u32 s8, s8, restore_bb 2144 // s_addc_u32 s9, s9, 0 2145 // s_setpc_b64 s[8:9] 2146 // 2147 // skip_long_branch: 2148 // foo; 2149 // 2150 // ..... 2151 // 2152 // dest_bb_fallthrough_predecessor: 2153 // bar; 2154 // s_branch dest_bb 2155 // 2156 // restore_bb: 2157 // restore s[8:9] 2158 // fallthrough dest_bb 2159 /// 2160 // dest_bb: 2161 // buzz; 2162 2163 RS->enterBasicBlockEnd(MBB); 2164 Register Scav = RS->scavengeRegisterBackwards( 2165 AMDGPU::SReg_64RegClass, 2166 MachineBasicBlock::iterator(GetPC), false, 0); 2167 MRI.replaceRegWith(PCReg, Scav); 2168 MRI.clearVirtRegs(); 2169 RS->setRegUsed(Scav); 2170 2171 return 4 + 8 + 4 + 4; 2172 } 2173 2174 unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) { 2175 switch (Cond) { 2176 case SIInstrInfo::SCC_TRUE: 2177 return AMDGPU::S_CBRANCH_SCC1; 2178 case SIInstrInfo::SCC_FALSE: 2179 return AMDGPU::S_CBRANCH_SCC0; 2180 case SIInstrInfo::VCCNZ: 2181 return AMDGPU::S_CBRANCH_VCCNZ; 2182 case SIInstrInfo::VCCZ: 2183 return AMDGPU::S_CBRANCH_VCCZ; 2184 case SIInstrInfo::EXECNZ: 2185 return AMDGPU::S_CBRANCH_EXECNZ; 2186 case SIInstrInfo::EXECZ: 2187 return AMDGPU::S_CBRANCH_EXECZ; 2188 default: 2189 llvm_unreachable("invalid branch predicate"); 2190 } 2191 } 2192 2193 SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) { 2194 switch (Opcode) { 2195 case AMDGPU::S_CBRANCH_SCC0: 2196 return SCC_FALSE; 2197 case AMDGPU::S_CBRANCH_SCC1: 2198 return SCC_TRUE; 2199 case AMDGPU::S_CBRANCH_VCCNZ: 2200 return VCCNZ; 2201 case AMDGPU::S_CBRANCH_VCCZ: 2202 return VCCZ; 2203 case AMDGPU::S_CBRANCH_EXECNZ: 2204 return EXECNZ; 2205 case AMDGPU::S_CBRANCH_EXECZ: 2206 return EXECZ; 2207 default: 2208 return INVALID_BR; 2209 } 2210 } 2211 2212 bool SIInstrInfo::analyzeBranchImpl(MachineBasicBlock &MBB, 2213 MachineBasicBlock::iterator I, 2214 MachineBasicBlock *&TBB, 2215 MachineBasicBlock *&FBB, 2216 SmallVectorImpl<MachineOperand> &Cond, 2217 bool AllowModify) const { 2218 if (I->getOpcode() == AMDGPU::S_BRANCH) { 2219 // Unconditional Branch 2220 TBB = I->getOperand(0).getMBB(); 2221 return false; 2222 } 2223 2224 MachineBasicBlock *CondBB = nullptr; 2225 2226 if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { 2227 CondBB = I->getOperand(1).getMBB(); 2228 Cond.push_back(I->getOperand(0)); 2229 } else { 2230 BranchPredicate Pred = getBranchPredicate(I->getOpcode()); 2231 if (Pred == INVALID_BR) 2232 return true; 2233 2234 CondBB = I->getOperand(0).getMBB(); 2235 Cond.push_back(MachineOperand::CreateImm(Pred)); 2236 Cond.push_back(I->getOperand(1)); // Save the branch register. 2237 } 2238 ++I; 2239 2240 if (I == MBB.end()) { 2241 // Conditional branch followed by fall-through. 2242 TBB = CondBB; 2243 return false; 2244 } 2245 2246 if (I->getOpcode() == AMDGPU::S_BRANCH) { 2247 TBB = CondBB; 2248 FBB = I->getOperand(0).getMBB(); 2249 return false; 2250 } 2251 2252 return true; 2253 } 2254 2255 bool SIInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, 2256 MachineBasicBlock *&FBB, 2257 SmallVectorImpl<MachineOperand> &Cond, 2258 bool AllowModify) const { 2259 MachineBasicBlock::iterator I = MBB.getFirstTerminator(); 2260 auto E = MBB.end(); 2261 if (I == E) 2262 return false; 2263 2264 // Skip over the instructions that are artificially terminators for special 2265 // exec management. 2266 while (I != E && !I->isBranch() && !I->isReturn() && 2267 I->getOpcode() != AMDGPU::SI_MASK_BRANCH) { 2268 switch (I->getOpcode()) { 2269 case AMDGPU::SI_MASK_BRANCH: 2270 case AMDGPU::S_MOV_B64_term: 2271 case AMDGPU::S_XOR_B64_term: 2272 case AMDGPU::S_OR_B64_term: 2273 case AMDGPU::S_ANDN2_B64_term: 2274 case AMDGPU::S_MOV_B32_term: 2275 case AMDGPU::S_XOR_B32_term: 2276 case AMDGPU::S_OR_B32_term: 2277 case AMDGPU::S_ANDN2_B32_term: 2278 break; 2279 case AMDGPU::SI_IF: 2280 case AMDGPU::SI_ELSE: 2281 case AMDGPU::SI_KILL_I1_TERMINATOR: 2282 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR: 2283 // FIXME: It's messy that these need to be considered here at all. 2284 return true; 2285 default: 2286 llvm_unreachable("unexpected non-branch terminator inst"); 2287 } 2288 2289 ++I; 2290 } 2291 2292 if (I == E) 2293 return false; 2294 2295 if (I->getOpcode() != AMDGPU::SI_MASK_BRANCH) 2296 return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify); 2297 2298 ++I; 2299 2300 // TODO: Should be able to treat as fallthrough? 2301 if (I == MBB.end()) 2302 return true; 2303 2304 if (analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify)) 2305 return true; 2306 2307 MachineBasicBlock *MaskBrDest = I->getOperand(0).getMBB(); 2308 2309 // Specifically handle the case where the conditional branch is to the same 2310 // destination as the mask branch. e.g. 2311 // 2312 // si_mask_branch BB8 2313 // s_cbranch_execz BB8 2314 // s_cbranch BB9 2315 // 2316 // This is required to understand divergent loops which may need the branches 2317 // to be relaxed. 2318 if (TBB != MaskBrDest || Cond.empty()) 2319 return true; 2320 2321 auto Pred = Cond[0].getImm(); 2322 return (Pred != EXECZ && Pred != EXECNZ); 2323 } 2324 2325 unsigned SIInstrInfo::removeBranch(MachineBasicBlock &MBB, 2326 int *BytesRemoved) const { 2327 MachineBasicBlock::iterator I = MBB.getFirstTerminator(); 2328 2329 unsigned Count = 0; 2330 unsigned RemovedSize = 0; 2331 while (I != MBB.end()) { 2332 MachineBasicBlock::iterator Next = std::next(I); 2333 if (I->getOpcode() == AMDGPU::SI_MASK_BRANCH) { 2334 I = Next; 2335 continue; 2336 } 2337 2338 RemovedSize += getInstSizeInBytes(*I); 2339 I->eraseFromParent(); 2340 ++Count; 2341 I = Next; 2342 } 2343 2344 if (BytesRemoved) 2345 *BytesRemoved = RemovedSize; 2346 2347 return Count; 2348 } 2349 2350 // Copy the flags onto the implicit condition register operand. 2351 static void preserveCondRegFlags(MachineOperand &CondReg, 2352 const MachineOperand &OrigCond) { 2353 CondReg.setIsUndef(OrigCond.isUndef()); 2354 CondReg.setIsKill(OrigCond.isKill()); 2355 } 2356 2357 unsigned SIInstrInfo::insertBranch(MachineBasicBlock &MBB, 2358 MachineBasicBlock *TBB, 2359 MachineBasicBlock *FBB, 2360 ArrayRef<MachineOperand> Cond, 2361 const DebugLoc &DL, 2362 int *BytesAdded) const { 2363 if (!FBB && Cond.empty()) { 2364 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH)) 2365 .addMBB(TBB); 2366 if (BytesAdded) 2367 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4; 2368 return 1; 2369 } 2370 2371 if(Cond.size() == 1 && Cond[0].isReg()) { 2372 BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO)) 2373 .add(Cond[0]) 2374 .addMBB(TBB); 2375 return 1; 2376 } 2377 2378 assert(TBB && Cond[0].isImm()); 2379 2380 unsigned Opcode 2381 = getBranchOpcode(static_cast<BranchPredicate>(Cond[0].getImm())); 2382 2383 if (!FBB) { 2384 Cond[1].isUndef(); 2385 MachineInstr *CondBr = 2386 BuildMI(&MBB, DL, get(Opcode)) 2387 .addMBB(TBB); 2388 2389 // Copy the flags onto the implicit condition register operand. 2390 preserveCondRegFlags(CondBr->getOperand(1), Cond[1]); 2391 fixImplicitOperands(*CondBr); 2392 2393 if (BytesAdded) 2394 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4; 2395 return 1; 2396 } 2397 2398 assert(TBB && FBB); 2399 2400 MachineInstr *CondBr = 2401 BuildMI(&MBB, DL, get(Opcode)) 2402 .addMBB(TBB); 2403 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH)) 2404 .addMBB(FBB); 2405 2406 MachineOperand &CondReg = CondBr->getOperand(1); 2407 CondReg.setIsUndef(Cond[1].isUndef()); 2408 CondReg.setIsKill(Cond[1].isKill()); 2409 2410 if (BytesAdded) 2411 *BytesAdded = ST.hasOffset3fBug() ? 16 : 8; 2412 2413 return 2; 2414 } 2415 2416 bool SIInstrInfo::reverseBranchCondition( 2417 SmallVectorImpl<MachineOperand> &Cond) const { 2418 if (Cond.size() != 2) { 2419 return true; 2420 } 2421 2422 if (Cond[0].isImm()) { 2423 Cond[0].setImm(-Cond[0].getImm()); 2424 return false; 2425 } 2426 2427 return true; 2428 } 2429 2430 bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB, 2431 ArrayRef<MachineOperand> Cond, 2432 Register DstReg, Register TrueReg, 2433 Register FalseReg, int &CondCycles, 2434 int &TrueCycles, int &FalseCycles) const { 2435 switch (Cond[0].getImm()) { 2436 case VCCNZ: 2437 case VCCZ: { 2438 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 2439 const TargetRegisterClass *RC = MRI.getRegClass(TrueReg); 2440 if (MRI.getRegClass(FalseReg) != RC) 2441 return false; 2442 2443 int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32; 2444 CondCycles = TrueCycles = FalseCycles = NumInsts; // ??? 2445 2446 // Limit to equal cost for branch vs. N v_cndmask_b32s. 2447 return RI.hasVGPRs(RC) && NumInsts <= 6; 2448 } 2449 case SCC_TRUE: 2450 case SCC_FALSE: { 2451 // FIXME: We could insert for VGPRs if we could replace the original compare 2452 // with a vector one. 2453 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 2454 const TargetRegisterClass *RC = MRI.getRegClass(TrueReg); 2455 if (MRI.getRegClass(FalseReg) != RC) 2456 return false; 2457 2458 int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32; 2459 2460 // Multiples of 8 can do s_cselect_b64 2461 if (NumInsts % 2 == 0) 2462 NumInsts /= 2; 2463 2464 CondCycles = TrueCycles = FalseCycles = NumInsts; // ??? 2465 return RI.isSGPRClass(RC); 2466 } 2467 default: 2468 return false; 2469 } 2470 } 2471 2472 void SIInstrInfo::insertSelect(MachineBasicBlock &MBB, 2473 MachineBasicBlock::iterator I, const DebugLoc &DL, 2474 Register DstReg, ArrayRef<MachineOperand> Cond, 2475 Register TrueReg, Register FalseReg) const { 2476 BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm()); 2477 if (Pred == VCCZ || Pred == SCC_FALSE) { 2478 Pred = static_cast<BranchPredicate>(-Pred); 2479 std::swap(TrueReg, FalseReg); 2480 } 2481 2482 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 2483 const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg); 2484 unsigned DstSize = RI.getRegSizeInBits(*DstRC); 2485 2486 if (DstSize == 32) { 2487 MachineInstr *Select; 2488 if (Pred == SCC_TRUE) { 2489 Select = BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B32), DstReg) 2490 .addReg(TrueReg) 2491 .addReg(FalseReg); 2492 } else { 2493 // Instruction's operands are backwards from what is expected. 2494 Select = BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e32), DstReg) 2495 .addReg(FalseReg) 2496 .addReg(TrueReg); 2497 } 2498 2499 preserveCondRegFlags(Select->getOperand(3), Cond[1]); 2500 return; 2501 } 2502 2503 if (DstSize == 64 && Pred == SCC_TRUE) { 2504 MachineInstr *Select = 2505 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg) 2506 .addReg(TrueReg) 2507 .addReg(FalseReg); 2508 2509 preserveCondRegFlags(Select->getOperand(3), Cond[1]); 2510 return; 2511 } 2512 2513 static const int16_t Sub0_15[] = { 2514 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, 2515 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7, 2516 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11, 2517 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15, 2518 }; 2519 2520 static const int16_t Sub0_15_64[] = { 2521 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3, 2522 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7, 2523 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11, 2524 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15, 2525 }; 2526 2527 unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32; 2528 const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass; 2529 const int16_t *SubIndices = Sub0_15; 2530 int NElts = DstSize / 32; 2531 2532 // 64-bit select is only available for SALU. 2533 // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit. 2534 if (Pred == SCC_TRUE) { 2535 if (NElts % 2) { 2536 SelOp = AMDGPU::S_CSELECT_B32; 2537 EltRC = &AMDGPU::SGPR_32RegClass; 2538 } else { 2539 SelOp = AMDGPU::S_CSELECT_B64; 2540 EltRC = &AMDGPU::SGPR_64RegClass; 2541 SubIndices = Sub0_15_64; 2542 NElts /= 2; 2543 } 2544 } 2545 2546 MachineInstrBuilder MIB = BuildMI( 2547 MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg); 2548 2549 I = MIB->getIterator(); 2550 2551 SmallVector<Register, 8> Regs; 2552 for (int Idx = 0; Idx != NElts; ++Idx) { 2553 Register DstElt = MRI.createVirtualRegister(EltRC); 2554 Regs.push_back(DstElt); 2555 2556 unsigned SubIdx = SubIndices[Idx]; 2557 2558 MachineInstr *Select; 2559 if (SelOp == AMDGPU::V_CNDMASK_B32_e32) { 2560 Select = 2561 BuildMI(MBB, I, DL, get(SelOp), DstElt) 2562 .addReg(FalseReg, 0, SubIdx) 2563 .addReg(TrueReg, 0, SubIdx); 2564 } else { 2565 Select = 2566 BuildMI(MBB, I, DL, get(SelOp), DstElt) 2567 .addReg(TrueReg, 0, SubIdx) 2568 .addReg(FalseReg, 0, SubIdx); 2569 } 2570 2571 preserveCondRegFlags(Select->getOperand(3), Cond[1]); 2572 fixImplicitOperands(*Select); 2573 2574 MIB.addReg(DstElt) 2575 .addImm(SubIdx); 2576 } 2577 } 2578 2579 bool SIInstrInfo::isFoldableCopy(const MachineInstr &MI) const { 2580 switch (MI.getOpcode()) { 2581 case AMDGPU::V_MOV_B32_e32: 2582 case AMDGPU::V_MOV_B32_e64: 2583 case AMDGPU::V_MOV_B64_PSEUDO: { 2584 // If there are additional implicit register operands, this may be used for 2585 // register indexing so the source register operand isn't simply copied. 2586 unsigned NumOps = MI.getDesc().getNumOperands() + 2587 MI.getDesc().getNumImplicitUses(); 2588 2589 return MI.getNumOperands() == NumOps; 2590 } 2591 case AMDGPU::S_MOV_B32: 2592 case AMDGPU::S_MOV_B64: 2593 case AMDGPU::COPY: 2594 case AMDGPU::V_ACCVGPR_WRITE_B32_e64: 2595 case AMDGPU::V_ACCVGPR_READ_B32_e64: 2596 return true; 2597 default: 2598 return false; 2599 } 2600 } 2601 2602 unsigned SIInstrInfo::getAddressSpaceForPseudoSourceKind( 2603 unsigned Kind) const { 2604 switch(Kind) { 2605 case PseudoSourceValue::Stack: 2606 case PseudoSourceValue::FixedStack: 2607 return AMDGPUAS::PRIVATE_ADDRESS; 2608 case PseudoSourceValue::ConstantPool: 2609 case PseudoSourceValue::GOT: 2610 case PseudoSourceValue::JumpTable: 2611 case PseudoSourceValue::GlobalValueCallEntry: 2612 case PseudoSourceValue::ExternalSymbolCallEntry: 2613 case PseudoSourceValue::TargetCustom: 2614 return AMDGPUAS::CONSTANT_ADDRESS; 2615 } 2616 return AMDGPUAS::FLAT_ADDRESS; 2617 } 2618 2619 static void removeModOperands(MachineInstr &MI) { 2620 unsigned Opc = MI.getOpcode(); 2621 int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc, 2622 AMDGPU::OpName::src0_modifiers); 2623 int Src1ModIdx = AMDGPU::getNamedOperandIdx(Opc, 2624 AMDGPU::OpName::src1_modifiers); 2625 int Src2ModIdx = AMDGPU::getNamedOperandIdx(Opc, 2626 AMDGPU::OpName::src2_modifiers); 2627 2628 MI.RemoveOperand(Src2ModIdx); 2629 MI.RemoveOperand(Src1ModIdx); 2630 MI.RemoveOperand(Src0ModIdx); 2631 } 2632 2633 bool SIInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, 2634 Register Reg, MachineRegisterInfo *MRI) const { 2635 if (!MRI->hasOneNonDBGUse(Reg)) 2636 return false; 2637 2638 switch (DefMI.getOpcode()) { 2639 default: 2640 return false; 2641 case AMDGPU::S_MOV_B64: 2642 // TODO: We could fold 64-bit immediates, but this get compilicated 2643 // when there are sub-registers. 2644 return false; 2645 2646 case AMDGPU::V_MOV_B32_e32: 2647 case AMDGPU::S_MOV_B32: 2648 case AMDGPU::V_ACCVGPR_WRITE_B32_e64: 2649 break; 2650 } 2651 2652 const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0); 2653 assert(ImmOp); 2654 // FIXME: We could handle FrameIndex values here. 2655 if (!ImmOp->isImm()) 2656 return false; 2657 2658 unsigned Opc = UseMI.getOpcode(); 2659 if (Opc == AMDGPU::COPY) { 2660 Register DstReg = UseMI.getOperand(0).getReg(); 2661 bool Is16Bit = getOpSize(UseMI, 0) == 2; 2662 bool isVGPRCopy = RI.isVGPR(*MRI, DstReg); 2663 unsigned NewOpc = isVGPRCopy ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32; 2664 APInt Imm(32, ImmOp->getImm()); 2665 2666 if (UseMI.getOperand(1).getSubReg() == AMDGPU::hi16) 2667 Imm = Imm.ashr(16); 2668 2669 if (RI.isAGPR(*MRI, DstReg)) { 2670 if (!isInlineConstant(Imm)) 2671 return false; 2672 NewOpc = AMDGPU::V_ACCVGPR_WRITE_B32_e64; 2673 } 2674 2675 if (Is16Bit) { 2676 if (isVGPRCopy) 2677 return false; // Do not clobber vgpr_hi16 2678 2679 if (DstReg.isVirtual() && 2680 UseMI.getOperand(0).getSubReg() != AMDGPU::lo16) 2681 return false; 2682 2683 UseMI.getOperand(0).setSubReg(0); 2684 if (DstReg.isPhysical()) { 2685 DstReg = RI.get32BitRegister(DstReg); 2686 UseMI.getOperand(0).setReg(DstReg); 2687 } 2688 assert(UseMI.getOperand(1).getReg().isVirtual()); 2689 } 2690 2691 UseMI.setDesc(get(NewOpc)); 2692 UseMI.getOperand(1).ChangeToImmediate(Imm.getSExtValue()); 2693 UseMI.addImplicitDefUseOperands(*UseMI.getParent()->getParent()); 2694 return true; 2695 } 2696 2697 if (Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 || 2698 Opc == AMDGPU::V_MAD_F16_e64 || Opc == AMDGPU::V_MAC_F16_e64 || 2699 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 || 2700 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64) { 2701 // Don't fold if we are using source or output modifiers. The new VOP2 2702 // instructions don't have them. 2703 if (hasAnyModifiersSet(UseMI)) 2704 return false; 2705 2706 // If this is a free constant, there's no reason to do this. 2707 // TODO: We could fold this here instead of letting SIFoldOperands do it 2708 // later. 2709 MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0); 2710 2711 // Any src operand can be used for the legality check. 2712 if (isInlineConstant(UseMI, *Src0, *ImmOp)) 2713 return false; 2714 2715 bool IsF32 = Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 || 2716 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64; 2717 bool IsFMA = Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 || 2718 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64; 2719 MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1); 2720 MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2); 2721 2722 // Multiplied part is the constant: Use v_madmk_{f16, f32}. 2723 // We should only expect these to be on src0 due to canonicalizations. 2724 if (Src0->isReg() && Src0->getReg() == Reg) { 2725 if (!Src1->isReg() || RI.isSGPRClass(MRI->getRegClass(Src1->getReg()))) 2726 return false; 2727 2728 if (!Src2->isReg() || RI.isSGPRClass(MRI->getRegClass(Src2->getReg()))) 2729 return false; 2730 2731 unsigned NewOpc = 2732 IsFMA ? (IsF32 ? AMDGPU::V_FMAMK_F32 : AMDGPU::V_FMAMK_F16) 2733 : (IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16); 2734 if (pseudoToMCOpcode(NewOpc) == -1) 2735 return false; 2736 2737 // We need to swap operands 0 and 1 since madmk constant is at operand 1. 2738 2739 const int64_t Imm = ImmOp->getImm(); 2740 2741 // FIXME: This would be a lot easier if we could return a new instruction 2742 // instead of having to modify in place. 2743 2744 // Remove these first since they are at the end. 2745 UseMI.RemoveOperand( 2746 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod)); 2747 UseMI.RemoveOperand( 2748 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp)); 2749 2750 Register Src1Reg = Src1->getReg(); 2751 unsigned Src1SubReg = Src1->getSubReg(); 2752 Src0->setReg(Src1Reg); 2753 Src0->setSubReg(Src1SubReg); 2754 Src0->setIsKill(Src1->isKill()); 2755 2756 if (Opc == AMDGPU::V_MAC_F32_e64 || 2757 Opc == AMDGPU::V_MAC_F16_e64 || 2758 Opc == AMDGPU::V_FMAC_F32_e64 || 2759 Opc == AMDGPU::V_FMAC_F16_e64) 2760 UseMI.untieRegOperand( 2761 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)); 2762 2763 Src1->ChangeToImmediate(Imm); 2764 2765 removeModOperands(UseMI); 2766 UseMI.setDesc(get(NewOpc)); 2767 2768 bool DeleteDef = MRI->hasOneNonDBGUse(Reg); 2769 if (DeleteDef) 2770 DefMI.eraseFromParent(); 2771 2772 return true; 2773 } 2774 2775 // Added part is the constant: Use v_madak_{f16, f32}. 2776 if (Src2->isReg() && Src2->getReg() == Reg) { 2777 // Not allowed to use constant bus for another operand. 2778 // We can however allow an inline immediate as src0. 2779 bool Src0Inlined = false; 2780 if (Src0->isReg()) { 2781 // Try to inline constant if possible. 2782 // If the Def moves immediate and the use is single 2783 // We are saving VGPR here. 2784 MachineInstr *Def = MRI->getUniqueVRegDef(Src0->getReg()); 2785 if (Def && Def->isMoveImmediate() && 2786 isInlineConstant(Def->getOperand(1)) && 2787 MRI->hasOneUse(Src0->getReg())) { 2788 Src0->ChangeToImmediate(Def->getOperand(1).getImm()); 2789 Src0Inlined = true; 2790 } else if ((Src0->getReg().isPhysical() && 2791 (ST.getConstantBusLimit(Opc) <= 1 && 2792 RI.isSGPRClass(RI.getPhysRegClass(Src0->getReg())))) || 2793 (Src0->getReg().isVirtual() && 2794 (ST.getConstantBusLimit(Opc) <= 1 && 2795 RI.isSGPRClass(MRI->getRegClass(Src0->getReg()))))) 2796 return false; 2797 // VGPR is okay as Src0 - fallthrough 2798 } 2799 2800 if (Src1->isReg() && !Src0Inlined ) { 2801 // We have one slot for inlinable constant so far - try to fill it 2802 MachineInstr *Def = MRI->getUniqueVRegDef(Src1->getReg()); 2803 if (Def && Def->isMoveImmediate() && 2804 isInlineConstant(Def->getOperand(1)) && 2805 MRI->hasOneUse(Src1->getReg()) && 2806 commuteInstruction(UseMI)) { 2807 Src0->ChangeToImmediate(Def->getOperand(1).getImm()); 2808 } else if ((Src1->getReg().isPhysical() && 2809 RI.isSGPRClass(RI.getPhysRegClass(Src1->getReg()))) || 2810 (Src1->getReg().isVirtual() && 2811 RI.isSGPRClass(MRI->getRegClass(Src1->getReg())))) 2812 return false; 2813 // VGPR is okay as Src1 - fallthrough 2814 } 2815 2816 unsigned NewOpc = 2817 IsFMA ? (IsF32 ? AMDGPU::V_FMAAK_F32 : AMDGPU::V_FMAAK_F16) 2818 : (IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16); 2819 if (pseudoToMCOpcode(NewOpc) == -1) 2820 return false; 2821 2822 const int64_t Imm = ImmOp->getImm(); 2823 2824 // FIXME: This would be a lot easier if we could return a new instruction 2825 // instead of having to modify in place. 2826 2827 // Remove these first since they are at the end. 2828 UseMI.RemoveOperand( 2829 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod)); 2830 UseMI.RemoveOperand( 2831 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp)); 2832 2833 if (Opc == AMDGPU::V_MAC_F32_e64 || 2834 Opc == AMDGPU::V_MAC_F16_e64 || 2835 Opc == AMDGPU::V_FMAC_F32_e64 || 2836 Opc == AMDGPU::V_FMAC_F16_e64) 2837 UseMI.untieRegOperand( 2838 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)); 2839 2840 // ChangingToImmediate adds Src2 back to the instruction. 2841 Src2->ChangeToImmediate(Imm); 2842 2843 // These come before src2. 2844 removeModOperands(UseMI); 2845 UseMI.setDesc(get(NewOpc)); 2846 // It might happen that UseMI was commuted 2847 // and we now have SGPR as SRC1. If so 2 inlined 2848 // constant and SGPR are illegal. 2849 legalizeOperands(UseMI); 2850 2851 bool DeleteDef = MRI->hasOneNonDBGUse(Reg); 2852 if (DeleteDef) 2853 DefMI.eraseFromParent(); 2854 2855 return true; 2856 } 2857 } 2858 2859 return false; 2860 } 2861 2862 static bool 2863 memOpsHaveSameBaseOperands(ArrayRef<const MachineOperand *> BaseOps1, 2864 ArrayRef<const MachineOperand *> BaseOps2) { 2865 if (BaseOps1.size() != BaseOps2.size()) 2866 return false; 2867 for (size_t I = 0, E = BaseOps1.size(); I < E; ++I) { 2868 if (!BaseOps1[I]->isIdenticalTo(*BaseOps2[I])) 2869 return false; 2870 } 2871 return true; 2872 } 2873 2874 static bool offsetsDoNotOverlap(int WidthA, int OffsetA, 2875 int WidthB, int OffsetB) { 2876 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB; 2877 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA; 2878 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB; 2879 return LowOffset + LowWidth <= HighOffset; 2880 } 2881 2882 bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa, 2883 const MachineInstr &MIb) const { 2884 SmallVector<const MachineOperand *, 4> BaseOps0, BaseOps1; 2885 int64_t Offset0, Offset1; 2886 unsigned Dummy0, Dummy1; 2887 bool Offset0IsScalable, Offset1IsScalable; 2888 if (!getMemOperandsWithOffsetWidth(MIa, BaseOps0, Offset0, Offset0IsScalable, 2889 Dummy0, &RI) || 2890 !getMemOperandsWithOffsetWidth(MIb, BaseOps1, Offset1, Offset1IsScalable, 2891 Dummy1, &RI)) 2892 return false; 2893 2894 if (!memOpsHaveSameBaseOperands(BaseOps0, BaseOps1)) 2895 return false; 2896 2897 if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) { 2898 // FIXME: Handle ds_read2 / ds_write2. 2899 return false; 2900 } 2901 unsigned Width0 = MIa.memoperands().front()->getSize(); 2902 unsigned Width1 = MIb.memoperands().front()->getSize(); 2903 return offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1); 2904 } 2905 2906 bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, 2907 const MachineInstr &MIb) const { 2908 assert(MIa.mayLoadOrStore() && 2909 "MIa must load from or modify a memory location"); 2910 assert(MIb.mayLoadOrStore() && 2911 "MIb must load from or modify a memory location"); 2912 2913 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects()) 2914 return false; 2915 2916 // XXX - Can we relax this between address spaces? 2917 if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef()) 2918 return false; 2919 2920 // TODO: Should we check the address space from the MachineMemOperand? That 2921 // would allow us to distinguish objects we know don't alias based on the 2922 // underlying address space, even if it was lowered to a different one, 2923 // e.g. private accesses lowered to use MUBUF instructions on a scratch 2924 // buffer. 2925 if (isDS(MIa)) { 2926 if (isDS(MIb)) 2927 return checkInstOffsetsDoNotOverlap(MIa, MIb); 2928 2929 return !isFLAT(MIb) || isSegmentSpecificFLAT(MIb); 2930 } 2931 2932 if (isMUBUF(MIa) || isMTBUF(MIa)) { 2933 if (isMUBUF(MIb) || isMTBUF(MIb)) 2934 return checkInstOffsetsDoNotOverlap(MIa, MIb); 2935 2936 return !isFLAT(MIb) && !isSMRD(MIb); 2937 } 2938 2939 if (isSMRD(MIa)) { 2940 if (isSMRD(MIb)) 2941 return checkInstOffsetsDoNotOverlap(MIa, MIb); 2942 2943 return !isFLAT(MIb) && !isMUBUF(MIb) && !isMTBUF(MIb); 2944 } 2945 2946 if (isFLAT(MIa)) { 2947 if (isFLAT(MIb)) 2948 return checkInstOffsetsDoNotOverlap(MIa, MIb); 2949 2950 return false; 2951 } 2952 2953 return false; 2954 } 2955 2956 static int64_t getFoldableImm(const MachineOperand* MO) { 2957 if (!MO->isReg()) 2958 return false; 2959 const MachineFunction *MF = MO->getParent()->getParent()->getParent(); 2960 const MachineRegisterInfo &MRI = MF->getRegInfo(); 2961 auto Def = MRI.getUniqueVRegDef(MO->getReg()); 2962 if (Def && Def->getOpcode() == AMDGPU::V_MOV_B32_e32 && 2963 Def->getOperand(1).isImm()) 2964 return Def->getOperand(1).getImm(); 2965 return AMDGPU::NoRegister; 2966 } 2967 2968 static void updateLiveVariables(LiveVariables *LV, MachineInstr &MI, 2969 MachineInstr &NewMI) { 2970 if (LV) { 2971 unsigned NumOps = MI.getNumOperands(); 2972 for (unsigned I = 1; I < NumOps; ++I) { 2973 MachineOperand &Op = MI.getOperand(I); 2974 if (Op.isReg() && Op.isKill()) 2975 LV->replaceKillInstruction(Op.getReg(), MI, NewMI); 2976 } 2977 } 2978 } 2979 2980 MachineInstr *SIInstrInfo::convertToThreeAddress(MachineFunction::iterator &MBB, 2981 MachineInstr &MI, 2982 LiveVariables *LV) const { 2983 unsigned Opc = MI.getOpcode(); 2984 bool IsF16 = false; 2985 bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64 || 2986 Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64; 2987 2988 switch (Opc) { 2989 default: 2990 return nullptr; 2991 case AMDGPU::V_MAC_F16_e64: 2992 case AMDGPU::V_FMAC_F16_e64: 2993 IsF16 = true; 2994 LLVM_FALLTHROUGH; 2995 case AMDGPU::V_MAC_F32_e64: 2996 case AMDGPU::V_FMAC_F32_e64: 2997 break; 2998 case AMDGPU::V_MAC_F16_e32: 2999 case AMDGPU::V_FMAC_F16_e32: 3000 IsF16 = true; 3001 LLVM_FALLTHROUGH; 3002 case AMDGPU::V_MAC_F32_e32: 3003 case AMDGPU::V_FMAC_F32_e32: { 3004 int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), 3005 AMDGPU::OpName::src0); 3006 const MachineOperand *Src0 = &MI.getOperand(Src0Idx); 3007 if (!Src0->isReg() && !Src0->isImm()) 3008 return nullptr; 3009 3010 if (Src0->isImm() && !isInlineConstant(MI, Src0Idx, *Src0)) 3011 return nullptr; 3012 3013 break; 3014 } 3015 } 3016 3017 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); 3018 const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0); 3019 const MachineOperand *Src0Mods = 3020 getNamedOperand(MI, AMDGPU::OpName::src0_modifiers); 3021 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); 3022 const MachineOperand *Src1Mods = 3023 getNamedOperand(MI, AMDGPU::OpName::src1_modifiers); 3024 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); 3025 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp); 3026 const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod); 3027 MachineInstrBuilder MIB; 3028 3029 if (!Src0Mods && !Src1Mods && !Clamp && !Omod && 3030 // If we have an SGPR input, we will violate the constant bus restriction. 3031 (ST.getConstantBusLimit(Opc) > 1 || !Src0->isReg() || 3032 !RI.isSGPRReg(MBB->getParent()->getRegInfo(), Src0->getReg()))) { 3033 if (auto Imm = getFoldableImm(Src2)) { 3034 unsigned NewOpc = 3035 IsFMA ? (IsF16 ? AMDGPU::V_FMAAK_F16 : AMDGPU::V_FMAAK_F32) 3036 : (IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32); 3037 if (pseudoToMCOpcode(NewOpc) != -1) { 3038 MIB = BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3039 .add(*Dst) 3040 .add(*Src0) 3041 .add(*Src1) 3042 .addImm(Imm); 3043 updateLiveVariables(LV, MI, *MIB); 3044 return MIB; 3045 } 3046 } 3047 unsigned NewOpc = IsFMA 3048 ? (IsF16 ? AMDGPU::V_FMAMK_F16 : AMDGPU::V_FMAMK_F32) 3049 : (IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32); 3050 if (auto Imm = getFoldableImm(Src1)) { 3051 if (pseudoToMCOpcode(NewOpc) != -1) { 3052 MIB = BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3053 .add(*Dst) 3054 .add(*Src0) 3055 .addImm(Imm) 3056 .add(*Src2); 3057 updateLiveVariables(LV, MI, *MIB); 3058 return MIB; 3059 } 3060 } 3061 if (auto Imm = getFoldableImm(Src0)) { 3062 if (pseudoToMCOpcode(NewOpc) != -1 && 3063 isOperandLegal( 3064 MI, AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::src0), 3065 Src1)) { 3066 MIB = BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3067 .add(*Dst) 3068 .add(*Src1) 3069 .addImm(Imm) 3070 .add(*Src2); 3071 updateLiveVariables(LV, MI, *MIB); 3072 return MIB; 3073 } 3074 } 3075 } 3076 3077 unsigned NewOpc = IsFMA ? (IsF16 ? AMDGPU::V_FMA_F16_e64 : AMDGPU::V_FMA_F32_e64) 3078 : (IsF16 ? AMDGPU::V_MAD_F16_e64 : AMDGPU::V_MAD_F32_e64); 3079 if (pseudoToMCOpcode(NewOpc) == -1) 3080 return nullptr; 3081 3082 MIB = BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3083 .add(*Dst) 3084 .addImm(Src0Mods ? Src0Mods->getImm() : 0) 3085 .add(*Src0) 3086 .addImm(Src1Mods ? Src1Mods->getImm() : 0) 3087 .add(*Src1) 3088 .addImm(0) // Src mods 3089 .add(*Src2) 3090 .addImm(Clamp ? Clamp->getImm() : 0) 3091 .addImm(Omod ? Omod->getImm() : 0); 3092 updateLiveVariables(LV, MI, *MIB); 3093 return MIB; 3094 } 3095 3096 // It's not generally safe to move VALU instructions across these since it will 3097 // start using the register as a base index rather than directly. 3098 // XXX - Why isn't hasSideEffects sufficient for these? 3099 static bool changesVGPRIndexingMode(const MachineInstr &MI) { 3100 switch (MI.getOpcode()) { 3101 case AMDGPU::S_SET_GPR_IDX_ON: 3102 case AMDGPU::S_SET_GPR_IDX_MODE: 3103 case AMDGPU::S_SET_GPR_IDX_OFF: 3104 return true; 3105 default: 3106 return false; 3107 } 3108 } 3109 3110 bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI, 3111 const MachineBasicBlock *MBB, 3112 const MachineFunction &MF) const { 3113 // Skipping the check for SP writes in the base implementation. The reason it 3114 // was added was apparently due to compile time concerns. 3115 // 3116 // TODO: Do we really want this barrier? It triggers unnecessary hazard nops 3117 // but is probably avoidable. 3118 3119 // Copied from base implementation. 3120 // Terminators and labels can't be scheduled around. 3121 if (MI.isTerminator() || MI.isPosition()) 3122 return true; 3123 3124 // INLINEASM_BR can jump to another block 3125 if (MI.getOpcode() == TargetOpcode::INLINEASM_BR) 3126 return true; 3127 3128 // Target-independent instructions do not have an implicit-use of EXEC, even 3129 // when they operate on VGPRs. Treating EXEC modifications as scheduling 3130 // boundaries prevents incorrect movements of such instructions. 3131 return MI.modifiesRegister(AMDGPU::EXEC, &RI) || 3132 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 || 3133 MI.getOpcode() == AMDGPU::S_SETREG_B32 || 3134 changesVGPRIndexingMode(MI); 3135 } 3136 3137 bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const { 3138 return Opcode == AMDGPU::DS_ORDERED_COUNT || 3139 Opcode == AMDGPU::DS_GWS_INIT || 3140 Opcode == AMDGPU::DS_GWS_SEMA_V || 3141 Opcode == AMDGPU::DS_GWS_SEMA_BR || 3142 Opcode == AMDGPU::DS_GWS_SEMA_P || 3143 Opcode == AMDGPU::DS_GWS_SEMA_RELEASE_ALL || 3144 Opcode == AMDGPU::DS_GWS_BARRIER; 3145 } 3146 3147 bool SIInstrInfo::modifiesModeRegister(const MachineInstr &MI) { 3148 // Skip the full operand and register alias search modifiesRegister 3149 // does. There's only a handful of instructions that touch this, it's only an 3150 // implicit def, and doesn't alias any other registers. 3151 if (const MCPhysReg *ImpDef = MI.getDesc().getImplicitDefs()) { 3152 for (; ImpDef && *ImpDef; ++ImpDef) { 3153 if (*ImpDef == AMDGPU::MODE) 3154 return true; 3155 } 3156 } 3157 3158 return false; 3159 } 3160 3161 bool SIInstrInfo::hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const { 3162 unsigned Opcode = MI.getOpcode(); 3163 3164 if (MI.mayStore() && isSMRD(MI)) 3165 return true; // scalar store or atomic 3166 3167 // This will terminate the function when other lanes may need to continue. 3168 if (MI.isReturn()) 3169 return true; 3170 3171 // These instructions cause shader I/O that may cause hardware lockups 3172 // when executed with an empty EXEC mask. 3173 // 3174 // Note: exp with VM = DONE = 0 is automatically skipped by hardware when 3175 // EXEC = 0, but checking for that case here seems not worth it 3176 // given the typical code patterns. 3177 if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT || 3178 isEXP(Opcode) || 3179 Opcode == AMDGPU::DS_ORDERED_COUNT || Opcode == AMDGPU::S_TRAP || 3180 Opcode == AMDGPU::DS_GWS_INIT || Opcode == AMDGPU::DS_GWS_BARRIER) 3181 return true; 3182 3183 if (MI.isCall() || MI.isInlineAsm()) 3184 return true; // conservative assumption 3185 3186 // A mode change is a scalar operation that influences vector instructions. 3187 if (modifiesModeRegister(MI)) 3188 return true; 3189 3190 // These are like SALU instructions in terms of effects, so it's questionable 3191 // whether we should return true for those. 3192 // 3193 // However, executing them with EXEC = 0 causes them to operate on undefined 3194 // data, which we avoid by returning true here. 3195 if (Opcode == AMDGPU::V_READFIRSTLANE_B32 || 3196 Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32) 3197 return true; 3198 3199 return false; 3200 } 3201 3202 bool SIInstrInfo::mayReadEXEC(const MachineRegisterInfo &MRI, 3203 const MachineInstr &MI) const { 3204 if (MI.isMetaInstruction()) 3205 return false; 3206 3207 // This won't read exec if this is an SGPR->SGPR copy. 3208 if (MI.isCopyLike()) { 3209 if (!RI.isSGPRReg(MRI, MI.getOperand(0).getReg())) 3210 return true; 3211 3212 // Make sure this isn't copying exec as a normal operand 3213 return MI.readsRegister(AMDGPU::EXEC, &RI); 3214 } 3215 3216 // Make a conservative assumption about the callee. 3217 if (MI.isCall()) 3218 return true; 3219 3220 // Be conservative with any unhandled generic opcodes. 3221 if (!isTargetSpecificOpcode(MI.getOpcode())) 3222 return true; 3223 3224 return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI); 3225 } 3226 3227 bool SIInstrInfo::isInlineConstant(const APInt &Imm) const { 3228 switch (Imm.getBitWidth()) { 3229 case 1: // This likely will be a condition code mask. 3230 return true; 3231 3232 case 32: 3233 return AMDGPU::isInlinableLiteral32(Imm.getSExtValue(), 3234 ST.hasInv2PiInlineImm()); 3235 case 64: 3236 return AMDGPU::isInlinableLiteral64(Imm.getSExtValue(), 3237 ST.hasInv2PiInlineImm()); 3238 case 16: 3239 return ST.has16BitInsts() && 3240 AMDGPU::isInlinableLiteral16(Imm.getSExtValue(), 3241 ST.hasInv2PiInlineImm()); 3242 default: 3243 llvm_unreachable("invalid bitwidth"); 3244 } 3245 } 3246 3247 bool SIInstrInfo::isInlineConstant(const MachineOperand &MO, 3248 uint8_t OperandType) const { 3249 if (!MO.isImm() || 3250 OperandType < AMDGPU::OPERAND_SRC_FIRST || 3251 OperandType > AMDGPU::OPERAND_SRC_LAST) 3252 return false; 3253 3254 // MachineOperand provides no way to tell the true operand size, since it only 3255 // records a 64-bit value. We need to know the size to determine if a 32-bit 3256 // floating point immediate bit pattern is legal for an integer immediate. It 3257 // would be for any 32-bit integer operand, but would not be for a 64-bit one. 3258 3259 int64_t Imm = MO.getImm(); 3260 switch (OperandType) { 3261 case AMDGPU::OPERAND_REG_IMM_INT32: 3262 case AMDGPU::OPERAND_REG_IMM_FP32: 3263 case AMDGPU::OPERAND_REG_INLINE_C_INT32: 3264 case AMDGPU::OPERAND_REG_INLINE_C_FP32: 3265 case AMDGPU::OPERAND_REG_INLINE_AC_INT32: 3266 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: { 3267 int32_t Trunc = static_cast<int32_t>(Imm); 3268 return AMDGPU::isInlinableLiteral32(Trunc, ST.hasInv2PiInlineImm()); 3269 } 3270 case AMDGPU::OPERAND_REG_IMM_INT64: 3271 case AMDGPU::OPERAND_REG_IMM_FP64: 3272 case AMDGPU::OPERAND_REG_INLINE_C_INT64: 3273 case AMDGPU::OPERAND_REG_INLINE_C_FP64: 3274 return AMDGPU::isInlinableLiteral64(MO.getImm(), 3275 ST.hasInv2PiInlineImm()); 3276 case AMDGPU::OPERAND_REG_IMM_INT16: 3277 case AMDGPU::OPERAND_REG_INLINE_C_INT16: 3278 case AMDGPU::OPERAND_REG_INLINE_AC_INT16: 3279 // We would expect inline immediates to not be concerned with an integer/fp 3280 // distinction. However, in the case of 16-bit integer operations, the 3281 // "floating point" values appear to not work. It seems read the low 16-bits 3282 // of 32-bit immediates, which happens to always work for the integer 3283 // values. 3284 // 3285 // See llvm bugzilla 46302. 3286 // 3287 // TODO: Theoretically we could use op-sel to use the high bits of the 3288 // 32-bit FP values. 3289 return AMDGPU::isInlinableIntLiteral(Imm); 3290 case AMDGPU::OPERAND_REG_IMM_V2INT16: 3291 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: 3292 case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16: 3293 // This suffers the same problem as the scalar 16-bit cases. 3294 return AMDGPU::isInlinableIntLiteralV216(Imm); 3295 case AMDGPU::OPERAND_REG_IMM_FP16: 3296 case AMDGPU::OPERAND_REG_INLINE_C_FP16: 3297 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: { 3298 if (isInt<16>(Imm) || isUInt<16>(Imm)) { 3299 // A few special case instructions have 16-bit operands on subtargets 3300 // where 16-bit instructions are not legal. 3301 // TODO: Do the 32-bit immediates work? We shouldn't really need to handle 3302 // constants in these cases 3303 int16_t Trunc = static_cast<int16_t>(Imm); 3304 return ST.has16BitInsts() && 3305 AMDGPU::isInlinableLiteral16(Trunc, ST.hasInv2PiInlineImm()); 3306 } 3307 3308 return false; 3309 } 3310 case AMDGPU::OPERAND_REG_IMM_V2FP16: 3311 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: 3312 case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: { 3313 uint32_t Trunc = static_cast<uint32_t>(Imm); 3314 return AMDGPU::isInlinableLiteralV216(Trunc, ST.hasInv2PiInlineImm()); 3315 } 3316 default: 3317 llvm_unreachable("invalid bitwidth"); 3318 } 3319 } 3320 3321 bool SIInstrInfo::isLiteralConstantLike(const MachineOperand &MO, 3322 const MCOperandInfo &OpInfo) const { 3323 switch (MO.getType()) { 3324 case MachineOperand::MO_Register: 3325 return false; 3326 case MachineOperand::MO_Immediate: 3327 return !isInlineConstant(MO, OpInfo); 3328 case MachineOperand::MO_FrameIndex: 3329 case MachineOperand::MO_MachineBasicBlock: 3330 case MachineOperand::MO_ExternalSymbol: 3331 case MachineOperand::MO_GlobalAddress: 3332 case MachineOperand::MO_MCSymbol: 3333 return true; 3334 default: 3335 llvm_unreachable("unexpected operand type"); 3336 } 3337 } 3338 3339 static bool compareMachineOp(const MachineOperand &Op0, 3340 const MachineOperand &Op1) { 3341 if (Op0.getType() != Op1.getType()) 3342 return false; 3343 3344 switch (Op0.getType()) { 3345 case MachineOperand::MO_Register: 3346 return Op0.getReg() == Op1.getReg(); 3347 case MachineOperand::MO_Immediate: 3348 return Op0.getImm() == Op1.getImm(); 3349 default: 3350 llvm_unreachable("Didn't expect to be comparing these operand types"); 3351 } 3352 } 3353 3354 bool SIInstrInfo::isImmOperandLegal(const MachineInstr &MI, unsigned OpNo, 3355 const MachineOperand &MO) const { 3356 const MCInstrDesc &InstDesc = MI.getDesc(); 3357 const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpNo]; 3358 3359 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal()); 3360 3361 if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE) 3362 return true; 3363 3364 if (OpInfo.RegClass < 0) 3365 return false; 3366 3367 if (MO.isImm() && isInlineConstant(MO, OpInfo)) { 3368 if (isMAI(MI) && ST.hasMFMAInlineLiteralBug() && 3369 OpNo ==(unsigned)AMDGPU::getNamedOperandIdx(MI.getOpcode(), 3370 AMDGPU::OpName::src2)) 3371 return false; 3372 return RI.opCanUseInlineConstant(OpInfo.OperandType); 3373 } 3374 3375 if (!RI.opCanUseLiteralConstant(OpInfo.OperandType)) 3376 return false; 3377 3378 if (!isVOP3(MI) || !AMDGPU::isSISrcOperand(InstDesc, OpNo)) 3379 return true; 3380 3381 return ST.hasVOP3Literal(); 3382 } 3383 3384 bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const { 3385 int Op32 = AMDGPU::getVOPe32(Opcode); 3386 if (Op32 == -1) 3387 return false; 3388 3389 return pseudoToMCOpcode(Op32) != -1; 3390 } 3391 3392 bool SIInstrInfo::hasModifiers(unsigned Opcode) const { 3393 // The src0_modifier operand is present on all instructions 3394 // that have modifiers. 3395 3396 return AMDGPU::getNamedOperandIdx(Opcode, 3397 AMDGPU::OpName::src0_modifiers) != -1; 3398 } 3399 3400 bool SIInstrInfo::hasModifiersSet(const MachineInstr &MI, 3401 unsigned OpName) const { 3402 const MachineOperand *Mods = getNamedOperand(MI, OpName); 3403 return Mods && Mods->getImm(); 3404 } 3405 3406 bool SIInstrInfo::hasAnyModifiersSet(const MachineInstr &MI) const { 3407 return hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) || 3408 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) || 3409 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers) || 3410 hasModifiersSet(MI, AMDGPU::OpName::clamp) || 3411 hasModifiersSet(MI, AMDGPU::OpName::omod); 3412 } 3413 3414 bool SIInstrInfo::canShrink(const MachineInstr &MI, 3415 const MachineRegisterInfo &MRI) const { 3416 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); 3417 // Can't shrink instruction with three operands. 3418 // FIXME: v_cndmask_b32 has 3 operands and is shrinkable, but we need to add 3419 // a special case for it. It can only be shrunk if the third operand 3420 // is vcc, and src0_modifiers and src1_modifiers are not set. 3421 // We should handle this the same way we handle vopc, by addding 3422 // a register allocation hint pre-regalloc and then do the shrinking 3423 // post-regalloc. 3424 if (Src2) { 3425 switch (MI.getOpcode()) { 3426 default: return false; 3427 3428 case AMDGPU::V_ADDC_U32_e64: 3429 case AMDGPU::V_SUBB_U32_e64: 3430 case AMDGPU::V_SUBBREV_U32_e64: { 3431 const MachineOperand *Src1 3432 = getNamedOperand(MI, AMDGPU::OpName::src1); 3433 if (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg())) 3434 return false; 3435 // Additional verification is needed for sdst/src2. 3436 return true; 3437 } 3438 case AMDGPU::V_MAC_F32_e64: 3439 case AMDGPU::V_MAC_F16_e64: 3440 case AMDGPU::V_FMAC_F32_e64: 3441 case AMDGPU::V_FMAC_F16_e64: 3442 if (!Src2->isReg() || !RI.isVGPR(MRI, Src2->getReg()) || 3443 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers)) 3444 return false; 3445 break; 3446 3447 case AMDGPU::V_CNDMASK_B32_e64: 3448 break; 3449 } 3450 } 3451 3452 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); 3453 if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()) || 3454 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers))) 3455 return false; 3456 3457 // We don't need to check src0, all input types are legal, so just make sure 3458 // src0 isn't using any modifiers. 3459 if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers)) 3460 return false; 3461 3462 // Can it be shrunk to a valid 32 bit opcode? 3463 if (!hasVALU32BitEncoding(MI.getOpcode())) 3464 return false; 3465 3466 // Check output modifiers 3467 return !hasModifiersSet(MI, AMDGPU::OpName::omod) && 3468 !hasModifiersSet(MI, AMDGPU::OpName::clamp); 3469 } 3470 3471 // Set VCC operand with all flags from \p Orig, except for setting it as 3472 // implicit. 3473 static void copyFlagsToImplicitVCC(MachineInstr &MI, 3474 const MachineOperand &Orig) { 3475 3476 for (MachineOperand &Use : MI.implicit_operands()) { 3477 if (Use.isUse() && 3478 (Use.getReg() == AMDGPU::VCC || Use.getReg() == AMDGPU::VCC_LO)) { 3479 Use.setIsUndef(Orig.isUndef()); 3480 Use.setIsKill(Orig.isKill()); 3481 return; 3482 } 3483 } 3484 } 3485 3486 MachineInstr *SIInstrInfo::buildShrunkInst(MachineInstr &MI, 3487 unsigned Op32) const { 3488 MachineBasicBlock *MBB = MI.getParent();; 3489 MachineInstrBuilder Inst32 = 3490 BuildMI(*MBB, MI, MI.getDebugLoc(), get(Op32)) 3491 .setMIFlags(MI.getFlags()); 3492 3493 // Add the dst operand if the 32-bit encoding also has an explicit $vdst. 3494 // For VOPC instructions, this is replaced by an implicit def of vcc. 3495 int Op32DstIdx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::vdst); 3496 if (Op32DstIdx != -1) { 3497 // dst 3498 Inst32.add(MI.getOperand(0)); 3499 } else { 3500 assert(((MI.getOperand(0).getReg() == AMDGPU::VCC) || 3501 (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) && 3502 "Unexpected case"); 3503 } 3504 3505 Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0)); 3506 3507 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); 3508 if (Src1) 3509 Inst32.add(*Src1); 3510 3511 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); 3512 3513 if (Src2) { 3514 int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2); 3515 if (Op32Src2Idx != -1) { 3516 Inst32.add(*Src2); 3517 } else { 3518 // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is 3519 // replaced with an implicit read of vcc or vcc_lo. The implicit read 3520 // of vcc was already added during the initial BuildMI, but we 3521 // 1) may need to change vcc to vcc_lo to preserve the original register 3522 // 2) have to preserve the original flags. 3523 fixImplicitOperands(*Inst32); 3524 copyFlagsToImplicitVCC(*Inst32, *Src2); 3525 } 3526 } 3527 3528 return Inst32; 3529 } 3530 3531 bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo &MRI, 3532 const MachineOperand &MO, 3533 const MCOperandInfo &OpInfo) const { 3534 // Literal constants use the constant bus. 3535 //if (isLiteralConstantLike(MO, OpInfo)) 3536 // return true; 3537 if (MO.isImm()) 3538 return !isInlineConstant(MO, OpInfo); 3539 3540 if (!MO.isReg()) 3541 return true; // Misc other operands like FrameIndex 3542 3543 if (!MO.isUse()) 3544 return false; 3545 3546 if (MO.getReg().isVirtual()) 3547 return RI.isSGPRClass(MRI.getRegClass(MO.getReg())); 3548 3549 // Null is free 3550 if (MO.getReg() == AMDGPU::SGPR_NULL) 3551 return false; 3552 3553 // SGPRs use the constant bus 3554 if (MO.isImplicit()) { 3555 return MO.getReg() == AMDGPU::M0 || 3556 MO.getReg() == AMDGPU::VCC || 3557 MO.getReg() == AMDGPU::VCC_LO; 3558 } else { 3559 return AMDGPU::SReg_32RegClass.contains(MO.getReg()) || 3560 AMDGPU::SReg_64RegClass.contains(MO.getReg()); 3561 } 3562 } 3563 3564 static Register findImplicitSGPRRead(const MachineInstr &MI) { 3565 for (const MachineOperand &MO : MI.implicit_operands()) { 3566 // We only care about reads. 3567 if (MO.isDef()) 3568 continue; 3569 3570 switch (MO.getReg()) { 3571 case AMDGPU::VCC: 3572 case AMDGPU::VCC_LO: 3573 case AMDGPU::VCC_HI: 3574 case AMDGPU::M0: 3575 case AMDGPU::FLAT_SCR: 3576 return MO.getReg(); 3577 3578 default: 3579 break; 3580 } 3581 } 3582 3583 return AMDGPU::NoRegister; 3584 } 3585 3586 static bool shouldReadExec(const MachineInstr &MI) { 3587 if (SIInstrInfo::isVALU(MI)) { 3588 switch (MI.getOpcode()) { 3589 case AMDGPU::V_READLANE_B32: 3590 case AMDGPU::V_WRITELANE_B32: 3591 return false; 3592 } 3593 3594 return true; 3595 } 3596 3597 if (MI.isPreISelOpcode() || 3598 SIInstrInfo::isGenericOpcode(MI.getOpcode()) || 3599 SIInstrInfo::isSALU(MI) || 3600 SIInstrInfo::isSMRD(MI)) 3601 return false; 3602 3603 return true; 3604 } 3605 3606 static bool isSubRegOf(const SIRegisterInfo &TRI, 3607 const MachineOperand &SuperVec, 3608 const MachineOperand &SubReg) { 3609 if (SubReg.getReg().isPhysical()) 3610 return TRI.isSubRegister(SuperVec.getReg(), SubReg.getReg()); 3611 3612 return SubReg.getSubReg() != AMDGPU::NoSubRegister && 3613 SubReg.getReg() == SuperVec.getReg(); 3614 } 3615 3616 bool SIInstrInfo::verifyInstruction(const MachineInstr &MI, 3617 StringRef &ErrInfo) const { 3618 uint16_t Opcode = MI.getOpcode(); 3619 if (SIInstrInfo::isGenericOpcode(MI.getOpcode())) 3620 return true; 3621 3622 const MachineFunction *MF = MI.getParent()->getParent(); 3623 const MachineRegisterInfo &MRI = MF->getRegInfo(); 3624 3625 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0); 3626 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1); 3627 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2); 3628 3629 // Make sure the number of operands is correct. 3630 const MCInstrDesc &Desc = get(Opcode); 3631 if (!Desc.isVariadic() && 3632 Desc.getNumOperands() != MI.getNumExplicitOperands()) { 3633 ErrInfo = "Instruction has wrong number of operands."; 3634 return false; 3635 } 3636 3637 if (MI.isInlineAsm()) { 3638 // Verify register classes for inlineasm constraints. 3639 for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands(); 3640 I != E; ++I) { 3641 const TargetRegisterClass *RC = MI.getRegClassConstraint(I, this, &RI); 3642 if (!RC) 3643 continue; 3644 3645 const MachineOperand &Op = MI.getOperand(I); 3646 if (!Op.isReg()) 3647 continue; 3648 3649 Register Reg = Op.getReg(); 3650 if (!Reg.isVirtual() && !RC->contains(Reg)) { 3651 ErrInfo = "inlineasm operand has incorrect register class."; 3652 return false; 3653 } 3654 } 3655 3656 return true; 3657 } 3658 3659 if (isMIMG(MI) && MI.memoperands_empty() && MI.mayLoadOrStore()) { 3660 ErrInfo = "missing memory operand from MIMG instruction."; 3661 return false; 3662 } 3663 3664 // Make sure the register classes are correct. 3665 for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) { 3666 if (MI.getOperand(i).isFPImm()) { 3667 ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast " 3668 "all fp values to integers."; 3669 return false; 3670 } 3671 3672 int RegClass = Desc.OpInfo[i].RegClass; 3673 3674 switch (Desc.OpInfo[i].OperandType) { 3675 case MCOI::OPERAND_REGISTER: 3676 if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) { 3677 ErrInfo = "Illegal immediate value for operand."; 3678 return false; 3679 } 3680 break; 3681 case AMDGPU::OPERAND_REG_IMM_INT32: 3682 case AMDGPU::OPERAND_REG_IMM_FP32: 3683 break; 3684 case AMDGPU::OPERAND_REG_INLINE_C_INT32: 3685 case AMDGPU::OPERAND_REG_INLINE_C_FP32: 3686 case AMDGPU::OPERAND_REG_INLINE_C_INT64: 3687 case AMDGPU::OPERAND_REG_INLINE_C_FP64: 3688 case AMDGPU::OPERAND_REG_INLINE_C_INT16: 3689 case AMDGPU::OPERAND_REG_INLINE_C_FP16: 3690 case AMDGPU::OPERAND_REG_INLINE_AC_INT32: 3691 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: 3692 case AMDGPU::OPERAND_REG_INLINE_AC_INT16: 3693 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: { 3694 const MachineOperand &MO = MI.getOperand(i); 3695 if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, i))) { 3696 ErrInfo = "Illegal immediate value for operand."; 3697 return false; 3698 } 3699 break; 3700 } 3701 case MCOI::OPERAND_IMMEDIATE: 3702 case AMDGPU::OPERAND_KIMM32: 3703 // Check if this operand is an immediate. 3704 // FrameIndex operands will be replaced by immediates, so they are 3705 // allowed. 3706 if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) { 3707 ErrInfo = "Expected immediate, but got non-immediate"; 3708 return false; 3709 } 3710 LLVM_FALLTHROUGH; 3711 default: 3712 continue; 3713 } 3714 3715 if (!MI.getOperand(i).isReg()) 3716 continue; 3717 3718 if (RegClass != -1) { 3719 Register Reg = MI.getOperand(i).getReg(); 3720 if (Reg == AMDGPU::NoRegister || Reg.isVirtual()) 3721 continue; 3722 3723 const TargetRegisterClass *RC = RI.getRegClass(RegClass); 3724 if (!RC->contains(Reg)) { 3725 ErrInfo = "Operand has incorrect register class."; 3726 return false; 3727 } 3728 } 3729 } 3730 3731 // Verify SDWA 3732 if (isSDWA(MI)) { 3733 if (!ST.hasSDWA()) { 3734 ErrInfo = "SDWA is not supported on this target"; 3735 return false; 3736 } 3737 3738 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst); 3739 3740 const int OpIndicies[] = { DstIdx, Src0Idx, Src1Idx, Src2Idx }; 3741 3742 for (int OpIdx: OpIndicies) { 3743 if (OpIdx == -1) 3744 continue; 3745 const MachineOperand &MO = MI.getOperand(OpIdx); 3746 3747 if (!ST.hasSDWAScalar()) { 3748 // Only VGPRS on VI 3749 if (!MO.isReg() || !RI.hasVGPRs(RI.getRegClassForReg(MRI, MO.getReg()))) { 3750 ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI"; 3751 return false; 3752 } 3753 } else { 3754 // No immediates on GFX9 3755 if (!MO.isReg()) { 3756 ErrInfo = 3757 "Only reg allowed as operands in SDWA instructions on GFX9+"; 3758 return false; 3759 } 3760 } 3761 } 3762 3763 if (!ST.hasSDWAOmod()) { 3764 // No omod allowed on VI 3765 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod); 3766 if (OMod != nullptr && 3767 (!OMod->isImm() || OMod->getImm() != 0)) { 3768 ErrInfo = "OMod not allowed in SDWA instructions on VI"; 3769 return false; 3770 } 3771 } 3772 3773 uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode); 3774 if (isVOPC(BasicOpcode)) { 3775 if (!ST.hasSDWASdst() && DstIdx != -1) { 3776 // Only vcc allowed as dst on VI for VOPC 3777 const MachineOperand &Dst = MI.getOperand(DstIdx); 3778 if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) { 3779 ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI"; 3780 return false; 3781 } 3782 } else if (!ST.hasSDWAOutModsVOPC()) { 3783 // No clamp allowed on GFX9 for VOPC 3784 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp); 3785 if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) { 3786 ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI"; 3787 return false; 3788 } 3789 3790 // No omod allowed on GFX9 for VOPC 3791 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod); 3792 if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) { 3793 ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI"; 3794 return false; 3795 } 3796 } 3797 } 3798 3799 const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused); 3800 if (DstUnused && DstUnused->isImm() && 3801 DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) { 3802 const MachineOperand &Dst = MI.getOperand(DstIdx); 3803 if (!Dst.isReg() || !Dst.isTied()) { 3804 ErrInfo = "Dst register should have tied register"; 3805 return false; 3806 } 3807 3808 const MachineOperand &TiedMO = 3809 MI.getOperand(MI.findTiedOperandIdx(DstIdx)); 3810 if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) { 3811 ErrInfo = 3812 "Dst register should be tied to implicit use of preserved register"; 3813 return false; 3814 } else if (TiedMO.getReg().isPhysical() && 3815 Dst.getReg() != TiedMO.getReg()) { 3816 ErrInfo = "Dst register should use same physical register as preserved"; 3817 return false; 3818 } 3819 } 3820 } 3821 3822 // Verify MIMG 3823 if (isMIMG(MI.getOpcode()) && !MI.mayStore()) { 3824 // Ensure that the return type used is large enough for all the options 3825 // being used TFE/LWE require an extra result register. 3826 const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask); 3827 if (DMask) { 3828 uint64_t DMaskImm = DMask->getImm(); 3829 uint32_t RegCount = 3830 isGather4(MI.getOpcode()) ? 4 : countPopulation(DMaskImm); 3831 const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe); 3832 const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe); 3833 const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16); 3834 3835 // Adjust for packed 16 bit values 3836 if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem()) 3837 RegCount >>= 1; 3838 3839 // Adjust if using LWE or TFE 3840 if ((LWE && LWE->getImm()) || (TFE && TFE->getImm())) 3841 RegCount += 1; 3842 3843 const uint32_t DstIdx = 3844 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata); 3845 const MachineOperand &Dst = MI.getOperand(DstIdx); 3846 if (Dst.isReg()) { 3847 const TargetRegisterClass *DstRC = getOpRegClass(MI, DstIdx); 3848 uint32_t DstSize = RI.getRegSizeInBits(*DstRC) / 32; 3849 if (RegCount > DstSize) { 3850 ErrInfo = "MIMG instruction returns too many registers for dst " 3851 "register class"; 3852 return false; 3853 } 3854 } 3855 } 3856 } 3857 3858 // Verify VOP*. Ignore multiple sgpr operands on writelane. 3859 if (Desc.getOpcode() != AMDGPU::V_WRITELANE_B32 3860 && (isVOP1(MI) || isVOP2(MI) || isVOP3(MI) || isVOPC(MI) || isSDWA(MI))) { 3861 // Only look at the true operands. Only a real operand can use the constant 3862 // bus, and we don't want to check pseudo-operands like the source modifier 3863 // flags. 3864 const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx }; 3865 3866 unsigned ConstantBusCount = 0; 3867 unsigned LiteralCount = 0; 3868 3869 if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1) 3870 ++ConstantBusCount; 3871 3872 SmallVector<Register, 2> SGPRsUsed; 3873 Register SGPRUsed; 3874 3875 for (int OpIdx : OpIndices) { 3876 if (OpIdx == -1) 3877 break; 3878 const MachineOperand &MO = MI.getOperand(OpIdx); 3879 if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) { 3880 if (MO.isReg()) { 3881 SGPRUsed = MO.getReg(); 3882 if (llvm::all_of(SGPRsUsed, [SGPRUsed](unsigned SGPR) { 3883 return SGPRUsed != SGPR; 3884 })) { 3885 ++ConstantBusCount; 3886 SGPRsUsed.push_back(SGPRUsed); 3887 } 3888 } else { 3889 ++ConstantBusCount; 3890 ++LiteralCount; 3891 } 3892 } 3893 } 3894 3895 SGPRUsed = findImplicitSGPRRead(MI); 3896 if (SGPRUsed != AMDGPU::NoRegister) { 3897 // Implicit uses may safely overlap true overands 3898 if (llvm::all_of(SGPRsUsed, [this, SGPRUsed](unsigned SGPR) { 3899 return !RI.regsOverlap(SGPRUsed, SGPR); 3900 })) { 3901 ++ConstantBusCount; 3902 SGPRsUsed.push_back(SGPRUsed); 3903 } 3904 } 3905 3906 // v_writelane_b32 is an exception from constant bus restriction: 3907 // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const 3908 if (ConstantBusCount > ST.getConstantBusLimit(Opcode) && 3909 Opcode != AMDGPU::V_WRITELANE_B32) { 3910 ErrInfo = "VOP* instruction violates constant bus restriction"; 3911 return false; 3912 } 3913 3914 if (isVOP3(MI) && LiteralCount) { 3915 if (!ST.hasVOP3Literal()) { 3916 ErrInfo = "VOP3 instruction uses literal"; 3917 return false; 3918 } 3919 if (LiteralCount > 1) { 3920 ErrInfo = "VOP3 instruction uses more than one literal"; 3921 return false; 3922 } 3923 } 3924 } 3925 3926 // Special case for writelane - this can break the multiple constant bus rule, 3927 // but still can't use more than one SGPR register 3928 if (Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) { 3929 unsigned SGPRCount = 0; 3930 Register SGPRUsed = AMDGPU::NoRegister; 3931 3932 for (int OpIdx : {Src0Idx, Src1Idx, Src2Idx}) { 3933 if (OpIdx == -1) 3934 break; 3935 3936 const MachineOperand &MO = MI.getOperand(OpIdx); 3937 3938 if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) { 3939 if (MO.isReg() && MO.getReg() != AMDGPU::M0) { 3940 if (MO.getReg() != SGPRUsed) 3941 ++SGPRCount; 3942 SGPRUsed = MO.getReg(); 3943 } 3944 } 3945 if (SGPRCount > ST.getConstantBusLimit(Opcode)) { 3946 ErrInfo = "WRITELANE instruction violates constant bus restriction"; 3947 return false; 3948 } 3949 } 3950 } 3951 3952 // Verify misc. restrictions on specific instructions. 3953 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32_e64 || 3954 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64_e64) { 3955 const MachineOperand &Src0 = MI.getOperand(Src0Idx); 3956 const MachineOperand &Src1 = MI.getOperand(Src1Idx); 3957 const MachineOperand &Src2 = MI.getOperand(Src2Idx); 3958 if (Src0.isReg() && Src1.isReg() && Src2.isReg()) { 3959 if (!compareMachineOp(Src0, Src1) && 3960 !compareMachineOp(Src0, Src2)) { 3961 ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2"; 3962 return false; 3963 } 3964 } 3965 if ((getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm() & 3966 SISrcMods::ABS) || 3967 (getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm() & 3968 SISrcMods::ABS) || 3969 (getNamedOperand(MI, AMDGPU::OpName::src2_modifiers)->getImm() & 3970 SISrcMods::ABS)) { 3971 ErrInfo = "ABS not allowed in VOP3B instructions"; 3972 return false; 3973 } 3974 } 3975 3976 if (isSOP2(MI) || isSOPC(MI)) { 3977 const MachineOperand &Src0 = MI.getOperand(Src0Idx); 3978 const MachineOperand &Src1 = MI.getOperand(Src1Idx); 3979 unsigned Immediates = 0; 3980 3981 if (!Src0.isReg() && 3982 !isInlineConstant(Src0, Desc.OpInfo[Src0Idx].OperandType)) 3983 Immediates++; 3984 if (!Src1.isReg() && 3985 !isInlineConstant(Src1, Desc.OpInfo[Src1Idx].OperandType)) 3986 Immediates++; 3987 3988 if (Immediates > 1) { 3989 ErrInfo = "SOP2/SOPC instruction requires too many immediate constants"; 3990 return false; 3991 } 3992 } 3993 3994 if (isSOPK(MI)) { 3995 auto Op = getNamedOperand(MI, AMDGPU::OpName::simm16); 3996 if (Desc.isBranch()) { 3997 if (!Op->isMBB()) { 3998 ErrInfo = "invalid branch target for SOPK instruction"; 3999 return false; 4000 } 4001 } else { 4002 uint64_t Imm = Op->getImm(); 4003 if (sopkIsZext(MI)) { 4004 if (!isUInt<16>(Imm)) { 4005 ErrInfo = "invalid immediate for SOPK instruction"; 4006 return false; 4007 } 4008 } else { 4009 if (!isInt<16>(Imm)) { 4010 ErrInfo = "invalid immediate for SOPK instruction"; 4011 return false; 4012 } 4013 } 4014 } 4015 } 4016 4017 if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 || 4018 Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 || 4019 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 || 4020 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) { 4021 const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 || 4022 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64; 4023 4024 const unsigned StaticNumOps = Desc.getNumOperands() + 4025 Desc.getNumImplicitUses(); 4026 const unsigned NumImplicitOps = IsDst ? 2 : 1; 4027 4028 // Allow additional implicit operands. This allows a fixup done by the post 4029 // RA scheduler where the main implicit operand is killed and implicit-defs 4030 // are added for sub-registers that remain live after this instruction. 4031 if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) { 4032 ErrInfo = "missing implicit register operands"; 4033 return false; 4034 } 4035 4036 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); 4037 if (IsDst) { 4038 if (!Dst->isUse()) { 4039 ErrInfo = "v_movreld_b32 vdst should be a use operand"; 4040 return false; 4041 } 4042 4043 unsigned UseOpIdx; 4044 if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) || 4045 UseOpIdx != StaticNumOps + 1) { 4046 ErrInfo = "movrel implicit operands should be tied"; 4047 return false; 4048 } 4049 } 4050 4051 const MachineOperand &Src0 = MI.getOperand(Src0Idx); 4052 const MachineOperand &ImpUse 4053 = MI.getOperand(StaticNumOps + NumImplicitOps - 1); 4054 if (!ImpUse.isReg() || !ImpUse.isUse() || 4055 !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) { 4056 ErrInfo = "src0 should be subreg of implicit vector use"; 4057 return false; 4058 } 4059 } 4060 4061 // Make sure we aren't losing exec uses in the td files. This mostly requires 4062 // being careful when using let Uses to try to add other use registers. 4063 if (shouldReadExec(MI)) { 4064 if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) { 4065 ErrInfo = "VALU instruction does not implicitly read exec mask"; 4066 return false; 4067 } 4068 } 4069 4070 if (isSMRD(MI)) { 4071 if (MI.mayStore()) { 4072 // The register offset form of scalar stores may only use m0 as the 4073 // soffset register. 4074 const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soff); 4075 if (Soff && Soff->getReg() != AMDGPU::M0) { 4076 ErrInfo = "scalar stores must use m0 as offset register"; 4077 return false; 4078 } 4079 } 4080 } 4081 4082 if (isFLAT(MI) && !ST.hasFlatInstOffsets()) { 4083 const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset); 4084 if (Offset->getImm() != 0) { 4085 ErrInfo = "subtarget does not support offsets in flat instructions"; 4086 return false; 4087 } 4088 } 4089 4090 if (isMIMG(MI)) { 4091 const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim); 4092 if (DimOp) { 4093 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode, 4094 AMDGPU::OpName::vaddr0); 4095 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc); 4096 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opcode); 4097 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = 4098 AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode); 4099 const AMDGPU::MIMGDimInfo *Dim = 4100 AMDGPU::getMIMGDimInfoByEncoding(DimOp->getImm()); 4101 4102 if (!Dim) { 4103 ErrInfo = "dim is out of range"; 4104 return false; 4105 } 4106 4107 bool IsA16 = false; 4108 if (ST.hasR128A16()) { 4109 const MachineOperand *R128A16 = getNamedOperand(MI, AMDGPU::OpName::r128); 4110 IsA16 = R128A16->getImm() != 0; 4111 } else if (ST.hasGFX10A16()) { 4112 const MachineOperand *A16 = getNamedOperand(MI, AMDGPU::OpName::a16); 4113 IsA16 = A16->getImm() != 0; 4114 } 4115 4116 bool PackDerivatives = IsA16 || BaseOpcode->G16; 4117 bool IsNSA = SRsrcIdx - VAddr0Idx > 1; 4118 4119 unsigned AddrWords = BaseOpcode->NumExtraArgs; 4120 unsigned AddrComponents = (BaseOpcode->Coordinates ? Dim->NumCoords : 0) + 4121 (BaseOpcode->LodOrClampOrMip ? 1 : 0); 4122 if (IsA16) 4123 AddrWords += (AddrComponents + 1) / 2; 4124 else 4125 AddrWords += AddrComponents; 4126 4127 if (BaseOpcode->Gradients) { 4128 if (PackDerivatives) 4129 // There are two gradients per coordinate, we pack them separately. 4130 // For the 3d case, we get (dy/du, dx/du) (-, dz/du) (dy/dv, dx/dv) (-, dz/dv) 4131 AddrWords += (Dim->NumGradients / 2 + 1) / 2 * 2; 4132 else 4133 AddrWords += Dim->NumGradients; 4134 } 4135 4136 unsigned VAddrWords; 4137 if (IsNSA) { 4138 VAddrWords = SRsrcIdx - VAddr0Idx; 4139 } else { 4140 const TargetRegisterClass *RC = getOpRegClass(MI, VAddr0Idx); 4141 VAddrWords = MRI.getTargetRegisterInfo()->getRegSizeInBits(*RC) / 32; 4142 if (AddrWords > 8) 4143 AddrWords = 16; 4144 else if (AddrWords > 4) 4145 AddrWords = 8; 4146 else if (AddrWords == 4) 4147 AddrWords = 4; 4148 else if (AddrWords == 3) 4149 AddrWords = 3; 4150 } 4151 4152 if (VAddrWords != AddrWords) { 4153 LLVM_DEBUG(dbgs() << "bad vaddr size, expected " << AddrWords 4154 << " but got " << VAddrWords << "\n"); 4155 ErrInfo = "bad vaddr size"; 4156 return false; 4157 } 4158 } 4159 } 4160 4161 const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl); 4162 if (DppCt) { 4163 using namespace AMDGPU::DPP; 4164 4165 unsigned DC = DppCt->getImm(); 4166 if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 || 4167 DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST || 4168 (DC >= DppCtrl::DPP_UNUSED4_FIRST && DC <= DppCtrl::DPP_UNUSED4_LAST) || 4169 (DC >= DppCtrl::DPP_UNUSED5_FIRST && DC <= DppCtrl::DPP_UNUSED5_LAST) || 4170 (DC >= DppCtrl::DPP_UNUSED6_FIRST && DC <= DppCtrl::DPP_UNUSED6_LAST) || 4171 (DC >= DppCtrl::DPP_UNUSED7_FIRST && DC <= DppCtrl::DPP_UNUSED7_LAST) || 4172 (DC >= DppCtrl::DPP_UNUSED8_FIRST && DC <= DppCtrl::DPP_UNUSED8_LAST)) { 4173 ErrInfo = "Invalid dpp_ctrl value"; 4174 return false; 4175 } 4176 if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 && 4177 ST.getGeneration() >= AMDGPUSubtarget::GFX10) { 4178 ErrInfo = "Invalid dpp_ctrl value: " 4179 "wavefront shifts are not supported on GFX10+"; 4180 return false; 4181 } 4182 if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 && 4183 ST.getGeneration() >= AMDGPUSubtarget::GFX10) { 4184 ErrInfo = "Invalid dpp_ctrl value: " 4185 "broadcasts are not supported on GFX10+"; 4186 return false; 4187 } 4188 if (DC >= DppCtrl::ROW_SHARE_FIRST && DC <= DppCtrl::ROW_XMASK_LAST && 4189 ST.getGeneration() < AMDGPUSubtarget::GFX10) { 4190 ErrInfo = "Invalid dpp_ctrl value: " 4191 "row_share and row_xmask are not supported before GFX10"; 4192 return false; 4193 } 4194 } 4195 4196 return true; 4197 } 4198 4199 unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const { 4200 switch (MI.getOpcode()) { 4201 default: return AMDGPU::INSTRUCTION_LIST_END; 4202 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE; 4203 case AMDGPU::COPY: return AMDGPU::COPY; 4204 case AMDGPU::PHI: return AMDGPU::PHI; 4205 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG; 4206 case AMDGPU::WQM: return AMDGPU::WQM; 4207 case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM; 4208 case AMDGPU::WWM: return AMDGPU::WWM; 4209 case AMDGPU::S_MOV_B32: { 4210 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 4211 return MI.getOperand(1).isReg() || 4212 RI.isAGPR(MRI, MI.getOperand(0).getReg()) ? 4213 AMDGPU::COPY : AMDGPU::V_MOV_B32_e32; 4214 } 4215 case AMDGPU::S_ADD_I32: 4216 return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32; 4217 case AMDGPU::S_ADDC_U32: 4218 return AMDGPU::V_ADDC_U32_e32; 4219 case AMDGPU::S_SUB_I32: 4220 return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_CO_U32_e32; 4221 // FIXME: These are not consistently handled, and selected when the carry is 4222 // used. 4223 case AMDGPU::S_ADD_U32: 4224 return AMDGPU::V_ADD_CO_U32_e32; 4225 case AMDGPU::S_SUB_U32: 4226 return AMDGPU::V_SUB_CO_U32_e32; 4227 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32; 4228 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32_e64; 4229 case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32_e64; 4230 case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32_e64; 4231 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64; 4232 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64; 4233 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64; 4234 case AMDGPU::S_XNOR_B32: 4235 return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END; 4236 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64; 4237 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64; 4238 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64; 4239 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64; 4240 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32; 4241 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64_e64; 4242 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32; 4243 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64_e64; 4244 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32; 4245 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64_e64; 4246 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32_e64; 4247 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32_e64; 4248 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32_e64; 4249 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32_e64; 4250 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64; 4251 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32; 4252 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32; 4253 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32; 4254 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e32; 4255 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e32; 4256 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e32; 4257 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e32; 4258 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e32; 4259 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e32; 4260 case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e32; 4261 case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e32; 4262 case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e32; 4263 case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e32; 4264 case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e32; 4265 case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e32; 4266 case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e32; 4267 case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e32; 4268 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64; 4269 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32; 4270 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32; 4271 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64; 4272 case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ; 4273 case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ; 4274 } 4275 llvm_unreachable( 4276 "Unexpected scalar opcode without corresponding vector one!"); 4277 } 4278 4279 const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI, 4280 unsigned OpNo) const { 4281 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 4282 const MCInstrDesc &Desc = get(MI.getOpcode()); 4283 if (MI.isVariadic() || OpNo >= Desc.getNumOperands() || 4284 Desc.OpInfo[OpNo].RegClass == -1) { 4285 Register Reg = MI.getOperand(OpNo).getReg(); 4286 4287 if (Reg.isVirtual()) 4288 return MRI.getRegClass(Reg); 4289 return RI.getPhysRegClass(Reg); 4290 } 4291 4292 unsigned RCID = Desc.OpInfo[OpNo].RegClass; 4293 return RI.getRegClass(RCID); 4294 } 4295 4296 void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const { 4297 MachineBasicBlock::iterator I = MI; 4298 MachineBasicBlock *MBB = MI.getParent(); 4299 MachineOperand &MO = MI.getOperand(OpIdx); 4300 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 4301 unsigned RCID = get(MI.getOpcode()).OpInfo[OpIdx].RegClass; 4302 const TargetRegisterClass *RC = RI.getRegClass(RCID); 4303 unsigned Size = RI.getRegSizeInBits(*RC); 4304 unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO : AMDGPU::V_MOV_B32_e32; 4305 if (MO.isReg()) 4306 Opcode = AMDGPU::COPY; 4307 else if (RI.isSGPRClass(RC)) 4308 Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32; 4309 4310 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(RC); 4311 if (RI.getCommonSubClass(&AMDGPU::VReg_64RegClass, VRC)) 4312 VRC = &AMDGPU::VReg_64RegClass; 4313 else 4314 VRC = &AMDGPU::VGPR_32RegClass; 4315 4316 Register Reg = MRI.createVirtualRegister(VRC); 4317 DebugLoc DL = MBB->findDebugLoc(I); 4318 BuildMI(*MI.getParent(), I, DL, get(Opcode), Reg).add(MO); 4319 MO.ChangeToRegister(Reg, false); 4320 } 4321 4322 unsigned SIInstrInfo::buildExtractSubReg(MachineBasicBlock::iterator MI, 4323 MachineRegisterInfo &MRI, 4324 MachineOperand &SuperReg, 4325 const TargetRegisterClass *SuperRC, 4326 unsigned SubIdx, 4327 const TargetRegisterClass *SubRC) 4328 const { 4329 MachineBasicBlock *MBB = MI->getParent(); 4330 DebugLoc DL = MI->getDebugLoc(); 4331 Register SubReg = MRI.createVirtualRegister(SubRC); 4332 4333 if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) { 4334 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg) 4335 .addReg(SuperReg.getReg(), 0, SubIdx); 4336 return SubReg; 4337 } 4338 4339 // Just in case the super register is itself a sub-register, copy it to a new 4340 // value so we don't need to worry about merging its subreg index with the 4341 // SubIdx passed to this function. The register coalescer should be able to 4342 // eliminate this extra copy. 4343 Register NewSuperReg = MRI.createVirtualRegister(SuperRC); 4344 4345 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), NewSuperReg) 4346 .addReg(SuperReg.getReg(), 0, SuperReg.getSubReg()); 4347 4348 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg) 4349 .addReg(NewSuperReg, 0, SubIdx); 4350 4351 return SubReg; 4352 } 4353 4354 MachineOperand SIInstrInfo::buildExtractSubRegOrImm( 4355 MachineBasicBlock::iterator MII, 4356 MachineRegisterInfo &MRI, 4357 MachineOperand &Op, 4358 const TargetRegisterClass *SuperRC, 4359 unsigned SubIdx, 4360 const TargetRegisterClass *SubRC) const { 4361 if (Op.isImm()) { 4362 if (SubIdx == AMDGPU::sub0) 4363 return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm())); 4364 if (SubIdx == AMDGPU::sub1) 4365 return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm() >> 32)); 4366 4367 llvm_unreachable("Unhandled register index for immediate"); 4368 } 4369 4370 unsigned SubReg = buildExtractSubReg(MII, MRI, Op, SuperRC, 4371 SubIdx, SubRC); 4372 return MachineOperand::CreateReg(SubReg, false); 4373 } 4374 4375 // Change the order of operands from (0, 1, 2) to (0, 2, 1) 4376 void SIInstrInfo::swapOperands(MachineInstr &Inst) const { 4377 assert(Inst.getNumExplicitOperands() == 3); 4378 MachineOperand Op1 = Inst.getOperand(1); 4379 Inst.RemoveOperand(1); 4380 Inst.addOperand(Op1); 4381 } 4382 4383 bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo &MRI, 4384 const MCOperandInfo &OpInfo, 4385 const MachineOperand &MO) const { 4386 if (!MO.isReg()) 4387 return false; 4388 4389 Register Reg = MO.getReg(); 4390 4391 const TargetRegisterClass *DRC = RI.getRegClass(OpInfo.RegClass); 4392 if (Reg.isPhysical()) 4393 return DRC->contains(Reg); 4394 4395 const TargetRegisterClass *RC = MRI.getRegClass(Reg); 4396 4397 if (MO.getSubReg()) { 4398 const MachineFunction *MF = MO.getParent()->getParent()->getParent(); 4399 const TargetRegisterClass *SuperRC = RI.getLargestLegalSuperClass(RC, *MF); 4400 if (!SuperRC) 4401 return false; 4402 4403 DRC = RI.getMatchingSuperRegClass(SuperRC, DRC, MO.getSubReg()); 4404 if (!DRC) 4405 return false; 4406 } 4407 return RC->hasSuperClassEq(DRC); 4408 } 4409 4410 bool SIInstrInfo::isLegalVSrcOperand(const MachineRegisterInfo &MRI, 4411 const MCOperandInfo &OpInfo, 4412 const MachineOperand &MO) const { 4413 if (MO.isReg()) 4414 return isLegalRegOperand(MRI, OpInfo, MO); 4415 4416 // Handle non-register types that are treated like immediates. 4417 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal()); 4418 return true; 4419 } 4420 4421 bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx, 4422 const MachineOperand *MO) const { 4423 const MachineFunction &MF = *MI.getParent()->getParent(); 4424 const MachineRegisterInfo &MRI = MF.getRegInfo(); 4425 const MCInstrDesc &InstDesc = MI.getDesc(); 4426 const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpIdx]; 4427 const TargetRegisterClass *DefinedRC = 4428 OpInfo.RegClass != -1 ? RI.getRegClass(OpInfo.RegClass) : nullptr; 4429 if (!MO) 4430 MO = &MI.getOperand(OpIdx); 4431 4432 int ConstantBusLimit = ST.getConstantBusLimit(MI.getOpcode()); 4433 int VOP3LiteralLimit = ST.hasVOP3Literal() ? 1 : 0; 4434 if (isVALU(MI) && usesConstantBus(MRI, *MO, OpInfo)) { 4435 if (isVOP3(MI) && isLiteralConstantLike(*MO, OpInfo) && !VOP3LiteralLimit--) 4436 return false; 4437 4438 SmallDenseSet<RegSubRegPair> SGPRsUsed; 4439 if (MO->isReg()) 4440 SGPRsUsed.insert(RegSubRegPair(MO->getReg(), MO->getSubReg())); 4441 4442 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 4443 if (i == OpIdx) 4444 continue; 4445 const MachineOperand &Op = MI.getOperand(i); 4446 if (Op.isReg()) { 4447 RegSubRegPair SGPR(Op.getReg(), Op.getSubReg()); 4448 if (!SGPRsUsed.count(SGPR) && 4449 usesConstantBus(MRI, Op, InstDesc.OpInfo[i])) { 4450 if (--ConstantBusLimit <= 0) 4451 return false; 4452 SGPRsUsed.insert(SGPR); 4453 } 4454 } else if (InstDesc.OpInfo[i].OperandType == AMDGPU::OPERAND_KIMM32) { 4455 if (--ConstantBusLimit <= 0) 4456 return false; 4457 } else if (isVOP3(MI) && AMDGPU::isSISrcOperand(InstDesc, i) && 4458 isLiteralConstantLike(Op, InstDesc.OpInfo[i])) { 4459 if (!VOP3LiteralLimit--) 4460 return false; 4461 if (--ConstantBusLimit <= 0) 4462 return false; 4463 } 4464 } 4465 } 4466 4467 if (MO->isReg()) { 4468 assert(DefinedRC); 4469 return isLegalRegOperand(MRI, OpInfo, *MO); 4470 } 4471 4472 // Handle non-register types that are treated like immediates. 4473 assert(MO->isImm() || MO->isTargetIndex() || MO->isFI() || MO->isGlobal()); 4474 4475 if (!DefinedRC) { 4476 // This operand expects an immediate. 4477 return true; 4478 } 4479 4480 return isImmOperandLegal(MI, OpIdx, *MO); 4481 } 4482 4483 void SIInstrInfo::legalizeOperandsVOP2(MachineRegisterInfo &MRI, 4484 MachineInstr &MI) const { 4485 unsigned Opc = MI.getOpcode(); 4486 const MCInstrDesc &InstrDesc = get(Opc); 4487 4488 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); 4489 MachineOperand &Src0 = MI.getOperand(Src0Idx); 4490 4491 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); 4492 MachineOperand &Src1 = MI.getOperand(Src1Idx); 4493 4494 // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32 4495 // we need to only have one constant bus use before GFX10. 4496 bool HasImplicitSGPR = findImplicitSGPRRead(MI) != AMDGPU::NoRegister; 4497 if (HasImplicitSGPR && ST.getConstantBusLimit(Opc) <= 1 && 4498 Src0.isReg() && (RI.isSGPRReg(MRI, Src0.getReg()) || 4499 isLiteralConstantLike(Src0, InstrDesc.OpInfo[Src0Idx]))) 4500 legalizeOpWithMove(MI, Src0Idx); 4501 4502 // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for 4503 // both the value to write (src0) and lane select (src1). Fix up non-SGPR 4504 // src0/src1 with V_READFIRSTLANE. 4505 if (Opc == AMDGPU::V_WRITELANE_B32) { 4506 const DebugLoc &DL = MI.getDebugLoc(); 4507 if (Src0.isReg() && RI.isVGPR(MRI, Src0.getReg())) { 4508 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 4509 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 4510 .add(Src0); 4511 Src0.ChangeToRegister(Reg, false); 4512 } 4513 if (Src1.isReg() && RI.isVGPR(MRI, Src1.getReg())) { 4514 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 4515 const DebugLoc &DL = MI.getDebugLoc(); 4516 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 4517 .add(Src1); 4518 Src1.ChangeToRegister(Reg, false); 4519 } 4520 return; 4521 } 4522 4523 // No VOP2 instructions support AGPRs. 4524 if (Src0.isReg() && RI.isAGPR(MRI, Src0.getReg())) 4525 legalizeOpWithMove(MI, Src0Idx); 4526 4527 if (Src1.isReg() && RI.isAGPR(MRI, Src1.getReg())) 4528 legalizeOpWithMove(MI, Src1Idx); 4529 4530 // VOP2 src0 instructions support all operand types, so we don't need to check 4531 // their legality. If src1 is already legal, we don't need to do anything. 4532 if (isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src1)) 4533 return; 4534 4535 // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for 4536 // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane 4537 // select is uniform. 4538 if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() && 4539 RI.isVGPR(MRI, Src1.getReg())) { 4540 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 4541 const DebugLoc &DL = MI.getDebugLoc(); 4542 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 4543 .add(Src1); 4544 Src1.ChangeToRegister(Reg, false); 4545 return; 4546 } 4547 4548 // We do not use commuteInstruction here because it is too aggressive and will 4549 // commute if it is possible. We only want to commute here if it improves 4550 // legality. This can be called a fairly large number of times so don't waste 4551 // compile time pointlessly swapping and checking legality again. 4552 if (HasImplicitSGPR || !MI.isCommutable()) { 4553 legalizeOpWithMove(MI, Src1Idx); 4554 return; 4555 } 4556 4557 // If src0 can be used as src1, commuting will make the operands legal. 4558 // Otherwise we have to give up and insert a move. 4559 // 4560 // TODO: Other immediate-like operand kinds could be commuted if there was a 4561 // MachineOperand::ChangeTo* for them. 4562 if ((!Src1.isImm() && !Src1.isReg()) || 4563 !isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src0)) { 4564 legalizeOpWithMove(MI, Src1Idx); 4565 return; 4566 } 4567 4568 int CommutedOpc = commuteOpcode(MI); 4569 if (CommutedOpc == -1) { 4570 legalizeOpWithMove(MI, Src1Idx); 4571 return; 4572 } 4573 4574 MI.setDesc(get(CommutedOpc)); 4575 4576 Register Src0Reg = Src0.getReg(); 4577 unsigned Src0SubReg = Src0.getSubReg(); 4578 bool Src0Kill = Src0.isKill(); 4579 4580 if (Src1.isImm()) 4581 Src0.ChangeToImmediate(Src1.getImm()); 4582 else if (Src1.isReg()) { 4583 Src0.ChangeToRegister(Src1.getReg(), false, false, Src1.isKill()); 4584 Src0.setSubReg(Src1.getSubReg()); 4585 } else 4586 llvm_unreachable("Should only have register or immediate operands"); 4587 4588 Src1.ChangeToRegister(Src0Reg, false, false, Src0Kill); 4589 Src1.setSubReg(Src0SubReg); 4590 fixImplicitOperands(MI); 4591 } 4592 4593 // Legalize VOP3 operands. All operand types are supported for any operand 4594 // but only one literal constant and only starting from GFX10. 4595 void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI, 4596 MachineInstr &MI) const { 4597 unsigned Opc = MI.getOpcode(); 4598 4599 int VOP3Idx[3] = { 4600 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0), 4601 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1), 4602 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2) 4603 }; 4604 4605 if (Opc == AMDGPU::V_PERMLANE16_B32_e64 || 4606 Opc == AMDGPU::V_PERMLANEX16_B32_e64) { 4607 // src1 and src2 must be scalar 4608 MachineOperand &Src1 = MI.getOperand(VOP3Idx[1]); 4609 MachineOperand &Src2 = MI.getOperand(VOP3Idx[2]); 4610 const DebugLoc &DL = MI.getDebugLoc(); 4611 if (Src1.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src1.getReg()))) { 4612 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 4613 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 4614 .add(Src1); 4615 Src1.ChangeToRegister(Reg, false); 4616 } 4617 if (Src2.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src2.getReg()))) { 4618 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 4619 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 4620 .add(Src2); 4621 Src2.ChangeToRegister(Reg, false); 4622 } 4623 } 4624 4625 // Find the one SGPR operand we are allowed to use. 4626 int ConstantBusLimit = ST.getConstantBusLimit(Opc); 4627 int LiteralLimit = ST.hasVOP3Literal() ? 1 : 0; 4628 SmallDenseSet<unsigned> SGPRsUsed; 4629 Register SGPRReg = findUsedSGPR(MI, VOP3Idx); 4630 if (SGPRReg != AMDGPU::NoRegister) { 4631 SGPRsUsed.insert(SGPRReg); 4632 --ConstantBusLimit; 4633 } 4634 4635 for (unsigned i = 0; i < 3; ++i) { 4636 int Idx = VOP3Idx[i]; 4637 if (Idx == -1) 4638 break; 4639 MachineOperand &MO = MI.getOperand(Idx); 4640 4641 if (!MO.isReg()) { 4642 if (!isLiteralConstantLike(MO, get(Opc).OpInfo[Idx])) 4643 continue; 4644 4645 if (LiteralLimit > 0 && ConstantBusLimit > 0) { 4646 --LiteralLimit; 4647 --ConstantBusLimit; 4648 continue; 4649 } 4650 4651 --LiteralLimit; 4652 --ConstantBusLimit; 4653 legalizeOpWithMove(MI, Idx); 4654 continue; 4655 } 4656 4657 if (RI.hasAGPRs(MRI.getRegClass(MO.getReg())) && 4658 !isOperandLegal(MI, Idx, &MO)) { 4659 legalizeOpWithMove(MI, Idx); 4660 continue; 4661 } 4662 4663 if (!RI.isSGPRClass(MRI.getRegClass(MO.getReg()))) 4664 continue; // VGPRs are legal 4665 4666 // We can use one SGPR in each VOP3 instruction prior to GFX10 4667 // and two starting from GFX10. 4668 if (SGPRsUsed.count(MO.getReg())) 4669 continue; 4670 if (ConstantBusLimit > 0) { 4671 SGPRsUsed.insert(MO.getReg()); 4672 --ConstantBusLimit; 4673 continue; 4674 } 4675 4676 // If we make it this far, then the operand is not legal and we must 4677 // legalize it. 4678 legalizeOpWithMove(MI, Idx); 4679 } 4680 } 4681 4682 Register SIInstrInfo::readlaneVGPRToSGPR(Register SrcReg, MachineInstr &UseMI, 4683 MachineRegisterInfo &MRI) const { 4684 const TargetRegisterClass *VRC = MRI.getRegClass(SrcReg); 4685 const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC); 4686 Register DstReg = MRI.createVirtualRegister(SRC); 4687 unsigned SubRegs = RI.getRegSizeInBits(*VRC) / 32; 4688 4689 if (RI.hasAGPRs(VRC)) { 4690 VRC = RI.getEquivalentVGPRClass(VRC); 4691 Register NewSrcReg = MRI.createVirtualRegister(VRC); 4692 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), 4693 get(TargetOpcode::COPY), NewSrcReg) 4694 .addReg(SrcReg); 4695 SrcReg = NewSrcReg; 4696 } 4697 4698 if (SubRegs == 1) { 4699 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), 4700 get(AMDGPU::V_READFIRSTLANE_B32), DstReg) 4701 .addReg(SrcReg); 4702 return DstReg; 4703 } 4704 4705 SmallVector<unsigned, 8> SRegs; 4706 for (unsigned i = 0; i < SubRegs; ++i) { 4707 Register SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 4708 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), 4709 get(AMDGPU::V_READFIRSTLANE_B32), SGPR) 4710 .addReg(SrcReg, 0, RI.getSubRegFromChannel(i)); 4711 SRegs.push_back(SGPR); 4712 } 4713 4714 MachineInstrBuilder MIB = 4715 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), 4716 get(AMDGPU::REG_SEQUENCE), DstReg); 4717 for (unsigned i = 0; i < SubRegs; ++i) { 4718 MIB.addReg(SRegs[i]); 4719 MIB.addImm(RI.getSubRegFromChannel(i)); 4720 } 4721 return DstReg; 4722 } 4723 4724 void SIInstrInfo::legalizeOperandsSMRD(MachineRegisterInfo &MRI, 4725 MachineInstr &MI) const { 4726 4727 // If the pointer is store in VGPRs, then we need to move them to 4728 // SGPRs using v_readfirstlane. This is safe because we only select 4729 // loads with uniform pointers to SMRD instruction so we know the 4730 // pointer value is uniform. 4731 MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase); 4732 if (SBase && !RI.isSGPRClass(MRI.getRegClass(SBase->getReg()))) { 4733 Register SGPR = readlaneVGPRToSGPR(SBase->getReg(), MI, MRI); 4734 SBase->setReg(SGPR); 4735 } 4736 MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soff); 4737 if (SOff && !RI.isSGPRClass(MRI.getRegClass(SOff->getReg()))) { 4738 Register SGPR = readlaneVGPRToSGPR(SOff->getReg(), MI, MRI); 4739 SOff->setReg(SGPR); 4740 } 4741 } 4742 4743 // FIXME: Remove this when SelectionDAG is obsoleted. 4744 void SIInstrInfo::legalizeOperandsFLAT(MachineRegisterInfo &MRI, 4745 MachineInstr &MI) const { 4746 if (!isSegmentSpecificFLAT(MI)) 4747 return; 4748 4749 // Fixup SGPR operands in VGPRs. We only select these when the DAG divergence 4750 // thinks they are uniform, so a readfirstlane should be valid. 4751 MachineOperand *SAddr = getNamedOperand(MI, AMDGPU::OpName::saddr); 4752 if (!SAddr || RI.isSGPRClass(MRI.getRegClass(SAddr->getReg()))) 4753 return; 4754 4755 Register ToSGPR = readlaneVGPRToSGPR(SAddr->getReg(), MI, MRI); 4756 SAddr->setReg(ToSGPR); 4757 } 4758 4759 void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB, 4760 MachineBasicBlock::iterator I, 4761 const TargetRegisterClass *DstRC, 4762 MachineOperand &Op, 4763 MachineRegisterInfo &MRI, 4764 const DebugLoc &DL) const { 4765 Register OpReg = Op.getReg(); 4766 unsigned OpSubReg = Op.getSubReg(); 4767 4768 const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg( 4769 RI.getRegClassForReg(MRI, OpReg), OpSubReg); 4770 4771 // Check if operand is already the correct register class. 4772 if (DstRC == OpRC) 4773 return; 4774 4775 Register DstReg = MRI.createVirtualRegister(DstRC); 4776 MachineInstr *Copy = 4777 BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).add(Op); 4778 4779 Op.setReg(DstReg); 4780 Op.setSubReg(0); 4781 4782 MachineInstr *Def = MRI.getVRegDef(OpReg); 4783 if (!Def) 4784 return; 4785 4786 // Try to eliminate the copy if it is copying an immediate value. 4787 if (Def->isMoveImmediate() && DstRC != &AMDGPU::VReg_1RegClass) 4788 FoldImmediate(*Copy, *Def, OpReg, &MRI); 4789 4790 bool ImpDef = Def->isImplicitDef(); 4791 while (!ImpDef && Def && Def->isCopy()) { 4792 if (Def->getOperand(1).getReg().isPhysical()) 4793 break; 4794 Def = MRI.getUniqueVRegDef(Def->getOperand(1).getReg()); 4795 ImpDef = Def && Def->isImplicitDef(); 4796 } 4797 if (!RI.isSGPRClass(DstRC) && !Copy->readsRegister(AMDGPU::EXEC, &RI) && 4798 !ImpDef) 4799 Copy->addOperand(MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); 4800 } 4801 4802 // Emit the actual waterfall loop, executing the wrapped instruction for each 4803 // unique value of \p Rsrc across all lanes. In the best case we execute 1 4804 // iteration, in the worst case we execute 64 (once per lane). 4805 static void 4806 emitLoadSRsrcFromVGPRLoop(const SIInstrInfo &TII, MachineRegisterInfo &MRI, 4807 MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB, 4808 const DebugLoc &DL, MachineOperand &Rsrc) { 4809 MachineFunction &MF = *OrigBB.getParent(); 4810 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 4811 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 4812 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 4813 unsigned SaveExecOpc = 4814 ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64; 4815 unsigned XorTermOpc = 4816 ST.isWave32() ? AMDGPU::S_XOR_B32_term : AMDGPU::S_XOR_B64_term; 4817 unsigned AndOpc = 4818 ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64; 4819 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 4820 4821 MachineBasicBlock::iterator I = LoopBB.begin(); 4822 4823 SmallVector<Register, 8> ReadlanePieces; 4824 Register CondReg = AMDGPU::NoRegister; 4825 4826 Register VRsrc = Rsrc.getReg(); 4827 unsigned VRsrcUndef = getUndefRegState(Rsrc.isUndef()); 4828 4829 unsigned RegSize = TRI->getRegSizeInBits(Rsrc.getReg(), MRI); 4830 unsigned NumSubRegs = RegSize / 32; 4831 assert(NumSubRegs % 2 == 0 && NumSubRegs <= 32 && "Unhandled register size"); 4832 4833 for (unsigned Idx = 0; Idx < NumSubRegs; Idx += 2) { 4834 4835 Register CurRegLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 4836 Register CurRegHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 4837 4838 // Read the next variant <- also loop target. 4839 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegLo) 4840 .addReg(VRsrc, VRsrcUndef, TRI->getSubRegFromChannel(Idx)); 4841 4842 // Read the next variant <- also loop target. 4843 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegHi) 4844 .addReg(VRsrc, VRsrcUndef, TRI->getSubRegFromChannel(Idx + 1)); 4845 4846 ReadlanePieces.push_back(CurRegLo); 4847 ReadlanePieces.push_back(CurRegHi); 4848 4849 // Comparison is to be done as 64-bit. 4850 Register CurReg = MRI.createVirtualRegister(&AMDGPU::SGPR_64RegClass); 4851 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), CurReg) 4852 .addReg(CurRegLo) 4853 .addImm(AMDGPU::sub0) 4854 .addReg(CurRegHi) 4855 .addImm(AMDGPU::sub1); 4856 4857 Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC); 4858 auto Cmp = 4859 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), NewCondReg) 4860 .addReg(CurReg); 4861 if (NumSubRegs <= 2) 4862 Cmp.addReg(VRsrc); 4863 else 4864 Cmp.addReg(VRsrc, VRsrcUndef, TRI->getSubRegFromChannel(Idx, 2)); 4865 4866 // Combine the comparision results with AND. 4867 if (CondReg == AMDGPU::NoRegister) // First. 4868 CondReg = NewCondReg; 4869 else { // If not the first, we create an AND. 4870 Register AndReg = MRI.createVirtualRegister(BoolXExecRC); 4871 BuildMI(LoopBB, I, DL, TII.get(AndOpc), AndReg) 4872 .addReg(CondReg) 4873 .addReg(NewCondReg); 4874 CondReg = AndReg; 4875 } 4876 } // End for loop. 4877 4878 auto SRsrcRC = TRI->getEquivalentSGPRClass(MRI.getRegClass(VRsrc)); 4879 Register SRsrc = MRI.createVirtualRegister(SRsrcRC); 4880 4881 // Build scalar Rsrc. 4882 auto Merge = BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SRsrc); 4883 unsigned Channel = 0; 4884 for (Register Piece : ReadlanePieces) { 4885 Merge.addReg(Piece) 4886 .addImm(TRI->getSubRegFromChannel(Channel++)); 4887 } 4888 4889 // Update Rsrc operand to use the SGPR Rsrc. 4890 Rsrc.setReg(SRsrc); 4891 Rsrc.setIsKill(true); 4892 4893 Register SaveExec = MRI.createVirtualRegister(BoolXExecRC); 4894 MRI.setSimpleHint(SaveExec, CondReg); 4895 4896 // Update EXEC to matching lanes, saving original to SaveExec. 4897 BuildMI(LoopBB, I, DL, TII.get(SaveExecOpc), SaveExec) 4898 .addReg(CondReg, RegState::Kill); 4899 4900 // The original instruction is here; we insert the terminators after it. 4901 I = LoopBB.end(); 4902 4903 // Update EXEC, switch all done bits to 0 and all todo bits to 1. 4904 BuildMI(LoopBB, I, DL, TII.get(XorTermOpc), Exec) 4905 .addReg(Exec) 4906 .addReg(SaveExec); 4907 4908 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_CBRANCH_EXECNZ)).addMBB(&LoopBB); 4909 } 4910 4911 // Build a waterfall loop around \p MI, replacing the VGPR \p Rsrc register 4912 // with SGPRs by iterating over all unique values across all lanes. 4913 // Returns the loop basic block that now contains \p MI. 4914 static MachineBasicBlock * 4915 loadSRsrcFromVGPR(const SIInstrInfo &TII, MachineInstr &MI, 4916 MachineOperand &Rsrc, MachineDominatorTree *MDT, 4917 MachineBasicBlock::iterator Begin = nullptr, 4918 MachineBasicBlock::iterator End = nullptr) { 4919 MachineBasicBlock &MBB = *MI.getParent(); 4920 MachineFunction &MF = *MBB.getParent(); 4921 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 4922 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 4923 MachineRegisterInfo &MRI = MF.getRegInfo(); 4924 if (!Begin.isValid()) 4925 Begin = &MI; 4926 if (!End.isValid()) { 4927 End = &MI; 4928 ++End; 4929 } 4930 const DebugLoc &DL = MI.getDebugLoc(); 4931 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 4932 unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; 4933 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 4934 4935 Register SaveExec = MRI.createVirtualRegister(BoolXExecRC); 4936 4937 // Save the EXEC mask 4938 BuildMI(MBB, Begin, DL, TII.get(MovExecOpc), SaveExec).addReg(Exec); 4939 4940 // Killed uses in the instruction we are waterfalling around will be 4941 // incorrect due to the added control-flow. 4942 MachineBasicBlock::iterator AfterMI = MI; 4943 ++AfterMI; 4944 for (auto I = Begin; I != AfterMI; I++) { 4945 for (auto &MO : I->uses()) { 4946 if (MO.isReg() && MO.isUse()) { 4947 MRI.clearKillFlags(MO.getReg()); 4948 } 4949 } 4950 } 4951 4952 // To insert the loop we need to split the block. Move everything after this 4953 // point to a new block, and insert a new empty block between the two. 4954 MachineBasicBlock *LoopBB = MF.CreateMachineBasicBlock(); 4955 MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock(); 4956 MachineFunction::iterator MBBI(MBB); 4957 ++MBBI; 4958 4959 MF.insert(MBBI, LoopBB); 4960 MF.insert(MBBI, RemainderBB); 4961 4962 LoopBB->addSuccessor(LoopBB); 4963 LoopBB->addSuccessor(RemainderBB); 4964 4965 // Move Begin to MI to the LoopBB, and the remainder of the block to 4966 // RemainderBB. 4967 RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB); 4968 RemainderBB->splice(RemainderBB->begin(), &MBB, End, MBB.end()); 4969 LoopBB->splice(LoopBB->begin(), &MBB, Begin, MBB.end()); 4970 4971 MBB.addSuccessor(LoopBB); 4972 4973 // Update dominators. We know that MBB immediately dominates LoopBB, that 4974 // LoopBB immediately dominates RemainderBB, and that RemainderBB immediately 4975 // dominates all of the successors transferred to it from MBB that MBB used 4976 // to properly dominate. 4977 if (MDT) { 4978 MDT->addNewBlock(LoopBB, &MBB); 4979 MDT->addNewBlock(RemainderBB, LoopBB); 4980 for (auto &Succ : RemainderBB->successors()) { 4981 if (MDT->properlyDominates(&MBB, Succ)) { 4982 MDT->changeImmediateDominator(Succ, RemainderBB); 4983 } 4984 } 4985 } 4986 4987 emitLoadSRsrcFromVGPRLoop(TII, MRI, MBB, *LoopBB, DL, Rsrc); 4988 4989 // Restore the EXEC mask 4990 MachineBasicBlock::iterator First = RemainderBB->begin(); 4991 BuildMI(*RemainderBB, First, DL, TII.get(MovExecOpc), Exec).addReg(SaveExec); 4992 return LoopBB; 4993 } 4994 4995 // Extract pointer from Rsrc and return a zero-value Rsrc replacement. 4996 static std::tuple<unsigned, unsigned> 4997 extractRsrcPtr(const SIInstrInfo &TII, MachineInstr &MI, MachineOperand &Rsrc) { 4998 MachineBasicBlock &MBB = *MI.getParent(); 4999 MachineFunction &MF = *MBB.getParent(); 5000 MachineRegisterInfo &MRI = MF.getRegInfo(); 5001 5002 // Extract the ptr from the resource descriptor. 5003 unsigned RsrcPtr = 5004 TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass, 5005 AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass); 5006 5007 // Create an empty resource descriptor 5008 Register Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); 5009 Register SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 5010 Register SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 5011 Register NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass); 5012 uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat(); 5013 5014 // Zero64 = 0 5015 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64) 5016 .addImm(0); 5017 5018 // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0} 5019 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo) 5020 .addImm(RsrcDataFormat & 0xFFFFFFFF); 5021 5022 // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32} 5023 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi) 5024 .addImm(RsrcDataFormat >> 32); 5025 5026 // NewSRsrc = {Zero64, SRsrcFormat} 5027 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc) 5028 .addReg(Zero64) 5029 .addImm(AMDGPU::sub0_sub1) 5030 .addReg(SRsrcFormatLo) 5031 .addImm(AMDGPU::sub2) 5032 .addReg(SRsrcFormatHi) 5033 .addImm(AMDGPU::sub3); 5034 5035 return std::make_tuple(RsrcPtr, NewSRsrc); 5036 } 5037 5038 MachineBasicBlock * 5039 SIInstrInfo::legalizeOperands(MachineInstr &MI, 5040 MachineDominatorTree *MDT) const { 5041 MachineFunction &MF = *MI.getParent()->getParent(); 5042 MachineRegisterInfo &MRI = MF.getRegInfo(); 5043 MachineBasicBlock *CreatedBB = nullptr; 5044 5045 // Legalize VOP2 5046 if (isVOP2(MI) || isVOPC(MI)) { 5047 legalizeOperandsVOP2(MRI, MI); 5048 return CreatedBB; 5049 } 5050 5051 // Legalize VOP3 5052 if (isVOP3(MI)) { 5053 legalizeOperandsVOP3(MRI, MI); 5054 return CreatedBB; 5055 } 5056 5057 // Legalize SMRD 5058 if (isSMRD(MI)) { 5059 legalizeOperandsSMRD(MRI, MI); 5060 return CreatedBB; 5061 } 5062 5063 // Legalize FLAT 5064 if (isFLAT(MI)) { 5065 legalizeOperandsFLAT(MRI, MI); 5066 return CreatedBB; 5067 } 5068 5069 // Legalize REG_SEQUENCE and PHI 5070 // The register class of the operands much be the same type as the register 5071 // class of the output. 5072 if (MI.getOpcode() == AMDGPU::PHI) { 5073 const TargetRegisterClass *RC = nullptr, *SRC = nullptr, *VRC = nullptr; 5074 for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) { 5075 if (!MI.getOperand(i).isReg() || !MI.getOperand(i).getReg().isVirtual()) 5076 continue; 5077 const TargetRegisterClass *OpRC = 5078 MRI.getRegClass(MI.getOperand(i).getReg()); 5079 if (RI.hasVectorRegisters(OpRC)) { 5080 VRC = OpRC; 5081 } else { 5082 SRC = OpRC; 5083 } 5084 } 5085 5086 // If any of the operands are VGPR registers, then they all most be 5087 // otherwise we will create illegal VGPR->SGPR copies when legalizing 5088 // them. 5089 if (VRC || !RI.isSGPRClass(getOpRegClass(MI, 0))) { 5090 if (!VRC) { 5091 assert(SRC); 5092 if (getOpRegClass(MI, 0) == &AMDGPU::VReg_1RegClass) { 5093 VRC = &AMDGPU::VReg_1RegClass; 5094 } else 5095 VRC = RI.hasAGPRs(getOpRegClass(MI, 0)) 5096 ? RI.getEquivalentAGPRClass(SRC) 5097 : RI.getEquivalentVGPRClass(SRC); 5098 } else { 5099 VRC = RI.hasAGPRs(getOpRegClass(MI, 0)) 5100 ? RI.getEquivalentAGPRClass(VRC) 5101 : RI.getEquivalentVGPRClass(VRC); 5102 } 5103 RC = VRC; 5104 } else { 5105 RC = SRC; 5106 } 5107 5108 // Update all the operands so they have the same type. 5109 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { 5110 MachineOperand &Op = MI.getOperand(I); 5111 if (!Op.isReg() || !Op.getReg().isVirtual()) 5112 continue; 5113 5114 // MI is a PHI instruction. 5115 MachineBasicBlock *InsertBB = MI.getOperand(I + 1).getMBB(); 5116 MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator(); 5117 5118 // Avoid creating no-op copies with the same src and dst reg class. These 5119 // confuse some of the machine passes. 5120 legalizeGenericOperand(*InsertBB, Insert, RC, Op, MRI, MI.getDebugLoc()); 5121 } 5122 } 5123 5124 // REG_SEQUENCE doesn't really require operand legalization, but if one has a 5125 // VGPR dest type and SGPR sources, insert copies so all operands are 5126 // VGPRs. This seems to help operand folding / the register coalescer. 5127 if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) { 5128 MachineBasicBlock *MBB = MI.getParent(); 5129 const TargetRegisterClass *DstRC = getOpRegClass(MI, 0); 5130 if (RI.hasVGPRs(DstRC)) { 5131 // Update all the operands so they are VGPR register classes. These may 5132 // not be the same register class because REG_SEQUENCE supports mixing 5133 // subregister index types e.g. sub0_sub1 + sub2 + sub3 5134 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { 5135 MachineOperand &Op = MI.getOperand(I); 5136 if (!Op.isReg() || !Op.getReg().isVirtual()) 5137 continue; 5138 5139 const TargetRegisterClass *OpRC = MRI.getRegClass(Op.getReg()); 5140 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(OpRC); 5141 if (VRC == OpRC) 5142 continue; 5143 5144 legalizeGenericOperand(*MBB, MI, VRC, Op, MRI, MI.getDebugLoc()); 5145 Op.setIsKill(); 5146 } 5147 } 5148 5149 return CreatedBB; 5150 } 5151 5152 // Legalize INSERT_SUBREG 5153 // src0 must have the same register class as dst 5154 if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) { 5155 Register Dst = MI.getOperand(0).getReg(); 5156 Register Src0 = MI.getOperand(1).getReg(); 5157 const TargetRegisterClass *DstRC = MRI.getRegClass(Dst); 5158 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0); 5159 if (DstRC != Src0RC) { 5160 MachineBasicBlock *MBB = MI.getParent(); 5161 MachineOperand &Op = MI.getOperand(1); 5162 legalizeGenericOperand(*MBB, MI, DstRC, Op, MRI, MI.getDebugLoc()); 5163 } 5164 return CreatedBB; 5165 } 5166 5167 // Legalize SI_INIT_M0 5168 if (MI.getOpcode() == AMDGPU::SI_INIT_M0) { 5169 MachineOperand &Src = MI.getOperand(0); 5170 if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg()))) 5171 Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI)); 5172 return CreatedBB; 5173 } 5174 5175 // Legalize MIMG and MUBUF/MTBUF for shaders. 5176 // 5177 // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via 5178 // scratch memory access. In both cases, the legalization never involves 5179 // conversion to the addr64 form. 5180 if (isMIMG(MI) || (AMDGPU::isGraphics(MF.getFunction().getCallingConv()) && 5181 (isMUBUF(MI) || isMTBUF(MI)))) { 5182 MachineOperand *SRsrc = getNamedOperand(MI, AMDGPU::OpName::srsrc); 5183 if (SRsrc && !RI.isSGPRClass(MRI.getRegClass(SRsrc->getReg()))) 5184 CreatedBB = loadSRsrcFromVGPR(*this, MI, *SRsrc, MDT); 5185 5186 MachineOperand *SSamp = getNamedOperand(MI, AMDGPU::OpName::ssamp); 5187 if (SSamp && !RI.isSGPRClass(MRI.getRegClass(SSamp->getReg()))) 5188 CreatedBB = loadSRsrcFromVGPR(*this, MI, *SSamp, MDT); 5189 5190 return CreatedBB; 5191 } 5192 5193 // Legalize SI_CALL 5194 if (MI.getOpcode() == AMDGPU::SI_CALL_ISEL) { 5195 MachineOperand *Dest = &MI.getOperand(0); 5196 if (!RI.isSGPRClass(MRI.getRegClass(Dest->getReg()))) { 5197 // Move everything between ADJCALLSTACKUP and ADJCALLSTACKDOWN and 5198 // following copies, we also need to move copies from and to physical 5199 // registers into the loop block. 5200 unsigned FrameSetupOpcode = getCallFrameSetupOpcode(); 5201 unsigned FrameDestroyOpcode = getCallFrameDestroyOpcode(); 5202 5203 // Also move the copies to physical registers into the loop block 5204 MachineBasicBlock &MBB = *MI.getParent(); 5205 MachineBasicBlock::iterator Start(&MI); 5206 while (Start->getOpcode() != FrameSetupOpcode) 5207 --Start; 5208 MachineBasicBlock::iterator End(&MI); 5209 while (End->getOpcode() != FrameDestroyOpcode) 5210 ++End; 5211 // Also include following copies of the return value 5212 ++End; 5213 while (End != MBB.end() && End->isCopy() && End->getOperand(1).isReg() && 5214 MI.definesRegister(End->getOperand(1).getReg())) 5215 ++End; 5216 CreatedBB = loadSRsrcFromVGPR(*this, MI, *Dest, MDT, Start, End); 5217 } 5218 } 5219 5220 // Legalize MUBUF* instructions. 5221 int RsrcIdx = 5222 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc); 5223 if (RsrcIdx != -1) { 5224 // We have an MUBUF instruction 5225 MachineOperand *Rsrc = &MI.getOperand(RsrcIdx); 5226 unsigned RsrcRC = get(MI.getOpcode()).OpInfo[RsrcIdx].RegClass; 5227 if (RI.getCommonSubClass(MRI.getRegClass(Rsrc->getReg()), 5228 RI.getRegClass(RsrcRC))) { 5229 // The operands are legal. 5230 // FIXME: We may need to legalize operands besided srsrc. 5231 return CreatedBB; 5232 } 5233 5234 // Legalize a VGPR Rsrc. 5235 // 5236 // If the instruction is _ADDR64, we can avoid a waterfall by extracting 5237 // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using 5238 // a zero-value SRsrc. 5239 // 5240 // If the instruction is _OFFSET (both idxen and offen disabled), and we 5241 // support ADDR64 instructions, we can convert to ADDR64 and do the same as 5242 // above. 5243 // 5244 // Otherwise we are on non-ADDR64 hardware, and/or we have 5245 // idxen/offen/bothen and we fall back to a waterfall loop. 5246 5247 MachineBasicBlock &MBB = *MI.getParent(); 5248 5249 MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr); 5250 if (VAddr && AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -1) { 5251 // This is already an ADDR64 instruction so we need to add the pointer 5252 // extracted from the resource descriptor to the current value of VAddr. 5253 Register NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 5254 Register NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 5255 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 5256 5257 const auto *BoolXExecRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 5258 Register CondReg0 = MRI.createVirtualRegister(BoolXExecRC); 5259 Register CondReg1 = MRI.createVirtualRegister(BoolXExecRC); 5260 5261 unsigned RsrcPtr, NewSRsrc; 5262 std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc); 5263 5264 // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0 5265 const DebugLoc &DL = MI.getDebugLoc(); 5266 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_CO_U32_e64), NewVAddrLo) 5267 .addDef(CondReg0) 5268 .addReg(RsrcPtr, 0, AMDGPU::sub0) 5269 .addReg(VAddr->getReg(), 0, AMDGPU::sub0) 5270 .addImm(0); 5271 5272 // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1 5273 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e64), NewVAddrHi) 5274 .addDef(CondReg1, RegState::Dead) 5275 .addReg(RsrcPtr, 0, AMDGPU::sub1) 5276 .addReg(VAddr->getReg(), 0, AMDGPU::sub1) 5277 .addReg(CondReg0, RegState::Kill) 5278 .addImm(0); 5279 5280 // NewVaddr = {NewVaddrHi, NewVaddrLo} 5281 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr) 5282 .addReg(NewVAddrLo) 5283 .addImm(AMDGPU::sub0) 5284 .addReg(NewVAddrHi) 5285 .addImm(AMDGPU::sub1); 5286 5287 VAddr->setReg(NewVAddr); 5288 Rsrc->setReg(NewSRsrc); 5289 } else if (!VAddr && ST.hasAddr64()) { 5290 // This instructions is the _OFFSET variant, so we need to convert it to 5291 // ADDR64. 5292 assert(ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS && 5293 "FIXME: Need to emit flat atomics here"); 5294 5295 unsigned RsrcPtr, NewSRsrc; 5296 std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc); 5297 5298 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 5299 MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata); 5300 MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset); 5301 MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset); 5302 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode()); 5303 5304 // Atomics rith return have have an additional tied operand and are 5305 // missing some of the special bits. 5306 MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in); 5307 MachineInstr *Addr64; 5308 5309 if (!VDataIn) { 5310 // Regular buffer load / store. 5311 MachineInstrBuilder MIB = 5312 BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode)) 5313 .add(*VData) 5314 .addReg(NewVAddr) 5315 .addReg(NewSRsrc) 5316 .add(*SOffset) 5317 .add(*Offset); 5318 5319 // Atomics do not have this operand. 5320 if (const MachineOperand *GLC = 5321 getNamedOperand(MI, AMDGPU::OpName::glc)) { 5322 MIB.addImm(GLC->getImm()); 5323 } 5324 if (const MachineOperand *DLC = 5325 getNamedOperand(MI, AMDGPU::OpName::dlc)) { 5326 MIB.addImm(DLC->getImm()); 5327 } 5328 5329 MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc)); 5330 5331 if (const MachineOperand *TFE = 5332 getNamedOperand(MI, AMDGPU::OpName::tfe)) { 5333 MIB.addImm(TFE->getImm()); 5334 } 5335 5336 MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::swz)); 5337 5338 MIB.cloneMemRefs(MI); 5339 Addr64 = MIB; 5340 } else { 5341 // Atomics with return. 5342 Addr64 = BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode)) 5343 .add(*VData) 5344 .add(*VDataIn) 5345 .addReg(NewVAddr) 5346 .addReg(NewSRsrc) 5347 .add(*SOffset) 5348 .add(*Offset) 5349 .addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc)) 5350 .cloneMemRefs(MI); 5351 } 5352 5353 MI.removeFromParent(); 5354 5355 // NewVaddr = {NewVaddrHi, NewVaddrLo} 5356 BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE), 5357 NewVAddr) 5358 .addReg(RsrcPtr, 0, AMDGPU::sub0) 5359 .addImm(AMDGPU::sub0) 5360 .addReg(RsrcPtr, 0, AMDGPU::sub1) 5361 .addImm(AMDGPU::sub1); 5362 } else { 5363 // This is another variant; legalize Rsrc with waterfall loop from VGPRs 5364 // to SGPRs. 5365 CreatedBB = loadSRsrcFromVGPR(*this, MI, *Rsrc, MDT); 5366 return CreatedBB; 5367 } 5368 } 5369 return CreatedBB; 5370 } 5371 5372 MachineBasicBlock *SIInstrInfo::moveToVALU(MachineInstr &TopInst, 5373 MachineDominatorTree *MDT) const { 5374 SetVectorType Worklist; 5375 Worklist.insert(&TopInst); 5376 MachineBasicBlock *CreatedBB = nullptr; 5377 MachineBasicBlock *CreatedBBTmp = nullptr; 5378 5379 while (!Worklist.empty()) { 5380 MachineInstr &Inst = *Worklist.pop_back_val(); 5381 MachineBasicBlock *MBB = Inst.getParent(); 5382 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 5383 5384 unsigned Opcode = Inst.getOpcode(); 5385 unsigned NewOpcode = getVALUOp(Inst); 5386 5387 // Handle some special cases 5388 switch (Opcode) { 5389 default: 5390 break; 5391 case AMDGPU::S_ADD_U64_PSEUDO: 5392 case AMDGPU::S_SUB_U64_PSEUDO: 5393 splitScalar64BitAddSub(Worklist, Inst, MDT); 5394 Inst.eraseFromParent(); 5395 continue; 5396 case AMDGPU::S_ADD_I32: 5397 case AMDGPU::S_SUB_I32: { 5398 // FIXME: The u32 versions currently selected use the carry. 5399 bool Changed; 5400 std::tie(Changed, CreatedBBTmp) = moveScalarAddSub(Worklist, Inst, MDT); 5401 if (CreatedBBTmp && TopInst.getParent() == CreatedBBTmp) 5402 CreatedBB = CreatedBBTmp; 5403 if (Changed) 5404 continue; 5405 5406 // Default handling 5407 break; 5408 } 5409 case AMDGPU::S_AND_B64: 5410 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT); 5411 Inst.eraseFromParent(); 5412 continue; 5413 5414 case AMDGPU::S_OR_B64: 5415 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT); 5416 Inst.eraseFromParent(); 5417 continue; 5418 5419 case AMDGPU::S_XOR_B64: 5420 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT); 5421 Inst.eraseFromParent(); 5422 continue; 5423 5424 case AMDGPU::S_NAND_B64: 5425 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT); 5426 Inst.eraseFromParent(); 5427 continue; 5428 5429 case AMDGPU::S_NOR_B64: 5430 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT); 5431 Inst.eraseFromParent(); 5432 continue; 5433 5434 case AMDGPU::S_XNOR_B64: 5435 if (ST.hasDLInsts()) 5436 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT); 5437 else 5438 splitScalar64BitXnor(Worklist, Inst, MDT); 5439 Inst.eraseFromParent(); 5440 continue; 5441 5442 case AMDGPU::S_ANDN2_B64: 5443 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT); 5444 Inst.eraseFromParent(); 5445 continue; 5446 5447 case AMDGPU::S_ORN2_B64: 5448 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT); 5449 Inst.eraseFromParent(); 5450 continue; 5451 5452 case AMDGPU::S_NOT_B64: 5453 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32); 5454 Inst.eraseFromParent(); 5455 continue; 5456 5457 case AMDGPU::S_BCNT1_I32_B64: 5458 splitScalar64BitBCNT(Worklist, Inst); 5459 Inst.eraseFromParent(); 5460 continue; 5461 5462 case AMDGPU::S_BFE_I64: 5463 splitScalar64BitBFE(Worklist, Inst); 5464 Inst.eraseFromParent(); 5465 continue; 5466 5467 case AMDGPU::S_LSHL_B32: 5468 if (ST.hasOnlyRevVALUShifts()) { 5469 NewOpcode = AMDGPU::V_LSHLREV_B32_e64; 5470 swapOperands(Inst); 5471 } 5472 break; 5473 case AMDGPU::S_ASHR_I32: 5474 if (ST.hasOnlyRevVALUShifts()) { 5475 NewOpcode = AMDGPU::V_ASHRREV_I32_e64; 5476 swapOperands(Inst); 5477 } 5478 break; 5479 case AMDGPU::S_LSHR_B32: 5480 if (ST.hasOnlyRevVALUShifts()) { 5481 NewOpcode = AMDGPU::V_LSHRREV_B32_e64; 5482 swapOperands(Inst); 5483 } 5484 break; 5485 case AMDGPU::S_LSHL_B64: 5486 if (ST.hasOnlyRevVALUShifts()) { 5487 NewOpcode = AMDGPU::V_LSHLREV_B64_e64; 5488 swapOperands(Inst); 5489 } 5490 break; 5491 case AMDGPU::S_ASHR_I64: 5492 if (ST.hasOnlyRevVALUShifts()) { 5493 NewOpcode = AMDGPU::V_ASHRREV_I64_e64; 5494 swapOperands(Inst); 5495 } 5496 break; 5497 case AMDGPU::S_LSHR_B64: 5498 if (ST.hasOnlyRevVALUShifts()) { 5499 NewOpcode = AMDGPU::V_LSHRREV_B64_e64; 5500 swapOperands(Inst); 5501 } 5502 break; 5503 5504 case AMDGPU::S_ABS_I32: 5505 lowerScalarAbs(Worklist, Inst); 5506 Inst.eraseFromParent(); 5507 continue; 5508 5509 case AMDGPU::S_CBRANCH_SCC0: 5510 case AMDGPU::S_CBRANCH_SCC1: 5511 // Clear unused bits of vcc 5512 if (ST.isWave32()) 5513 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B32), 5514 AMDGPU::VCC_LO) 5515 .addReg(AMDGPU::EXEC_LO) 5516 .addReg(AMDGPU::VCC_LO); 5517 else 5518 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B64), 5519 AMDGPU::VCC) 5520 .addReg(AMDGPU::EXEC) 5521 .addReg(AMDGPU::VCC); 5522 break; 5523 5524 case AMDGPU::S_BFE_U64: 5525 case AMDGPU::S_BFM_B64: 5526 llvm_unreachable("Moving this op to VALU not implemented"); 5527 5528 case AMDGPU::S_PACK_LL_B32_B16: 5529 case AMDGPU::S_PACK_LH_B32_B16: 5530 case AMDGPU::S_PACK_HH_B32_B16: 5531 movePackToVALU(Worklist, MRI, Inst); 5532 Inst.eraseFromParent(); 5533 continue; 5534 5535 case AMDGPU::S_XNOR_B32: 5536 lowerScalarXnor(Worklist, Inst); 5537 Inst.eraseFromParent(); 5538 continue; 5539 5540 case AMDGPU::S_NAND_B32: 5541 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32); 5542 Inst.eraseFromParent(); 5543 continue; 5544 5545 case AMDGPU::S_NOR_B32: 5546 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32); 5547 Inst.eraseFromParent(); 5548 continue; 5549 5550 case AMDGPU::S_ANDN2_B32: 5551 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32); 5552 Inst.eraseFromParent(); 5553 continue; 5554 5555 case AMDGPU::S_ORN2_B32: 5556 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32); 5557 Inst.eraseFromParent(); 5558 continue; 5559 5560 // TODO: remove as soon as everything is ready 5561 // to replace VGPR to SGPR copy with V_READFIRSTLANEs. 5562 // S_ADD/SUB_CO_PSEUDO as well as S_UADDO/USUBO_PSEUDO 5563 // can only be selected from the uniform SDNode. 5564 case AMDGPU::S_ADD_CO_PSEUDO: 5565 case AMDGPU::S_SUB_CO_PSEUDO: { 5566 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO) 5567 ? AMDGPU::V_ADDC_U32_e64 5568 : AMDGPU::V_SUBB_U32_e64; 5569 const auto *CarryRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 5570 5571 Register CarryInReg = Inst.getOperand(4).getReg(); 5572 if (!MRI.constrainRegClass(CarryInReg, CarryRC)) { 5573 Register NewCarryReg = MRI.createVirtualRegister(CarryRC); 5574 BuildMI(*MBB, &Inst, Inst.getDebugLoc(), get(AMDGPU::COPY), NewCarryReg) 5575 .addReg(CarryInReg); 5576 } 5577 5578 Register CarryOutReg = Inst.getOperand(1).getReg(); 5579 5580 Register DestReg = MRI.createVirtualRegister(RI.getEquivalentVGPRClass( 5581 MRI.getRegClass(Inst.getOperand(0).getReg()))); 5582 MachineInstr *CarryOp = 5583 BuildMI(*MBB, &Inst, Inst.getDebugLoc(), get(Opc), DestReg) 5584 .addReg(CarryOutReg, RegState::Define) 5585 .add(Inst.getOperand(2)) 5586 .add(Inst.getOperand(3)) 5587 .addReg(CarryInReg) 5588 .addImm(0); 5589 CreatedBBTmp = legalizeOperands(*CarryOp); 5590 if (CreatedBBTmp && TopInst.getParent() == CreatedBBTmp) 5591 CreatedBB = CreatedBBTmp; 5592 MRI.replaceRegWith(Inst.getOperand(0).getReg(), DestReg); 5593 addUsersToMoveToVALUWorklist(DestReg, MRI, Worklist); 5594 Inst.eraseFromParent(); 5595 } 5596 continue; 5597 case AMDGPU::S_UADDO_PSEUDO: 5598 case AMDGPU::S_USUBO_PSEUDO: { 5599 const DebugLoc &DL = Inst.getDebugLoc(); 5600 MachineOperand &Dest0 = Inst.getOperand(0); 5601 MachineOperand &Dest1 = Inst.getOperand(1); 5602 MachineOperand &Src0 = Inst.getOperand(2); 5603 MachineOperand &Src1 = Inst.getOperand(3); 5604 5605 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_UADDO_PSEUDO) 5606 ? AMDGPU::V_ADD_CO_U32_e64 5607 : AMDGPU::V_SUB_CO_U32_e64; 5608 const TargetRegisterClass *NewRC = 5609 RI.getEquivalentVGPRClass(MRI.getRegClass(Dest0.getReg())); 5610 Register DestReg = MRI.createVirtualRegister(NewRC); 5611 MachineInstr *NewInstr = BuildMI(*MBB, &Inst, DL, get(Opc), DestReg) 5612 .addReg(Dest1.getReg(), RegState::Define) 5613 .add(Src0) 5614 .add(Src1) 5615 .addImm(0); // clamp bit 5616 5617 CreatedBBTmp = legalizeOperands(*NewInstr, MDT); 5618 if (CreatedBBTmp && TopInst.getParent() == CreatedBBTmp) 5619 CreatedBB = CreatedBBTmp; 5620 5621 MRI.replaceRegWith(Dest0.getReg(), DestReg); 5622 addUsersToMoveToVALUWorklist(NewInstr->getOperand(0).getReg(), MRI, 5623 Worklist); 5624 Inst.eraseFromParent(); 5625 } 5626 continue; 5627 5628 case AMDGPU::S_CSELECT_B32: 5629 case AMDGPU::S_CSELECT_B64: 5630 lowerSelect(Worklist, Inst, MDT); 5631 Inst.eraseFromParent(); 5632 continue; 5633 } 5634 5635 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) { 5636 // We cannot move this instruction to the VALU, so we should try to 5637 // legalize its operands instead. 5638 CreatedBBTmp = legalizeOperands(Inst, MDT); 5639 if (CreatedBBTmp && TopInst.getParent() == CreatedBBTmp) 5640 CreatedBB = CreatedBBTmp; 5641 continue; 5642 } 5643 5644 // Use the new VALU Opcode. 5645 const MCInstrDesc &NewDesc = get(NewOpcode); 5646 Inst.setDesc(NewDesc); 5647 5648 // Remove any references to SCC. Vector instructions can't read from it, and 5649 // We're just about to add the implicit use / defs of VCC, and we don't want 5650 // both. 5651 for (unsigned i = Inst.getNumOperands() - 1; i > 0; --i) { 5652 MachineOperand &Op = Inst.getOperand(i); 5653 if (Op.isReg() && Op.getReg() == AMDGPU::SCC) { 5654 // Only propagate through live-def of SCC. 5655 if (Op.isDef() && !Op.isDead()) 5656 addSCCDefUsersToVALUWorklist(Op, Inst, Worklist); 5657 Inst.RemoveOperand(i); 5658 } 5659 } 5660 5661 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) { 5662 // We are converting these to a BFE, so we need to add the missing 5663 // operands for the size and offset. 5664 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16; 5665 Inst.addOperand(MachineOperand::CreateImm(0)); 5666 Inst.addOperand(MachineOperand::CreateImm(Size)); 5667 5668 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) { 5669 // The VALU version adds the second operand to the result, so insert an 5670 // extra 0 operand. 5671 Inst.addOperand(MachineOperand::CreateImm(0)); 5672 } 5673 5674 Inst.addImplicitDefUseOperands(*Inst.getParent()->getParent()); 5675 fixImplicitOperands(Inst); 5676 5677 if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) { 5678 const MachineOperand &OffsetWidthOp = Inst.getOperand(2); 5679 // If we need to move this to VGPRs, we need to unpack the second operand 5680 // back into the 2 separate ones for bit offset and width. 5681 assert(OffsetWidthOp.isImm() && 5682 "Scalar BFE is only implemented for constant width and offset"); 5683 uint32_t Imm = OffsetWidthOp.getImm(); 5684 5685 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0]. 5686 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16]. 5687 Inst.RemoveOperand(2); // Remove old immediate. 5688 Inst.addOperand(MachineOperand::CreateImm(Offset)); 5689 Inst.addOperand(MachineOperand::CreateImm(BitWidth)); 5690 } 5691 5692 bool HasDst = Inst.getOperand(0).isReg() && Inst.getOperand(0).isDef(); 5693 unsigned NewDstReg = AMDGPU::NoRegister; 5694 if (HasDst) { 5695 Register DstReg = Inst.getOperand(0).getReg(); 5696 if (DstReg.isPhysical()) 5697 continue; 5698 5699 // Update the destination register class. 5700 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst); 5701 if (!NewDstRC) 5702 continue; 5703 5704 if (Inst.isCopy() && Inst.getOperand(1).getReg().isVirtual() && 5705 NewDstRC == RI.getRegClassForReg(MRI, Inst.getOperand(1).getReg())) { 5706 // Instead of creating a copy where src and dst are the same register 5707 // class, we just replace all uses of dst with src. These kinds of 5708 // copies interfere with the heuristics MachineSink uses to decide 5709 // whether or not to split a critical edge. Since the pass assumes 5710 // that copies will end up as machine instructions and not be 5711 // eliminated. 5712 addUsersToMoveToVALUWorklist(DstReg, MRI, Worklist); 5713 MRI.replaceRegWith(DstReg, Inst.getOperand(1).getReg()); 5714 MRI.clearKillFlags(Inst.getOperand(1).getReg()); 5715 Inst.getOperand(0).setReg(DstReg); 5716 5717 // Make sure we don't leave around a dead VGPR->SGPR copy. Normally 5718 // these are deleted later, but at -O0 it would leave a suspicious 5719 // looking illegal copy of an undef register. 5720 for (unsigned I = Inst.getNumOperands() - 1; I != 0; --I) 5721 Inst.RemoveOperand(I); 5722 Inst.setDesc(get(AMDGPU::IMPLICIT_DEF)); 5723 continue; 5724 } 5725 5726 NewDstReg = MRI.createVirtualRegister(NewDstRC); 5727 MRI.replaceRegWith(DstReg, NewDstReg); 5728 } 5729 5730 // Legalize the operands 5731 CreatedBBTmp = legalizeOperands(Inst, MDT); 5732 if (CreatedBBTmp && TopInst.getParent() == CreatedBBTmp) 5733 CreatedBB = CreatedBBTmp; 5734 5735 if (HasDst) 5736 addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist); 5737 } 5738 return CreatedBB; 5739 } 5740 5741 // Add/sub require special handling to deal with carry outs. 5742 std::pair<bool, MachineBasicBlock *> 5743 SIInstrInfo::moveScalarAddSub(SetVectorType &Worklist, MachineInstr &Inst, 5744 MachineDominatorTree *MDT) const { 5745 if (ST.hasAddNoCarry()) { 5746 // Assume there is no user of scc since we don't select this in that case. 5747 // Since scc isn't used, it doesn't really matter if the i32 or u32 variant 5748 // is used. 5749 5750 MachineBasicBlock &MBB = *Inst.getParent(); 5751 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 5752 5753 Register OldDstReg = Inst.getOperand(0).getReg(); 5754 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 5755 5756 unsigned Opc = Inst.getOpcode(); 5757 assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32); 5758 5759 unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ? 5760 AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64; 5761 5762 assert(Inst.getOperand(3).getReg() == AMDGPU::SCC); 5763 Inst.RemoveOperand(3); 5764 5765 Inst.setDesc(get(NewOpc)); 5766 Inst.addOperand(MachineOperand::CreateImm(0)); // clamp bit 5767 Inst.addImplicitDefUseOperands(*MBB.getParent()); 5768 MRI.replaceRegWith(OldDstReg, ResultReg); 5769 MachineBasicBlock *NewBB = legalizeOperands(Inst, MDT); 5770 5771 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 5772 return std::make_pair(true, NewBB); 5773 } 5774 5775 return std::make_pair(false, nullptr); 5776 } 5777 5778 void SIInstrInfo::lowerSelect(SetVectorType &Worklist, MachineInstr &Inst, 5779 MachineDominatorTree *MDT) const { 5780 5781 MachineBasicBlock &MBB = *Inst.getParent(); 5782 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 5783 MachineBasicBlock::iterator MII = Inst; 5784 DebugLoc DL = Inst.getDebugLoc(); 5785 5786 MachineOperand &Dest = Inst.getOperand(0); 5787 MachineOperand &Src0 = Inst.getOperand(1); 5788 MachineOperand &Src1 = Inst.getOperand(2); 5789 MachineOperand &Cond = Inst.getOperand(3); 5790 5791 Register SCCSource = Cond.getReg(); 5792 // Find SCC def, and if that is a copy (SCC = COPY reg) then use reg instead. 5793 if (!Cond.isUndef()) { 5794 for (MachineInstr &CandI : 5795 make_range(std::next(MachineBasicBlock::reverse_iterator(Inst)), 5796 Inst.getParent()->rend())) { 5797 if (CandI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != 5798 -1) { 5799 if (CandI.isCopy() && CandI.getOperand(0).getReg() == AMDGPU::SCC) { 5800 SCCSource = CandI.getOperand(1).getReg(); 5801 } 5802 break; 5803 } 5804 } 5805 } 5806 5807 // If this is a trivial select where the condition is effectively not SCC 5808 // (SCCSource is a source of copy to SCC), then the select is semantically 5809 // equivalent to copying SCCSource. Hence, there is no need to create 5810 // V_CNDMASK, we can just use that and bail out. 5811 if ((SCCSource != AMDGPU::SCC) && Src0.isImm() && (Src0.getImm() == -1) && 5812 Src1.isImm() && (Src1.getImm() == 0)) { 5813 MRI.replaceRegWith(Dest.getReg(), SCCSource); 5814 return; 5815 } 5816 5817 const TargetRegisterClass *TC = ST.getWavefrontSize() == 64 5818 ? &AMDGPU::SReg_64_XEXECRegClass 5819 : &AMDGPU::SReg_32_XM0_XEXECRegClass; 5820 Register CopySCC = MRI.createVirtualRegister(TC); 5821 5822 if (SCCSource == AMDGPU::SCC) { 5823 // Insert a trivial select instead of creating a copy, because a copy from 5824 // SCC would semantically mean just copying a single bit, but we may need 5825 // the result to be a vector condition mask that needs preserving. 5826 unsigned Opcode = (ST.getWavefrontSize() == 64) ? AMDGPU::S_CSELECT_B64 5827 : AMDGPU::S_CSELECT_B32; 5828 auto NewSelect = 5829 BuildMI(MBB, MII, DL, get(Opcode), CopySCC).addImm(-1).addImm(0); 5830 NewSelect->getOperand(3).setIsUndef(Cond.isUndef()); 5831 } else { 5832 BuildMI(MBB, MII, DL, get(AMDGPU::COPY), CopySCC).addReg(SCCSource); 5833 } 5834 5835 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 5836 5837 auto UpdatedInst = 5838 BuildMI(MBB, MII, DL, get(AMDGPU::V_CNDMASK_B32_e64), ResultReg) 5839 .addImm(0) 5840 .add(Src1) // False 5841 .addImm(0) 5842 .add(Src0) // True 5843 .addReg(CopySCC); 5844 5845 MRI.replaceRegWith(Dest.getReg(), ResultReg); 5846 legalizeOperands(*UpdatedInst, MDT); 5847 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 5848 } 5849 5850 void SIInstrInfo::lowerScalarAbs(SetVectorType &Worklist, 5851 MachineInstr &Inst) const { 5852 MachineBasicBlock &MBB = *Inst.getParent(); 5853 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 5854 MachineBasicBlock::iterator MII = Inst; 5855 DebugLoc DL = Inst.getDebugLoc(); 5856 5857 MachineOperand &Dest = Inst.getOperand(0); 5858 MachineOperand &Src = Inst.getOperand(1); 5859 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 5860 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 5861 5862 unsigned SubOp = ST.hasAddNoCarry() ? 5863 AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_CO_U32_e32; 5864 5865 BuildMI(MBB, MII, DL, get(SubOp), TmpReg) 5866 .addImm(0) 5867 .addReg(Src.getReg()); 5868 5869 BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg) 5870 .addReg(Src.getReg()) 5871 .addReg(TmpReg); 5872 5873 MRI.replaceRegWith(Dest.getReg(), ResultReg); 5874 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 5875 } 5876 5877 void SIInstrInfo::lowerScalarXnor(SetVectorType &Worklist, 5878 MachineInstr &Inst) const { 5879 MachineBasicBlock &MBB = *Inst.getParent(); 5880 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 5881 MachineBasicBlock::iterator MII = Inst; 5882 const DebugLoc &DL = Inst.getDebugLoc(); 5883 5884 MachineOperand &Dest = Inst.getOperand(0); 5885 MachineOperand &Src0 = Inst.getOperand(1); 5886 MachineOperand &Src1 = Inst.getOperand(2); 5887 5888 if (ST.hasDLInsts()) { 5889 Register NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 5890 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL); 5891 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL); 5892 5893 BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest) 5894 .add(Src0) 5895 .add(Src1); 5896 5897 MRI.replaceRegWith(Dest.getReg(), NewDest); 5898 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist); 5899 } else { 5900 // Using the identity !(x ^ y) == (!x ^ y) == (x ^ !y), we can 5901 // invert either source and then perform the XOR. If either source is a 5902 // scalar register, then we can leave the inversion on the scalar unit to 5903 // acheive a better distrubution of scalar and vector instructions. 5904 bool Src0IsSGPR = Src0.isReg() && 5905 RI.isSGPRClass(MRI.getRegClass(Src0.getReg())); 5906 bool Src1IsSGPR = Src1.isReg() && 5907 RI.isSGPRClass(MRI.getRegClass(Src1.getReg())); 5908 MachineInstr *Xor; 5909 Register Temp = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 5910 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 5911 5912 // Build a pair of scalar instructions and add them to the work list. 5913 // The next iteration over the work list will lower these to the vector 5914 // unit as necessary. 5915 if (Src0IsSGPR) { 5916 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src0); 5917 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest) 5918 .addReg(Temp) 5919 .add(Src1); 5920 } else if (Src1IsSGPR) { 5921 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src1); 5922 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest) 5923 .add(Src0) 5924 .addReg(Temp); 5925 } else { 5926 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), Temp) 5927 .add(Src0) 5928 .add(Src1); 5929 MachineInstr *Not = 5930 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest).addReg(Temp); 5931 Worklist.insert(Not); 5932 } 5933 5934 MRI.replaceRegWith(Dest.getReg(), NewDest); 5935 5936 Worklist.insert(Xor); 5937 5938 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist); 5939 } 5940 } 5941 5942 void SIInstrInfo::splitScalarNotBinop(SetVectorType &Worklist, 5943 MachineInstr &Inst, 5944 unsigned Opcode) const { 5945 MachineBasicBlock &MBB = *Inst.getParent(); 5946 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 5947 MachineBasicBlock::iterator MII = Inst; 5948 const DebugLoc &DL = Inst.getDebugLoc(); 5949 5950 MachineOperand &Dest = Inst.getOperand(0); 5951 MachineOperand &Src0 = Inst.getOperand(1); 5952 MachineOperand &Src1 = Inst.getOperand(2); 5953 5954 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 5955 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 5956 5957 MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), Interm) 5958 .add(Src0) 5959 .add(Src1); 5960 5961 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest) 5962 .addReg(Interm); 5963 5964 Worklist.insert(&Op); 5965 Worklist.insert(&Not); 5966 5967 MRI.replaceRegWith(Dest.getReg(), NewDest); 5968 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist); 5969 } 5970 5971 void SIInstrInfo::splitScalarBinOpN2(SetVectorType& Worklist, 5972 MachineInstr &Inst, 5973 unsigned Opcode) const { 5974 MachineBasicBlock &MBB = *Inst.getParent(); 5975 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 5976 MachineBasicBlock::iterator MII = Inst; 5977 const DebugLoc &DL = Inst.getDebugLoc(); 5978 5979 MachineOperand &Dest = Inst.getOperand(0); 5980 MachineOperand &Src0 = Inst.getOperand(1); 5981 MachineOperand &Src1 = Inst.getOperand(2); 5982 5983 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 5984 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 5985 5986 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Interm) 5987 .add(Src1); 5988 5989 MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), NewDest) 5990 .add(Src0) 5991 .addReg(Interm); 5992 5993 Worklist.insert(&Not); 5994 Worklist.insert(&Op); 5995 5996 MRI.replaceRegWith(Dest.getReg(), NewDest); 5997 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist); 5998 } 5999 6000 void SIInstrInfo::splitScalar64BitUnaryOp( 6001 SetVectorType &Worklist, MachineInstr &Inst, 6002 unsigned Opcode) const { 6003 MachineBasicBlock &MBB = *Inst.getParent(); 6004 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6005 6006 MachineOperand &Dest = Inst.getOperand(0); 6007 MachineOperand &Src0 = Inst.getOperand(1); 6008 DebugLoc DL = Inst.getDebugLoc(); 6009 6010 MachineBasicBlock::iterator MII = Inst; 6011 6012 const MCInstrDesc &InstDesc = get(Opcode); 6013 const TargetRegisterClass *Src0RC = Src0.isReg() ? 6014 MRI.getRegClass(Src0.getReg()) : 6015 &AMDGPU::SGPR_32RegClass; 6016 6017 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0); 6018 6019 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 6020 AMDGPU::sub0, Src0SubRC); 6021 6022 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg()); 6023 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC); 6024 const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0); 6025 6026 Register DestSub0 = MRI.createVirtualRegister(NewDestSubRC); 6027 MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0).add(SrcReg0Sub0); 6028 6029 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 6030 AMDGPU::sub1, Src0SubRC); 6031 6032 Register DestSub1 = MRI.createVirtualRegister(NewDestSubRC); 6033 MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1).add(SrcReg0Sub1); 6034 6035 Register FullDestReg = MRI.createVirtualRegister(NewDestRC); 6036 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg) 6037 .addReg(DestSub0) 6038 .addImm(AMDGPU::sub0) 6039 .addReg(DestSub1) 6040 .addImm(AMDGPU::sub1); 6041 6042 MRI.replaceRegWith(Dest.getReg(), FullDestReg); 6043 6044 Worklist.insert(&LoHalf); 6045 Worklist.insert(&HiHalf); 6046 6047 // We don't need to legalizeOperands here because for a single operand, src0 6048 // will support any kind of input. 6049 6050 // Move all users of this moved value. 6051 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist); 6052 } 6053 6054 void SIInstrInfo::splitScalar64BitAddSub(SetVectorType &Worklist, 6055 MachineInstr &Inst, 6056 MachineDominatorTree *MDT) const { 6057 bool IsAdd = (Inst.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO); 6058 6059 MachineBasicBlock &MBB = *Inst.getParent(); 6060 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6061 const auto *CarryRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 6062 6063 Register FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 6064 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6065 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6066 6067 Register CarryReg = MRI.createVirtualRegister(CarryRC); 6068 Register DeadCarryReg = MRI.createVirtualRegister(CarryRC); 6069 6070 MachineOperand &Dest = Inst.getOperand(0); 6071 MachineOperand &Src0 = Inst.getOperand(1); 6072 MachineOperand &Src1 = Inst.getOperand(2); 6073 const DebugLoc &DL = Inst.getDebugLoc(); 6074 MachineBasicBlock::iterator MII = Inst; 6075 6076 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0.getReg()); 6077 const TargetRegisterClass *Src1RC = MRI.getRegClass(Src1.getReg()); 6078 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0); 6079 const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0); 6080 6081 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 6082 AMDGPU::sub0, Src0SubRC); 6083 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, 6084 AMDGPU::sub0, Src1SubRC); 6085 6086 6087 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 6088 AMDGPU::sub1, Src0SubRC); 6089 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, 6090 AMDGPU::sub1, Src1SubRC); 6091 6092 unsigned LoOpc = IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64; 6093 MachineInstr *LoHalf = 6094 BuildMI(MBB, MII, DL, get(LoOpc), DestSub0) 6095 .addReg(CarryReg, RegState::Define) 6096 .add(SrcReg0Sub0) 6097 .add(SrcReg1Sub0) 6098 .addImm(0); // clamp bit 6099 6100 unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64; 6101 MachineInstr *HiHalf = 6102 BuildMI(MBB, MII, DL, get(HiOpc), DestSub1) 6103 .addReg(DeadCarryReg, RegState::Define | RegState::Dead) 6104 .add(SrcReg0Sub1) 6105 .add(SrcReg1Sub1) 6106 .addReg(CarryReg, RegState::Kill) 6107 .addImm(0); // clamp bit 6108 6109 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg) 6110 .addReg(DestSub0) 6111 .addImm(AMDGPU::sub0) 6112 .addReg(DestSub1) 6113 .addImm(AMDGPU::sub1); 6114 6115 MRI.replaceRegWith(Dest.getReg(), FullDestReg); 6116 6117 // Try to legalize the operands in case we need to swap the order to keep it 6118 // valid. 6119 legalizeOperands(*LoHalf, MDT); 6120 legalizeOperands(*HiHalf, MDT); 6121 6122 // Move all users of this moved vlaue. 6123 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist); 6124 } 6125 6126 void SIInstrInfo::splitScalar64BitBinaryOp(SetVectorType &Worklist, 6127 MachineInstr &Inst, unsigned Opcode, 6128 MachineDominatorTree *MDT) const { 6129 MachineBasicBlock &MBB = *Inst.getParent(); 6130 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6131 6132 MachineOperand &Dest = Inst.getOperand(0); 6133 MachineOperand &Src0 = Inst.getOperand(1); 6134 MachineOperand &Src1 = Inst.getOperand(2); 6135 DebugLoc DL = Inst.getDebugLoc(); 6136 6137 MachineBasicBlock::iterator MII = Inst; 6138 6139 const MCInstrDesc &InstDesc = get(Opcode); 6140 const TargetRegisterClass *Src0RC = Src0.isReg() ? 6141 MRI.getRegClass(Src0.getReg()) : 6142 &AMDGPU::SGPR_32RegClass; 6143 6144 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0); 6145 const TargetRegisterClass *Src1RC = Src1.isReg() ? 6146 MRI.getRegClass(Src1.getReg()) : 6147 &AMDGPU::SGPR_32RegClass; 6148 6149 const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0); 6150 6151 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 6152 AMDGPU::sub0, Src0SubRC); 6153 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, 6154 AMDGPU::sub0, Src1SubRC); 6155 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 6156 AMDGPU::sub1, Src0SubRC); 6157 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, 6158 AMDGPU::sub1, Src1SubRC); 6159 6160 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg()); 6161 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC); 6162 const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0); 6163 6164 Register DestSub0 = MRI.createVirtualRegister(NewDestSubRC); 6165 MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0) 6166 .add(SrcReg0Sub0) 6167 .add(SrcReg1Sub0); 6168 6169 Register DestSub1 = MRI.createVirtualRegister(NewDestSubRC); 6170 MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1) 6171 .add(SrcReg0Sub1) 6172 .add(SrcReg1Sub1); 6173 6174 Register FullDestReg = MRI.createVirtualRegister(NewDestRC); 6175 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg) 6176 .addReg(DestSub0) 6177 .addImm(AMDGPU::sub0) 6178 .addReg(DestSub1) 6179 .addImm(AMDGPU::sub1); 6180 6181 MRI.replaceRegWith(Dest.getReg(), FullDestReg); 6182 6183 Worklist.insert(&LoHalf); 6184 Worklist.insert(&HiHalf); 6185 6186 // Move all users of this moved vlaue. 6187 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist); 6188 } 6189 6190 void SIInstrInfo::splitScalar64BitXnor(SetVectorType &Worklist, 6191 MachineInstr &Inst, 6192 MachineDominatorTree *MDT) const { 6193 MachineBasicBlock &MBB = *Inst.getParent(); 6194 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6195 6196 MachineOperand &Dest = Inst.getOperand(0); 6197 MachineOperand &Src0 = Inst.getOperand(1); 6198 MachineOperand &Src1 = Inst.getOperand(2); 6199 const DebugLoc &DL = Inst.getDebugLoc(); 6200 6201 MachineBasicBlock::iterator MII = Inst; 6202 6203 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg()); 6204 6205 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); 6206 6207 MachineOperand* Op0; 6208 MachineOperand* Op1; 6209 6210 if (Src0.isReg() && RI.isSGPRReg(MRI, Src0.getReg())) { 6211 Op0 = &Src0; 6212 Op1 = &Src1; 6213 } else { 6214 Op0 = &Src1; 6215 Op1 = &Src0; 6216 } 6217 6218 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B64), Interm) 6219 .add(*Op0); 6220 6221 Register NewDest = MRI.createVirtualRegister(DestRC); 6222 6223 MachineInstr &Xor = *BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B64), NewDest) 6224 .addReg(Interm) 6225 .add(*Op1); 6226 6227 MRI.replaceRegWith(Dest.getReg(), NewDest); 6228 6229 Worklist.insert(&Xor); 6230 } 6231 6232 void SIInstrInfo::splitScalar64BitBCNT( 6233 SetVectorType &Worklist, MachineInstr &Inst) const { 6234 MachineBasicBlock &MBB = *Inst.getParent(); 6235 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6236 6237 MachineBasicBlock::iterator MII = Inst; 6238 const DebugLoc &DL = Inst.getDebugLoc(); 6239 6240 MachineOperand &Dest = Inst.getOperand(0); 6241 MachineOperand &Src = Inst.getOperand(1); 6242 6243 const MCInstrDesc &InstDesc = get(AMDGPU::V_BCNT_U32_B32_e64); 6244 const TargetRegisterClass *SrcRC = Src.isReg() ? 6245 MRI.getRegClass(Src.getReg()) : 6246 &AMDGPU::SGPR_32RegClass; 6247 6248 Register MidReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6249 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6250 6251 const TargetRegisterClass *SrcSubRC = RI.getSubRegClass(SrcRC, AMDGPU::sub0); 6252 6253 MachineOperand SrcRegSub0 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, 6254 AMDGPU::sub0, SrcSubRC); 6255 MachineOperand SrcRegSub1 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, 6256 AMDGPU::sub1, SrcSubRC); 6257 6258 BuildMI(MBB, MII, DL, InstDesc, MidReg).add(SrcRegSub0).addImm(0); 6259 6260 BuildMI(MBB, MII, DL, InstDesc, ResultReg).add(SrcRegSub1).addReg(MidReg); 6261 6262 MRI.replaceRegWith(Dest.getReg(), ResultReg); 6263 6264 // We don't need to legalize operands here. src0 for etiher instruction can be 6265 // an SGPR, and the second input is unused or determined here. 6266 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 6267 } 6268 6269 void SIInstrInfo::splitScalar64BitBFE(SetVectorType &Worklist, 6270 MachineInstr &Inst) const { 6271 MachineBasicBlock &MBB = *Inst.getParent(); 6272 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6273 MachineBasicBlock::iterator MII = Inst; 6274 const DebugLoc &DL = Inst.getDebugLoc(); 6275 6276 MachineOperand &Dest = Inst.getOperand(0); 6277 uint32_t Imm = Inst.getOperand(2).getImm(); 6278 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0]. 6279 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16]. 6280 6281 (void) Offset; 6282 6283 // Only sext_inreg cases handled. 6284 assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 && 6285 Offset == 0 && "Not implemented"); 6286 6287 if (BitWidth < 32) { 6288 Register MidRegLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6289 Register MidRegHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6290 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 6291 6292 BuildMI(MBB, MII, DL, get(AMDGPU::V_BFE_I32_e64), MidRegLo) 6293 .addReg(Inst.getOperand(1).getReg(), 0, AMDGPU::sub0) 6294 .addImm(0) 6295 .addImm(BitWidth); 6296 6297 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e32), MidRegHi) 6298 .addImm(31) 6299 .addReg(MidRegLo); 6300 6301 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg) 6302 .addReg(MidRegLo) 6303 .addImm(AMDGPU::sub0) 6304 .addReg(MidRegHi) 6305 .addImm(AMDGPU::sub1); 6306 6307 MRI.replaceRegWith(Dest.getReg(), ResultReg); 6308 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 6309 return; 6310 } 6311 6312 MachineOperand &Src = Inst.getOperand(1); 6313 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6314 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 6315 6316 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e64), TmpReg) 6317 .addImm(31) 6318 .addReg(Src.getReg(), 0, AMDGPU::sub0); 6319 6320 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg) 6321 .addReg(Src.getReg(), 0, AMDGPU::sub0) 6322 .addImm(AMDGPU::sub0) 6323 .addReg(TmpReg) 6324 .addImm(AMDGPU::sub1); 6325 6326 MRI.replaceRegWith(Dest.getReg(), ResultReg); 6327 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 6328 } 6329 6330 void SIInstrInfo::addUsersToMoveToVALUWorklist( 6331 Register DstReg, 6332 MachineRegisterInfo &MRI, 6333 SetVectorType &Worklist) const { 6334 for (MachineRegisterInfo::use_iterator I = MRI.use_begin(DstReg), 6335 E = MRI.use_end(); I != E;) { 6336 MachineInstr &UseMI = *I->getParent(); 6337 6338 unsigned OpNo = 0; 6339 6340 switch (UseMI.getOpcode()) { 6341 case AMDGPU::COPY: 6342 case AMDGPU::WQM: 6343 case AMDGPU::SOFT_WQM: 6344 case AMDGPU::WWM: 6345 case AMDGPU::REG_SEQUENCE: 6346 case AMDGPU::PHI: 6347 case AMDGPU::INSERT_SUBREG: 6348 break; 6349 default: 6350 OpNo = I.getOperandNo(); 6351 break; 6352 } 6353 6354 if (!RI.hasVectorRegisters(getOpRegClass(UseMI, OpNo))) { 6355 Worklist.insert(&UseMI); 6356 6357 do { 6358 ++I; 6359 } while (I != E && I->getParent() == &UseMI); 6360 } else { 6361 ++I; 6362 } 6363 } 6364 } 6365 6366 void SIInstrInfo::movePackToVALU(SetVectorType &Worklist, 6367 MachineRegisterInfo &MRI, 6368 MachineInstr &Inst) const { 6369 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6370 MachineBasicBlock *MBB = Inst.getParent(); 6371 MachineOperand &Src0 = Inst.getOperand(1); 6372 MachineOperand &Src1 = Inst.getOperand(2); 6373 const DebugLoc &DL = Inst.getDebugLoc(); 6374 6375 switch (Inst.getOpcode()) { 6376 case AMDGPU::S_PACK_LL_B32_B16: { 6377 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6378 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6379 6380 // FIXME: Can do a lot better if we know the high bits of src0 or src1 are 6381 // 0. 6382 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) 6383 .addImm(0xffff); 6384 6385 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_B32_e64), TmpReg) 6386 .addReg(ImmReg, RegState::Kill) 6387 .add(Src0); 6388 6389 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32_e64), ResultReg) 6390 .add(Src1) 6391 .addImm(16) 6392 .addReg(TmpReg, RegState::Kill); 6393 break; 6394 } 6395 case AMDGPU::S_PACK_LH_B32_B16: { 6396 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6397 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) 6398 .addImm(0xffff); 6399 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_BFI_B32_e64), ResultReg) 6400 .addReg(ImmReg, RegState::Kill) 6401 .add(Src0) 6402 .add(Src1); 6403 break; 6404 } 6405 case AMDGPU::S_PACK_HH_B32_B16: { 6406 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6407 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6408 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg) 6409 .addImm(16) 6410 .add(Src0); 6411 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) 6412 .addImm(0xffff0000); 6413 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_OR_B32_e64), ResultReg) 6414 .add(Src1) 6415 .addReg(ImmReg, RegState::Kill) 6416 .addReg(TmpReg, RegState::Kill); 6417 break; 6418 } 6419 default: 6420 llvm_unreachable("unhandled s_pack_* instruction"); 6421 } 6422 6423 MachineOperand &Dest = Inst.getOperand(0); 6424 MRI.replaceRegWith(Dest.getReg(), ResultReg); 6425 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 6426 } 6427 6428 void SIInstrInfo::addSCCDefUsersToVALUWorklist(MachineOperand &Op, 6429 MachineInstr &SCCDefInst, 6430 SetVectorType &Worklist) const { 6431 bool SCCUsedImplicitly = false; 6432 6433 // Ensure that def inst defines SCC, which is still live. 6434 assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isDef() && 6435 !Op.isDead() && Op.getParent() == &SCCDefInst); 6436 SmallVector<MachineInstr *, 4> CopyToDelete; 6437 // This assumes that all the users of SCC are in the same block 6438 // as the SCC def. 6439 for (MachineInstr &MI : // Skip the def inst itself. 6440 make_range(std::next(MachineBasicBlock::iterator(SCCDefInst)), 6441 SCCDefInst.getParent()->end())) { 6442 // Check if SCC is used first. 6443 if (MI.findRegisterUseOperandIdx(AMDGPU::SCC, false, &RI) != -1) { 6444 if (MI.isCopy()) { 6445 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 6446 Register DestReg = MI.getOperand(0).getReg(); 6447 6448 for (auto &User : MRI.use_nodbg_instructions(DestReg)) { 6449 if ((User.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO) || 6450 (User.getOpcode() == AMDGPU::S_SUB_CO_PSEUDO)) { 6451 User.getOperand(4).setReg(RI.getVCC()); 6452 Worklist.insert(&User); 6453 } else if (User.getOpcode() == AMDGPU::V_CNDMASK_B32_e64) { 6454 User.getOperand(5).setReg(RI.getVCC()); 6455 // No need to add to Worklist. 6456 } 6457 } 6458 CopyToDelete.push_back(&MI); 6459 } else { 6460 if (MI.getOpcode() == AMDGPU::S_CSELECT_B32 || 6461 MI.getOpcode() == AMDGPU::S_CSELECT_B64) { 6462 // This is an implicit use of SCC and it is really expected by 6463 // the SCC users to handle. 6464 // We cannot preserve the edge to the user so add the explicit 6465 // copy: SCC = COPY VCC. 6466 // The copy will be cleaned up during the processing of the user 6467 // in lowerSelect. 6468 SCCUsedImplicitly = true; 6469 } 6470 6471 Worklist.insert(&MI); 6472 } 6473 } 6474 // Exit if we find another SCC def. 6475 if (MI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != -1) 6476 break; 6477 } 6478 for (auto &Copy : CopyToDelete) 6479 Copy->eraseFromParent(); 6480 6481 if (SCCUsedImplicitly) { 6482 BuildMI(*SCCDefInst.getParent(), std::next(SCCDefInst.getIterator()), 6483 SCCDefInst.getDebugLoc(), get(AMDGPU::COPY), AMDGPU::SCC) 6484 .addReg(RI.getVCC()); 6485 } 6486 } 6487 6488 const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass( 6489 const MachineInstr &Inst) const { 6490 const TargetRegisterClass *NewDstRC = getOpRegClass(Inst, 0); 6491 6492 switch (Inst.getOpcode()) { 6493 // For target instructions, getOpRegClass just returns the virtual register 6494 // class associated with the operand, so we need to find an equivalent VGPR 6495 // register class in order to move the instruction to the VALU. 6496 case AMDGPU::COPY: 6497 case AMDGPU::PHI: 6498 case AMDGPU::REG_SEQUENCE: 6499 case AMDGPU::INSERT_SUBREG: 6500 case AMDGPU::WQM: 6501 case AMDGPU::SOFT_WQM: 6502 case AMDGPU::WWM: { 6503 const TargetRegisterClass *SrcRC = getOpRegClass(Inst, 1); 6504 if (RI.hasAGPRs(SrcRC)) { 6505 if (RI.hasAGPRs(NewDstRC)) 6506 return nullptr; 6507 6508 switch (Inst.getOpcode()) { 6509 case AMDGPU::PHI: 6510 case AMDGPU::REG_SEQUENCE: 6511 case AMDGPU::INSERT_SUBREG: 6512 NewDstRC = RI.getEquivalentAGPRClass(NewDstRC); 6513 break; 6514 default: 6515 NewDstRC = RI.getEquivalentVGPRClass(NewDstRC); 6516 } 6517 6518 if (!NewDstRC) 6519 return nullptr; 6520 } else { 6521 if (RI.hasVGPRs(NewDstRC) || NewDstRC == &AMDGPU::VReg_1RegClass) 6522 return nullptr; 6523 6524 NewDstRC = RI.getEquivalentVGPRClass(NewDstRC); 6525 if (!NewDstRC) 6526 return nullptr; 6527 } 6528 6529 return NewDstRC; 6530 } 6531 default: 6532 return NewDstRC; 6533 } 6534 } 6535 6536 // Find the one SGPR operand we are allowed to use. 6537 Register SIInstrInfo::findUsedSGPR(const MachineInstr &MI, 6538 int OpIndices[3]) const { 6539 const MCInstrDesc &Desc = MI.getDesc(); 6540 6541 // Find the one SGPR operand we are allowed to use. 6542 // 6543 // First we need to consider the instruction's operand requirements before 6544 // legalizing. Some operands are required to be SGPRs, such as implicit uses 6545 // of VCC, but we are still bound by the constant bus requirement to only use 6546 // one. 6547 // 6548 // If the operand's class is an SGPR, we can never move it. 6549 6550 Register SGPRReg = findImplicitSGPRRead(MI); 6551 if (SGPRReg != AMDGPU::NoRegister) 6552 return SGPRReg; 6553 6554 Register UsedSGPRs[3] = { AMDGPU::NoRegister }; 6555 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 6556 6557 for (unsigned i = 0; i < 3; ++i) { 6558 int Idx = OpIndices[i]; 6559 if (Idx == -1) 6560 break; 6561 6562 const MachineOperand &MO = MI.getOperand(Idx); 6563 if (!MO.isReg()) 6564 continue; 6565 6566 // Is this operand statically required to be an SGPR based on the operand 6567 // constraints? 6568 const TargetRegisterClass *OpRC = RI.getRegClass(Desc.OpInfo[Idx].RegClass); 6569 bool IsRequiredSGPR = RI.isSGPRClass(OpRC); 6570 if (IsRequiredSGPR) 6571 return MO.getReg(); 6572 6573 // If this could be a VGPR or an SGPR, Check the dynamic register class. 6574 Register Reg = MO.getReg(); 6575 const TargetRegisterClass *RegRC = MRI.getRegClass(Reg); 6576 if (RI.isSGPRClass(RegRC)) 6577 UsedSGPRs[i] = Reg; 6578 } 6579 6580 // We don't have a required SGPR operand, so we have a bit more freedom in 6581 // selecting operands to move. 6582 6583 // Try to select the most used SGPR. If an SGPR is equal to one of the 6584 // others, we choose that. 6585 // 6586 // e.g. 6587 // V_FMA_F32 v0, s0, s0, s0 -> No moves 6588 // V_FMA_F32 v0, s0, s1, s0 -> Move s1 6589 6590 // TODO: If some of the operands are 64-bit SGPRs and some 32, we should 6591 // prefer those. 6592 6593 if (UsedSGPRs[0] != AMDGPU::NoRegister) { 6594 if (UsedSGPRs[0] == UsedSGPRs[1] || UsedSGPRs[0] == UsedSGPRs[2]) 6595 SGPRReg = UsedSGPRs[0]; 6596 } 6597 6598 if (SGPRReg == AMDGPU::NoRegister && UsedSGPRs[1] != AMDGPU::NoRegister) { 6599 if (UsedSGPRs[1] == UsedSGPRs[2]) 6600 SGPRReg = UsedSGPRs[1]; 6601 } 6602 6603 return SGPRReg; 6604 } 6605 6606 MachineOperand *SIInstrInfo::getNamedOperand(MachineInstr &MI, 6607 unsigned OperandName) const { 6608 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OperandName); 6609 if (Idx == -1) 6610 return nullptr; 6611 6612 return &MI.getOperand(Idx); 6613 } 6614 6615 uint64_t SIInstrInfo::getDefaultRsrcDataFormat() const { 6616 if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) { 6617 return (22ULL << 44) | // IMG_FORMAT_32_FLOAT 6618 (1ULL << 56) | // RESOURCE_LEVEL = 1 6619 (3ULL << 60); // OOB_SELECT = 3 6620 } 6621 6622 uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT; 6623 if (ST.isAmdHsaOS()) { 6624 // Set ATC = 1. GFX9 doesn't have this bit. 6625 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) 6626 RsrcDataFormat |= (1ULL << 56); 6627 6628 // Set MTYPE = 2 (MTYPE_UC = uncached). GFX9 doesn't have this. 6629 // BTW, it disables TC L2 and therefore decreases performance. 6630 if (ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS) 6631 RsrcDataFormat |= (2ULL << 59); 6632 } 6633 6634 return RsrcDataFormat; 6635 } 6636 6637 uint64_t SIInstrInfo::getScratchRsrcWords23() const { 6638 uint64_t Rsrc23 = getDefaultRsrcDataFormat() | 6639 AMDGPU::RSRC_TID_ENABLE | 6640 0xffffffff; // Size; 6641 6642 // GFX9 doesn't have ELEMENT_SIZE. 6643 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) { 6644 uint64_t EltSizeValue = Log2_32(ST.getMaxPrivateElementSize(true)) - 1; 6645 Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT; 6646 } 6647 6648 // IndexStride = 64 / 32. 6649 uint64_t IndexStride = ST.getWavefrontSize() == 64 ? 3 : 2; 6650 Rsrc23 |= IndexStride << AMDGPU::RSRC_INDEX_STRIDE_SHIFT; 6651 6652 // If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17]. 6653 // Clear them unless we want a huge stride. 6654 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS && 6655 ST.getGeneration() <= AMDGPUSubtarget::GFX9) 6656 Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT; 6657 6658 return Rsrc23; 6659 } 6660 6661 bool SIInstrInfo::isLowLatencyInstruction(const MachineInstr &MI) const { 6662 unsigned Opc = MI.getOpcode(); 6663 6664 return isSMRD(Opc); 6665 } 6666 6667 bool SIInstrInfo::isHighLatencyDef(int Opc) const { 6668 return get(Opc).mayLoad() && 6669 (isMUBUF(Opc) || isMTBUF(Opc) || isMIMG(Opc) || isFLAT(Opc)); 6670 } 6671 6672 unsigned SIInstrInfo::isStackAccess(const MachineInstr &MI, 6673 int &FrameIndex) const { 6674 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::vaddr); 6675 if (!Addr || !Addr->isFI()) 6676 return AMDGPU::NoRegister; 6677 6678 assert(!MI.memoperands_empty() && 6679 (*MI.memoperands_begin())->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS); 6680 6681 FrameIndex = Addr->getIndex(); 6682 return getNamedOperand(MI, AMDGPU::OpName::vdata)->getReg(); 6683 } 6684 6685 unsigned SIInstrInfo::isSGPRStackAccess(const MachineInstr &MI, 6686 int &FrameIndex) const { 6687 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::addr); 6688 assert(Addr && Addr->isFI()); 6689 FrameIndex = Addr->getIndex(); 6690 return getNamedOperand(MI, AMDGPU::OpName::data)->getReg(); 6691 } 6692 6693 unsigned SIInstrInfo::isLoadFromStackSlot(const MachineInstr &MI, 6694 int &FrameIndex) const { 6695 if (!MI.mayLoad()) 6696 return AMDGPU::NoRegister; 6697 6698 if (isMUBUF(MI) || isVGPRSpill(MI)) 6699 return isStackAccess(MI, FrameIndex); 6700 6701 if (isSGPRSpill(MI)) 6702 return isSGPRStackAccess(MI, FrameIndex); 6703 6704 return AMDGPU::NoRegister; 6705 } 6706 6707 unsigned SIInstrInfo::isStoreToStackSlot(const MachineInstr &MI, 6708 int &FrameIndex) const { 6709 if (!MI.mayStore()) 6710 return AMDGPU::NoRegister; 6711 6712 if (isMUBUF(MI) || isVGPRSpill(MI)) 6713 return isStackAccess(MI, FrameIndex); 6714 6715 if (isSGPRSpill(MI)) 6716 return isSGPRStackAccess(MI, FrameIndex); 6717 6718 return AMDGPU::NoRegister; 6719 } 6720 6721 unsigned SIInstrInfo::getInstBundleSize(const MachineInstr &MI) const { 6722 unsigned Size = 0; 6723 MachineBasicBlock::const_instr_iterator I = MI.getIterator(); 6724 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end(); 6725 while (++I != E && I->isInsideBundle()) { 6726 assert(!I->isBundle() && "No nested bundle!"); 6727 Size += getInstSizeInBytes(*I); 6728 } 6729 6730 return Size; 6731 } 6732 6733 unsigned SIInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { 6734 unsigned Opc = MI.getOpcode(); 6735 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opc); 6736 unsigned DescSize = Desc.getSize(); 6737 6738 // If we have a definitive size, we can use it. Otherwise we need to inspect 6739 // the operands to know the size. 6740 if (isFixedSize(MI)) { 6741 unsigned Size = DescSize; 6742 6743 // If we hit the buggy offset, an extra nop will be inserted in MC so 6744 // estimate the worst case. 6745 if (MI.isBranch() && ST.hasOffset3fBug()) 6746 Size += 4; 6747 6748 return Size; 6749 } 6750 6751 // 4-byte instructions may have a 32-bit literal encoded after them. Check 6752 // operands that coud ever be literals. 6753 if (isVALU(MI) || isSALU(MI)) { 6754 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); 6755 if (Src0Idx == -1) 6756 return DescSize; // No operands. 6757 6758 if (isLiteralConstantLike(MI.getOperand(Src0Idx), Desc.OpInfo[Src0Idx])) 6759 return isVOP3(MI) ? 12 : (DescSize + 4); 6760 6761 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); 6762 if (Src1Idx == -1) 6763 return DescSize; 6764 6765 if (isLiteralConstantLike(MI.getOperand(Src1Idx), Desc.OpInfo[Src1Idx])) 6766 return isVOP3(MI) ? 12 : (DescSize + 4); 6767 6768 int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); 6769 if (Src2Idx == -1) 6770 return DescSize; 6771 6772 if (isLiteralConstantLike(MI.getOperand(Src2Idx), Desc.OpInfo[Src2Idx])) 6773 return isVOP3(MI) ? 12 : (DescSize + 4); 6774 6775 return DescSize; 6776 } 6777 6778 // Check whether we have extra NSA words. 6779 if (isMIMG(MI)) { 6780 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0); 6781 if (VAddr0Idx < 0) 6782 return 8; 6783 6784 int RSrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc); 6785 return 8 + 4 * ((RSrcIdx - VAddr0Idx + 2) / 4); 6786 } 6787 6788 switch (Opc) { 6789 case TargetOpcode::IMPLICIT_DEF: 6790 case TargetOpcode::KILL: 6791 case TargetOpcode::DBG_VALUE: 6792 case TargetOpcode::EH_LABEL: 6793 return 0; 6794 case TargetOpcode::BUNDLE: 6795 return getInstBundleSize(MI); 6796 case TargetOpcode::INLINEASM: 6797 case TargetOpcode::INLINEASM_BR: { 6798 const MachineFunction *MF = MI.getParent()->getParent(); 6799 const char *AsmStr = MI.getOperand(0).getSymbolName(); 6800 return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo(), &ST); 6801 } 6802 default: 6803 return DescSize; 6804 } 6805 } 6806 6807 bool SIInstrInfo::mayAccessFlatAddressSpace(const MachineInstr &MI) const { 6808 if (!isFLAT(MI)) 6809 return false; 6810 6811 if (MI.memoperands_empty()) 6812 return true; 6813 6814 for (const MachineMemOperand *MMO : MI.memoperands()) { 6815 if (MMO->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS) 6816 return true; 6817 } 6818 return false; 6819 } 6820 6821 bool SIInstrInfo::isNonUniformBranchInstr(MachineInstr &Branch) const { 6822 return Branch.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO; 6823 } 6824 6825 void SIInstrInfo::convertNonUniformIfRegion(MachineBasicBlock *IfEntry, 6826 MachineBasicBlock *IfEnd) const { 6827 MachineBasicBlock::iterator TI = IfEntry->getFirstTerminator(); 6828 assert(TI != IfEntry->end()); 6829 6830 MachineInstr *Branch = &(*TI); 6831 MachineFunction *MF = IfEntry->getParent(); 6832 MachineRegisterInfo &MRI = IfEntry->getParent()->getRegInfo(); 6833 6834 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { 6835 Register DstReg = MRI.createVirtualRegister(RI.getBoolRC()); 6836 MachineInstr *SIIF = 6837 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_IF), DstReg) 6838 .add(Branch->getOperand(0)) 6839 .add(Branch->getOperand(1)); 6840 MachineInstr *SIEND = 6841 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_END_CF)) 6842 .addReg(DstReg); 6843 6844 IfEntry->erase(TI); 6845 IfEntry->insert(IfEntry->end(), SIIF); 6846 IfEnd->insert(IfEnd->getFirstNonPHI(), SIEND); 6847 } 6848 } 6849 6850 void SIInstrInfo::convertNonUniformLoopRegion( 6851 MachineBasicBlock *LoopEntry, MachineBasicBlock *LoopEnd) const { 6852 MachineBasicBlock::iterator TI = LoopEnd->getFirstTerminator(); 6853 // We expect 2 terminators, one conditional and one unconditional. 6854 assert(TI != LoopEnd->end()); 6855 6856 MachineInstr *Branch = &(*TI); 6857 MachineFunction *MF = LoopEnd->getParent(); 6858 MachineRegisterInfo &MRI = LoopEnd->getParent()->getRegInfo(); 6859 6860 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { 6861 6862 Register DstReg = MRI.createVirtualRegister(RI.getBoolRC()); 6863 Register BackEdgeReg = MRI.createVirtualRegister(RI.getBoolRC()); 6864 MachineInstrBuilder HeaderPHIBuilder = 6865 BuildMI(*(MF), Branch->getDebugLoc(), get(TargetOpcode::PHI), DstReg); 6866 for (MachineBasicBlock::pred_iterator PI = LoopEntry->pred_begin(), 6867 E = LoopEntry->pred_end(); 6868 PI != E; ++PI) { 6869 if (*PI == LoopEnd) { 6870 HeaderPHIBuilder.addReg(BackEdgeReg); 6871 } else { 6872 MachineBasicBlock *PMBB = *PI; 6873 Register ZeroReg = MRI.createVirtualRegister(RI.getBoolRC()); 6874 materializeImmediate(*PMBB, PMBB->getFirstTerminator(), DebugLoc(), 6875 ZeroReg, 0); 6876 HeaderPHIBuilder.addReg(ZeroReg); 6877 } 6878 HeaderPHIBuilder.addMBB(*PI); 6879 } 6880 MachineInstr *HeaderPhi = HeaderPHIBuilder; 6881 MachineInstr *SIIFBREAK = BuildMI(*(MF), Branch->getDebugLoc(), 6882 get(AMDGPU::SI_IF_BREAK), BackEdgeReg) 6883 .addReg(DstReg) 6884 .add(Branch->getOperand(0)); 6885 MachineInstr *SILOOP = 6886 BuildMI(*(MF), Branch->getDebugLoc(), get(AMDGPU::SI_LOOP)) 6887 .addReg(BackEdgeReg) 6888 .addMBB(LoopEntry); 6889 6890 LoopEntry->insert(LoopEntry->begin(), HeaderPhi); 6891 LoopEnd->erase(TI); 6892 LoopEnd->insert(LoopEnd->end(), SIIFBREAK); 6893 LoopEnd->insert(LoopEnd->end(), SILOOP); 6894 } 6895 } 6896 6897 ArrayRef<std::pair<int, const char *>> 6898 SIInstrInfo::getSerializableTargetIndices() const { 6899 static const std::pair<int, const char *> TargetIndices[] = { 6900 {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"}, 6901 {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"}, 6902 {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"}, 6903 {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"}, 6904 {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}}; 6905 return makeArrayRef(TargetIndices); 6906 } 6907 6908 /// This is used by the post-RA scheduler (SchedulePostRAList.cpp). The 6909 /// post-RA version of misched uses CreateTargetMIHazardRecognizer. 6910 ScheduleHazardRecognizer * 6911 SIInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II, 6912 const ScheduleDAG *DAG) const { 6913 return new GCNHazardRecognizer(DAG->MF); 6914 } 6915 6916 /// This is the hazard recognizer used at -O0 by the PostRAHazardRecognizer 6917 /// pass. 6918 ScheduleHazardRecognizer * 6919 SIInstrInfo::CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const { 6920 return new GCNHazardRecognizer(MF); 6921 } 6922 6923 std::pair<unsigned, unsigned> 6924 SIInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const { 6925 return std::make_pair(TF & MO_MASK, TF & ~MO_MASK); 6926 } 6927 6928 ArrayRef<std::pair<unsigned, const char *>> 6929 SIInstrInfo::getSerializableDirectMachineOperandTargetFlags() const { 6930 static const std::pair<unsigned, const char *> TargetFlags[] = { 6931 { MO_GOTPCREL, "amdgpu-gotprel" }, 6932 { MO_GOTPCREL32_LO, "amdgpu-gotprel32-lo" }, 6933 { MO_GOTPCREL32_HI, "amdgpu-gotprel32-hi" }, 6934 { MO_REL32_LO, "amdgpu-rel32-lo" }, 6935 { MO_REL32_HI, "amdgpu-rel32-hi" }, 6936 { MO_ABS32_LO, "amdgpu-abs32-lo" }, 6937 { MO_ABS32_HI, "amdgpu-abs32-hi" }, 6938 }; 6939 6940 return makeArrayRef(TargetFlags); 6941 } 6942 6943 bool SIInstrInfo::isBasicBlockPrologue(const MachineInstr &MI) const { 6944 return !MI.isTerminator() && MI.getOpcode() != AMDGPU::COPY && 6945 MI.modifiesRegister(AMDGPU::EXEC, &RI); 6946 } 6947 6948 MachineInstrBuilder 6949 SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB, 6950 MachineBasicBlock::iterator I, 6951 const DebugLoc &DL, 6952 Register DestReg) const { 6953 if (ST.hasAddNoCarry()) 6954 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e64), DestReg); 6955 6956 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6957 Register UnusedCarry = MRI.createVirtualRegister(RI.getBoolRC()); 6958 MRI.setRegAllocationHint(UnusedCarry, 0, RI.getVCC()); 6959 6960 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg) 6961 .addReg(UnusedCarry, RegState::Define | RegState::Dead); 6962 } 6963 6964 MachineInstrBuilder SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB, 6965 MachineBasicBlock::iterator I, 6966 const DebugLoc &DL, 6967 Register DestReg, 6968 RegScavenger &RS) const { 6969 if (ST.hasAddNoCarry()) 6970 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e32), DestReg); 6971 6972 // If available, prefer to use vcc. 6973 Register UnusedCarry = !RS.isRegUsed(AMDGPU::VCC) 6974 ? Register(RI.getVCC()) 6975 : RS.scavengeRegister(RI.getBoolRC(), I, 0, false); 6976 6977 // TODO: Users need to deal with this. 6978 if (!UnusedCarry.isValid()) 6979 return MachineInstrBuilder(); 6980 6981 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg) 6982 .addReg(UnusedCarry, RegState::Define | RegState::Dead); 6983 } 6984 6985 bool SIInstrInfo::isKillTerminator(unsigned Opcode) { 6986 switch (Opcode) { 6987 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR: 6988 case AMDGPU::SI_KILL_I1_TERMINATOR: 6989 return true; 6990 default: 6991 return false; 6992 } 6993 } 6994 6995 const MCInstrDesc &SIInstrInfo::getKillTerminatorFromPseudo(unsigned Opcode) const { 6996 switch (Opcode) { 6997 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO: 6998 return get(AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR); 6999 case AMDGPU::SI_KILL_I1_PSEUDO: 7000 return get(AMDGPU::SI_KILL_I1_TERMINATOR); 7001 default: 7002 llvm_unreachable("invalid opcode, expected SI_KILL_*_PSEUDO"); 7003 } 7004 } 7005 7006 void SIInstrInfo::fixImplicitOperands(MachineInstr &MI) const { 7007 if (!ST.isWave32()) 7008 return; 7009 7010 for (auto &Op : MI.implicit_operands()) { 7011 if (Op.isReg() && Op.getReg() == AMDGPU::VCC) 7012 Op.setReg(AMDGPU::VCC_LO); 7013 } 7014 } 7015 7016 bool SIInstrInfo::isBufferSMRD(const MachineInstr &MI) const { 7017 if (!isSMRD(MI)) 7018 return false; 7019 7020 // Check that it is using a buffer resource. 7021 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::sbase); 7022 if (Idx == -1) // e.g. s_memtime 7023 return false; 7024 7025 const auto RCID = MI.getDesc().OpInfo[Idx].RegClass; 7026 return RI.getRegClass(RCID)->hasSubClassEq(&AMDGPU::SGPR_128RegClass); 7027 } 7028 7029 bool SIInstrInfo::isLegalFLATOffset(int64_t Offset, unsigned AddrSpace, 7030 bool Signed) const { 7031 // TODO: Should 0 be special cased? 7032 if (!ST.hasFlatInstOffsets()) 7033 return false; 7034 7035 if (ST.hasFlatSegmentOffsetBug() && AddrSpace == AMDGPUAS::FLAT_ADDRESS) 7036 return false; 7037 7038 unsigned N = AMDGPU::getNumFlatOffsetBits(ST, Signed); 7039 return Signed ? isIntN(N, Offset) : isUIntN(N, Offset); 7040 } 7041 7042 std::pair<int64_t, int64_t> SIInstrInfo::splitFlatOffset(int64_t COffsetVal, 7043 unsigned AddrSpace, 7044 bool IsSigned) const { 7045 int64_t RemainderOffset = COffsetVal; 7046 int64_t ImmField = 0; 7047 const unsigned NumBits = AMDGPU::getNumFlatOffsetBits(ST, IsSigned); 7048 if (IsSigned) { 7049 // Use signed division by a power of two to truncate towards 0. 7050 int64_t D = 1LL << (NumBits - 1); 7051 RemainderOffset = (COffsetVal / D) * D; 7052 ImmField = COffsetVal - RemainderOffset; 7053 } else if (COffsetVal >= 0) { 7054 ImmField = COffsetVal & maskTrailingOnes<uint64_t>(NumBits); 7055 RemainderOffset = COffsetVal - ImmField; 7056 } 7057 7058 assert(isLegalFLATOffset(ImmField, AddrSpace, IsSigned)); 7059 assert(RemainderOffset + ImmField == COffsetVal); 7060 return {ImmField, RemainderOffset}; 7061 } 7062 7063 // This must be kept in sync with the SIEncodingFamily class in SIInstrInfo.td 7064 enum SIEncodingFamily { 7065 SI = 0, 7066 VI = 1, 7067 SDWA = 2, 7068 SDWA9 = 3, 7069 GFX80 = 4, 7070 GFX9 = 5, 7071 GFX10 = 6, 7072 SDWA10 = 7 7073 }; 7074 7075 static SIEncodingFamily subtargetEncodingFamily(const GCNSubtarget &ST) { 7076 switch (ST.getGeneration()) { 7077 default: 7078 break; 7079 case AMDGPUSubtarget::SOUTHERN_ISLANDS: 7080 case AMDGPUSubtarget::SEA_ISLANDS: 7081 return SIEncodingFamily::SI; 7082 case AMDGPUSubtarget::VOLCANIC_ISLANDS: 7083 case AMDGPUSubtarget::GFX9: 7084 return SIEncodingFamily::VI; 7085 case AMDGPUSubtarget::GFX10: 7086 return SIEncodingFamily::GFX10; 7087 } 7088 llvm_unreachable("Unknown subtarget generation!"); 7089 } 7090 7091 bool SIInstrInfo::isAsmOnlyOpcode(int MCOp) const { 7092 switch(MCOp) { 7093 // These opcodes use indirect register addressing so 7094 // they need special handling by codegen (currently missing). 7095 // Therefore it is too risky to allow these opcodes 7096 // to be selected by dpp combiner or sdwa peepholer. 7097 case AMDGPU::V_MOVRELS_B32_dpp_gfx10: 7098 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10: 7099 case AMDGPU::V_MOVRELD_B32_dpp_gfx10: 7100 case AMDGPU::V_MOVRELD_B32_sdwa_gfx10: 7101 case AMDGPU::V_MOVRELSD_B32_dpp_gfx10: 7102 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10: 7103 case AMDGPU::V_MOVRELSD_2_B32_dpp_gfx10: 7104 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10: 7105 return true; 7106 default: 7107 return false; 7108 } 7109 } 7110 7111 int SIInstrInfo::pseudoToMCOpcode(int Opcode) const { 7112 SIEncodingFamily Gen = subtargetEncodingFamily(ST); 7113 7114 if ((get(Opcode).TSFlags & SIInstrFlags::renamedInGFX9) != 0 && 7115 ST.getGeneration() == AMDGPUSubtarget::GFX9) 7116 Gen = SIEncodingFamily::GFX9; 7117 7118 // Adjust the encoding family to GFX80 for D16 buffer instructions when the 7119 // subtarget has UnpackedD16VMem feature. 7120 // TODO: remove this when we discard GFX80 encoding. 7121 if (ST.hasUnpackedD16VMem() && (get(Opcode).TSFlags & SIInstrFlags::D16Buf)) 7122 Gen = SIEncodingFamily::GFX80; 7123 7124 if (get(Opcode).TSFlags & SIInstrFlags::SDWA) { 7125 switch (ST.getGeneration()) { 7126 default: 7127 Gen = SIEncodingFamily::SDWA; 7128 break; 7129 case AMDGPUSubtarget::GFX9: 7130 Gen = SIEncodingFamily::SDWA9; 7131 break; 7132 case AMDGPUSubtarget::GFX10: 7133 Gen = SIEncodingFamily::SDWA10; 7134 break; 7135 } 7136 } 7137 7138 int MCOp = AMDGPU::getMCOpcode(Opcode, Gen); 7139 7140 // -1 means that Opcode is already a native instruction. 7141 if (MCOp == -1) 7142 return Opcode; 7143 7144 // (uint16_t)-1 means that Opcode is a pseudo instruction that has 7145 // no encoding in the given subtarget generation. 7146 if (MCOp == (uint16_t)-1) 7147 return -1; 7148 7149 if (isAsmOnlyOpcode(MCOp)) 7150 return -1; 7151 7152 return MCOp; 7153 } 7154 7155 static 7156 TargetInstrInfo::RegSubRegPair getRegOrUndef(const MachineOperand &RegOpnd) { 7157 assert(RegOpnd.isReg()); 7158 return RegOpnd.isUndef() ? TargetInstrInfo::RegSubRegPair() : 7159 getRegSubRegPair(RegOpnd); 7160 } 7161 7162 TargetInstrInfo::RegSubRegPair 7163 llvm::getRegSequenceSubReg(MachineInstr &MI, unsigned SubReg) { 7164 assert(MI.isRegSequence()); 7165 for (unsigned I = 0, E = (MI.getNumOperands() - 1)/ 2; I < E; ++I) 7166 if (MI.getOperand(1 + 2 * I + 1).getImm() == SubReg) { 7167 auto &RegOp = MI.getOperand(1 + 2 * I); 7168 return getRegOrUndef(RegOp); 7169 } 7170 return TargetInstrInfo::RegSubRegPair(); 7171 } 7172 7173 // Try to find the definition of reg:subreg in subreg-manipulation pseudos 7174 // Following a subreg of reg:subreg isn't supported 7175 static bool followSubRegDef(MachineInstr &MI, 7176 TargetInstrInfo::RegSubRegPair &RSR) { 7177 if (!RSR.SubReg) 7178 return false; 7179 switch (MI.getOpcode()) { 7180 default: break; 7181 case AMDGPU::REG_SEQUENCE: 7182 RSR = getRegSequenceSubReg(MI, RSR.SubReg); 7183 return true; 7184 // EXTRACT_SUBREG ins't supported as this would follow a subreg of subreg 7185 case AMDGPU::INSERT_SUBREG: 7186 if (RSR.SubReg == (unsigned)MI.getOperand(3).getImm()) 7187 // inserted the subreg we're looking for 7188 RSR = getRegOrUndef(MI.getOperand(2)); 7189 else { // the subreg in the rest of the reg 7190 auto R1 = getRegOrUndef(MI.getOperand(1)); 7191 if (R1.SubReg) // subreg of subreg isn't supported 7192 return false; 7193 RSR.Reg = R1.Reg; 7194 } 7195 return true; 7196 } 7197 return false; 7198 } 7199 7200 MachineInstr *llvm::getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P, 7201 MachineRegisterInfo &MRI) { 7202 assert(MRI.isSSA()); 7203 if (!P.Reg.isVirtual()) 7204 return nullptr; 7205 7206 auto RSR = P; 7207 auto *DefInst = MRI.getVRegDef(RSR.Reg); 7208 while (auto *MI = DefInst) { 7209 DefInst = nullptr; 7210 switch (MI->getOpcode()) { 7211 case AMDGPU::COPY: 7212 case AMDGPU::V_MOV_B32_e32: { 7213 auto &Op1 = MI->getOperand(1); 7214 if (Op1.isReg() && Op1.getReg().isVirtual()) { 7215 if (Op1.isUndef()) 7216 return nullptr; 7217 RSR = getRegSubRegPair(Op1); 7218 DefInst = MRI.getVRegDef(RSR.Reg); 7219 } 7220 break; 7221 } 7222 default: 7223 if (followSubRegDef(*MI, RSR)) { 7224 if (!RSR.Reg) 7225 return nullptr; 7226 DefInst = MRI.getVRegDef(RSR.Reg); 7227 } 7228 } 7229 if (!DefInst) 7230 return MI; 7231 } 7232 return nullptr; 7233 } 7234 7235 bool llvm::execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI, 7236 Register VReg, 7237 const MachineInstr &DefMI, 7238 const MachineInstr &UseMI) { 7239 assert(MRI.isSSA() && "Must be run on SSA"); 7240 7241 auto *TRI = MRI.getTargetRegisterInfo(); 7242 auto *DefBB = DefMI.getParent(); 7243 7244 // Don't bother searching between blocks, although it is possible this block 7245 // doesn't modify exec. 7246 if (UseMI.getParent() != DefBB) 7247 return true; 7248 7249 const int MaxInstScan = 20; 7250 int NumInst = 0; 7251 7252 // Stop scan at the use. 7253 auto E = UseMI.getIterator(); 7254 for (auto I = std::next(DefMI.getIterator()); I != E; ++I) { 7255 if (I->isDebugInstr()) 7256 continue; 7257 7258 if (++NumInst > MaxInstScan) 7259 return true; 7260 7261 if (I->modifiesRegister(AMDGPU::EXEC, TRI)) 7262 return true; 7263 } 7264 7265 return false; 7266 } 7267 7268 bool llvm::execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI, 7269 Register VReg, 7270 const MachineInstr &DefMI) { 7271 assert(MRI.isSSA() && "Must be run on SSA"); 7272 7273 auto *TRI = MRI.getTargetRegisterInfo(); 7274 auto *DefBB = DefMI.getParent(); 7275 7276 const int MaxUseScan = 10; 7277 int NumUse = 0; 7278 7279 for (auto &Use : MRI.use_nodbg_operands(VReg)) { 7280 auto &UseInst = *Use.getParent(); 7281 // Don't bother searching between blocks, although it is possible this block 7282 // doesn't modify exec. 7283 if (UseInst.getParent() != DefBB) 7284 return true; 7285 7286 if (++NumUse > MaxUseScan) 7287 return true; 7288 } 7289 7290 if (NumUse == 0) 7291 return false; 7292 7293 const int MaxInstScan = 20; 7294 int NumInst = 0; 7295 7296 // Stop scan when we have seen all the uses. 7297 for (auto I = std::next(DefMI.getIterator()); ; ++I) { 7298 assert(I != DefBB->end()); 7299 7300 if (I->isDebugInstr()) 7301 continue; 7302 7303 if (++NumInst > MaxInstScan) 7304 return true; 7305 7306 for (const MachineOperand &Op : I->operands()) { 7307 // We don't check reg masks here as they're used only on calls: 7308 // 1. EXEC is only considered const within one BB 7309 // 2. Call should be a terminator instruction if present in a BB 7310 7311 if (!Op.isReg()) 7312 continue; 7313 7314 Register Reg = Op.getReg(); 7315 if (Op.isUse()) { 7316 if (Reg == VReg && --NumUse == 0) 7317 return false; 7318 } else if (TRI->regsOverlap(Reg, AMDGPU::EXEC)) 7319 return true; 7320 } 7321 } 7322 } 7323 7324 MachineInstr *SIInstrInfo::createPHIDestinationCopy( 7325 MachineBasicBlock &MBB, MachineBasicBlock::iterator LastPHIIt, 7326 const DebugLoc &DL, Register Src, Register Dst) const { 7327 auto Cur = MBB.begin(); 7328 if (Cur != MBB.end()) 7329 do { 7330 if (!Cur->isPHI() && Cur->readsRegister(Dst)) 7331 return BuildMI(MBB, Cur, DL, get(TargetOpcode::COPY), Dst).addReg(Src); 7332 ++Cur; 7333 } while (Cur != MBB.end() && Cur != LastPHIIt); 7334 7335 return TargetInstrInfo::createPHIDestinationCopy(MBB, LastPHIIt, DL, Src, 7336 Dst); 7337 } 7338 7339 MachineInstr *SIInstrInfo::createPHISourceCopy( 7340 MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt, 7341 const DebugLoc &DL, Register Src, unsigned SrcSubReg, Register Dst) const { 7342 if (InsPt != MBB.end() && 7343 (InsPt->getOpcode() == AMDGPU::SI_IF || 7344 InsPt->getOpcode() == AMDGPU::SI_ELSE || 7345 InsPt->getOpcode() == AMDGPU::SI_IF_BREAK) && 7346 InsPt->definesRegister(Src)) { 7347 InsPt++; 7348 return BuildMI(MBB, InsPt, DL, 7349 get(ST.isWave32() ? AMDGPU::S_MOV_B32_term 7350 : AMDGPU::S_MOV_B64_term), 7351 Dst) 7352 .addReg(Src, 0, SrcSubReg) 7353 .addReg(AMDGPU::EXEC, RegState::Implicit); 7354 } 7355 return TargetInstrInfo::createPHISourceCopy(MBB, InsPt, DL, Src, SrcSubReg, 7356 Dst); 7357 } 7358 7359 bool llvm::SIInstrInfo::isWave32() const { return ST.isWave32(); } 7360 7361 MachineInstr *SIInstrInfo::foldMemoryOperandImpl( 7362 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, 7363 MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS, 7364 VirtRegMap *VRM) const { 7365 // This is a bit of a hack (copied from AArch64). Consider this instruction: 7366 // 7367 // %0:sreg_32 = COPY $m0 7368 // 7369 // We explicitly chose SReg_32 for the virtual register so such a copy might 7370 // be eliminated by RegisterCoalescer. However, that may not be possible, and 7371 // %0 may even spill. We can't spill $m0 normally (it would require copying to 7372 // a numbered SGPR anyway), and since it is in the SReg_32 register class, 7373 // TargetInstrInfo::foldMemoryOperand() is going to try. 7374 // A similar issue also exists with spilling and reloading $exec registers. 7375 // 7376 // To prevent that, constrain the %0 register class here. 7377 if (MI.isFullCopy()) { 7378 Register DstReg = MI.getOperand(0).getReg(); 7379 Register SrcReg = MI.getOperand(1).getReg(); 7380 if ((DstReg.isVirtual() || SrcReg.isVirtual()) && 7381 (DstReg.isVirtual() != SrcReg.isVirtual())) { 7382 MachineRegisterInfo &MRI = MF.getRegInfo(); 7383 Register VirtReg = DstReg.isVirtual() ? DstReg : SrcReg; 7384 const TargetRegisterClass *RC = MRI.getRegClass(VirtReg); 7385 if (RC->hasSuperClassEq(&AMDGPU::SReg_32RegClass)) { 7386 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); 7387 return nullptr; 7388 } else if (RC->hasSuperClassEq(&AMDGPU::SReg_64RegClass)) { 7389 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_64_XEXECRegClass); 7390 return nullptr; 7391 } 7392 } 7393 } 7394 7395 return nullptr; 7396 } 7397 7398 unsigned SIInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, 7399 const MachineInstr &MI, 7400 unsigned *PredCost) const { 7401 if (MI.isBundle()) { 7402 MachineBasicBlock::const_instr_iterator I(MI.getIterator()); 7403 MachineBasicBlock::const_instr_iterator E(MI.getParent()->instr_end()); 7404 unsigned Lat = 0, Count = 0; 7405 for (++I; I != E && I->isBundledWithPred(); ++I) { 7406 ++Count; 7407 Lat = std::max(Lat, SchedModel.computeInstrLatency(&*I)); 7408 } 7409 return Lat + Count - 1; 7410 } 7411 7412 return SchedModel.computeInstrLatency(&MI); 7413 } 7414 7415 unsigned SIInstrInfo::getDSShaderTypeValue(const MachineFunction &MF) { 7416 switch (MF.getFunction().getCallingConv()) { 7417 case CallingConv::AMDGPU_PS: 7418 return 1; 7419 case CallingConv::AMDGPU_VS: 7420 return 2; 7421 case CallingConv::AMDGPU_GS: 7422 return 3; 7423 case CallingConv::AMDGPU_HS: 7424 case CallingConv::AMDGPU_LS: 7425 case CallingConv::AMDGPU_ES: 7426 report_fatal_error("ds_ordered_count unsupported for this calling conv"); 7427 case CallingConv::AMDGPU_CS: 7428 case CallingConv::AMDGPU_KERNEL: 7429 case CallingConv::C: 7430 case CallingConv::Fast: 7431 default: 7432 // Assume other calling conventions are various compute callable functions 7433 return 0; 7434 } 7435 } 7436