1 //===-- AMDGPUISelDAGToDAG.cpp - A dag to dag inst selector for AMDGPU ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //==-----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// Defines an instruction selector for the AMDGPU target. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "AMDGPU.h" 15 #include "AMDGPUTargetMachine.h" 16 #include "SIMachineFunctionInfo.h" 17 #include "llvm/Analysis/LegacyDivergenceAnalysis.h" 18 #include "llvm/Analysis/ValueTracking.h" 19 #include "llvm/CodeGen/FunctionLoweringInfo.h" 20 #include "llvm/CodeGen/SelectionDAG.h" 21 #include "llvm/CodeGen/SelectionDAGISel.h" 22 #include "llvm/CodeGen/SelectionDAGNodes.h" 23 #include "llvm/IR/IntrinsicsAMDGPU.h" 24 #include "llvm/InitializePasses.h" 25 26 #ifdef EXPENSIVE_CHECKS 27 #include "llvm/Analysis/LoopInfo.h" 28 #include "llvm/IR/Dominators.h" 29 #endif 30 31 #define DEBUG_TYPE "isel" 32 33 using namespace llvm; 34 35 namespace llvm { 36 37 class R600InstrInfo; 38 39 } // end namespace llvm 40 41 //===----------------------------------------------------------------------===// 42 // Instruction Selector Implementation 43 //===----------------------------------------------------------------------===// 44 45 namespace { 46 47 static bool isNullConstantOrUndef(SDValue V) { 48 if (V.isUndef()) 49 return true; 50 51 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V); 52 return Const != nullptr && Const->isNullValue(); 53 } 54 55 static bool getConstantValue(SDValue N, uint32_t &Out) { 56 // This is only used for packed vectors, where ussing 0 for undef should 57 // always be good. 58 if (N.isUndef()) { 59 Out = 0; 60 return true; 61 } 62 63 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) { 64 Out = C->getAPIntValue().getSExtValue(); 65 return true; 66 } 67 68 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) { 69 Out = C->getValueAPF().bitcastToAPInt().getSExtValue(); 70 return true; 71 } 72 73 return false; 74 } 75 76 // TODO: Handle undef as zero 77 static SDNode *packConstantV2I16(const SDNode *N, SelectionDAG &DAG, 78 bool Negate = false) { 79 assert(N->getOpcode() == ISD::BUILD_VECTOR && N->getNumOperands() == 2); 80 uint32_t LHSVal, RHSVal; 81 if (getConstantValue(N->getOperand(0), LHSVal) && 82 getConstantValue(N->getOperand(1), RHSVal)) { 83 SDLoc SL(N); 84 uint32_t K = Negate ? 85 (-LHSVal & 0xffff) | (-RHSVal << 16) : 86 (LHSVal & 0xffff) | (RHSVal << 16); 87 return DAG.getMachineNode(AMDGPU::S_MOV_B32, SL, N->getValueType(0), 88 DAG.getTargetConstant(K, SL, MVT::i32)); 89 } 90 91 return nullptr; 92 } 93 94 static SDNode *packNegConstantV2I16(const SDNode *N, SelectionDAG &DAG) { 95 return packConstantV2I16(N, DAG, true); 96 } 97 98 /// AMDGPU specific code to select AMDGPU machine instructions for 99 /// SelectionDAG operations. 100 class AMDGPUDAGToDAGISel : public SelectionDAGISel { 101 // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can 102 // make the right decision when generating code for different targets. 103 const GCNSubtarget *Subtarget; 104 105 // Default FP mode for the current function. 106 AMDGPU::SIModeRegisterDefaults Mode; 107 108 bool EnableLateStructurizeCFG; 109 110 // Instructions that will be lowered with a final instruction that zeros the 111 // high result bits. 112 bool fp16SrcZerosHighBits(unsigned Opc) const; 113 114 public: 115 explicit AMDGPUDAGToDAGISel(TargetMachine *TM = nullptr, 116 CodeGenOpt::Level OptLevel = CodeGenOpt::Default) 117 : SelectionDAGISel(*TM, OptLevel) { 118 EnableLateStructurizeCFG = AMDGPUTargetMachine::EnableLateStructurizeCFG; 119 } 120 ~AMDGPUDAGToDAGISel() override = default; 121 122 void getAnalysisUsage(AnalysisUsage &AU) const override { 123 AU.addRequired<AMDGPUArgumentUsageInfo>(); 124 AU.addRequired<LegacyDivergenceAnalysis>(); 125 #ifdef EXPENSIVE_CHECKS 126 AU.addRequired<DominatorTreeWrapperPass>(); 127 AU.addRequired<LoopInfoWrapperPass>(); 128 #endif 129 SelectionDAGISel::getAnalysisUsage(AU); 130 } 131 132 bool matchLoadD16FromBuildVector(SDNode *N) const; 133 134 bool runOnMachineFunction(MachineFunction &MF) override; 135 void PreprocessISelDAG() override; 136 void Select(SDNode *N) override; 137 StringRef getPassName() const override; 138 void PostprocessISelDAG() override; 139 140 protected: 141 void SelectBuildVector(SDNode *N, unsigned RegClassID); 142 143 private: 144 std::pair<SDValue, SDValue> foldFrameIndex(SDValue N) const; 145 bool isNoNanSrc(SDValue N) const; 146 bool isInlineImmediate(const SDNode *N, bool Negated = false) const; 147 bool isNegInlineImmediate(const SDNode *N) const { 148 return isInlineImmediate(N, true); 149 } 150 151 bool isInlineImmediate16(int64_t Imm) const { 152 return AMDGPU::isInlinableLiteral16(Imm, Subtarget->hasInv2PiInlineImm()); 153 } 154 155 bool isInlineImmediate32(int64_t Imm) const { 156 return AMDGPU::isInlinableLiteral32(Imm, Subtarget->hasInv2PiInlineImm()); 157 } 158 159 bool isInlineImmediate64(int64_t Imm) const { 160 return AMDGPU::isInlinableLiteral64(Imm, Subtarget->hasInv2PiInlineImm()); 161 } 162 163 bool isInlineImmediate(const APFloat &Imm) const { 164 return Subtarget->getInstrInfo()->isInlineConstant(Imm); 165 } 166 167 bool isVGPRImm(const SDNode *N) const; 168 bool isUniformLoad(const SDNode *N) const; 169 bool isUniformBr(const SDNode *N) const; 170 171 bool isBaseWithConstantOffset64(SDValue Addr, SDValue &LHS, 172 SDValue &RHS) const; 173 174 MachineSDNode *buildSMovImm64(SDLoc &DL, uint64_t Val, EVT VT) const; 175 176 SDNode *glueCopyToOp(SDNode *N, SDValue NewChain, SDValue Glue) const; 177 SDNode *glueCopyToM0(SDNode *N, SDValue Val) const; 178 SDNode *glueCopyToM0LDSInit(SDNode *N) const; 179 180 const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const; 181 virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset); 182 virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset); 183 bool isDSOffsetLegal(SDValue Base, unsigned Offset) const; 184 bool isDSOffset2Legal(SDValue Base, unsigned Offset0, unsigned Offset1, 185 unsigned Size) const; 186 bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const; 187 bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0, 188 SDValue &Offset1) const; 189 bool SelectDS128Bit8ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0, 190 SDValue &Offset1) const; 191 bool SelectDSReadWrite2(SDValue Ptr, SDValue &Base, SDValue &Offset0, 192 SDValue &Offset1, unsigned Size) const; 193 bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr, 194 SDValue &SOffset, SDValue &Offset, SDValue &Offen, 195 SDValue &Idxen, SDValue &Addr64) const; 196 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr, 197 SDValue &SOffset, SDValue &Offset) const; 198 bool SelectMUBUFScratchOffen(SDNode *Parent, 199 SDValue Addr, SDValue &RSrc, SDValue &VAddr, 200 SDValue &SOffset, SDValue &ImmOffset) const; 201 bool SelectMUBUFScratchOffset(SDNode *Parent, 202 SDValue Addr, SDValue &SRsrc, SDValue &Soffset, 203 SDValue &Offset) const; 204 205 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset, 206 SDValue &Offset) const; 207 208 bool SelectFlatOffsetImpl(SDNode *N, SDValue Addr, SDValue &VAddr, 209 SDValue &Offset, uint64_t FlatVariant) const; 210 bool SelectFlatOffset(SDNode *N, SDValue Addr, SDValue &VAddr, 211 SDValue &Offset) const; 212 bool SelectGlobalOffset(SDNode *N, SDValue Addr, SDValue &VAddr, 213 SDValue &Offset) const; 214 bool SelectScratchOffset(SDNode *N, SDValue Addr, SDValue &VAddr, 215 SDValue &Offset) const; 216 bool SelectGlobalSAddr(SDNode *N, SDValue Addr, SDValue &SAddr, 217 SDValue &VOffset, SDValue &Offset) const; 218 bool SelectScratchSAddr(SDNode *N, SDValue Addr, SDValue &SAddr, 219 SDValue &Offset) const; 220 221 bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue &Offset, 222 bool &Imm) const; 223 SDValue Expand32BitAddress(SDValue Addr) const; 224 bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue &Offset, 225 bool &Imm) const; 226 bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const; 227 bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const; 228 bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &Offset) const; 229 bool SelectSMRDBufferImm(SDValue Addr, SDValue &Offset) const; 230 bool SelectSMRDBufferImm32(SDValue Addr, SDValue &Offset) const; 231 bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const; 232 233 bool SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, SDValue &SrcMods) const; 234 bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods, 235 bool AllowAbs = true) const; 236 bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const; 237 bool SelectVOP3BMods(SDValue In, SDValue &Src, SDValue &SrcMods) const; 238 bool SelectVOP3NoMods(SDValue In, SDValue &Src) const; 239 bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods, 240 SDValue &Clamp, SDValue &Omod) const; 241 bool SelectVOP3BMods0(SDValue In, SDValue &Src, SDValue &SrcMods, 242 SDValue &Clamp, SDValue &Omod) const; 243 bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods, 244 SDValue &Clamp, SDValue &Omod) const; 245 246 bool SelectVOP3OMods(SDValue In, SDValue &Src, 247 SDValue &Clamp, SDValue &Omod) const; 248 249 bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods) const; 250 251 bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const; 252 253 bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const; 254 bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, unsigned &Mods) const; 255 bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const; 256 257 SDValue getHi16Elt(SDValue In) const; 258 259 SDValue getMaterializedScalarImm32(int64_t Val, const SDLoc &DL) const; 260 261 void SelectADD_SUB_I64(SDNode *N); 262 void SelectAddcSubb(SDNode *N); 263 void SelectUADDO_USUBO(SDNode *N); 264 void SelectDIV_SCALE(SDNode *N); 265 void SelectMAD_64_32(SDNode *N); 266 void SelectFMA_W_CHAIN(SDNode *N); 267 void SelectFMUL_W_CHAIN(SDNode *N); 268 269 SDNode *getS_BFE(unsigned Opcode, const SDLoc &DL, SDValue Val, 270 uint32_t Offset, uint32_t Width); 271 void SelectS_BFEFromShifts(SDNode *N); 272 void SelectS_BFE(SDNode *N); 273 bool isCBranchSCC(const SDNode *N) const; 274 void SelectBRCOND(SDNode *N); 275 void SelectFMAD_FMA(SDNode *N); 276 void SelectATOMIC_CMP_SWAP(SDNode *N); 277 void SelectDSAppendConsume(SDNode *N, unsigned IntrID); 278 void SelectDS_GWS(SDNode *N, unsigned IntrID); 279 void SelectInterpP1F16(SDNode *N); 280 void SelectINTRINSIC_W_CHAIN(SDNode *N); 281 void SelectINTRINSIC_WO_CHAIN(SDNode *N); 282 void SelectINTRINSIC_VOID(SDNode *N); 283 284 protected: 285 // Include the pieces autogenerated from the target description. 286 #include "AMDGPUGenDAGISel.inc" 287 }; 288 289 class R600DAGToDAGISel : public AMDGPUDAGToDAGISel { 290 const R600Subtarget *Subtarget; 291 292 bool isConstantLoad(const MemSDNode *N, int cbID) const; 293 bool SelectGlobalValueConstantOffset(SDValue Addr, SDValue& IntPtr); 294 bool SelectGlobalValueVariableOffset(SDValue Addr, SDValue &BaseReg, 295 SDValue& Offset); 296 public: 297 explicit R600DAGToDAGISel(TargetMachine *TM, CodeGenOpt::Level OptLevel) : 298 AMDGPUDAGToDAGISel(TM, OptLevel) {} 299 300 void Select(SDNode *N) override; 301 302 bool SelectADDRIndirect(SDValue Addr, SDValue &Base, 303 SDValue &Offset) override; 304 bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, 305 SDValue &Offset) override; 306 307 bool runOnMachineFunction(MachineFunction &MF) override; 308 309 void PreprocessISelDAG() override {} 310 311 protected: 312 // Include the pieces autogenerated from the target description. 313 #include "R600GenDAGISel.inc" 314 }; 315 316 static SDValue stripBitcast(SDValue Val) { 317 return Val.getOpcode() == ISD::BITCAST ? Val.getOperand(0) : Val; 318 } 319 320 // Figure out if this is really an extract of the high 16-bits of a dword. 321 static bool isExtractHiElt(SDValue In, SDValue &Out) { 322 In = stripBitcast(In); 323 324 if (In.getOpcode() == ISD::EXTRACT_VECTOR_ELT) { 325 if (ConstantSDNode *Idx = dyn_cast<ConstantSDNode>(In.getOperand(1))) { 326 if (!Idx->isOne()) 327 return false; 328 Out = In.getOperand(0); 329 return true; 330 } 331 } 332 333 if (In.getOpcode() != ISD::TRUNCATE) 334 return false; 335 336 SDValue Srl = In.getOperand(0); 337 if (Srl.getOpcode() == ISD::SRL) { 338 if (ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(Srl.getOperand(1))) { 339 if (ShiftAmt->getZExtValue() == 16) { 340 Out = stripBitcast(Srl.getOperand(0)); 341 return true; 342 } 343 } 344 } 345 346 return false; 347 } 348 349 // Look through operations that obscure just looking at the low 16-bits of the 350 // same register. 351 static SDValue stripExtractLoElt(SDValue In) { 352 if (In.getOpcode() == ISD::EXTRACT_VECTOR_ELT) { 353 if (ConstantSDNode *Idx = dyn_cast<ConstantSDNode>(In.getOperand(1))) { 354 if (Idx->isNullValue() && In.getValueSizeInBits() <= 32) 355 return In.getOperand(0); 356 } 357 } 358 359 if (In.getOpcode() == ISD::TRUNCATE) { 360 SDValue Src = In.getOperand(0); 361 if (Src.getValueType().getSizeInBits() == 32) 362 return stripBitcast(Src); 363 } 364 365 return In; 366 } 367 368 } // end anonymous namespace 369 370 INITIALIZE_PASS_BEGIN(AMDGPUDAGToDAGISel, "amdgpu-isel", 371 "AMDGPU DAG->DAG Pattern Instruction Selection", false, false) 372 INITIALIZE_PASS_DEPENDENCY(AMDGPUArgumentUsageInfo) 373 INITIALIZE_PASS_DEPENDENCY(AMDGPUPerfHintAnalysis) 374 INITIALIZE_PASS_DEPENDENCY(LegacyDivergenceAnalysis) 375 #ifdef EXPENSIVE_CHECKS 376 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) 377 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) 378 #endif 379 INITIALIZE_PASS_END(AMDGPUDAGToDAGISel, "amdgpu-isel", 380 "AMDGPU DAG->DAG Pattern Instruction Selection", false, false) 381 382 /// This pass converts a legalized DAG into a AMDGPU-specific 383 // DAG, ready for instruction scheduling. 384 FunctionPass *llvm::createAMDGPUISelDag(TargetMachine *TM, 385 CodeGenOpt::Level OptLevel) { 386 return new AMDGPUDAGToDAGISel(TM, OptLevel); 387 } 388 389 /// This pass converts a legalized DAG into a R600-specific 390 // DAG, ready for instruction scheduling. 391 FunctionPass *llvm::createR600ISelDag(TargetMachine *TM, 392 CodeGenOpt::Level OptLevel) { 393 return new R600DAGToDAGISel(TM, OptLevel); 394 } 395 396 bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) { 397 #ifdef EXPENSIVE_CHECKS 398 DominatorTree & DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree(); 399 LoopInfo * LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(); 400 for (auto &L : LI->getLoopsInPreorder()) { 401 assert(L->isLCSSAForm(DT)); 402 } 403 #endif 404 Subtarget = &MF.getSubtarget<GCNSubtarget>(); 405 Mode = AMDGPU::SIModeRegisterDefaults(MF.getFunction()); 406 return SelectionDAGISel::runOnMachineFunction(MF); 407 } 408 409 bool AMDGPUDAGToDAGISel::fp16SrcZerosHighBits(unsigned Opc) const { 410 // XXX - only need to list legal operations. 411 switch (Opc) { 412 case ISD::FADD: 413 case ISD::FSUB: 414 case ISD::FMUL: 415 case ISD::FDIV: 416 case ISD::FREM: 417 case ISD::FCANONICALIZE: 418 case ISD::UINT_TO_FP: 419 case ISD::SINT_TO_FP: 420 case ISD::FABS: 421 // Fabs is lowered to a bit operation, but it's an and which will clear the 422 // high bits anyway. 423 case ISD::FSQRT: 424 case ISD::FSIN: 425 case ISD::FCOS: 426 case ISD::FPOWI: 427 case ISD::FPOW: 428 case ISD::FLOG: 429 case ISD::FLOG2: 430 case ISD::FLOG10: 431 case ISD::FEXP: 432 case ISD::FEXP2: 433 case ISD::FCEIL: 434 case ISD::FTRUNC: 435 case ISD::FRINT: 436 case ISD::FNEARBYINT: 437 case ISD::FROUND: 438 case ISD::FFLOOR: 439 case ISD::FMINNUM: 440 case ISD::FMAXNUM: 441 case AMDGPUISD::FRACT: 442 case AMDGPUISD::CLAMP: 443 case AMDGPUISD::COS_HW: 444 case AMDGPUISD::SIN_HW: 445 case AMDGPUISD::FMIN3: 446 case AMDGPUISD::FMAX3: 447 case AMDGPUISD::FMED3: 448 case AMDGPUISD::FMAD_FTZ: 449 case AMDGPUISD::RCP: 450 case AMDGPUISD::RSQ: 451 case AMDGPUISD::RCP_IFLAG: 452 case AMDGPUISD::LDEXP: 453 // On gfx10, all 16-bit instructions preserve the high bits. 454 return Subtarget->getGeneration() <= AMDGPUSubtarget::GFX9; 455 case ISD::FP_ROUND: 456 // We may select fptrunc (fma/mad) to mad_mixlo, which does not zero the 457 // high bits on gfx9. 458 // TODO: If we had the source node we could see if the source was fma/mad 459 return Subtarget->getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS; 460 case ISD::FMA: 461 case ISD::FMAD: 462 case AMDGPUISD::DIV_FIXUP: 463 return Subtarget->getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS; 464 default: 465 // fcopysign, select and others may be lowered to 32-bit bit operations 466 // which don't zero the high bits. 467 return false; 468 } 469 } 470 471 bool AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(SDNode *N) const { 472 assert(Subtarget->d16PreservesUnusedBits()); 473 MVT VT = N->getValueType(0).getSimpleVT(); 474 if (VT != MVT::v2i16 && VT != MVT::v2f16) 475 return false; 476 477 SDValue Lo = N->getOperand(0); 478 SDValue Hi = N->getOperand(1); 479 480 LoadSDNode *LdHi = dyn_cast<LoadSDNode>(stripBitcast(Hi)); 481 482 // build_vector lo, (load ptr) -> load_d16_hi ptr, lo 483 // build_vector lo, (zextload ptr from i8) -> load_d16_hi_u8 ptr, lo 484 // build_vector lo, (sextload ptr from i8) -> load_d16_hi_i8 ptr, lo 485 486 // Need to check for possible indirect dependencies on the other half of the 487 // vector to avoid introducing a cycle. 488 if (LdHi && Hi.hasOneUse() && !LdHi->isPredecessorOf(Lo.getNode())) { 489 SDVTList VTList = CurDAG->getVTList(VT, MVT::Other); 490 491 SDValue TiedIn = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Lo); 492 SDValue Ops[] = { 493 LdHi->getChain(), LdHi->getBasePtr(), TiedIn 494 }; 495 496 unsigned LoadOp = AMDGPUISD::LOAD_D16_HI; 497 if (LdHi->getMemoryVT() == MVT::i8) { 498 LoadOp = LdHi->getExtensionType() == ISD::SEXTLOAD ? 499 AMDGPUISD::LOAD_D16_HI_I8 : AMDGPUISD::LOAD_D16_HI_U8; 500 } else { 501 assert(LdHi->getMemoryVT() == MVT::i16); 502 } 503 504 SDValue NewLoadHi = 505 CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdHi), VTList, 506 Ops, LdHi->getMemoryVT(), 507 LdHi->getMemOperand()); 508 509 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadHi); 510 CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdHi, 1), NewLoadHi.getValue(1)); 511 return true; 512 } 513 514 // build_vector (load ptr), hi -> load_d16_lo ptr, hi 515 // build_vector (zextload ptr from i8), hi -> load_d16_lo_u8 ptr, hi 516 // build_vector (sextload ptr from i8), hi -> load_d16_lo_i8 ptr, hi 517 LoadSDNode *LdLo = dyn_cast<LoadSDNode>(stripBitcast(Lo)); 518 if (LdLo && Lo.hasOneUse()) { 519 SDValue TiedIn = getHi16Elt(Hi); 520 if (!TiedIn || LdLo->isPredecessorOf(TiedIn.getNode())) 521 return false; 522 523 SDVTList VTList = CurDAG->getVTList(VT, MVT::Other); 524 unsigned LoadOp = AMDGPUISD::LOAD_D16_LO; 525 if (LdLo->getMemoryVT() == MVT::i8) { 526 LoadOp = LdLo->getExtensionType() == ISD::SEXTLOAD ? 527 AMDGPUISD::LOAD_D16_LO_I8 : AMDGPUISD::LOAD_D16_LO_U8; 528 } else { 529 assert(LdLo->getMemoryVT() == MVT::i16); 530 } 531 532 TiedIn = CurDAG->getNode(ISD::BITCAST, SDLoc(N), VT, TiedIn); 533 534 SDValue Ops[] = { 535 LdLo->getChain(), LdLo->getBasePtr(), TiedIn 536 }; 537 538 SDValue NewLoadLo = 539 CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdLo), VTList, 540 Ops, LdLo->getMemoryVT(), 541 LdLo->getMemOperand()); 542 543 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadLo); 544 CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdLo, 1), NewLoadLo.getValue(1)); 545 return true; 546 } 547 548 return false; 549 } 550 551 void AMDGPUDAGToDAGISel::PreprocessISelDAG() { 552 if (!Subtarget->d16PreservesUnusedBits()) 553 return; 554 555 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); 556 557 bool MadeChange = false; 558 while (Position != CurDAG->allnodes_begin()) { 559 SDNode *N = &*--Position; 560 if (N->use_empty()) 561 continue; 562 563 switch (N->getOpcode()) { 564 case ISD::BUILD_VECTOR: 565 MadeChange |= matchLoadD16FromBuildVector(N); 566 break; 567 default: 568 break; 569 } 570 } 571 572 if (MadeChange) { 573 CurDAG->RemoveDeadNodes(); 574 LLVM_DEBUG(dbgs() << "After PreProcess:\n"; 575 CurDAG->dump();); 576 } 577 } 578 579 bool AMDGPUDAGToDAGISel::isNoNanSrc(SDValue N) const { 580 if (TM.Options.NoNaNsFPMath) 581 return true; 582 583 // TODO: Move into isKnownNeverNaN 584 if (N->getFlags().hasNoNaNs()) 585 return true; 586 587 return CurDAG->isKnownNeverNaN(N); 588 } 589 590 bool AMDGPUDAGToDAGISel::isInlineImmediate(const SDNode *N, 591 bool Negated) const { 592 if (N->isUndef()) 593 return true; 594 595 const SIInstrInfo *TII = Subtarget->getInstrInfo(); 596 if (Negated) { 597 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) 598 return TII->isInlineConstant(-C->getAPIntValue()); 599 600 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) 601 return TII->isInlineConstant(-C->getValueAPF().bitcastToAPInt()); 602 603 } else { 604 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) 605 return TII->isInlineConstant(C->getAPIntValue()); 606 607 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) 608 return TII->isInlineConstant(C->getValueAPF().bitcastToAPInt()); 609 } 610 611 return false; 612 } 613 614 /// Determine the register class for \p OpNo 615 /// \returns The register class of the virtual register that will be used for 616 /// the given operand number \OpNo or NULL if the register class cannot be 617 /// determined. 618 const TargetRegisterClass *AMDGPUDAGToDAGISel::getOperandRegClass(SDNode *N, 619 unsigned OpNo) const { 620 if (!N->isMachineOpcode()) { 621 if (N->getOpcode() == ISD::CopyToReg) { 622 Register Reg = cast<RegisterSDNode>(N->getOperand(1))->getReg(); 623 if (Reg.isVirtual()) { 624 MachineRegisterInfo &MRI = CurDAG->getMachineFunction().getRegInfo(); 625 return MRI.getRegClass(Reg); 626 } 627 628 const SIRegisterInfo *TRI 629 = static_cast<const GCNSubtarget *>(Subtarget)->getRegisterInfo(); 630 return TRI->getPhysRegClass(Reg); 631 } 632 633 return nullptr; 634 } 635 636 switch (N->getMachineOpcode()) { 637 default: { 638 const MCInstrDesc &Desc = 639 Subtarget->getInstrInfo()->get(N->getMachineOpcode()); 640 unsigned OpIdx = Desc.getNumDefs() + OpNo; 641 if (OpIdx >= Desc.getNumOperands()) 642 return nullptr; 643 int RegClass = Desc.OpInfo[OpIdx].RegClass; 644 if (RegClass == -1) 645 return nullptr; 646 647 return Subtarget->getRegisterInfo()->getRegClass(RegClass); 648 } 649 case AMDGPU::REG_SEQUENCE: { 650 unsigned RCID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 651 const TargetRegisterClass *SuperRC = 652 Subtarget->getRegisterInfo()->getRegClass(RCID); 653 654 SDValue SubRegOp = N->getOperand(OpNo + 1); 655 unsigned SubRegIdx = cast<ConstantSDNode>(SubRegOp)->getZExtValue(); 656 return Subtarget->getRegisterInfo()->getSubClassWithSubReg(SuperRC, 657 SubRegIdx); 658 } 659 } 660 } 661 662 SDNode *AMDGPUDAGToDAGISel::glueCopyToOp(SDNode *N, SDValue NewChain, 663 SDValue Glue) const { 664 SmallVector <SDValue, 8> Ops; 665 Ops.push_back(NewChain); // Replace the chain. 666 for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i) 667 Ops.push_back(N->getOperand(i)); 668 669 Ops.push_back(Glue); 670 return CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), Ops); 671 } 672 673 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N, SDValue Val) const { 674 const SITargetLowering& Lowering = 675 *static_cast<const SITargetLowering*>(getTargetLowering()); 676 677 assert(N->getOperand(0).getValueType() == MVT::Other && "Expected chain"); 678 679 SDValue M0 = Lowering.copyToM0(*CurDAG, N->getOperand(0), SDLoc(N), Val); 680 return glueCopyToOp(N, M0, M0.getValue(1)); 681 } 682 683 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0LDSInit(SDNode *N) const { 684 unsigned AS = cast<MemSDNode>(N)->getAddressSpace(); 685 if (AS == AMDGPUAS::LOCAL_ADDRESS) { 686 if (Subtarget->ldsRequiresM0Init()) 687 return glueCopyToM0(N, CurDAG->getTargetConstant(-1, SDLoc(N), MVT::i32)); 688 } else if (AS == AMDGPUAS::REGION_ADDRESS) { 689 MachineFunction &MF = CurDAG->getMachineFunction(); 690 unsigned Value = MF.getInfo<SIMachineFunctionInfo>()->getGDSSize(); 691 return 692 glueCopyToM0(N, CurDAG->getTargetConstant(Value, SDLoc(N), MVT::i32)); 693 } 694 return N; 695 } 696 697 MachineSDNode *AMDGPUDAGToDAGISel::buildSMovImm64(SDLoc &DL, uint64_t Imm, 698 EVT VT) const { 699 SDNode *Lo = CurDAG->getMachineNode( 700 AMDGPU::S_MOV_B32, DL, MVT::i32, 701 CurDAG->getTargetConstant(Imm & 0xFFFFFFFF, DL, MVT::i32)); 702 SDNode *Hi = 703 CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, 704 CurDAG->getTargetConstant(Imm >> 32, DL, MVT::i32)); 705 const SDValue Ops[] = { 706 CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32), 707 SDValue(Lo, 0), CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32), 708 SDValue(Hi, 0), CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)}; 709 710 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, VT, Ops); 711 } 712 713 void AMDGPUDAGToDAGISel::SelectBuildVector(SDNode *N, unsigned RegClassID) { 714 EVT VT = N->getValueType(0); 715 unsigned NumVectorElts = VT.getVectorNumElements(); 716 EVT EltVT = VT.getVectorElementType(); 717 SDLoc DL(N); 718 SDValue RegClass = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32); 719 720 if (NumVectorElts == 1) { 721 CurDAG->SelectNodeTo(N, AMDGPU::COPY_TO_REGCLASS, EltVT, N->getOperand(0), 722 RegClass); 723 return; 724 } 725 726 assert(NumVectorElts <= 32 && "Vectors with more than 32 elements not " 727 "supported yet"); 728 // 32 = Max Num Vector Elements 729 // 2 = 2 REG_SEQUENCE operands per element (value, subreg index) 730 // 1 = Vector Register Class 731 SmallVector<SDValue, 32 * 2 + 1> RegSeqArgs(NumVectorElts * 2 + 1); 732 733 bool IsGCN = CurDAG->getSubtarget().getTargetTriple().getArch() == 734 Triple::amdgcn; 735 RegSeqArgs[0] = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32); 736 bool IsRegSeq = true; 737 unsigned NOps = N->getNumOperands(); 738 for (unsigned i = 0; i < NOps; i++) { 739 // XXX: Why is this here? 740 if (isa<RegisterSDNode>(N->getOperand(i))) { 741 IsRegSeq = false; 742 break; 743 } 744 unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(i) 745 : R600RegisterInfo::getSubRegFromChannel(i); 746 RegSeqArgs[1 + (2 * i)] = N->getOperand(i); 747 RegSeqArgs[1 + (2 * i) + 1] = CurDAG->getTargetConstant(Sub, DL, MVT::i32); 748 } 749 if (NOps != NumVectorElts) { 750 // Fill in the missing undef elements if this was a scalar_to_vector. 751 assert(N->getOpcode() == ISD::SCALAR_TO_VECTOR && NOps < NumVectorElts); 752 MachineSDNode *ImpDef = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, 753 DL, EltVT); 754 for (unsigned i = NOps; i < NumVectorElts; ++i) { 755 unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(i) 756 : R600RegisterInfo::getSubRegFromChannel(i); 757 RegSeqArgs[1 + (2 * i)] = SDValue(ImpDef, 0); 758 RegSeqArgs[1 + (2 * i) + 1] = 759 CurDAG->getTargetConstant(Sub, DL, MVT::i32); 760 } 761 } 762 763 if (!IsRegSeq) 764 SelectCode(N); 765 CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->getVTList(), RegSeqArgs); 766 } 767 768 void AMDGPUDAGToDAGISel::Select(SDNode *N) { 769 unsigned int Opc = N->getOpcode(); 770 if (N->isMachineOpcode()) { 771 N->setNodeId(-1); 772 return; // Already selected. 773 } 774 775 // isa<MemSDNode> almost works but is slightly too permissive for some DS 776 // intrinsics. 777 if (Opc == ISD::LOAD || Opc == ISD::STORE || isa<AtomicSDNode>(N) || 778 (Opc == AMDGPUISD::ATOMIC_INC || Opc == AMDGPUISD::ATOMIC_DEC || 779 Opc == ISD::ATOMIC_LOAD_FADD || 780 Opc == AMDGPUISD::ATOMIC_LOAD_FMIN || 781 Opc == AMDGPUISD::ATOMIC_LOAD_FMAX)) { 782 N = glueCopyToM0LDSInit(N); 783 SelectCode(N); 784 return; 785 } 786 787 switch (Opc) { 788 default: 789 break; 790 // We are selecting i64 ADD here instead of custom lower it during 791 // DAG legalization, so we can fold some i64 ADDs used for address 792 // calculation into the LOAD and STORE instructions. 793 case ISD::ADDC: 794 case ISD::ADDE: 795 case ISD::SUBC: 796 case ISD::SUBE: { 797 if (N->getValueType(0) != MVT::i64) 798 break; 799 800 SelectADD_SUB_I64(N); 801 return; 802 } 803 case ISD::ADDCARRY: 804 case ISD::SUBCARRY: 805 if (N->getValueType(0) != MVT::i32) 806 break; 807 808 SelectAddcSubb(N); 809 return; 810 case ISD::UADDO: 811 case ISD::USUBO: { 812 SelectUADDO_USUBO(N); 813 return; 814 } 815 case AMDGPUISD::FMUL_W_CHAIN: { 816 SelectFMUL_W_CHAIN(N); 817 return; 818 } 819 case AMDGPUISD::FMA_W_CHAIN: { 820 SelectFMA_W_CHAIN(N); 821 return; 822 } 823 824 case ISD::SCALAR_TO_VECTOR: 825 case ISD::BUILD_VECTOR: { 826 EVT VT = N->getValueType(0); 827 unsigned NumVectorElts = VT.getVectorNumElements(); 828 if (VT.getScalarSizeInBits() == 16) { 829 if (Opc == ISD::BUILD_VECTOR && NumVectorElts == 2) { 830 if (SDNode *Packed = packConstantV2I16(N, *CurDAG)) { 831 ReplaceNode(N, Packed); 832 return; 833 } 834 } 835 836 break; 837 } 838 839 assert(VT.getVectorElementType().bitsEq(MVT::i32)); 840 unsigned RegClassID = 841 SIRegisterInfo::getSGPRClassForBitWidth(NumVectorElts * 32)->getID(); 842 SelectBuildVector(N, RegClassID); 843 return; 844 } 845 case ISD::BUILD_PAIR: { 846 SDValue RC, SubReg0, SubReg1; 847 SDLoc DL(N); 848 if (N->getValueType(0) == MVT::i128) { 849 RC = CurDAG->getTargetConstant(AMDGPU::SGPR_128RegClassID, DL, MVT::i32); 850 SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32); 851 SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32); 852 } else if (N->getValueType(0) == MVT::i64) { 853 RC = CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32); 854 SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32); 855 SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32); 856 } else { 857 llvm_unreachable("Unhandled value type for BUILD_PAIR"); 858 } 859 const SDValue Ops[] = { RC, N->getOperand(0), SubReg0, 860 N->getOperand(1), SubReg1 }; 861 ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, 862 N->getValueType(0), Ops)); 863 return; 864 } 865 866 case ISD::Constant: 867 case ISD::ConstantFP: { 868 if (N->getValueType(0).getSizeInBits() != 64 || isInlineImmediate(N)) 869 break; 870 871 uint64_t Imm; 872 if (ConstantFPSDNode *FP = dyn_cast<ConstantFPSDNode>(N)) 873 Imm = FP->getValueAPF().bitcastToAPInt().getZExtValue(); 874 else { 875 ConstantSDNode *C = cast<ConstantSDNode>(N); 876 Imm = C->getZExtValue(); 877 } 878 879 SDLoc DL(N); 880 ReplaceNode(N, buildSMovImm64(DL, Imm, N->getValueType(0))); 881 return; 882 } 883 case AMDGPUISD::BFE_I32: 884 case AMDGPUISD::BFE_U32: { 885 // There is a scalar version available, but unlike the vector version which 886 // has a separate operand for the offset and width, the scalar version packs 887 // the width and offset into a single operand. Try to move to the scalar 888 // version if the offsets are constant, so that we can try to keep extended 889 // loads of kernel arguments in SGPRs. 890 891 // TODO: Technically we could try to pattern match scalar bitshifts of 892 // dynamic values, but it's probably not useful. 893 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1)); 894 if (!Offset) 895 break; 896 897 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2)); 898 if (!Width) 899 break; 900 901 bool Signed = Opc == AMDGPUISD::BFE_I32; 902 903 uint32_t OffsetVal = Offset->getZExtValue(); 904 uint32_t WidthVal = Width->getZExtValue(); 905 906 ReplaceNode(N, getS_BFE(Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32, 907 SDLoc(N), N->getOperand(0), OffsetVal, WidthVal)); 908 return; 909 } 910 case AMDGPUISD::DIV_SCALE: { 911 SelectDIV_SCALE(N); 912 return; 913 } 914 case AMDGPUISD::MAD_I64_I32: 915 case AMDGPUISD::MAD_U64_U32: { 916 SelectMAD_64_32(N); 917 return; 918 } 919 case ISD::CopyToReg: { 920 const SITargetLowering& Lowering = 921 *static_cast<const SITargetLowering*>(getTargetLowering()); 922 N = Lowering.legalizeTargetIndependentNode(N, *CurDAG); 923 break; 924 } 925 case ISD::AND: 926 case ISD::SRL: 927 case ISD::SRA: 928 case ISD::SIGN_EXTEND_INREG: 929 if (N->getValueType(0) != MVT::i32) 930 break; 931 932 SelectS_BFE(N); 933 return; 934 case ISD::BRCOND: 935 SelectBRCOND(N); 936 return; 937 case ISD::FMAD: 938 case ISD::FMA: 939 SelectFMAD_FMA(N); 940 return; 941 case AMDGPUISD::ATOMIC_CMP_SWAP: 942 SelectATOMIC_CMP_SWAP(N); 943 return; 944 case AMDGPUISD::CVT_PKRTZ_F16_F32: 945 case AMDGPUISD::CVT_PKNORM_I16_F32: 946 case AMDGPUISD::CVT_PKNORM_U16_F32: 947 case AMDGPUISD::CVT_PK_U16_U32: 948 case AMDGPUISD::CVT_PK_I16_I32: { 949 // Hack around using a legal type if f16 is illegal. 950 if (N->getValueType(0) == MVT::i32) { 951 MVT NewVT = Opc == AMDGPUISD::CVT_PKRTZ_F16_F32 ? MVT::v2f16 : MVT::v2i16; 952 N = CurDAG->MorphNodeTo(N, N->getOpcode(), CurDAG->getVTList(NewVT), 953 { N->getOperand(0), N->getOperand(1) }); 954 SelectCode(N); 955 return; 956 } 957 958 break; 959 } 960 case ISD::INTRINSIC_W_CHAIN: { 961 SelectINTRINSIC_W_CHAIN(N); 962 return; 963 } 964 case ISD::INTRINSIC_WO_CHAIN: { 965 SelectINTRINSIC_WO_CHAIN(N); 966 return; 967 } 968 case ISD::INTRINSIC_VOID: { 969 SelectINTRINSIC_VOID(N); 970 return; 971 } 972 } 973 974 SelectCode(N); 975 } 976 977 bool AMDGPUDAGToDAGISel::isUniformBr(const SDNode *N) const { 978 const BasicBlock *BB = FuncInfo->MBB->getBasicBlock(); 979 const Instruction *Term = BB->getTerminator(); 980 return Term->getMetadata("amdgpu.uniform") || 981 Term->getMetadata("structurizecfg.uniform"); 982 } 983 984 static bool getBaseWithOffsetUsingSplitOR(SelectionDAG &DAG, SDValue Addr, 985 SDValue &N0, SDValue &N1) { 986 if (Addr.getValueType() == MVT::i64 && Addr.getOpcode() == ISD::BITCAST && 987 Addr.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) { 988 // As we split 64-bit `or` earlier, it's complicated pattern to match, i.e. 989 // (i64 (bitcast (v2i32 (build_vector 990 // (or (extract_vector_elt V, 0), OFFSET), 991 // (extract_vector_elt V, 1))))) 992 SDValue Lo = Addr.getOperand(0).getOperand(0); 993 if (Lo.getOpcode() == ISD::OR && DAG.isBaseWithConstantOffset(Lo)) { 994 SDValue BaseLo = Lo.getOperand(0); 995 SDValue BaseHi = Addr.getOperand(0).getOperand(1); 996 // Check that split base (Lo and Hi) are extracted from the same one. 997 if (BaseLo.getOpcode() == ISD::EXTRACT_VECTOR_ELT && 998 BaseHi.getOpcode() == ISD::EXTRACT_VECTOR_ELT && 999 BaseLo.getOperand(0) == BaseHi.getOperand(0) && 1000 // Lo is statically extracted from index 0. 1001 isa<ConstantSDNode>(BaseLo.getOperand(1)) && 1002 BaseLo.getConstantOperandVal(1) == 0 && 1003 // Hi is statically extracted from index 0. 1004 isa<ConstantSDNode>(BaseHi.getOperand(1)) && 1005 BaseHi.getConstantOperandVal(1) == 1) { 1006 N0 = BaseLo.getOperand(0).getOperand(0); 1007 N1 = Lo.getOperand(1); 1008 return true; 1009 } 1010 } 1011 } 1012 return false; 1013 } 1014 1015 bool AMDGPUDAGToDAGISel::isBaseWithConstantOffset64(SDValue Addr, SDValue &LHS, 1016 SDValue &RHS) const { 1017 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1018 LHS = Addr.getOperand(0); 1019 RHS = Addr.getOperand(1); 1020 return true; 1021 } 1022 1023 if (getBaseWithOffsetUsingSplitOR(*CurDAG, Addr, LHS, RHS)) { 1024 assert(LHS && RHS && isa<ConstantSDNode>(RHS)); 1025 return true; 1026 } 1027 1028 return false; 1029 } 1030 1031 StringRef AMDGPUDAGToDAGISel::getPassName() const { 1032 return "AMDGPU DAG->DAG Pattern Instruction Selection"; 1033 } 1034 1035 //===----------------------------------------------------------------------===// 1036 // Complex Patterns 1037 //===----------------------------------------------------------------------===// 1038 1039 bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base, 1040 SDValue &Offset) { 1041 return false; 1042 } 1043 1044 bool AMDGPUDAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base, 1045 SDValue &Offset) { 1046 ConstantSDNode *C; 1047 SDLoc DL(Addr); 1048 1049 if ((C = dyn_cast<ConstantSDNode>(Addr))) { 1050 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32); 1051 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 1052 } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) && 1053 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) { 1054 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32); 1055 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 1056 } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) && 1057 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) { 1058 Base = Addr.getOperand(0); 1059 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 1060 } else { 1061 Base = Addr; 1062 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32); 1063 } 1064 1065 return true; 1066 } 1067 1068 SDValue AMDGPUDAGToDAGISel::getMaterializedScalarImm32(int64_t Val, 1069 const SDLoc &DL) const { 1070 SDNode *Mov = CurDAG->getMachineNode( 1071 AMDGPU::S_MOV_B32, DL, MVT::i32, 1072 CurDAG->getTargetConstant(Val, DL, MVT::i32)); 1073 return SDValue(Mov, 0); 1074 } 1075 1076 // FIXME: Should only handle addcarry/subcarry 1077 void AMDGPUDAGToDAGISel::SelectADD_SUB_I64(SDNode *N) { 1078 SDLoc DL(N); 1079 SDValue LHS = N->getOperand(0); 1080 SDValue RHS = N->getOperand(1); 1081 1082 unsigned Opcode = N->getOpcode(); 1083 bool ConsumeCarry = (Opcode == ISD::ADDE || Opcode == ISD::SUBE); 1084 bool ProduceCarry = 1085 ConsumeCarry || Opcode == ISD::ADDC || Opcode == ISD::SUBC; 1086 bool IsAdd = Opcode == ISD::ADD || Opcode == ISD::ADDC || Opcode == ISD::ADDE; 1087 1088 SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32); 1089 SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32); 1090 1091 SDNode *Lo0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 1092 DL, MVT::i32, LHS, Sub0); 1093 SDNode *Hi0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 1094 DL, MVT::i32, LHS, Sub1); 1095 1096 SDNode *Lo1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 1097 DL, MVT::i32, RHS, Sub0); 1098 SDNode *Hi1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 1099 DL, MVT::i32, RHS, Sub1); 1100 1101 SDVTList VTList = CurDAG->getVTList(MVT::i32, MVT::Glue); 1102 1103 static const unsigned OpcMap[2][2][2] = { 1104 {{AMDGPU::S_SUB_U32, AMDGPU::S_ADD_U32}, 1105 {AMDGPU::V_SUB_CO_U32_e32, AMDGPU::V_ADD_CO_U32_e32}}, 1106 {{AMDGPU::S_SUBB_U32, AMDGPU::S_ADDC_U32}, 1107 {AMDGPU::V_SUBB_U32_e32, AMDGPU::V_ADDC_U32_e32}}}; 1108 1109 unsigned Opc = OpcMap[0][N->isDivergent()][IsAdd]; 1110 unsigned CarryOpc = OpcMap[1][N->isDivergent()][IsAdd]; 1111 1112 SDNode *AddLo; 1113 if (!ConsumeCarry) { 1114 SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0) }; 1115 AddLo = CurDAG->getMachineNode(Opc, DL, VTList, Args); 1116 } else { 1117 SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0), N->getOperand(2) }; 1118 AddLo = CurDAG->getMachineNode(CarryOpc, DL, VTList, Args); 1119 } 1120 SDValue AddHiArgs[] = { 1121 SDValue(Hi0, 0), 1122 SDValue(Hi1, 0), 1123 SDValue(AddLo, 1) 1124 }; 1125 SDNode *AddHi = CurDAG->getMachineNode(CarryOpc, DL, VTList, AddHiArgs); 1126 1127 SDValue RegSequenceArgs[] = { 1128 CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32), 1129 SDValue(AddLo,0), 1130 Sub0, 1131 SDValue(AddHi,0), 1132 Sub1, 1133 }; 1134 SDNode *RegSequence = CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL, 1135 MVT::i64, RegSequenceArgs); 1136 1137 if (ProduceCarry) { 1138 // Replace the carry-use 1139 ReplaceUses(SDValue(N, 1), SDValue(AddHi, 1)); 1140 } 1141 1142 // Replace the remaining uses. 1143 ReplaceNode(N, RegSequence); 1144 } 1145 1146 void AMDGPUDAGToDAGISel::SelectAddcSubb(SDNode *N) { 1147 SDLoc DL(N); 1148 SDValue LHS = N->getOperand(0); 1149 SDValue RHS = N->getOperand(1); 1150 SDValue CI = N->getOperand(2); 1151 1152 if (N->isDivergent()) { 1153 unsigned Opc = N->getOpcode() == ISD::ADDCARRY ? AMDGPU::V_ADDC_U32_e64 1154 : AMDGPU::V_SUBB_U32_e64; 1155 CurDAG->SelectNodeTo( 1156 N, Opc, N->getVTList(), 1157 {LHS, RHS, CI, 1158 CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/}); 1159 } else { 1160 unsigned Opc = N->getOpcode() == ISD::ADDCARRY ? AMDGPU::S_ADD_CO_PSEUDO 1161 : AMDGPU::S_SUB_CO_PSEUDO; 1162 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), {LHS, RHS, CI}); 1163 } 1164 } 1165 1166 void AMDGPUDAGToDAGISel::SelectUADDO_USUBO(SDNode *N) { 1167 // The name of the opcodes are misleading. v_add_i32/v_sub_i32 have unsigned 1168 // carry out despite the _i32 name. These were renamed in VI to _U32. 1169 // FIXME: We should probably rename the opcodes here. 1170 bool IsAdd = N->getOpcode() == ISD::UADDO; 1171 bool IsVALU = N->isDivergent(); 1172 1173 for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end(); UI != E; 1174 ++UI) 1175 if (UI.getUse().getResNo() == 1) { 1176 if ((IsAdd && (UI->getOpcode() != ISD::ADDCARRY)) || 1177 (!IsAdd && (UI->getOpcode() != ISD::SUBCARRY))) { 1178 IsVALU = true; 1179 break; 1180 } 1181 } 1182 1183 if (IsVALU) { 1184 unsigned Opc = IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64; 1185 1186 CurDAG->SelectNodeTo( 1187 N, Opc, N->getVTList(), 1188 {N->getOperand(0), N->getOperand(1), 1189 CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/}); 1190 } else { 1191 unsigned Opc = N->getOpcode() == ISD::UADDO ? AMDGPU::S_UADDO_PSEUDO 1192 : AMDGPU::S_USUBO_PSEUDO; 1193 1194 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), 1195 {N->getOperand(0), N->getOperand(1)}); 1196 } 1197 } 1198 1199 void AMDGPUDAGToDAGISel::SelectFMA_W_CHAIN(SDNode *N) { 1200 SDLoc SL(N); 1201 // src0_modifiers, src0, src1_modifiers, src1, src2_modifiers, src2, clamp, omod 1202 SDValue Ops[10]; 1203 1204 SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[6], Ops[7]); 1205 SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]); 1206 SelectVOP3Mods(N->getOperand(3), Ops[5], Ops[4]); 1207 Ops[8] = N->getOperand(0); 1208 Ops[9] = N->getOperand(4); 1209 1210 CurDAG->SelectNodeTo(N, AMDGPU::V_FMA_F32_e64, N->getVTList(), Ops); 1211 } 1212 1213 void AMDGPUDAGToDAGISel::SelectFMUL_W_CHAIN(SDNode *N) { 1214 SDLoc SL(N); 1215 // src0_modifiers, src0, src1_modifiers, src1, clamp, omod 1216 SDValue Ops[8]; 1217 1218 SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[4], Ops[5]); 1219 SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]); 1220 Ops[6] = N->getOperand(0); 1221 Ops[7] = N->getOperand(3); 1222 1223 CurDAG->SelectNodeTo(N, AMDGPU::V_MUL_F32_e64, N->getVTList(), Ops); 1224 } 1225 1226 // We need to handle this here because tablegen doesn't support matching 1227 // instructions with multiple outputs. 1228 void AMDGPUDAGToDAGISel::SelectDIV_SCALE(SDNode *N) { 1229 SDLoc SL(N); 1230 EVT VT = N->getValueType(0); 1231 1232 assert(VT == MVT::f32 || VT == MVT::f64); 1233 1234 unsigned Opc 1235 = (VT == MVT::f64) ? AMDGPU::V_DIV_SCALE_F64_e64 : AMDGPU::V_DIV_SCALE_F32_e64; 1236 1237 // src0_modifiers, src0, src1_modifiers, src1, src2_modifiers, src2, clamp, 1238 // omod 1239 SDValue Ops[8]; 1240 SelectVOP3BMods0(N->getOperand(0), Ops[1], Ops[0], Ops[6], Ops[7]); 1241 SelectVOP3BMods(N->getOperand(1), Ops[3], Ops[2]); 1242 SelectVOP3BMods(N->getOperand(2), Ops[5], Ops[4]); 1243 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); 1244 } 1245 1246 // We need to handle this here because tablegen doesn't support matching 1247 // instructions with multiple outputs. 1248 void AMDGPUDAGToDAGISel::SelectMAD_64_32(SDNode *N) { 1249 SDLoc SL(N); 1250 bool Signed = N->getOpcode() == AMDGPUISD::MAD_I64_I32; 1251 unsigned Opc = Signed ? AMDGPU::V_MAD_I64_I32_e64 : AMDGPU::V_MAD_U64_U32_e64; 1252 1253 SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1); 1254 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), 1255 Clamp }; 1256 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); 1257 } 1258 1259 bool AMDGPUDAGToDAGISel::isDSOffsetLegal(SDValue Base, unsigned Offset) const { 1260 if (!isUInt<16>(Offset)) 1261 return false; 1262 1263 if (!Base || Subtarget->hasUsableDSOffset() || 1264 Subtarget->unsafeDSOffsetFoldingEnabled()) 1265 return true; 1266 1267 // On Southern Islands instruction with a negative base value and an offset 1268 // don't seem to work. 1269 return CurDAG->SignBitIsZero(Base); 1270 } 1271 1272 bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base, 1273 SDValue &Offset) const { 1274 SDLoc DL(Addr); 1275 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1276 SDValue N0 = Addr.getOperand(0); 1277 SDValue N1 = Addr.getOperand(1); 1278 ConstantSDNode *C1 = cast<ConstantSDNode>(N1); 1279 if (isDSOffsetLegal(N0, C1->getSExtValue())) { 1280 // (add n0, c0) 1281 Base = N0; 1282 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16); 1283 return true; 1284 } 1285 } else if (Addr.getOpcode() == ISD::SUB) { 1286 // sub C, x -> add (sub 0, x), C 1287 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) { 1288 int64_t ByteOffset = C->getSExtValue(); 1289 if (isDSOffsetLegal(SDValue(), ByteOffset)) { 1290 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32); 1291 1292 // XXX - This is kind of hacky. Create a dummy sub node so we can check 1293 // the known bits in isDSOffsetLegal. We need to emit the selected node 1294 // here, so this is thrown away. 1295 SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32, 1296 Zero, Addr.getOperand(1)); 1297 1298 if (isDSOffsetLegal(Sub, ByteOffset)) { 1299 SmallVector<SDValue, 3> Opnds; 1300 Opnds.push_back(Zero); 1301 Opnds.push_back(Addr.getOperand(1)); 1302 1303 // FIXME: Select to VOP3 version for with-carry. 1304 unsigned SubOp = AMDGPU::V_SUB_CO_U32_e32; 1305 if (Subtarget->hasAddNoCarry()) { 1306 SubOp = AMDGPU::V_SUB_U32_e64; 1307 Opnds.push_back( 1308 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit 1309 } 1310 1311 MachineSDNode *MachineSub = 1312 CurDAG->getMachineNode(SubOp, DL, MVT::i32, Opnds); 1313 1314 Base = SDValue(MachineSub, 0); 1315 Offset = CurDAG->getTargetConstant(ByteOffset, DL, MVT::i16); 1316 return true; 1317 } 1318 } 1319 } 1320 } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) { 1321 // If we have a constant address, prefer to put the constant into the 1322 // offset. This can save moves to load the constant address since multiple 1323 // operations can share the zero base address register, and enables merging 1324 // into read2 / write2 instructions. 1325 1326 SDLoc DL(Addr); 1327 1328 if (isDSOffsetLegal(SDValue(), CAddr->getZExtValue())) { 1329 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32); 1330 MachineSDNode *MovZero = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, 1331 DL, MVT::i32, Zero); 1332 Base = SDValue(MovZero, 0); 1333 Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16); 1334 return true; 1335 } 1336 } 1337 1338 // default case 1339 Base = Addr; 1340 Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i16); 1341 return true; 1342 } 1343 1344 bool AMDGPUDAGToDAGISel::isDSOffset2Legal(SDValue Base, unsigned Offset0, 1345 unsigned Offset1, 1346 unsigned Size) const { 1347 if (Offset0 % Size != 0 || Offset1 % Size != 0) 1348 return false; 1349 if (!isUInt<8>(Offset0 / Size) || !isUInt<8>(Offset1 / Size)) 1350 return false; 1351 1352 if (!Base || Subtarget->hasUsableDSOffset() || 1353 Subtarget->unsafeDSOffsetFoldingEnabled()) 1354 return true; 1355 1356 // On Southern Islands instruction with a negative base value and an offset 1357 // don't seem to work. 1358 return CurDAG->SignBitIsZero(Base); 1359 } 1360 1361 // TODO: If offset is too big, put low 16-bit into offset. 1362 bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base, 1363 SDValue &Offset0, 1364 SDValue &Offset1) const { 1365 return SelectDSReadWrite2(Addr, Base, Offset0, Offset1, 4); 1366 } 1367 1368 bool AMDGPUDAGToDAGISel::SelectDS128Bit8ByteAligned(SDValue Addr, SDValue &Base, 1369 SDValue &Offset0, 1370 SDValue &Offset1) const { 1371 return SelectDSReadWrite2(Addr, Base, Offset0, Offset1, 8); 1372 } 1373 1374 bool AMDGPUDAGToDAGISel::SelectDSReadWrite2(SDValue Addr, SDValue &Base, 1375 SDValue &Offset0, SDValue &Offset1, 1376 unsigned Size) const { 1377 SDLoc DL(Addr); 1378 1379 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1380 SDValue N0 = Addr.getOperand(0); 1381 SDValue N1 = Addr.getOperand(1); 1382 ConstantSDNode *C1 = cast<ConstantSDNode>(N1); 1383 unsigned OffsetValue0 = C1->getZExtValue(); 1384 unsigned OffsetValue1 = OffsetValue0 + Size; 1385 1386 // (add n0, c0) 1387 if (isDSOffset2Legal(N0, OffsetValue0, OffsetValue1, Size)) { 1388 Base = N0; 1389 Offset0 = CurDAG->getTargetConstant(OffsetValue0 / Size, DL, MVT::i8); 1390 Offset1 = CurDAG->getTargetConstant(OffsetValue1 / Size, DL, MVT::i8); 1391 return true; 1392 } 1393 } else if (Addr.getOpcode() == ISD::SUB) { 1394 // sub C, x -> add (sub 0, x), C 1395 if (const ConstantSDNode *C = 1396 dyn_cast<ConstantSDNode>(Addr.getOperand(0))) { 1397 unsigned OffsetValue0 = C->getZExtValue(); 1398 unsigned OffsetValue1 = OffsetValue0 + Size; 1399 1400 if (isDSOffset2Legal(SDValue(), OffsetValue0, OffsetValue1, Size)) { 1401 SDLoc DL(Addr); 1402 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32); 1403 1404 // XXX - This is kind of hacky. Create a dummy sub node so we can check 1405 // the known bits in isDSOffsetLegal. We need to emit the selected node 1406 // here, so this is thrown away. 1407 SDValue Sub = 1408 CurDAG->getNode(ISD::SUB, DL, MVT::i32, Zero, Addr.getOperand(1)); 1409 1410 if (isDSOffset2Legal(Sub, OffsetValue0, OffsetValue1, Size)) { 1411 SmallVector<SDValue, 3> Opnds; 1412 Opnds.push_back(Zero); 1413 Opnds.push_back(Addr.getOperand(1)); 1414 unsigned SubOp = AMDGPU::V_SUB_CO_U32_e32; 1415 if (Subtarget->hasAddNoCarry()) { 1416 SubOp = AMDGPU::V_SUB_U32_e64; 1417 Opnds.push_back( 1418 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit 1419 } 1420 1421 MachineSDNode *MachineSub = CurDAG->getMachineNode( 1422 SubOp, DL, MVT::getIntegerVT(Size * 8), Opnds); 1423 1424 Base = SDValue(MachineSub, 0); 1425 Offset0 = CurDAG->getTargetConstant(OffsetValue0 / Size, DL, MVT::i8); 1426 Offset1 = CurDAG->getTargetConstant(OffsetValue1 / Size, DL, MVT::i8); 1427 return true; 1428 } 1429 } 1430 } 1431 } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) { 1432 unsigned OffsetValue0 = CAddr->getZExtValue(); 1433 unsigned OffsetValue1 = OffsetValue0 + Size; 1434 1435 if (isDSOffset2Legal(SDValue(), OffsetValue0, OffsetValue1, Size)) { 1436 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32); 1437 MachineSDNode *MovZero = 1438 CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, DL, MVT::i32, Zero); 1439 Base = SDValue(MovZero, 0); 1440 Offset0 = CurDAG->getTargetConstant(OffsetValue0 / Size, DL, MVT::i8); 1441 Offset1 = CurDAG->getTargetConstant(OffsetValue1 / Size, DL, MVT::i8); 1442 return true; 1443 } 1444 } 1445 1446 // default case 1447 1448 Base = Addr; 1449 Offset0 = CurDAG->getTargetConstant(0, DL, MVT::i8); 1450 Offset1 = CurDAG->getTargetConstant(1, DL, MVT::i8); 1451 return true; 1452 } 1453 1454 bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr, SDValue &VAddr, 1455 SDValue &SOffset, SDValue &Offset, 1456 SDValue &Offen, SDValue &Idxen, 1457 SDValue &Addr64) const { 1458 // Subtarget prefers to use flat instruction 1459 // FIXME: This should be a pattern predicate and not reach here 1460 if (Subtarget->useFlatForGlobal()) 1461 return false; 1462 1463 SDLoc DL(Addr); 1464 1465 Idxen = CurDAG->getTargetConstant(0, DL, MVT::i1); 1466 Offen = CurDAG->getTargetConstant(0, DL, MVT::i1); 1467 Addr64 = CurDAG->getTargetConstant(0, DL, MVT::i1); 1468 SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32); 1469 1470 ConstantSDNode *C1 = nullptr; 1471 SDValue N0 = Addr; 1472 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1473 C1 = cast<ConstantSDNode>(Addr.getOperand(1)); 1474 if (isUInt<32>(C1->getZExtValue())) 1475 N0 = Addr.getOperand(0); 1476 else 1477 C1 = nullptr; 1478 } 1479 1480 if (N0.getOpcode() == ISD::ADD) { 1481 // (add N2, N3) -> addr64, or 1482 // (add (add N2, N3), C1) -> addr64 1483 SDValue N2 = N0.getOperand(0); 1484 SDValue N3 = N0.getOperand(1); 1485 Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1); 1486 1487 if (N2->isDivergent()) { 1488 if (N3->isDivergent()) { 1489 // Both N2 and N3 are divergent. Use N0 (the result of the add) as the 1490 // addr64, and construct the resource from a 0 address. 1491 Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0); 1492 VAddr = N0; 1493 } else { 1494 // N2 is divergent, N3 is not. 1495 Ptr = N3; 1496 VAddr = N2; 1497 } 1498 } else { 1499 // N2 is not divergent. 1500 Ptr = N2; 1501 VAddr = N3; 1502 } 1503 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1504 } else if (N0->isDivergent()) { 1505 // N0 is divergent. Use it as the addr64, and construct the resource from a 1506 // 0 address. 1507 Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0); 1508 VAddr = N0; 1509 Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1); 1510 } else { 1511 // N0 -> offset, or 1512 // (N0 + C1) -> offset 1513 VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32); 1514 Ptr = N0; 1515 } 1516 1517 if (!C1) { 1518 // No offset. 1519 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1520 return true; 1521 } 1522 1523 if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue())) { 1524 // Legal offset for instruction. 1525 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16); 1526 return true; 1527 } 1528 1529 // Illegal offset, store it in soffset. 1530 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1531 SOffset = 1532 SDValue(CurDAG->getMachineNode( 1533 AMDGPU::S_MOV_B32, DL, MVT::i32, 1534 CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32)), 1535 0); 1536 return true; 1537 } 1538 1539 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, 1540 SDValue &VAddr, SDValue &SOffset, 1541 SDValue &Offset) const { 1542 SDValue Ptr, Offen, Idxen, Addr64; 1543 1544 // addr64 bit was removed for volcanic islands. 1545 // FIXME: This should be a pattern predicate and not reach here 1546 if (!Subtarget->hasAddr64()) 1547 return false; 1548 1549 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64)) 1550 return false; 1551 1552 ConstantSDNode *C = cast<ConstantSDNode>(Addr64); 1553 if (C->getSExtValue()) { 1554 SDLoc DL(Addr); 1555 1556 const SITargetLowering& Lowering = 1557 *static_cast<const SITargetLowering*>(getTargetLowering()); 1558 1559 SRsrc = SDValue(Lowering.wrapAddr64Rsrc(*CurDAG, DL, Ptr), 0); 1560 return true; 1561 } 1562 1563 return false; 1564 } 1565 1566 std::pair<SDValue, SDValue> AMDGPUDAGToDAGISel::foldFrameIndex(SDValue N) const { 1567 SDLoc DL(N); 1568 1569 auto *FI = dyn_cast<FrameIndexSDNode>(N); 1570 SDValue TFI = 1571 FI ? CurDAG->getTargetFrameIndex(FI->getIndex(), FI->getValueType(0)) : N; 1572 1573 // We rebase the base address into an absolute stack address and hence 1574 // use constant 0 for soffset. This value must be retained until 1575 // frame elimination and eliminateFrameIndex will choose the appropriate 1576 // frame register if need be. 1577 return std::make_pair(TFI, CurDAG->getTargetConstant(0, DL, MVT::i32)); 1578 } 1579 1580 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffen(SDNode *Parent, 1581 SDValue Addr, SDValue &Rsrc, 1582 SDValue &VAddr, SDValue &SOffset, 1583 SDValue &ImmOffset) const { 1584 1585 SDLoc DL(Addr); 1586 MachineFunction &MF = CurDAG->getMachineFunction(); 1587 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 1588 1589 Rsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32); 1590 1591 if (ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) { 1592 int64_t Imm = CAddr->getSExtValue(); 1593 const int64_t NullPtr = 1594 AMDGPUTargetMachine::getNullPointerValue(AMDGPUAS::PRIVATE_ADDRESS); 1595 // Don't fold null pointer. 1596 if (Imm != NullPtr) { 1597 SDValue HighBits = CurDAG->getTargetConstant(Imm & ~4095, DL, MVT::i32); 1598 MachineSDNode *MovHighBits = CurDAG->getMachineNode( 1599 AMDGPU::V_MOV_B32_e32, DL, MVT::i32, HighBits); 1600 VAddr = SDValue(MovHighBits, 0); 1601 1602 SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32); 1603 ImmOffset = CurDAG->getTargetConstant(Imm & 4095, DL, MVT::i16); 1604 return true; 1605 } 1606 } 1607 1608 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1609 // (add n0, c1) 1610 1611 SDValue N0 = Addr.getOperand(0); 1612 SDValue N1 = Addr.getOperand(1); 1613 1614 // Offsets in vaddr must be positive if range checking is enabled. 1615 // 1616 // The total computation of vaddr + soffset + offset must not overflow. If 1617 // vaddr is negative, even if offset is 0 the sgpr offset add will end up 1618 // overflowing. 1619 // 1620 // Prior to gfx9, MUBUF instructions with the vaddr offset enabled would 1621 // always perform a range check. If a negative vaddr base index was used, 1622 // this would fail the range check. The overall address computation would 1623 // compute a valid address, but this doesn't happen due to the range 1624 // check. For out-of-bounds MUBUF loads, a 0 is returned. 1625 // 1626 // Therefore it should be safe to fold any VGPR offset on gfx9 into the 1627 // MUBUF vaddr, but not on older subtargets which can only do this if the 1628 // sign bit is known 0. 1629 ConstantSDNode *C1 = cast<ConstantSDNode>(N1); 1630 if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue()) && 1631 (!Subtarget->privateMemoryResourceIsRangeChecked() || 1632 CurDAG->SignBitIsZero(N0))) { 1633 std::tie(VAddr, SOffset) = foldFrameIndex(N0); 1634 ImmOffset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16); 1635 return true; 1636 } 1637 } 1638 1639 // (node) 1640 std::tie(VAddr, SOffset) = foldFrameIndex(Addr); 1641 ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1642 return true; 1643 } 1644 1645 static bool IsCopyFromSGPR(const SIRegisterInfo &TRI, SDValue Val) { 1646 if (Val.getOpcode() != ISD::CopyFromReg) 1647 return false; 1648 auto RC = 1649 TRI.getPhysRegClass(cast<RegisterSDNode>(Val.getOperand(1))->getReg()); 1650 return RC && TRI.isSGPRClass(RC); 1651 } 1652 1653 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffset(SDNode *Parent, 1654 SDValue Addr, 1655 SDValue &SRsrc, 1656 SDValue &SOffset, 1657 SDValue &Offset) const { 1658 const SIRegisterInfo *TRI = 1659 static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo()); 1660 MachineFunction &MF = CurDAG->getMachineFunction(); 1661 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 1662 SDLoc DL(Addr); 1663 1664 // CopyFromReg <sgpr> 1665 if (IsCopyFromSGPR(*TRI, Addr)) { 1666 SRsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32); 1667 SOffset = Addr; 1668 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1669 return true; 1670 } 1671 1672 ConstantSDNode *CAddr; 1673 if (Addr.getOpcode() == ISD::ADD) { 1674 // Add (CopyFromReg <sgpr>) <constant> 1675 CAddr = dyn_cast<ConstantSDNode>(Addr.getOperand(1)); 1676 if (!CAddr || !SIInstrInfo::isLegalMUBUFImmOffset(CAddr->getZExtValue())) 1677 return false; 1678 if (!IsCopyFromSGPR(*TRI, Addr.getOperand(0))) 1679 return false; 1680 1681 SOffset = Addr.getOperand(0); 1682 } else if ((CAddr = dyn_cast<ConstantSDNode>(Addr)) && 1683 SIInstrInfo::isLegalMUBUFImmOffset(CAddr->getZExtValue())) { 1684 // <constant> 1685 SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32); 1686 } else { 1687 return false; 1688 } 1689 1690 SRsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32); 1691 1692 Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16); 1693 return true; 1694 } 1695 1696 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, 1697 SDValue &SOffset, SDValue &Offset 1698 ) const { 1699 SDValue Ptr, VAddr, Offen, Idxen, Addr64; 1700 const SIInstrInfo *TII = 1701 static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo()); 1702 1703 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64)) 1704 return false; 1705 1706 if (!cast<ConstantSDNode>(Offen)->getSExtValue() && 1707 !cast<ConstantSDNode>(Idxen)->getSExtValue() && 1708 !cast<ConstantSDNode>(Addr64)->getSExtValue()) { 1709 uint64_t Rsrc = TII->getDefaultRsrcDataFormat() | 1710 APInt::getAllOnesValue(32).getZExtValue(); // Size 1711 SDLoc DL(Addr); 1712 1713 const SITargetLowering& Lowering = 1714 *static_cast<const SITargetLowering*>(getTargetLowering()); 1715 1716 SRsrc = SDValue(Lowering.buildRSRC(*CurDAG, DL, Ptr, 0, Rsrc), 0); 1717 return true; 1718 } 1719 return false; 1720 } 1721 1722 // Find a load or store from corresponding pattern root. 1723 // Roots may be build_vector, bitconvert or their combinations. 1724 static MemSDNode* findMemSDNode(SDNode *N) { 1725 N = AMDGPUTargetLowering::stripBitcast(SDValue(N,0)).getNode(); 1726 if (MemSDNode *MN = dyn_cast<MemSDNode>(N)) 1727 return MN; 1728 assert(isa<BuildVectorSDNode>(N)); 1729 for (SDValue V : N->op_values()) 1730 if (MemSDNode *MN = 1731 dyn_cast<MemSDNode>(AMDGPUTargetLowering::stripBitcast(V))) 1732 return MN; 1733 llvm_unreachable("cannot find MemSDNode in the pattern!"); 1734 } 1735 1736 bool AMDGPUDAGToDAGISel::SelectFlatOffsetImpl(SDNode *N, SDValue Addr, 1737 SDValue &VAddr, SDValue &Offset, 1738 uint64_t FlatVariant) const { 1739 int64_t OffsetVal = 0; 1740 1741 unsigned AS = findMemSDNode(N)->getAddressSpace(); 1742 1743 bool CanHaveFlatSegmentOffsetBug = 1744 Subtarget->hasFlatSegmentOffsetBug() && 1745 FlatVariant == SIInstrFlags::FLAT && 1746 (AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::GLOBAL_ADDRESS); 1747 1748 if (Subtarget->hasFlatInstOffsets() && !CanHaveFlatSegmentOffsetBug) { 1749 SDValue N0, N1; 1750 if (isBaseWithConstantOffset64(Addr, N0, N1)) { 1751 int64_t COffsetVal = cast<ConstantSDNode>(N1)->getSExtValue(); 1752 1753 const SIInstrInfo *TII = Subtarget->getInstrInfo(); 1754 if (TII->isLegalFLATOffset(COffsetVal, AS, FlatVariant)) { 1755 Addr = N0; 1756 OffsetVal = COffsetVal; 1757 } else { 1758 // If the offset doesn't fit, put the low bits into the offset field and 1759 // add the rest. 1760 // 1761 // For a FLAT instruction the hardware decides whether to access 1762 // global/scratch/shared memory based on the high bits of vaddr, 1763 // ignoring the offset field, so we have to ensure that when we add 1764 // remainder to vaddr it still points into the same underlying object. 1765 // The easiest way to do that is to make sure that we split the offset 1766 // into two pieces that are both >= 0 or both <= 0. 1767 1768 SDLoc DL(N); 1769 uint64_t RemainderOffset; 1770 1771 std::tie(OffsetVal, RemainderOffset) = 1772 TII->splitFlatOffset(COffsetVal, AS, FlatVariant); 1773 1774 SDValue AddOffsetLo = 1775 getMaterializedScalarImm32(Lo_32(RemainderOffset), DL); 1776 SDValue Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1); 1777 1778 if (Addr.getValueType().getSizeInBits() == 32) { 1779 SmallVector<SDValue, 3> Opnds; 1780 Opnds.push_back(N0); 1781 Opnds.push_back(AddOffsetLo); 1782 unsigned AddOp = AMDGPU::V_ADD_CO_U32_e32; 1783 if (Subtarget->hasAddNoCarry()) { 1784 AddOp = AMDGPU::V_ADD_U32_e64; 1785 Opnds.push_back(Clamp); 1786 } 1787 Addr = SDValue(CurDAG->getMachineNode(AddOp, DL, MVT::i32, Opnds), 0); 1788 } else { 1789 // TODO: Should this try to use a scalar add pseudo if the base address 1790 // is uniform and saddr is usable? 1791 SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32); 1792 SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32); 1793 1794 SDNode *N0Lo = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 1795 DL, MVT::i32, N0, Sub0); 1796 SDNode *N0Hi = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 1797 DL, MVT::i32, N0, Sub1); 1798 1799 SDValue AddOffsetHi = 1800 getMaterializedScalarImm32(Hi_32(RemainderOffset), DL); 1801 1802 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i1); 1803 1804 SDNode *Add = 1805 CurDAG->getMachineNode(AMDGPU::V_ADD_CO_U32_e64, DL, VTs, 1806 {AddOffsetLo, SDValue(N0Lo, 0), Clamp}); 1807 1808 SDNode *Addc = CurDAG->getMachineNode( 1809 AMDGPU::V_ADDC_U32_e64, DL, VTs, 1810 {AddOffsetHi, SDValue(N0Hi, 0), SDValue(Add, 1), Clamp}); 1811 1812 SDValue RegSequenceArgs[] = { 1813 CurDAG->getTargetConstant(AMDGPU::VReg_64RegClassID, DL, MVT::i32), 1814 SDValue(Add, 0), Sub0, SDValue(Addc, 0), Sub1}; 1815 1816 Addr = SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL, 1817 MVT::i64, RegSequenceArgs), 1818 0); 1819 } 1820 } 1821 } 1822 } 1823 1824 VAddr = Addr; 1825 Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i16); 1826 return true; 1827 } 1828 1829 bool AMDGPUDAGToDAGISel::SelectFlatOffset(SDNode *N, SDValue Addr, 1830 SDValue &VAddr, 1831 SDValue &Offset) const { 1832 return SelectFlatOffsetImpl(N, Addr, VAddr, Offset, SIInstrFlags::FLAT); 1833 } 1834 1835 bool AMDGPUDAGToDAGISel::SelectGlobalOffset(SDNode *N, SDValue Addr, 1836 SDValue &VAddr, 1837 SDValue &Offset) const { 1838 return SelectFlatOffsetImpl(N, Addr, VAddr, Offset, SIInstrFlags::FlatGlobal); 1839 } 1840 1841 bool AMDGPUDAGToDAGISel::SelectScratchOffset(SDNode *N, SDValue Addr, 1842 SDValue &VAddr, 1843 SDValue &Offset) const { 1844 return SelectFlatOffsetImpl(N, Addr, VAddr, Offset, 1845 SIInstrFlags::FlatScratch); 1846 } 1847 1848 // If this matches zero_extend i32:x, return x 1849 static SDValue matchZExtFromI32(SDValue Op) { 1850 if (Op.getOpcode() != ISD::ZERO_EXTEND) 1851 return SDValue(); 1852 1853 SDValue ExtSrc = Op.getOperand(0); 1854 return (ExtSrc.getValueType() == MVT::i32) ? ExtSrc : SDValue(); 1855 } 1856 1857 // Match (64-bit SGPR base) + (zext vgpr offset) + sext(imm offset) 1858 bool AMDGPUDAGToDAGISel::SelectGlobalSAddr(SDNode *N, 1859 SDValue Addr, 1860 SDValue &SAddr, 1861 SDValue &VOffset, 1862 SDValue &Offset) const { 1863 int64_t ImmOffset = 0; 1864 1865 // Match the immediate offset first, which canonically is moved as low as 1866 // possible. 1867 1868 SDValue LHS, RHS; 1869 if (isBaseWithConstantOffset64(Addr, LHS, RHS)) { 1870 int64_t COffsetVal = cast<ConstantSDNode>(RHS)->getSExtValue(); 1871 const SIInstrInfo *TII = Subtarget->getInstrInfo(); 1872 1873 if (TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::GLOBAL_ADDRESS, 1874 SIInstrFlags::FlatGlobal)) { 1875 Addr = LHS; 1876 ImmOffset = COffsetVal; 1877 } else if (!LHS->isDivergent()) { 1878 if (COffsetVal > 0) { 1879 SDLoc SL(N); 1880 // saddr + large_offset -> saddr + 1881 // (voffset = large_offset & ~MaxOffset) + 1882 // (large_offset & MaxOffset); 1883 int64_t SplitImmOffset, RemainderOffset; 1884 std::tie(SplitImmOffset, RemainderOffset) = TII->splitFlatOffset( 1885 COffsetVal, AMDGPUAS::GLOBAL_ADDRESS, SIInstrFlags::FlatGlobal); 1886 1887 if (isUInt<32>(RemainderOffset)) { 1888 SDNode *VMov = CurDAG->getMachineNode( 1889 AMDGPU::V_MOV_B32_e32, SL, MVT::i32, 1890 CurDAG->getTargetConstant(RemainderOffset, SDLoc(), MVT::i32)); 1891 VOffset = SDValue(VMov, 0); 1892 SAddr = LHS; 1893 Offset = CurDAG->getTargetConstant(SplitImmOffset, SDLoc(), MVT::i16); 1894 return true; 1895 } 1896 } 1897 1898 // We are adding a 64 bit SGPR and a constant. If constant bus limit 1899 // is 1 we would need to perform 1 or 2 extra moves for each half of 1900 // the constant and it is better to do a scalar add and then issue a 1901 // single VALU instruction to materialize zero. Otherwise it is less 1902 // instructions to perform VALU adds with immediates or inline literals. 1903 unsigned NumLiterals = 1904 !TII->isInlineConstant(APInt(32, COffsetVal & 0xffffffff)) + 1905 !TII->isInlineConstant(APInt(32, COffsetVal >> 32)); 1906 if (Subtarget->getConstantBusLimit(AMDGPU::V_ADD_U32_e64) > NumLiterals) 1907 return false; 1908 } 1909 } 1910 1911 // Match the variable offset. 1912 if (Addr.getOpcode() == ISD::ADD) { 1913 LHS = Addr.getOperand(0); 1914 RHS = Addr.getOperand(1); 1915 1916 if (!LHS->isDivergent()) { 1917 // add (i64 sgpr), (zero_extend (i32 vgpr)) 1918 if (SDValue ZextRHS = matchZExtFromI32(RHS)) { 1919 SAddr = LHS; 1920 VOffset = ZextRHS; 1921 } 1922 } 1923 1924 if (!SAddr && !RHS->isDivergent()) { 1925 // add (zero_extend (i32 vgpr)), (i64 sgpr) 1926 if (SDValue ZextLHS = matchZExtFromI32(LHS)) { 1927 SAddr = RHS; 1928 VOffset = ZextLHS; 1929 } 1930 } 1931 1932 if (SAddr) { 1933 Offset = CurDAG->getTargetConstant(ImmOffset, SDLoc(), MVT::i16); 1934 return true; 1935 } 1936 } 1937 1938 if (Addr->isDivergent() || Addr.getOpcode() == ISD::UNDEF || 1939 isa<ConstantSDNode>(Addr)) 1940 return false; 1941 1942 // It's cheaper to materialize a single 32-bit zero for vaddr than the two 1943 // moves required to copy a 64-bit SGPR to VGPR. 1944 SAddr = Addr; 1945 SDNode *VMov = 1946 CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, SDLoc(Addr), MVT::i32, 1947 CurDAG->getTargetConstant(0, SDLoc(), MVT::i32)); 1948 VOffset = SDValue(VMov, 0); 1949 Offset = CurDAG->getTargetConstant(ImmOffset, SDLoc(), MVT::i16); 1950 return true; 1951 } 1952 1953 static SDValue SelectSAddrFI(SelectionDAG *CurDAG, SDValue SAddr) { 1954 if (auto FI = dyn_cast<FrameIndexSDNode>(SAddr)) { 1955 SAddr = CurDAG->getTargetFrameIndex(FI->getIndex(), FI->getValueType(0)); 1956 } else if (SAddr.getOpcode() == ISD::ADD && 1957 isa<FrameIndexSDNode>(SAddr.getOperand(0))) { 1958 // Materialize this into a scalar move for scalar address to avoid 1959 // readfirstlane. 1960 auto FI = cast<FrameIndexSDNode>(SAddr.getOperand(0)); 1961 SDValue TFI = CurDAG->getTargetFrameIndex(FI->getIndex(), 1962 FI->getValueType(0)); 1963 SAddr = SDValue(CurDAG->getMachineNode(AMDGPU::S_ADD_I32, SDLoc(SAddr), 1964 MVT::i32, TFI, SAddr.getOperand(1)), 1965 0); 1966 } 1967 1968 return SAddr; 1969 } 1970 1971 // Match (32-bit SGPR base) + sext(imm offset) 1972 bool AMDGPUDAGToDAGISel::SelectScratchSAddr(SDNode *Parent, SDValue Addr, 1973 SDValue &SAddr, 1974 SDValue &Offset) const { 1975 if (Addr->isDivergent()) 1976 return false; 1977 1978 SDLoc DL(Addr); 1979 1980 int64_t COffsetVal = 0; 1981 1982 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1983 COffsetVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue(); 1984 SAddr = Addr.getOperand(0); 1985 } else { 1986 SAddr = Addr; 1987 } 1988 1989 SAddr = SelectSAddrFI(CurDAG, SAddr); 1990 1991 const SIInstrInfo *TII = Subtarget->getInstrInfo(); 1992 1993 if (!TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::PRIVATE_ADDRESS, 1994 SIInstrFlags::FlatScratch)) { 1995 int64_t SplitImmOffset, RemainderOffset; 1996 std::tie(SplitImmOffset, RemainderOffset) = TII->splitFlatOffset( 1997 COffsetVal, AMDGPUAS::PRIVATE_ADDRESS, SIInstrFlags::FlatScratch); 1998 1999 COffsetVal = SplitImmOffset; 2000 2001 SDValue AddOffset = 2002 SAddr.getOpcode() == ISD::TargetFrameIndex 2003 ? getMaterializedScalarImm32(Lo_32(RemainderOffset), DL) 2004 : CurDAG->getTargetConstant(RemainderOffset, DL, MVT::i32); 2005 SAddr = SDValue(CurDAG->getMachineNode(AMDGPU::S_ADD_I32, DL, MVT::i32, 2006 SAddr, AddOffset), 2007 0); 2008 } 2009 2010 Offset = CurDAG->getTargetConstant(COffsetVal, DL, MVT::i16); 2011 2012 return true; 2013 } 2014 2015 bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDValue ByteOffsetNode, 2016 SDValue &Offset, bool &Imm) const { 2017 ConstantSDNode *C = dyn_cast<ConstantSDNode>(ByteOffsetNode); 2018 if (!C) { 2019 if (ByteOffsetNode.getValueType().isScalarInteger() && 2020 ByteOffsetNode.getValueType().getSizeInBits() == 32) { 2021 Offset = ByteOffsetNode; 2022 Imm = false; 2023 return true; 2024 } 2025 if (ByteOffsetNode.getOpcode() == ISD::ZERO_EXTEND) { 2026 if (ByteOffsetNode.getOperand(0).getValueType().getSizeInBits() == 32) { 2027 Offset = ByteOffsetNode.getOperand(0); 2028 Imm = false; 2029 return true; 2030 } 2031 } 2032 return false; 2033 } 2034 2035 SDLoc SL(ByteOffsetNode); 2036 // GFX9 and GFX10 have signed byte immediate offsets. 2037 int64_t ByteOffset = C->getSExtValue(); 2038 Optional<int64_t> EncodedOffset = 2039 AMDGPU::getSMRDEncodedOffset(*Subtarget, ByteOffset, false); 2040 if (EncodedOffset) { 2041 Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32); 2042 Imm = true; 2043 return true; 2044 } 2045 2046 // SGPR and literal offsets are unsigned. 2047 if (ByteOffset < 0) 2048 return false; 2049 2050 EncodedOffset = AMDGPU::getSMRDEncodedLiteralOffset32(*Subtarget, ByteOffset); 2051 if (EncodedOffset) { 2052 Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32); 2053 return true; 2054 } 2055 2056 if (!isUInt<32>(ByteOffset) && !isInt<32>(ByteOffset)) 2057 return false; 2058 2059 SDValue C32Bit = CurDAG->getTargetConstant(ByteOffset, SL, MVT::i32); 2060 Offset = SDValue( 2061 CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, C32Bit), 0); 2062 2063 return true; 2064 } 2065 2066 SDValue AMDGPUDAGToDAGISel::Expand32BitAddress(SDValue Addr) const { 2067 if (Addr.getValueType() != MVT::i32) 2068 return Addr; 2069 2070 // Zero-extend a 32-bit address. 2071 SDLoc SL(Addr); 2072 2073 const MachineFunction &MF = CurDAG->getMachineFunction(); 2074 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 2075 unsigned AddrHiVal = Info->get32BitAddressHighBits(); 2076 SDValue AddrHi = CurDAG->getTargetConstant(AddrHiVal, SL, MVT::i32); 2077 2078 const SDValue Ops[] = { 2079 CurDAG->getTargetConstant(AMDGPU::SReg_64_XEXECRegClassID, SL, MVT::i32), 2080 Addr, 2081 CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32), 2082 SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, AddrHi), 2083 0), 2084 CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32), 2085 }; 2086 2087 return SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, SL, MVT::i64, 2088 Ops), 0); 2089 } 2090 2091 bool AMDGPUDAGToDAGISel::SelectSMRD(SDValue Addr, SDValue &SBase, 2092 SDValue &Offset, bool &Imm) const { 2093 SDLoc SL(Addr); 2094 2095 // A 32-bit (address + offset) should not cause unsigned 32-bit integer 2096 // wraparound, because s_load instructions perform the addition in 64 bits. 2097 if ((Addr.getValueType() != MVT::i32 || 2098 Addr->getFlags().hasNoUnsignedWrap())) { 2099 SDValue N0, N1; 2100 // Extract the base and offset if possible. 2101 if (CurDAG->isBaseWithConstantOffset(Addr) || 2102 Addr.getOpcode() == ISD::ADD) { 2103 N0 = Addr.getOperand(0); 2104 N1 = Addr.getOperand(1); 2105 } else if (getBaseWithOffsetUsingSplitOR(*CurDAG, Addr, N0, N1)) { 2106 assert(N0 && N1 && isa<ConstantSDNode>(N1)); 2107 } 2108 if (N0 && N1) { 2109 if (SelectSMRDOffset(N1, Offset, Imm)) { 2110 SBase = Expand32BitAddress(N0); 2111 return true; 2112 } 2113 } 2114 } 2115 SBase = Expand32BitAddress(Addr); 2116 Offset = CurDAG->getTargetConstant(0, SL, MVT::i32); 2117 Imm = true; 2118 return true; 2119 } 2120 2121 bool AMDGPUDAGToDAGISel::SelectSMRDImm(SDValue Addr, SDValue &SBase, 2122 SDValue &Offset) const { 2123 bool Imm = false; 2124 return SelectSMRD(Addr, SBase, Offset, Imm) && Imm; 2125 } 2126 2127 bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase, 2128 SDValue &Offset) const { 2129 2130 assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS); 2131 2132 bool Imm = false; 2133 if (!SelectSMRD(Addr, SBase, Offset, Imm)) 2134 return false; 2135 2136 return !Imm && isa<ConstantSDNode>(Offset); 2137 } 2138 2139 bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDValue Addr, SDValue &SBase, 2140 SDValue &Offset) const { 2141 bool Imm = false; 2142 return SelectSMRD(Addr, SBase, Offset, Imm) && !Imm && 2143 !isa<ConstantSDNode>(Offset); 2144 } 2145 2146 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(SDValue Addr, 2147 SDValue &Offset) const { 2148 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr)) { 2149 // The immediate offset for S_BUFFER instructions is unsigned. 2150 if (auto Imm = 2151 AMDGPU::getSMRDEncodedOffset(*Subtarget, C->getZExtValue(), true)) { 2152 Offset = CurDAG->getTargetConstant(*Imm, SDLoc(Addr), MVT::i32); 2153 return true; 2154 } 2155 } 2156 2157 return false; 2158 } 2159 2160 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(SDValue Addr, 2161 SDValue &Offset) const { 2162 assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS); 2163 2164 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr)) { 2165 if (auto Imm = AMDGPU::getSMRDEncodedLiteralOffset32(*Subtarget, 2166 C->getZExtValue())) { 2167 Offset = CurDAG->getTargetConstant(*Imm, SDLoc(Addr), MVT::i32); 2168 return true; 2169 } 2170 } 2171 2172 return false; 2173 } 2174 2175 bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(SDValue Index, 2176 SDValue &Base, 2177 SDValue &Offset) const { 2178 SDLoc DL(Index); 2179 2180 if (CurDAG->isBaseWithConstantOffset(Index)) { 2181 SDValue N0 = Index.getOperand(0); 2182 SDValue N1 = Index.getOperand(1); 2183 ConstantSDNode *C1 = cast<ConstantSDNode>(N1); 2184 2185 // (add n0, c0) 2186 // Don't peel off the offset (c0) if doing so could possibly lead 2187 // the base (n0) to be negative. 2188 // (or n0, |c0|) can never change a sign given isBaseWithConstantOffset. 2189 if (C1->getSExtValue() <= 0 || CurDAG->SignBitIsZero(N0) || 2190 (Index->getOpcode() == ISD::OR && C1->getSExtValue() >= 0)) { 2191 Base = N0; 2192 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32); 2193 return true; 2194 } 2195 } 2196 2197 if (isa<ConstantSDNode>(Index)) 2198 return false; 2199 2200 Base = Index; 2201 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32); 2202 return true; 2203 } 2204 2205 SDNode *AMDGPUDAGToDAGISel::getS_BFE(unsigned Opcode, const SDLoc &DL, 2206 SDValue Val, uint32_t Offset, 2207 uint32_t Width) { 2208 // Transformation function, pack the offset and width of a BFE into 2209 // the format expected by the S_BFE_I32 / S_BFE_U32. In the second 2210 // source, bits [5:0] contain the offset and bits [22:16] the width. 2211 uint32_t PackedVal = Offset | (Width << 16); 2212 SDValue PackedConst = CurDAG->getTargetConstant(PackedVal, DL, MVT::i32); 2213 2214 return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, PackedConst); 2215 } 2216 2217 void AMDGPUDAGToDAGISel::SelectS_BFEFromShifts(SDNode *N) { 2218 // "(a << b) srl c)" ---> "BFE_U32 a, (c-b), (32-c) 2219 // "(a << b) sra c)" ---> "BFE_I32 a, (c-b), (32-c) 2220 // Predicate: 0 < b <= c < 32 2221 2222 const SDValue &Shl = N->getOperand(0); 2223 ConstantSDNode *B = dyn_cast<ConstantSDNode>(Shl->getOperand(1)); 2224 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1)); 2225 2226 if (B && C) { 2227 uint32_t BVal = B->getZExtValue(); 2228 uint32_t CVal = C->getZExtValue(); 2229 2230 if (0 < BVal && BVal <= CVal && CVal < 32) { 2231 bool Signed = N->getOpcode() == ISD::SRA; 2232 unsigned Opcode = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32; 2233 2234 ReplaceNode(N, getS_BFE(Opcode, SDLoc(N), Shl.getOperand(0), CVal - BVal, 2235 32 - CVal)); 2236 return; 2237 } 2238 } 2239 SelectCode(N); 2240 } 2241 2242 void AMDGPUDAGToDAGISel::SelectS_BFE(SDNode *N) { 2243 switch (N->getOpcode()) { 2244 case ISD::AND: 2245 if (N->getOperand(0).getOpcode() == ISD::SRL) { 2246 // "(a srl b) & mask" ---> "BFE_U32 a, b, popcount(mask)" 2247 // Predicate: isMask(mask) 2248 const SDValue &Srl = N->getOperand(0); 2249 ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(Srl.getOperand(1)); 2250 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1)); 2251 2252 if (Shift && Mask) { 2253 uint32_t ShiftVal = Shift->getZExtValue(); 2254 uint32_t MaskVal = Mask->getZExtValue(); 2255 2256 if (isMask_32(MaskVal)) { 2257 uint32_t WidthVal = countPopulation(MaskVal); 2258 2259 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N), 2260 Srl.getOperand(0), ShiftVal, WidthVal)); 2261 return; 2262 } 2263 } 2264 } 2265 break; 2266 case ISD::SRL: 2267 if (N->getOperand(0).getOpcode() == ISD::AND) { 2268 // "(a & mask) srl b)" ---> "BFE_U32 a, b, popcount(mask >> b)" 2269 // Predicate: isMask(mask >> b) 2270 const SDValue &And = N->getOperand(0); 2271 ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(N->getOperand(1)); 2272 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(And->getOperand(1)); 2273 2274 if (Shift && Mask) { 2275 uint32_t ShiftVal = Shift->getZExtValue(); 2276 uint32_t MaskVal = Mask->getZExtValue() >> ShiftVal; 2277 2278 if (isMask_32(MaskVal)) { 2279 uint32_t WidthVal = countPopulation(MaskVal); 2280 2281 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N), 2282 And.getOperand(0), ShiftVal, WidthVal)); 2283 return; 2284 } 2285 } 2286 } else if (N->getOperand(0).getOpcode() == ISD::SHL) { 2287 SelectS_BFEFromShifts(N); 2288 return; 2289 } 2290 break; 2291 case ISD::SRA: 2292 if (N->getOperand(0).getOpcode() == ISD::SHL) { 2293 SelectS_BFEFromShifts(N); 2294 return; 2295 } 2296 break; 2297 2298 case ISD::SIGN_EXTEND_INREG: { 2299 // sext_inreg (srl x, 16), i8 -> bfe_i32 x, 16, 8 2300 SDValue Src = N->getOperand(0); 2301 if (Src.getOpcode() != ISD::SRL) 2302 break; 2303 2304 const ConstantSDNode *Amt = dyn_cast<ConstantSDNode>(Src.getOperand(1)); 2305 if (!Amt) 2306 break; 2307 2308 unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits(); 2309 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_I32, SDLoc(N), Src.getOperand(0), 2310 Amt->getZExtValue(), Width)); 2311 return; 2312 } 2313 } 2314 2315 SelectCode(N); 2316 } 2317 2318 bool AMDGPUDAGToDAGISel::isCBranchSCC(const SDNode *N) const { 2319 assert(N->getOpcode() == ISD::BRCOND); 2320 if (!N->hasOneUse()) 2321 return false; 2322 2323 SDValue Cond = N->getOperand(1); 2324 if (Cond.getOpcode() == ISD::CopyToReg) 2325 Cond = Cond.getOperand(2); 2326 2327 if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse()) 2328 return false; 2329 2330 MVT VT = Cond.getOperand(0).getSimpleValueType(); 2331 if (VT == MVT::i32) 2332 return true; 2333 2334 if (VT == MVT::i64) { 2335 auto ST = static_cast<const GCNSubtarget *>(Subtarget); 2336 2337 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); 2338 return (CC == ISD::SETEQ || CC == ISD::SETNE) && ST->hasScalarCompareEq64(); 2339 } 2340 2341 return false; 2342 } 2343 2344 void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) { 2345 SDValue Cond = N->getOperand(1); 2346 2347 if (Cond.isUndef()) { 2348 CurDAG->SelectNodeTo(N, AMDGPU::SI_BR_UNDEF, MVT::Other, 2349 N->getOperand(2), N->getOperand(0)); 2350 return; 2351 } 2352 2353 const GCNSubtarget *ST = static_cast<const GCNSubtarget *>(Subtarget); 2354 const SIRegisterInfo *TRI = ST->getRegisterInfo(); 2355 2356 bool UseSCCBr = isCBranchSCC(N) && isUniformBr(N); 2357 unsigned BrOp = UseSCCBr ? AMDGPU::S_CBRANCH_SCC1 : AMDGPU::S_CBRANCH_VCCNZ; 2358 Register CondReg = UseSCCBr ? AMDGPU::SCC : TRI->getVCC(); 2359 SDLoc SL(N); 2360 2361 if (!UseSCCBr) { 2362 // This is the case that we are selecting to S_CBRANCH_VCCNZ. We have not 2363 // analyzed what generates the vcc value, so we do not know whether vcc 2364 // bits for disabled lanes are 0. Thus we need to mask out bits for 2365 // disabled lanes. 2366 // 2367 // For the case that we select S_CBRANCH_SCC1 and it gets 2368 // changed to S_CBRANCH_VCCNZ in SIFixSGPRCopies, SIFixSGPRCopies calls 2369 // SIInstrInfo::moveToVALU which inserts the S_AND). 2370 // 2371 // We could add an analysis of what generates the vcc value here and omit 2372 // the S_AND when is unnecessary. But it would be better to add a separate 2373 // pass after SIFixSGPRCopies to do the unnecessary S_AND removal, so it 2374 // catches both cases. 2375 Cond = SDValue(CurDAG->getMachineNode(ST->isWave32() ? AMDGPU::S_AND_B32 2376 : AMDGPU::S_AND_B64, 2377 SL, MVT::i1, 2378 CurDAG->getRegister(ST->isWave32() ? AMDGPU::EXEC_LO 2379 : AMDGPU::EXEC, 2380 MVT::i1), 2381 Cond), 2382 0); 2383 } 2384 2385 SDValue VCC = CurDAG->getCopyToReg(N->getOperand(0), SL, CondReg, Cond); 2386 CurDAG->SelectNodeTo(N, BrOp, MVT::Other, 2387 N->getOperand(2), // Basic Block 2388 VCC.getValue(0)); 2389 } 2390 2391 void AMDGPUDAGToDAGISel::SelectFMAD_FMA(SDNode *N) { 2392 MVT VT = N->getSimpleValueType(0); 2393 bool IsFMA = N->getOpcode() == ISD::FMA; 2394 if (VT != MVT::f32 || (!Subtarget->hasMadMixInsts() && 2395 !Subtarget->hasFmaMixInsts()) || 2396 ((IsFMA && Subtarget->hasMadMixInsts()) || 2397 (!IsFMA && Subtarget->hasFmaMixInsts()))) { 2398 SelectCode(N); 2399 return; 2400 } 2401 2402 SDValue Src0 = N->getOperand(0); 2403 SDValue Src1 = N->getOperand(1); 2404 SDValue Src2 = N->getOperand(2); 2405 unsigned Src0Mods, Src1Mods, Src2Mods; 2406 2407 // Avoid using v_mad_mix_f32/v_fma_mix_f32 unless there is actually an operand 2408 // using the conversion from f16. 2409 bool Sel0 = SelectVOP3PMadMixModsImpl(Src0, Src0, Src0Mods); 2410 bool Sel1 = SelectVOP3PMadMixModsImpl(Src1, Src1, Src1Mods); 2411 bool Sel2 = SelectVOP3PMadMixModsImpl(Src2, Src2, Src2Mods); 2412 2413 assert((IsFMA || !Mode.allFP32Denormals()) && 2414 "fmad selected with denormals enabled"); 2415 // TODO: We can select this with f32 denormals enabled if all the sources are 2416 // converted from f16 (in which case fmad isn't legal). 2417 2418 if (Sel0 || Sel1 || Sel2) { 2419 // For dummy operands. 2420 SDValue Zero = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32); 2421 SDValue Ops[] = { 2422 CurDAG->getTargetConstant(Src0Mods, SDLoc(), MVT::i32), Src0, 2423 CurDAG->getTargetConstant(Src1Mods, SDLoc(), MVT::i32), Src1, 2424 CurDAG->getTargetConstant(Src2Mods, SDLoc(), MVT::i32), Src2, 2425 CurDAG->getTargetConstant(0, SDLoc(), MVT::i1), 2426 Zero, Zero 2427 }; 2428 2429 CurDAG->SelectNodeTo(N, 2430 IsFMA ? AMDGPU::V_FMA_MIX_F32 : AMDGPU::V_MAD_MIX_F32, 2431 MVT::f32, Ops); 2432 } else { 2433 SelectCode(N); 2434 } 2435 } 2436 2437 // This is here because there isn't a way to use the generated sub0_sub1 as the 2438 // subreg index to EXTRACT_SUBREG in tablegen. 2439 void AMDGPUDAGToDAGISel::SelectATOMIC_CMP_SWAP(SDNode *N) { 2440 MemSDNode *Mem = cast<MemSDNode>(N); 2441 unsigned AS = Mem->getAddressSpace(); 2442 if (AS == AMDGPUAS::FLAT_ADDRESS) { 2443 SelectCode(N); 2444 return; 2445 } 2446 2447 MVT VT = N->getSimpleValueType(0); 2448 bool Is32 = (VT == MVT::i32); 2449 SDLoc SL(N); 2450 2451 MachineSDNode *CmpSwap = nullptr; 2452 if (Subtarget->hasAddr64()) { 2453 SDValue SRsrc, VAddr, SOffset, Offset; 2454 2455 if (SelectMUBUFAddr64(Mem->getBasePtr(), SRsrc, VAddr, SOffset, Offset)) { 2456 unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_ADDR64_RTN : 2457 AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_ADDR64_RTN; 2458 SDValue CmpVal = Mem->getOperand(2); 2459 SDValue CPol = CurDAG->getTargetConstant(AMDGPU::CPol::GLC, SL, MVT::i32); 2460 2461 // XXX - Do we care about glue operands? 2462 2463 SDValue Ops[] = {CmpVal, VAddr, SRsrc, SOffset, Offset, CPol, 2464 Mem->getChain()}; 2465 2466 CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops); 2467 } 2468 } 2469 2470 if (!CmpSwap) { 2471 SDValue SRsrc, SOffset, Offset; 2472 if (SelectMUBUFOffset(Mem->getBasePtr(), SRsrc, SOffset, Offset)) { 2473 unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN : 2474 AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_OFFSET_RTN; 2475 2476 SDValue CmpVal = Mem->getOperand(2); 2477 SDValue CPol = CurDAG->getTargetConstant(AMDGPU::CPol::GLC, SL, MVT::i32); 2478 SDValue Ops[] = {CmpVal, SRsrc, SOffset, Offset, CPol, Mem->getChain()}; 2479 2480 CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops); 2481 } 2482 } 2483 2484 if (!CmpSwap) { 2485 SelectCode(N); 2486 return; 2487 } 2488 2489 MachineMemOperand *MMO = Mem->getMemOperand(); 2490 CurDAG->setNodeMemRefs(CmpSwap, {MMO}); 2491 2492 unsigned SubReg = Is32 ? AMDGPU::sub0 : AMDGPU::sub0_sub1; 2493 SDValue Extract 2494 = CurDAG->getTargetExtractSubreg(SubReg, SL, VT, SDValue(CmpSwap, 0)); 2495 2496 ReplaceUses(SDValue(N, 0), Extract); 2497 ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 1)); 2498 CurDAG->RemoveDeadNode(N); 2499 } 2500 2501 void AMDGPUDAGToDAGISel::SelectDSAppendConsume(SDNode *N, unsigned IntrID) { 2502 // The address is assumed to be uniform, so if it ends up in a VGPR, it will 2503 // be copied to an SGPR with readfirstlane. 2504 unsigned Opc = IntrID == Intrinsic::amdgcn_ds_append ? 2505 AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME; 2506 2507 SDValue Chain = N->getOperand(0); 2508 SDValue Ptr = N->getOperand(2); 2509 MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N); 2510 MachineMemOperand *MMO = M->getMemOperand(); 2511 bool IsGDS = M->getAddressSpace() == AMDGPUAS::REGION_ADDRESS; 2512 2513 SDValue Offset; 2514 if (CurDAG->isBaseWithConstantOffset(Ptr)) { 2515 SDValue PtrBase = Ptr.getOperand(0); 2516 SDValue PtrOffset = Ptr.getOperand(1); 2517 2518 const APInt &OffsetVal = cast<ConstantSDNode>(PtrOffset)->getAPIntValue(); 2519 if (isDSOffsetLegal(PtrBase, OffsetVal.getZExtValue())) { 2520 N = glueCopyToM0(N, PtrBase); 2521 Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i32); 2522 } 2523 } 2524 2525 if (!Offset) { 2526 N = glueCopyToM0(N, Ptr); 2527 Offset = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32); 2528 } 2529 2530 SDValue Ops[] = { 2531 Offset, 2532 CurDAG->getTargetConstant(IsGDS, SDLoc(), MVT::i32), 2533 Chain, 2534 N->getOperand(N->getNumOperands() - 1) // New glue 2535 }; 2536 2537 SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); 2538 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO}); 2539 } 2540 2541 static unsigned gwsIntrinToOpcode(unsigned IntrID) { 2542 switch (IntrID) { 2543 case Intrinsic::amdgcn_ds_gws_init: 2544 return AMDGPU::DS_GWS_INIT; 2545 case Intrinsic::amdgcn_ds_gws_barrier: 2546 return AMDGPU::DS_GWS_BARRIER; 2547 case Intrinsic::amdgcn_ds_gws_sema_v: 2548 return AMDGPU::DS_GWS_SEMA_V; 2549 case Intrinsic::amdgcn_ds_gws_sema_br: 2550 return AMDGPU::DS_GWS_SEMA_BR; 2551 case Intrinsic::amdgcn_ds_gws_sema_p: 2552 return AMDGPU::DS_GWS_SEMA_P; 2553 case Intrinsic::amdgcn_ds_gws_sema_release_all: 2554 return AMDGPU::DS_GWS_SEMA_RELEASE_ALL; 2555 default: 2556 llvm_unreachable("not a gws intrinsic"); 2557 } 2558 } 2559 2560 void AMDGPUDAGToDAGISel::SelectDS_GWS(SDNode *N, unsigned IntrID) { 2561 if (IntrID == Intrinsic::amdgcn_ds_gws_sema_release_all && 2562 !Subtarget->hasGWSSemaReleaseAll()) { 2563 // Let this error. 2564 SelectCode(N); 2565 return; 2566 } 2567 2568 // Chain, intrinsic ID, vsrc, offset 2569 const bool HasVSrc = N->getNumOperands() == 4; 2570 assert(HasVSrc || N->getNumOperands() == 3); 2571 2572 SDLoc SL(N); 2573 SDValue BaseOffset = N->getOperand(HasVSrc ? 3 : 2); 2574 int ImmOffset = 0; 2575 MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N); 2576 MachineMemOperand *MMO = M->getMemOperand(); 2577 2578 // Don't worry if the offset ends up in a VGPR. Only one lane will have 2579 // effect, so SIFixSGPRCopies will validly insert readfirstlane. 2580 2581 // The resource id offset is computed as (<isa opaque base> + M0[21:16] + 2582 // offset field) % 64. Some versions of the programming guide omit the m0 2583 // part, or claim it's from offset 0. 2584 if (ConstantSDNode *ConstOffset = dyn_cast<ConstantSDNode>(BaseOffset)) { 2585 // If we have a constant offset, try to use the 0 in m0 as the base. 2586 // TODO: Look into changing the default m0 initialization value. If the 2587 // default -1 only set the low 16-bits, we could leave it as-is and add 1 to 2588 // the immediate offset. 2589 glueCopyToM0(N, CurDAG->getTargetConstant(0, SL, MVT::i32)); 2590 ImmOffset = ConstOffset->getZExtValue(); 2591 } else { 2592 if (CurDAG->isBaseWithConstantOffset(BaseOffset)) { 2593 ImmOffset = BaseOffset.getConstantOperandVal(1); 2594 BaseOffset = BaseOffset.getOperand(0); 2595 } 2596 2597 // Prefer to do the shift in an SGPR since it should be possible to use m0 2598 // as the result directly. If it's already an SGPR, it will be eliminated 2599 // later. 2600 SDNode *SGPROffset 2601 = CurDAG->getMachineNode(AMDGPU::V_READFIRSTLANE_B32, SL, MVT::i32, 2602 BaseOffset); 2603 // Shift to offset in m0 2604 SDNode *M0Base 2605 = CurDAG->getMachineNode(AMDGPU::S_LSHL_B32, SL, MVT::i32, 2606 SDValue(SGPROffset, 0), 2607 CurDAG->getTargetConstant(16, SL, MVT::i32)); 2608 glueCopyToM0(N, SDValue(M0Base, 0)); 2609 } 2610 2611 SDValue Chain = N->getOperand(0); 2612 SDValue OffsetField = CurDAG->getTargetConstant(ImmOffset, SL, MVT::i32); 2613 2614 const unsigned Opc = gwsIntrinToOpcode(IntrID); 2615 SmallVector<SDValue, 5> Ops; 2616 if (HasVSrc) 2617 Ops.push_back(N->getOperand(2)); 2618 Ops.push_back(OffsetField); 2619 Ops.push_back(Chain); 2620 2621 SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); 2622 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO}); 2623 } 2624 2625 void AMDGPUDAGToDAGISel::SelectInterpP1F16(SDNode *N) { 2626 if (Subtarget->getLDSBankCount() != 16) { 2627 // This is a single instruction with a pattern. 2628 SelectCode(N); 2629 return; 2630 } 2631 2632 SDLoc DL(N); 2633 2634 // This requires 2 instructions. It is possible to write a pattern to support 2635 // this, but the generated isel emitter doesn't correctly deal with multiple 2636 // output instructions using the same physical register input. The copy to m0 2637 // is incorrectly placed before the second instruction. 2638 // 2639 // TODO: Match source modifiers. 2640 // 2641 // def : Pat < 2642 // (int_amdgcn_interp_p1_f16 2643 // (VOP3Mods f32:$src0, i32:$src0_modifiers), 2644 // (i32 timm:$attrchan), (i32 timm:$attr), 2645 // (i1 timm:$high), M0), 2646 // (V_INTERP_P1LV_F16 $src0_modifiers, VGPR_32:$src0, timm:$attr, 2647 // timm:$attrchan, 0, 2648 // (V_INTERP_MOV_F32 2, timm:$attr, timm:$attrchan), timm:$high)> { 2649 // let Predicates = [has16BankLDS]; 2650 // } 2651 2652 // 16 bank LDS 2653 SDValue ToM0 = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL, AMDGPU::M0, 2654 N->getOperand(5), SDValue()); 2655 2656 SDVTList VTs = CurDAG->getVTList(MVT::f32, MVT::Other); 2657 2658 SDNode *InterpMov = 2659 CurDAG->getMachineNode(AMDGPU::V_INTERP_MOV_F32, DL, VTs, { 2660 CurDAG->getTargetConstant(2, DL, MVT::i32), // P0 2661 N->getOperand(3), // Attr 2662 N->getOperand(2), // Attrchan 2663 ToM0.getValue(1) // In glue 2664 }); 2665 2666 SDNode *InterpP1LV = 2667 CurDAG->getMachineNode(AMDGPU::V_INTERP_P1LV_F16, DL, MVT::f32, { 2668 CurDAG->getTargetConstant(0, DL, MVT::i32), // $src0_modifiers 2669 N->getOperand(1), // Src0 2670 N->getOperand(3), // Attr 2671 N->getOperand(2), // Attrchan 2672 CurDAG->getTargetConstant(0, DL, MVT::i32), // $src2_modifiers 2673 SDValue(InterpMov, 0), // Src2 - holds two f16 values selected by high 2674 N->getOperand(4), // high 2675 CurDAG->getTargetConstant(0, DL, MVT::i1), // $clamp 2676 CurDAG->getTargetConstant(0, DL, MVT::i32), // $omod 2677 SDValue(InterpMov, 1) 2678 }); 2679 2680 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), SDValue(InterpP1LV, 0)); 2681 } 2682 2683 void AMDGPUDAGToDAGISel::SelectINTRINSIC_W_CHAIN(SDNode *N) { 2684 unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 2685 switch (IntrID) { 2686 case Intrinsic::amdgcn_ds_append: 2687 case Intrinsic::amdgcn_ds_consume: { 2688 if (N->getValueType(0) != MVT::i32) 2689 break; 2690 SelectDSAppendConsume(N, IntrID); 2691 return; 2692 } 2693 } 2694 2695 SelectCode(N); 2696 } 2697 2698 void AMDGPUDAGToDAGISel::SelectINTRINSIC_WO_CHAIN(SDNode *N) { 2699 unsigned IntrID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 2700 unsigned Opcode; 2701 switch (IntrID) { 2702 case Intrinsic::amdgcn_wqm: 2703 Opcode = AMDGPU::WQM; 2704 break; 2705 case Intrinsic::amdgcn_softwqm: 2706 Opcode = AMDGPU::SOFT_WQM; 2707 break; 2708 case Intrinsic::amdgcn_wwm: 2709 case Intrinsic::amdgcn_strict_wwm: 2710 Opcode = AMDGPU::STRICT_WWM; 2711 break; 2712 case Intrinsic::amdgcn_strict_wqm: 2713 Opcode = AMDGPU::STRICT_WQM; 2714 break; 2715 case Intrinsic::amdgcn_interp_p1_f16: 2716 SelectInterpP1F16(N); 2717 return; 2718 default: 2719 SelectCode(N); 2720 return; 2721 } 2722 2723 SDValue Src = N->getOperand(1); 2724 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), {Src}); 2725 } 2726 2727 void AMDGPUDAGToDAGISel::SelectINTRINSIC_VOID(SDNode *N) { 2728 unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 2729 switch (IntrID) { 2730 case Intrinsic::amdgcn_ds_gws_init: 2731 case Intrinsic::amdgcn_ds_gws_barrier: 2732 case Intrinsic::amdgcn_ds_gws_sema_v: 2733 case Intrinsic::amdgcn_ds_gws_sema_br: 2734 case Intrinsic::amdgcn_ds_gws_sema_p: 2735 case Intrinsic::amdgcn_ds_gws_sema_release_all: 2736 SelectDS_GWS(N, IntrID); 2737 return; 2738 default: 2739 break; 2740 } 2741 2742 SelectCode(N); 2743 } 2744 2745 bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(SDValue In, SDValue &Src, 2746 unsigned &Mods, 2747 bool AllowAbs) const { 2748 Mods = 0; 2749 Src = In; 2750 2751 if (Src.getOpcode() == ISD::FNEG) { 2752 Mods |= SISrcMods::NEG; 2753 Src = Src.getOperand(0); 2754 } 2755 2756 if (AllowAbs && Src.getOpcode() == ISD::FABS) { 2757 Mods |= SISrcMods::ABS; 2758 Src = Src.getOperand(0); 2759 } 2760 2761 return true; 2762 } 2763 2764 bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src, 2765 SDValue &SrcMods) const { 2766 unsigned Mods; 2767 if (SelectVOP3ModsImpl(In, Src, Mods)) { 2768 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 2769 return true; 2770 } 2771 2772 return false; 2773 } 2774 2775 bool AMDGPUDAGToDAGISel::SelectVOP3BMods(SDValue In, SDValue &Src, 2776 SDValue &SrcMods) const { 2777 unsigned Mods; 2778 if (SelectVOP3ModsImpl(In, Src, Mods, /* AllowAbs */ false)) { 2779 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 2780 return true; 2781 } 2782 2783 return false; 2784 } 2785 2786 bool AMDGPUDAGToDAGISel::SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, 2787 SDValue &SrcMods) const { 2788 SelectVOP3Mods(In, Src, SrcMods); 2789 return isNoNanSrc(Src); 2790 } 2791 2792 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const { 2793 if (In.getOpcode() == ISD::FABS || In.getOpcode() == ISD::FNEG) 2794 return false; 2795 2796 Src = In; 2797 return true; 2798 } 2799 2800 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src, 2801 SDValue &SrcMods, SDValue &Clamp, 2802 SDValue &Omod) const { 2803 SDLoc DL(In); 2804 Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1); 2805 Omod = CurDAG->getTargetConstant(0, DL, MVT::i1); 2806 2807 return SelectVOP3Mods(In, Src, SrcMods); 2808 } 2809 2810 bool AMDGPUDAGToDAGISel::SelectVOP3BMods0(SDValue In, SDValue &Src, 2811 SDValue &SrcMods, SDValue &Clamp, 2812 SDValue &Omod) const { 2813 SDLoc DL(In); 2814 Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1); 2815 Omod = CurDAG->getTargetConstant(0, DL, MVT::i1); 2816 2817 return SelectVOP3BMods(In, Src, SrcMods); 2818 } 2819 2820 bool AMDGPUDAGToDAGISel::SelectVOP3OMods(SDValue In, SDValue &Src, 2821 SDValue &Clamp, SDValue &Omod) const { 2822 Src = In; 2823 2824 SDLoc DL(In); 2825 Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1); 2826 Omod = CurDAG->getTargetConstant(0, DL, MVT::i1); 2827 2828 return true; 2829 } 2830 2831 bool AMDGPUDAGToDAGISel::SelectVOP3PMods(SDValue In, SDValue &Src, 2832 SDValue &SrcMods) const { 2833 unsigned Mods = 0; 2834 Src = In; 2835 2836 if (Src.getOpcode() == ISD::FNEG) { 2837 Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI); 2838 Src = Src.getOperand(0); 2839 } 2840 2841 if (Src.getOpcode() == ISD::BUILD_VECTOR) { 2842 unsigned VecMods = Mods; 2843 2844 SDValue Lo = stripBitcast(Src.getOperand(0)); 2845 SDValue Hi = stripBitcast(Src.getOperand(1)); 2846 2847 if (Lo.getOpcode() == ISD::FNEG) { 2848 Lo = stripBitcast(Lo.getOperand(0)); 2849 Mods ^= SISrcMods::NEG; 2850 } 2851 2852 if (Hi.getOpcode() == ISD::FNEG) { 2853 Hi = stripBitcast(Hi.getOperand(0)); 2854 Mods ^= SISrcMods::NEG_HI; 2855 } 2856 2857 if (isExtractHiElt(Lo, Lo)) 2858 Mods |= SISrcMods::OP_SEL_0; 2859 2860 if (isExtractHiElt(Hi, Hi)) 2861 Mods |= SISrcMods::OP_SEL_1; 2862 2863 unsigned VecSize = Src.getValueSizeInBits(); 2864 Lo = stripExtractLoElt(Lo); 2865 Hi = stripExtractLoElt(Hi); 2866 2867 if (Lo.getValueSizeInBits() > VecSize) { 2868 Lo = CurDAG->getTargetExtractSubreg( 2869 (VecSize > 32) ? AMDGPU::sub0_sub1 : AMDGPU::sub0, SDLoc(In), 2870 MVT::getIntegerVT(VecSize), Lo); 2871 } 2872 2873 if (Hi.getValueSizeInBits() > VecSize) { 2874 Hi = CurDAG->getTargetExtractSubreg( 2875 (VecSize > 32) ? AMDGPU::sub0_sub1 : AMDGPU::sub0, SDLoc(In), 2876 MVT::getIntegerVT(VecSize), Hi); 2877 } 2878 2879 assert(Lo.getValueSizeInBits() <= VecSize && 2880 Hi.getValueSizeInBits() <= VecSize); 2881 2882 if (Lo == Hi && !isInlineImmediate(Lo.getNode())) { 2883 // Really a scalar input. Just select from the low half of the register to 2884 // avoid packing. 2885 2886 if (VecSize == 32 || VecSize == Lo.getValueSizeInBits()) { 2887 Src = Lo; 2888 } else { 2889 assert(Lo.getValueSizeInBits() == 32 && VecSize == 64); 2890 2891 SDLoc SL(In); 2892 SDValue Undef = SDValue( 2893 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, SL, 2894 Lo.getValueType()), 0); 2895 auto RC = Lo->isDivergent() ? AMDGPU::VReg_64RegClassID 2896 : AMDGPU::SReg_64RegClassID; 2897 const SDValue Ops[] = { 2898 CurDAG->getTargetConstant(RC, SL, MVT::i32), 2899 Lo, CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32), 2900 Undef, CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32) }; 2901 2902 Src = SDValue(CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, SL, 2903 Src.getValueType(), Ops), 0); 2904 } 2905 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 2906 return true; 2907 } 2908 2909 if (VecSize == 64 && Lo == Hi && isa<ConstantFPSDNode>(Lo)) { 2910 uint64_t Lit = cast<ConstantFPSDNode>(Lo)->getValueAPF() 2911 .bitcastToAPInt().getZExtValue(); 2912 if (AMDGPU::isInlinableLiteral32(Lit, Subtarget->hasInv2PiInlineImm())) { 2913 Src = CurDAG->getTargetConstant(Lit, SDLoc(In), MVT::i64);; 2914 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 2915 return true; 2916 } 2917 } 2918 2919 Mods = VecMods; 2920 } 2921 2922 // Packed instructions do not have abs modifiers. 2923 Mods |= SISrcMods::OP_SEL_1; 2924 2925 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 2926 return true; 2927 } 2928 2929 bool AMDGPUDAGToDAGISel::SelectVOP3OpSel(SDValue In, SDValue &Src, 2930 SDValue &SrcMods) const { 2931 Src = In; 2932 // FIXME: Handle op_sel 2933 SrcMods = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32); 2934 return true; 2935 } 2936 2937 bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods(SDValue In, SDValue &Src, 2938 SDValue &SrcMods) const { 2939 // FIXME: Handle op_sel 2940 return SelectVOP3Mods(In, Src, SrcMods); 2941 } 2942 2943 // The return value is not whether the match is possible (which it always is), 2944 // but whether or not it a conversion is really used. 2945 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, 2946 unsigned &Mods) const { 2947 Mods = 0; 2948 SelectVOP3ModsImpl(In, Src, Mods); 2949 2950 if (Src.getOpcode() == ISD::FP_EXTEND) { 2951 Src = Src.getOperand(0); 2952 assert(Src.getValueType() == MVT::f16); 2953 Src = stripBitcast(Src); 2954 2955 // Be careful about folding modifiers if we already have an abs. fneg is 2956 // applied last, so we don't want to apply an earlier fneg. 2957 if ((Mods & SISrcMods::ABS) == 0) { 2958 unsigned ModsTmp; 2959 SelectVOP3ModsImpl(Src, Src, ModsTmp); 2960 2961 if ((ModsTmp & SISrcMods::NEG) != 0) 2962 Mods ^= SISrcMods::NEG; 2963 2964 if ((ModsTmp & SISrcMods::ABS) != 0) 2965 Mods |= SISrcMods::ABS; 2966 } 2967 2968 // op_sel/op_sel_hi decide the source type and source. 2969 // If the source's op_sel_hi is set, it indicates to do a conversion from fp16. 2970 // If the sources's op_sel is set, it picks the high half of the source 2971 // register. 2972 2973 Mods |= SISrcMods::OP_SEL_1; 2974 if (isExtractHiElt(Src, Src)) { 2975 Mods |= SISrcMods::OP_SEL_0; 2976 2977 // TODO: Should we try to look for neg/abs here? 2978 } 2979 2980 return true; 2981 } 2982 2983 return false; 2984 } 2985 2986 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixMods(SDValue In, SDValue &Src, 2987 SDValue &SrcMods) const { 2988 unsigned Mods = 0; 2989 SelectVOP3PMadMixModsImpl(In, Src, Mods); 2990 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 2991 return true; 2992 } 2993 2994 SDValue AMDGPUDAGToDAGISel::getHi16Elt(SDValue In) const { 2995 if (In.isUndef()) 2996 return CurDAG->getUNDEF(MVT::i32); 2997 2998 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(In)) { 2999 SDLoc SL(In); 3000 return CurDAG->getConstant(C->getZExtValue() << 16, SL, MVT::i32); 3001 } 3002 3003 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(In)) { 3004 SDLoc SL(In); 3005 return CurDAG->getConstant( 3006 C->getValueAPF().bitcastToAPInt().getZExtValue() << 16, SL, MVT::i32); 3007 } 3008 3009 SDValue Src; 3010 if (isExtractHiElt(In, Src)) 3011 return Src; 3012 3013 return SDValue(); 3014 } 3015 3016 bool AMDGPUDAGToDAGISel::isVGPRImm(const SDNode * N) const { 3017 assert(CurDAG->getTarget().getTargetTriple().getArch() == Triple::amdgcn); 3018 3019 const SIRegisterInfo *SIRI = 3020 static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo()); 3021 const SIInstrInfo * SII = 3022 static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo()); 3023 3024 unsigned Limit = 0; 3025 bool AllUsesAcceptSReg = true; 3026 for (SDNode::use_iterator U = N->use_begin(), E = SDNode::use_end(); 3027 Limit < 10 && U != E; ++U, ++Limit) { 3028 const TargetRegisterClass *RC = getOperandRegClass(*U, U.getOperandNo()); 3029 3030 // If the register class is unknown, it could be an unknown 3031 // register class that needs to be an SGPR, e.g. an inline asm 3032 // constraint 3033 if (!RC || SIRI->isSGPRClass(RC)) 3034 return false; 3035 3036 if (RC != &AMDGPU::VS_32RegClass) { 3037 AllUsesAcceptSReg = false; 3038 SDNode * User = *U; 3039 if (User->isMachineOpcode()) { 3040 unsigned Opc = User->getMachineOpcode(); 3041 MCInstrDesc Desc = SII->get(Opc); 3042 if (Desc.isCommutable()) { 3043 unsigned OpIdx = Desc.getNumDefs() + U.getOperandNo(); 3044 unsigned CommuteIdx1 = TargetInstrInfo::CommuteAnyOperandIndex; 3045 if (SII->findCommutedOpIndices(Desc, OpIdx, CommuteIdx1)) { 3046 unsigned CommutedOpNo = CommuteIdx1 - Desc.getNumDefs(); 3047 const TargetRegisterClass *CommutedRC = getOperandRegClass(*U, CommutedOpNo); 3048 if (CommutedRC == &AMDGPU::VS_32RegClass) 3049 AllUsesAcceptSReg = true; 3050 } 3051 } 3052 } 3053 // If "AllUsesAcceptSReg == false" so far we haven't suceeded 3054 // commuting current user. This means have at least one use 3055 // that strictly require VGPR. Thus, we will not attempt to commute 3056 // other user instructions. 3057 if (!AllUsesAcceptSReg) 3058 break; 3059 } 3060 } 3061 return !AllUsesAcceptSReg && (Limit < 10); 3062 } 3063 3064 bool AMDGPUDAGToDAGISel::isUniformLoad(const SDNode * N) const { 3065 auto Ld = cast<LoadSDNode>(N); 3066 3067 return Ld->getAlignment() >= 4 && 3068 ( 3069 ( 3070 ( 3071 Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS || 3072 Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT 3073 ) 3074 && 3075 !N->isDivergent() 3076 ) 3077 || 3078 ( 3079 Subtarget->getScalarizeGlobalBehavior() && 3080 Ld->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS && 3081 Ld->isSimple() && 3082 !N->isDivergent() && 3083 static_cast<const SITargetLowering *>( 3084 getTargetLowering())->isMemOpHasNoClobberedMemOperand(N) 3085 ) 3086 ); 3087 } 3088 3089 void AMDGPUDAGToDAGISel::PostprocessISelDAG() { 3090 const AMDGPUTargetLowering& Lowering = 3091 *static_cast<const AMDGPUTargetLowering*>(getTargetLowering()); 3092 bool IsModified = false; 3093 do { 3094 IsModified = false; 3095 3096 // Go over all selected nodes and try to fold them a bit more 3097 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_begin(); 3098 while (Position != CurDAG->allnodes_end()) { 3099 SDNode *Node = &*Position++; 3100 MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(Node); 3101 if (!MachineNode) 3102 continue; 3103 3104 SDNode *ResNode = Lowering.PostISelFolding(MachineNode, *CurDAG); 3105 if (ResNode != Node) { 3106 if (ResNode) 3107 ReplaceUses(Node, ResNode); 3108 IsModified = true; 3109 } 3110 } 3111 CurDAG->RemoveDeadNodes(); 3112 } while (IsModified); 3113 } 3114 3115 bool R600DAGToDAGISel::runOnMachineFunction(MachineFunction &MF) { 3116 Subtarget = &MF.getSubtarget<R600Subtarget>(); 3117 return SelectionDAGISel::runOnMachineFunction(MF); 3118 } 3119 3120 bool R600DAGToDAGISel::isConstantLoad(const MemSDNode *N, int CbId) const { 3121 if (!N->readMem()) 3122 return false; 3123 if (CbId == -1) 3124 return N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS || 3125 N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT; 3126 3127 return N->getAddressSpace() == AMDGPUAS::CONSTANT_BUFFER_0 + CbId; 3128 } 3129 3130 bool R600DAGToDAGISel::SelectGlobalValueConstantOffset(SDValue Addr, 3131 SDValue& IntPtr) { 3132 if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Addr)) { 3133 IntPtr = CurDAG->getIntPtrConstant(Cst->getZExtValue() / 4, SDLoc(Addr), 3134 true); 3135 return true; 3136 } 3137 return false; 3138 } 3139 3140 bool R600DAGToDAGISel::SelectGlobalValueVariableOffset(SDValue Addr, 3141 SDValue& BaseReg, SDValue &Offset) { 3142 if (!isa<ConstantSDNode>(Addr)) { 3143 BaseReg = Addr; 3144 Offset = CurDAG->getIntPtrConstant(0, SDLoc(Addr), true); 3145 return true; 3146 } 3147 return false; 3148 } 3149 3150 void R600DAGToDAGISel::Select(SDNode *N) { 3151 unsigned int Opc = N->getOpcode(); 3152 if (N->isMachineOpcode()) { 3153 N->setNodeId(-1); 3154 return; // Already selected. 3155 } 3156 3157 switch (Opc) { 3158 default: break; 3159 case AMDGPUISD::BUILD_VERTICAL_VECTOR: 3160 case ISD::SCALAR_TO_VECTOR: 3161 case ISD::BUILD_VECTOR: { 3162 EVT VT = N->getValueType(0); 3163 unsigned NumVectorElts = VT.getVectorNumElements(); 3164 unsigned RegClassID; 3165 // BUILD_VECTOR was lowered into an IMPLICIT_DEF + 4 INSERT_SUBREG 3166 // that adds a 128 bits reg copy when going through TwoAddressInstructions 3167 // pass. We want to avoid 128 bits copies as much as possible because they 3168 // can't be bundled by our scheduler. 3169 switch(NumVectorElts) { 3170 case 2: RegClassID = R600::R600_Reg64RegClassID; break; 3171 case 4: 3172 if (Opc == AMDGPUISD::BUILD_VERTICAL_VECTOR) 3173 RegClassID = R600::R600_Reg128VerticalRegClassID; 3174 else 3175 RegClassID = R600::R600_Reg128RegClassID; 3176 break; 3177 default: llvm_unreachable("Do not know how to lower this BUILD_VECTOR"); 3178 } 3179 SelectBuildVector(N, RegClassID); 3180 return; 3181 } 3182 } 3183 3184 SelectCode(N); 3185 } 3186 3187 bool R600DAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base, 3188 SDValue &Offset) { 3189 ConstantSDNode *C; 3190 SDLoc DL(Addr); 3191 3192 if ((C = dyn_cast<ConstantSDNode>(Addr))) { 3193 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32); 3194 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 3195 } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) && 3196 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) { 3197 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32); 3198 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 3199 } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) && 3200 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) { 3201 Base = Addr.getOperand(0); 3202 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 3203 } else { 3204 Base = Addr; 3205 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32); 3206 } 3207 3208 return true; 3209 } 3210 3211 bool R600DAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base, 3212 SDValue &Offset) { 3213 ConstantSDNode *IMMOffset; 3214 3215 if (Addr.getOpcode() == ISD::ADD 3216 && (IMMOffset = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) 3217 && isInt<16>(IMMOffset->getZExtValue())) { 3218 3219 Base = Addr.getOperand(0); 3220 Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr), 3221 MVT::i32); 3222 return true; 3223 // If the pointer address is constant, we can move it to the offset field. 3224 } else if ((IMMOffset = dyn_cast<ConstantSDNode>(Addr)) 3225 && isInt<16>(IMMOffset->getZExtValue())) { 3226 Base = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), 3227 SDLoc(CurDAG->getEntryNode()), 3228 R600::ZERO, MVT::i32); 3229 Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr), 3230 MVT::i32); 3231 return true; 3232 } 3233 3234 // Default case, no offset 3235 Base = Addr; 3236 Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32); 3237 return true; 3238 } 3239