1 //===- AMDGPUInstructionSelector --------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// \file 9 /// This file declares the targeting of the InstructionSelector class for 10 /// AMDGPU. 11 //===----------------------------------------------------------------------===// 12 13 #ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUINSTRUCTIONSELECTOR_H 14 #define LLVM_LIB_TARGET_AMDGPU_AMDGPUINSTRUCTIONSELECTOR_H 15 16 #include "SIDefines.h" 17 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 18 #include "llvm/IR/InstrTypes.h" 19 20 namespace { 21 #define GET_GLOBALISEL_PREDICATE_BITSET 22 #define AMDGPUSubtarget GCNSubtarget 23 #include "AMDGPUGenGlobalISel.inc" 24 #undef GET_GLOBALISEL_PREDICATE_BITSET 25 #undef AMDGPUSubtarget 26 } 27 28 namespace llvm { 29 30 namespace AMDGPU { 31 struct ImageDimIntrinsicInfo; 32 } 33 34 class AMDGPURegisterBankInfo; 35 class AMDGPUTargetMachine; 36 class BlockFrequencyInfo; 37 class ProfileSummaryInfo; 38 class GCNSubtarget; 39 class MachineInstr; 40 class MachineIRBuilder; 41 class MachineOperand; 42 class MachineRegisterInfo; 43 class RegisterBank; 44 class SIInstrInfo; 45 class SIRegisterInfo; 46 class TargetRegisterClass; 47 48 class AMDGPUInstructionSelector final : public InstructionSelector { 49 private: 50 MachineRegisterInfo *MRI; 51 const GCNSubtarget *Subtarget; 52 53 public: 54 AMDGPUInstructionSelector(const GCNSubtarget &STI, 55 const AMDGPURegisterBankInfo &RBI, 56 const AMDGPUTargetMachine &TM); 57 58 bool select(MachineInstr &I) override; 59 static const char *getName(); 60 61 void setupMF(MachineFunction &MF, GISelValueTracking *VT, 62 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, 63 BlockFrequencyInfo *BFI) override; 64 65 private: 66 struct GEPInfo { 67 SmallVector<unsigned, 2> SgprParts; 68 SmallVector<unsigned, 2> VgprParts; 69 int64_t Imm = 0; 70 }; 71 72 bool isSGPR(Register Reg) const; 73 74 bool isInstrUniform(const MachineInstr &MI) const; 75 bool isVCC(Register Reg, const MachineRegisterInfo &MRI) const; 76 77 const RegisterBank *getArtifactRegBank( 78 Register Reg, const MachineRegisterInfo &MRI, 79 const TargetRegisterInfo &TRI) const; 80 81 /// tblgen-erated 'select' implementation. 82 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; 83 84 MachineOperand getSubOperand64(MachineOperand &MO, 85 const TargetRegisterClass &SubRC, 86 unsigned SubIdx) const; 87 88 bool constrainCopyLikeIntrin(MachineInstr &MI, unsigned NewOpc) const; 89 bool selectCOPY(MachineInstr &I) const; 90 bool selectCOPY_SCC_VCC(MachineInstr &I) const; 91 bool selectCOPY_VCC_SCC(MachineInstr &I) const; 92 bool selectReadAnyLane(MachineInstr &I) const; 93 bool selectPHI(MachineInstr &I) const; 94 bool selectG_TRUNC(MachineInstr &I) const; 95 bool selectG_SZA_EXT(MachineInstr &I) const; 96 bool selectG_FPEXT(MachineInstr &I) const; 97 bool selectG_FNEG(MachineInstr &I) const; 98 bool selectG_FABS(MachineInstr &I) const; 99 bool selectG_AND_OR_XOR(MachineInstr &I) const; 100 bool selectG_ADD_SUB(MachineInstr &I) const; 101 bool selectG_UADDO_USUBO_UADDE_USUBE(MachineInstr &I) const; 102 bool selectG_AMDGPU_MAD_64_32(MachineInstr &I) const; 103 bool selectG_EXTRACT(MachineInstr &I) const; 104 bool selectG_FMA_FMAD(MachineInstr &I) const; 105 bool selectG_MERGE_VALUES(MachineInstr &I) const; 106 bool selectG_UNMERGE_VALUES(MachineInstr &I) const; 107 bool selectG_BUILD_VECTOR(MachineInstr &I) const; 108 bool selectG_IMPLICIT_DEF(MachineInstr &I) const; 109 bool selectG_INSERT(MachineInstr &I) const; 110 bool selectG_SBFX_UBFX(MachineInstr &I) const; 111 112 bool selectInterpP1F16(MachineInstr &MI) const; 113 bool selectWritelane(MachineInstr &MI) const; 114 bool selectDivScale(MachineInstr &MI) const; 115 bool selectIntrinsicCmp(MachineInstr &MI) const; 116 bool selectBallot(MachineInstr &I) const; 117 bool selectRelocConstant(MachineInstr &I) const; 118 bool selectGroupStaticSize(MachineInstr &I) const; 119 bool selectReturnAddress(MachineInstr &I) const; 120 bool selectG_INTRINSIC(MachineInstr &I) const; 121 122 bool selectEndCfIntrinsic(MachineInstr &MI) const; 123 bool selectDSOrderedIntrinsic(MachineInstr &MI, Intrinsic::ID IID) const; 124 bool selectDSGWSIntrinsic(MachineInstr &MI, Intrinsic::ID IID) const; 125 bool selectDSAppendConsume(MachineInstr &MI, bool IsAppend) const; 126 bool selectInitWholeWave(MachineInstr &MI) const; 127 bool selectSBarrier(MachineInstr &MI) const; 128 bool selectDSBvhStackIntrinsic(MachineInstr &MI) const; 129 130 bool selectImageIntrinsic(MachineInstr &MI, 131 const AMDGPU::ImageDimIntrinsicInfo *Intr) const; 132 bool selectG_INTRINSIC_W_SIDE_EFFECTS(MachineInstr &I) const; 133 int getS_CMPOpcode(CmpInst::Predicate P, unsigned Size) const; 134 bool selectG_ICMP_or_FCMP(MachineInstr &I) const; 135 bool hasVgprParts(ArrayRef<GEPInfo> AddrInfo) const; 136 void getAddrModeInfo(const MachineInstr &Load, const MachineRegisterInfo &MRI, 137 SmallVectorImpl<GEPInfo> &AddrInfo) const; 138 139 void initM0(MachineInstr &I) const; 140 bool selectG_LOAD_STORE_ATOMICRMW(MachineInstr &I) const; 141 bool selectG_SELECT(MachineInstr &I) const; 142 bool selectG_BRCOND(MachineInstr &I) const; 143 bool selectG_GLOBAL_VALUE(MachineInstr &I) const; 144 bool selectG_PTRMASK(MachineInstr &I) const; 145 bool selectG_EXTRACT_VECTOR_ELT(MachineInstr &I) const; 146 bool selectG_INSERT_VECTOR_ELT(MachineInstr &I) const; 147 bool selectBufferLoadLds(MachineInstr &MI) const; 148 bool selectGlobalLoadLds(MachineInstr &MI) const; 149 bool selectBVHIntersectRayIntrinsic(MachineInstr &I) const; 150 bool selectSMFMACIntrin(MachineInstr &I) const; 151 bool selectPermlaneSwapIntrin(MachineInstr &I, Intrinsic::ID IntrID) const; 152 bool selectWaveAddress(MachineInstr &I) const; 153 bool selectBITOP3(MachineInstr &I) const; 154 bool selectStackRestore(MachineInstr &MI) const; 155 bool selectNamedBarrierInit(MachineInstr &I, Intrinsic::ID IID) const; 156 bool selectNamedBarrierInst(MachineInstr &I, Intrinsic::ID IID) const; 157 bool selectSBarrierSignalIsfirst(MachineInstr &I, Intrinsic::ID IID) const; 158 bool selectSGetBarrierState(MachineInstr &I, Intrinsic::ID IID) const; 159 160 std::pair<Register, unsigned> selectVOP3ModsImpl(Register Src, 161 bool IsCanonicalizing = true, 162 bool AllowAbs = true, 163 bool OpSel = false) const; 164 165 Register copyToVGPRIfSrcFolded(Register Src, unsigned Mods, 166 MachineOperand Root, MachineInstr *InsertPt, 167 bool ForceVGPR = false) const; 168 169 InstructionSelector::ComplexRendererFns 170 selectVCSRC(MachineOperand &Root) const; 171 172 InstructionSelector::ComplexRendererFns 173 selectVSRC0(MachineOperand &Root) const; 174 175 InstructionSelector::ComplexRendererFns 176 selectVOP3Mods0(MachineOperand &Root) const; 177 InstructionSelector::ComplexRendererFns 178 selectVOP3BMods0(MachineOperand &Root) const; 179 InstructionSelector::ComplexRendererFns 180 selectVOP3OMods(MachineOperand &Root) const; 181 InstructionSelector::ComplexRendererFns 182 selectVOP3Mods(MachineOperand &Root) const; 183 InstructionSelector::ComplexRendererFns 184 selectVOP3ModsNonCanonicalizing(MachineOperand &Root) const; 185 InstructionSelector::ComplexRendererFns 186 selectVOP3BMods(MachineOperand &Root) const; 187 188 ComplexRendererFns selectVOP3NoMods(MachineOperand &Root) const; 189 190 std::pair<Register, unsigned> 191 selectVOP3PModsImpl(Register RootReg, const MachineRegisterInfo &MRI, 192 bool IsDOT = false) const; 193 InstructionSelector::ComplexRendererFns 194 selectVOP3PRetHelper(MachineOperand &Root, bool IsDOT = false) const; 195 196 InstructionSelector::ComplexRendererFns 197 selectVOP3PMods(MachineOperand &Root) const; 198 199 InstructionSelector::ComplexRendererFns 200 selectVOP3PModsDOT(MachineOperand &Root) const; 201 202 InstructionSelector::ComplexRendererFns 203 selectVOP3PModsNeg(MachineOperand &Root) const; 204 205 InstructionSelector::ComplexRendererFns 206 selectWMMAOpSelVOP3PMods(MachineOperand &Root) const; 207 208 InstructionSelector::ComplexRendererFns 209 selectWMMAModsF32NegAbs(MachineOperand &Root) const; 210 InstructionSelector::ComplexRendererFns 211 selectWMMAModsF16Neg(MachineOperand &Root) const; 212 InstructionSelector::ComplexRendererFns 213 selectWMMAModsF16NegAbs(MachineOperand &Root) const; 214 InstructionSelector::ComplexRendererFns 215 selectWMMAVISrc(MachineOperand &Root) const; 216 InstructionSelector::ComplexRendererFns 217 selectSWMMACIndex8(MachineOperand &Root) const; 218 InstructionSelector::ComplexRendererFns 219 selectSWMMACIndex16(MachineOperand &Root) const; 220 221 InstructionSelector::ComplexRendererFns 222 selectVOP3OpSelMods(MachineOperand &Root) const; 223 224 InstructionSelector::ComplexRendererFns 225 selectVINTERPMods(MachineOperand &Root) const; 226 InstructionSelector::ComplexRendererFns 227 selectVINTERPModsHi(MachineOperand &Root) const; 228 229 bool selectSmrdOffset(MachineOperand &Root, Register &Base, Register *SOffset, 230 int64_t *Offset) const; 231 InstructionSelector::ComplexRendererFns 232 selectSmrdImm(MachineOperand &Root) const; 233 InstructionSelector::ComplexRendererFns 234 selectSmrdImm32(MachineOperand &Root) const; 235 InstructionSelector::ComplexRendererFns 236 selectSmrdSgpr(MachineOperand &Root) const; 237 InstructionSelector::ComplexRendererFns 238 selectSmrdSgprImm(MachineOperand &Root) const; 239 240 std::pair<Register, int> selectFlatOffsetImpl(MachineOperand &Root, 241 uint64_t FlatVariant) const; 242 243 InstructionSelector::ComplexRendererFns 244 selectFlatOffset(MachineOperand &Root) const; 245 InstructionSelector::ComplexRendererFns 246 selectGlobalOffset(MachineOperand &Root) const; 247 InstructionSelector::ComplexRendererFns 248 selectScratchOffset(MachineOperand &Root) const; 249 250 InstructionSelector::ComplexRendererFns 251 selectGlobalSAddr(MachineOperand &Root) const; 252 253 InstructionSelector::ComplexRendererFns 254 selectScratchSAddr(MachineOperand &Root) const; 255 bool checkFlatScratchSVSSwizzleBug(Register VAddr, Register SAddr, 256 uint64_t ImmOffset) const; 257 InstructionSelector::ComplexRendererFns 258 selectScratchSVAddr(MachineOperand &Root) const; 259 260 InstructionSelector::ComplexRendererFns 261 selectMUBUFScratchOffen(MachineOperand &Root) const; 262 InstructionSelector::ComplexRendererFns 263 selectMUBUFScratchOffset(MachineOperand &Root) const; 264 265 bool isDSOffsetLegal(Register Base, int64_t Offset) const; 266 bool isDSOffset2Legal(Register Base, int64_t Offset0, int64_t Offset1, 267 unsigned Size) const; 268 bool isFlatScratchBaseLegal(Register Addr) const; 269 bool isFlatScratchBaseLegalSV(Register Addr) const; 270 bool isFlatScratchBaseLegalSVImm(Register Addr) const; 271 272 std::pair<Register, unsigned> 273 selectDS1Addr1OffsetImpl(MachineOperand &Root) const; 274 InstructionSelector::ComplexRendererFns 275 selectDS1Addr1Offset(MachineOperand &Root) const; 276 277 InstructionSelector::ComplexRendererFns 278 selectDS64Bit4ByteAligned(MachineOperand &Root) const; 279 280 InstructionSelector::ComplexRendererFns 281 selectDS128Bit8ByteAligned(MachineOperand &Root) const; 282 283 std::pair<Register, unsigned> selectDSReadWrite2Impl(MachineOperand &Root, 284 unsigned size) const; 285 InstructionSelector::ComplexRendererFns 286 selectDSReadWrite2(MachineOperand &Root, unsigned size) const; 287 288 std::pair<Register, int64_t> 289 getPtrBaseWithConstantOffset(Register Root, 290 const MachineRegisterInfo &MRI) const; 291 292 // Parse out a chain of up to two g_ptr_add instructions. 293 // g_ptr_add (n0, _) 294 // g_ptr_add (n0, (n1 = g_ptr_add n2, n3)) 295 struct MUBUFAddressData { 296 Register N0, N2, N3; 297 int64_t Offset = 0; 298 }; 299 300 bool shouldUseAddr64(MUBUFAddressData AddrData) const; 301 302 void splitIllegalMUBUFOffset(MachineIRBuilder &B, 303 Register &SOffset, int64_t &ImmOffset) const; 304 305 MUBUFAddressData parseMUBUFAddress(Register Src) const; 306 307 bool selectMUBUFAddr64Impl(MachineOperand &Root, Register &VAddr, 308 Register &RSrcReg, Register &SOffset, 309 int64_t &Offset) const; 310 311 bool selectMUBUFOffsetImpl(MachineOperand &Root, Register &RSrcReg, 312 Register &SOffset, int64_t &Offset) const; 313 314 InstructionSelector::ComplexRendererFns 315 selectBUFSOffset(MachineOperand &Root) const; 316 317 InstructionSelector::ComplexRendererFns 318 selectMUBUFAddr64(MachineOperand &Root) const; 319 320 InstructionSelector::ComplexRendererFns 321 selectMUBUFOffset(MachineOperand &Root) const; 322 323 ComplexRendererFns selectSMRDBufferImm(MachineOperand &Root) const; 324 ComplexRendererFns selectSMRDBufferImm32(MachineOperand &Root) const; 325 ComplexRendererFns selectSMRDBufferSgprImm(MachineOperand &Root) const; 326 327 std::pair<Register, unsigned> selectVOP3PMadMixModsImpl(MachineOperand &Root, 328 bool &Matched) const; 329 ComplexRendererFns selectVOP3PMadMixModsExt(MachineOperand &Root) const; 330 ComplexRendererFns selectVOP3PMadMixMods(MachineOperand &Root) const; 331 332 void renderTruncImm32(MachineInstrBuilder &MIB, const MachineInstr &MI, 333 int OpIdx = -1) const; 334 335 void renderTruncTImm(MachineInstrBuilder &MIB, const MachineInstr &MI, 336 int OpIdx) const; 337 void renderZextBoolTImm(MachineInstrBuilder &MIB, const MachineInstr &MI, 338 int OpIdx) const; 339 340 void renderOpSelTImm(MachineInstrBuilder &MIB, const MachineInstr &MI, 341 int OpIdx) const; 342 343 void renderSrcAndDstSelToOpSelXForm_0_0(MachineInstrBuilder &MIB, 344 const MachineInstr &MI, 345 int OpIdx) const; 346 347 void renderSrcAndDstSelToOpSelXForm_0_1(MachineInstrBuilder &MIB, 348 const MachineInstr &MI, 349 int OpIdx) const; 350 351 void renderSrcAndDstSelToOpSelXForm_1_0(MachineInstrBuilder &MIB, 352 const MachineInstr &MI, 353 int OpIdx) const; 354 355 void renderSrcAndDstSelToOpSelXForm_1_1(MachineInstrBuilder &MIB, 356 const MachineInstr &MI, 357 int OpIdx) const; 358 359 void renderDstSelToOpSelXForm(MachineInstrBuilder &MIB, 360 const MachineInstr &MI, int OpIdx) const; 361 362 void renderSrcSelToOpSelXForm(MachineInstrBuilder &MIB, 363 const MachineInstr &MI, int OpIdx) const; 364 365 void renderSrcAndDstSelToOpSelXForm_2_0(MachineInstrBuilder &MIB, 366 const MachineInstr &MI, 367 int OpIdx) const; 368 369 void renderDstSelToOpSel3XFormXForm(MachineInstrBuilder &MIB, 370 const MachineInstr &MI, int OpIdx) const; 371 372 void renderNegateImm(MachineInstrBuilder &MIB, const MachineInstr &MI, 373 int OpIdx) const; 374 375 void renderBitcastFPImm(MachineInstrBuilder &MIB, const MachineInstr &MI, 376 int OpIdx) const; 377 378 void renderBitcastFPImm32(MachineInstrBuilder &MIB, const MachineInstr &MI, 379 int OpIdx) const { 380 renderBitcastFPImm(MIB, MI, OpIdx); 381 } 382 void renderBitcastFPImm64(MachineInstrBuilder &MIB, const MachineInstr &MI, 383 int OpIdx) const { 384 renderBitcastFPImm(MIB, MI, OpIdx); 385 } 386 387 void renderPopcntImm(MachineInstrBuilder &MIB, const MachineInstr &MI, 388 int OpIdx) const; 389 void renderExtractCPol(MachineInstrBuilder &MIB, const MachineInstr &MI, 390 int OpIdx) const; 391 void renderExtractSWZ(MachineInstrBuilder &MIB, const MachineInstr &MI, 392 int OpIdx) const; 393 void renderExtractCpolSetGLC(MachineInstrBuilder &MIB, const MachineInstr &MI, 394 int OpIdx) const; 395 396 void renderFrameIndex(MachineInstrBuilder &MIB, const MachineInstr &MI, 397 int OpIdx) const; 398 399 void renderFPPow2ToExponent(MachineInstrBuilder &MIB, const MachineInstr &MI, 400 int OpIdx) const; 401 402 void renderRoundMode(MachineInstrBuilder &MIB, const MachineInstr &MI, 403 int OpIdx) const; 404 void renderScaledMAIIntrinsicOperand(MachineInstrBuilder &MIB, 405 const MachineInstr &MI, int OpIdx) const; 406 407 bool isInlineImmediate(const APInt &Imm) const; 408 bool isInlineImmediate(const APFloat &Imm) const; 409 410 // Returns true if TargetOpcode::G_AND MachineInstr `MI`'s masking of the 411 // shift amount operand's `ShAmtBits` bits is unneeded. 412 bool isUnneededShiftMask(const MachineInstr &MI, unsigned ShAmtBits) const; 413 414 const SIInstrInfo &TII; 415 const SIRegisterInfo &TRI; 416 const AMDGPURegisterBankInfo &RBI; 417 const AMDGPUTargetMachine &TM; 418 const GCNSubtarget &STI; 419 #define GET_GLOBALISEL_PREDICATES_DECL 420 #define AMDGPUSubtarget GCNSubtarget 421 #include "AMDGPUGenGlobalISel.inc" 422 #undef GET_GLOBALISEL_PREDICATES_DECL 423 #undef AMDGPUSubtarget 424 425 #define GET_GLOBALISEL_TEMPORARIES_DECL 426 #include "AMDGPUGenGlobalISel.inc" 427 #undef GET_GLOBALISEL_TEMPORARIES_DECL 428 }; 429 430 } // End llvm namespace. 431 #endif 432