1 //===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "AMDGPUBaseInfo.h" 10 #include "AMDGPU.h" 11 #include "AMDGPUAsmUtils.h" 12 #include "AMDKernelCodeT.h" 13 #include "GCNSubtarget.h" 14 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 15 #include "llvm/BinaryFormat/ELF.h" 16 #include "llvm/IR/Attributes.h" 17 #include "llvm/IR/Function.h" 18 #include "llvm/IR/GlobalValue.h" 19 #include "llvm/IR/IntrinsicsAMDGPU.h" 20 #include "llvm/IR/IntrinsicsR600.h" 21 #include "llvm/IR/LLVMContext.h" 22 #include "llvm/MC/MCSubtargetInfo.h" 23 #include "llvm/Support/AMDHSAKernelDescriptor.h" 24 #include "llvm/Support/CommandLine.h" 25 #include "llvm/Support/TargetParser.h" 26 27 #define GET_INSTRINFO_NAMED_OPS 28 #define GET_INSTRMAP_INFO 29 #include "AMDGPUGenInstrInfo.inc" 30 31 static llvm::cl::opt<unsigned> AmdhsaCodeObjectVersion( 32 "amdhsa-code-object-version", llvm::cl::Hidden, 33 llvm::cl::desc("AMDHSA Code Object Version"), llvm::cl::init(4), 34 llvm::cl::ZeroOrMore); 35 36 namespace { 37 38 /// \returns Bit mask for given bit \p Shift and bit \p Width. 39 unsigned getBitMask(unsigned Shift, unsigned Width) { 40 return ((1 << Width) - 1) << Shift; 41 } 42 43 /// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width. 44 /// 45 /// \returns Packed \p Dst. 46 unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) { 47 Dst &= ~(1 << Shift) & ~getBitMask(Shift, Width); 48 Dst |= (Src << Shift) & getBitMask(Shift, Width); 49 return Dst; 50 } 51 52 /// Unpacks bits from \p Src for given bit \p Shift and bit \p Width. 53 /// 54 /// \returns Unpacked bits. 55 unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) { 56 return (Src & getBitMask(Shift, Width)) >> Shift; 57 } 58 59 /// \returns Vmcnt bit shift (lower bits). 60 unsigned getVmcntBitShiftLo() { return 0; } 61 62 /// \returns Vmcnt bit width (lower bits). 63 unsigned getVmcntBitWidthLo() { return 4; } 64 65 /// \returns Expcnt bit shift. 66 unsigned getExpcntBitShift() { return 4; } 67 68 /// \returns Expcnt bit width. 69 unsigned getExpcntBitWidth() { return 3; } 70 71 /// \returns Lgkmcnt bit shift. 72 unsigned getLgkmcntBitShift() { return 8; } 73 74 /// \returns Lgkmcnt bit width. 75 unsigned getLgkmcntBitWidth(unsigned VersionMajor) { 76 return (VersionMajor >= 10) ? 6 : 4; 77 } 78 79 /// \returns Vmcnt bit shift (higher bits). 80 unsigned getVmcntBitShiftHi() { return 14; } 81 82 /// \returns Vmcnt bit width (higher bits). 83 unsigned getVmcntBitWidthHi() { return 2; } 84 85 } // end namespace anonymous 86 87 namespace llvm { 88 89 namespace AMDGPU { 90 91 Optional<uint8_t> getHsaAbiVersion(const MCSubtargetInfo *STI) { 92 if (STI && STI->getTargetTriple().getOS() != Triple::AMDHSA) 93 return None; 94 95 switch (AmdhsaCodeObjectVersion) { 96 case 2: 97 return ELF::ELFABIVERSION_AMDGPU_HSA_V2; 98 case 3: 99 return ELF::ELFABIVERSION_AMDGPU_HSA_V3; 100 case 4: 101 return ELF::ELFABIVERSION_AMDGPU_HSA_V4; 102 default: 103 report_fatal_error(Twine("Unsupported AMDHSA Code Object Version ") + 104 Twine(AmdhsaCodeObjectVersion)); 105 } 106 } 107 108 bool isHsaAbiVersion2(const MCSubtargetInfo *STI) { 109 if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI)) 110 return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V2; 111 return false; 112 } 113 114 bool isHsaAbiVersion3(const MCSubtargetInfo *STI) { 115 if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI)) 116 return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V3; 117 return false; 118 } 119 120 bool isHsaAbiVersion4(const MCSubtargetInfo *STI) { 121 if (Optional<uint8_t> HsaAbiVer = getHsaAbiVersion(STI)) 122 return *HsaAbiVer == ELF::ELFABIVERSION_AMDGPU_HSA_V4; 123 return false; 124 } 125 126 bool isHsaAbiVersion3Or4(const MCSubtargetInfo *STI) { 127 return isHsaAbiVersion3(STI) || isHsaAbiVersion4(STI); 128 } 129 130 #define GET_MIMGBaseOpcodesTable_IMPL 131 #define GET_MIMGDimInfoTable_IMPL 132 #define GET_MIMGInfoTable_IMPL 133 #define GET_MIMGLZMappingTable_IMPL 134 #define GET_MIMGMIPMappingTable_IMPL 135 #define GET_MIMGG16MappingTable_IMPL 136 #include "AMDGPUGenSearchableTables.inc" 137 138 int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding, 139 unsigned VDataDwords, unsigned VAddrDwords) { 140 const MIMGInfo *Info = getMIMGOpcodeHelper(BaseOpcode, MIMGEncoding, 141 VDataDwords, VAddrDwords); 142 return Info ? Info->Opcode : -1; 143 } 144 145 const MIMGBaseOpcodeInfo *getMIMGBaseOpcode(unsigned Opc) { 146 const MIMGInfo *Info = getMIMGInfo(Opc); 147 return Info ? getMIMGBaseOpcodeInfo(Info->BaseOpcode) : nullptr; 148 } 149 150 int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) { 151 const MIMGInfo *OrigInfo = getMIMGInfo(Opc); 152 const MIMGInfo *NewInfo = 153 getMIMGOpcodeHelper(OrigInfo->BaseOpcode, OrigInfo->MIMGEncoding, 154 NewChannels, OrigInfo->VAddrDwords); 155 return NewInfo ? NewInfo->Opcode : -1; 156 } 157 158 unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode, 159 const MIMGDimInfo *Dim, bool IsA16, 160 bool IsG16Supported) { 161 unsigned AddrWords = BaseOpcode->NumExtraArgs; 162 unsigned AddrComponents = (BaseOpcode->Coordinates ? Dim->NumCoords : 0) + 163 (BaseOpcode->LodOrClampOrMip ? 1 : 0); 164 if (IsA16) 165 AddrWords += divideCeil(AddrComponents, 2); 166 else 167 AddrWords += AddrComponents; 168 169 // Note: For subtargets that support A16 but not G16, enabling A16 also 170 // enables 16 bit gradients. 171 // For subtargets that support A16 (operand) and G16 (done with a different 172 // instruction encoding), they are independent. 173 174 if (BaseOpcode->Gradients) { 175 if ((IsA16 && !IsG16Supported) || BaseOpcode->G16) 176 // There are two gradients per coordinate, we pack them separately. 177 // For the 3d case, 178 // we get (dy/du, dx/du) (-, dz/du) (dy/dv, dx/dv) (-, dz/dv) 179 AddrWords += alignTo<2>(Dim->NumGradients / 2); 180 else 181 AddrWords += Dim->NumGradients; 182 } 183 return AddrWords; 184 } 185 186 struct MUBUFInfo { 187 uint16_t Opcode; 188 uint16_t BaseOpcode; 189 uint8_t elements; 190 bool has_vaddr; 191 bool has_srsrc; 192 bool has_soffset; 193 bool IsBufferInv; 194 }; 195 196 struct MTBUFInfo { 197 uint16_t Opcode; 198 uint16_t BaseOpcode; 199 uint8_t elements; 200 bool has_vaddr; 201 bool has_srsrc; 202 bool has_soffset; 203 }; 204 205 struct SMInfo { 206 uint16_t Opcode; 207 bool IsBuffer; 208 }; 209 210 struct VOPInfo { 211 uint16_t Opcode; 212 bool IsSingle; 213 }; 214 215 #define GET_MTBUFInfoTable_DECL 216 #define GET_MTBUFInfoTable_IMPL 217 #define GET_MUBUFInfoTable_DECL 218 #define GET_MUBUFInfoTable_IMPL 219 #define GET_SMInfoTable_DECL 220 #define GET_SMInfoTable_IMPL 221 #define GET_VOP1InfoTable_DECL 222 #define GET_VOP1InfoTable_IMPL 223 #define GET_VOP2InfoTable_DECL 224 #define GET_VOP2InfoTable_IMPL 225 #define GET_VOP3InfoTable_DECL 226 #define GET_VOP3InfoTable_IMPL 227 #include "AMDGPUGenSearchableTables.inc" 228 229 int getMTBUFBaseOpcode(unsigned Opc) { 230 const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opc); 231 return Info ? Info->BaseOpcode : -1; 232 } 233 234 int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements) { 235 const MTBUFInfo *Info = getMTBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements); 236 return Info ? Info->Opcode : -1; 237 } 238 239 int getMTBUFElements(unsigned Opc) { 240 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 241 return Info ? Info->elements : 0; 242 } 243 244 bool getMTBUFHasVAddr(unsigned Opc) { 245 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 246 return Info ? Info->has_vaddr : false; 247 } 248 249 bool getMTBUFHasSrsrc(unsigned Opc) { 250 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 251 return Info ? Info->has_srsrc : false; 252 } 253 254 bool getMTBUFHasSoffset(unsigned Opc) { 255 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc); 256 return Info ? Info->has_soffset : false; 257 } 258 259 int getMUBUFBaseOpcode(unsigned Opc) { 260 const MUBUFInfo *Info = getMUBUFInfoFromOpcode(Opc); 261 return Info ? Info->BaseOpcode : -1; 262 } 263 264 int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements) { 265 const MUBUFInfo *Info = getMUBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements); 266 return Info ? Info->Opcode : -1; 267 } 268 269 int getMUBUFElements(unsigned Opc) { 270 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 271 return Info ? Info->elements : 0; 272 } 273 274 bool getMUBUFHasVAddr(unsigned Opc) { 275 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 276 return Info ? Info->has_vaddr : false; 277 } 278 279 bool getMUBUFHasSrsrc(unsigned Opc) { 280 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 281 return Info ? Info->has_srsrc : false; 282 } 283 284 bool getMUBUFHasSoffset(unsigned Opc) { 285 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 286 return Info ? Info->has_soffset : false; 287 } 288 289 bool getMUBUFIsBufferInv(unsigned Opc) { 290 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc); 291 return Info ? Info->IsBufferInv : false; 292 } 293 294 bool getSMEMIsBuffer(unsigned Opc) { 295 const SMInfo *Info = getSMEMOpcodeHelper(Opc); 296 return Info ? Info->IsBuffer : false; 297 } 298 299 bool getVOP1IsSingle(unsigned Opc) { 300 const VOPInfo *Info = getVOP1OpcodeHelper(Opc); 301 return Info ? Info->IsSingle : false; 302 } 303 304 bool getVOP2IsSingle(unsigned Opc) { 305 const VOPInfo *Info = getVOP2OpcodeHelper(Opc); 306 return Info ? Info->IsSingle : false; 307 } 308 309 bool getVOP3IsSingle(unsigned Opc) { 310 const VOPInfo *Info = getVOP3OpcodeHelper(Opc); 311 return Info ? Info->IsSingle : false; 312 } 313 314 // Wrapper for Tablegen'd function. enum Subtarget is not defined in any 315 // header files, so we need to wrap it in a function that takes unsigned 316 // instead. 317 int getMCOpcode(uint16_t Opcode, unsigned Gen) { 318 return getMCOpcodeGen(Opcode, static_cast<Subtarget>(Gen)); 319 } 320 321 namespace IsaInfo { 322 323 AMDGPUTargetID::AMDGPUTargetID(const MCSubtargetInfo &STI) 324 : STI(STI), XnackSetting(TargetIDSetting::Any), 325 SramEccSetting(TargetIDSetting::Any) { 326 if (!STI.getFeatureBits().test(FeatureSupportsXNACK)) 327 XnackSetting = TargetIDSetting::Unsupported; 328 if (!STI.getFeatureBits().test(FeatureSupportsSRAMECC)) 329 SramEccSetting = TargetIDSetting::Unsupported; 330 } 331 332 void AMDGPUTargetID::setTargetIDFromFeaturesString(StringRef FS) { 333 // Check if xnack or sramecc is explicitly enabled or disabled. In the 334 // absence of the target features we assume we must generate code that can run 335 // in any environment. 336 SubtargetFeatures Features(FS); 337 Optional<bool> XnackRequested; 338 Optional<bool> SramEccRequested; 339 340 for (const std::string &Feature : Features.getFeatures()) { 341 if (Feature == "+xnack") 342 XnackRequested = true; 343 else if (Feature == "-xnack") 344 XnackRequested = false; 345 else if (Feature == "+sramecc") 346 SramEccRequested = true; 347 else if (Feature == "-sramecc") 348 SramEccRequested = false; 349 } 350 351 bool XnackSupported = isXnackSupported(); 352 bool SramEccSupported = isSramEccSupported(); 353 354 if (XnackRequested) { 355 if (XnackSupported) { 356 XnackSetting = 357 *XnackRequested ? TargetIDSetting::On : TargetIDSetting::Off; 358 } else { 359 // If a specific xnack setting was requested and this GPU does not support 360 // xnack emit a warning. Setting will remain set to "Unsupported". 361 if (*XnackRequested) { 362 errs() << "warning: xnack 'On' was requested for a processor that does " 363 "not support it!\n"; 364 } else { 365 errs() << "warning: xnack 'Off' was requested for a processor that " 366 "does not support it!\n"; 367 } 368 } 369 } 370 371 if (SramEccRequested) { 372 if (SramEccSupported) { 373 SramEccSetting = 374 *SramEccRequested ? TargetIDSetting::On : TargetIDSetting::Off; 375 } else { 376 // If a specific sramecc setting was requested and this GPU does not 377 // support sramecc emit a warning. Setting will remain set to 378 // "Unsupported". 379 if (*SramEccRequested) { 380 errs() << "warning: sramecc 'On' was requested for a processor that " 381 "does not support it!\n"; 382 } else { 383 errs() << "warning: sramecc 'Off' was requested for a processor that " 384 "does not support it!\n"; 385 } 386 } 387 } 388 } 389 390 static TargetIDSetting 391 getTargetIDSettingFromFeatureString(StringRef FeatureString) { 392 if (FeatureString.endswith("-")) 393 return TargetIDSetting::Off; 394 if (FeatureString.endswith("+")) 395 return TargetIDSetting::On; 396 397 llvm_unreachable("Malformed feature string"); 398 } 399 400 void AMDGPUTargetID::setTargetIDFromTargetIDStream(StringRef TargetID) { 401 SmallVector<StringRef, 3> TargetIDSplit; 402 TargetID.split(TargetIDSplit, ':'); 403 404 for (const auto &FeatureString : TargetIDSplit) { 405 if (FeatureString.startswith("xnack")) 406 XnackSetting = getTargetIDSettingFromFeatureString(FeatureString); 407 if (FeatureString.startswith("sramecc")) 408 SramEccSetting = getTargetIDSettingFromFeatureString(FeatureString); 409 } 410 } 411 412 std::string AMDGPUTargetID::toString() const { 413 std::string StringRep = ""; 414 raw_string_ostream StreamRep(StringRep); 415 416 auto TargetTriple = STI.getTargetTriple(); 417 auto Version = getIsaVersion(STI.getCPU()); 418 419 StreamRep << TargetTriple.getArchName() << '-' 420 << TargetTriple.getVendorName() << '-' 421 << TargetTriple.getOSName() << '-' 422 << TargetTriple.getEnvironmentName() << '-'; 423 424 std::string Processor = ""; 425 // TODO: Following else statement is present here because we used various 426 // alias names for GPUs up until GFX9 (e.g. 'fiji' is same as 'gfx803'). 427 // Remove once all aliases are removed from GCNProcessors.td. 428 if (Version.Major >= 9) 429 Processor = STI.getCPU().str(); 430 else 431 Processor = (Twine("gfx") + Twine(Version.Major) + Twine(Version.Minor) + 432 Twine(Version.Stepping)) 433 .str(); 434 435 std::string Features = ""; 436 if (Optional<uint8_t> HsaAbiVersion = getHsaAbiVersion(&STI)) { 437 switch (*HsaAbiVersion) { 438 case ELF::ELFABIVERSION_AMDGPU_HSA_V2: 439 // Code object V2 only supported specific processors and had fixed 440 // settings for the XNACK. 441 if (Processor == "gfx600") { 442 } else if (Processor == "gfx601") { 443 } else if (Processor == "gfx602") { 444 } else if (Processor == "gfx700") { 445 } else if (Processor == "gfx701") { 446 } else if (Processor == "gfx702") { 447 } else if (Processor == "gfx703") { 448 } else if (Processor == "gfx704") { 449 } else if (Processor == "gfx705") { 450 } else if (Processor == "gfx801") { 451 if (!isXnackOnOrAny()) 452 report_fatal_error( 453 "AMD GPU code object V2 does not support processor " + 454 Twine(Processor) + " without XNACK"); 455 } else if (Processor == "gfx802") { 456 } else if (Processor == "gfx803") { 457 } else if (Processor == "gfx805") { 458 } else if (Processor == "gfx810") { 459 if (!isXnackOnOrAny()) 460 report_fatal_error( 461 "AMD GPU code object V2 does not support processor " + 462 Twine(Processor) + " without XNACK"); 463 } else if (Processor == "gfx900") { 464 if (isXnackOnOrAny()) 465 Processor = "gfx901"; 466 } else if (Processor == "gfx902") { 467 if (isXnackOnOrAny()) 468 Processor = "gfx903"; 469 } else if (Processor == "gfx904") { 470 if (isXnackOnOrAny()) 471 Processor = "gfx905"; 472 } else if (Processor == "gfx906") { 473 if (isXnackOnOrAny()) 474 Processor = "gfx907"; 475 } else if (Processor == "gfx90c") { 476 if (isXnackOnOrAny()) 477 report_fatal_error( 478 "AMD GPU code object V2 does not support processor " + 479 Twine(Processor) + " with XNACK being ON or ANY"); 480 } else { 481 report_fatal_error( 482 "AMD GPU code object V2 does not support processor " + 483 Twine(Processor)); 484 } 485 break; 486 case ELF::ELFABIVERSION_AMDGPU_HSA_V3: 487 // xnack. 488 if (isXnackOnOrAny()) 489 Features += "+xnack"; 490 // In code object v2 and v3, "sramecc" feature was spelled with a 491 // hyphen ("sram-ecc"). 492 if (isSramEccOnOrAny()) 493 Features += "+sram-ecc"; 494 break; 495 case ELF::ELFABIVERSION_AMDGPU_HSA_V4: 496 // sramecc. 497 if (getSramEccSetting() == TargetIDSetting::Off) 498 Features += ":sramecc-"; 499 else if (getSramEccSetting() == TargetIDSetting::On) 500 Features += ":sramecc+"; 501 // xnack. 502 if (getXnackSetting() == TargetIDSetting::Off) 503 Features += ":xnack-"; 504 else if (getXnackSetting() == TargetIDSetting::On) 505 Features += ":xnack+"; 506 break; 507 default: 508 break; 509 } 510 } 511 512 StreamRep << Processor << Features; 513 514 StreamRep.flush(); 515 return StringRep; 516 } 517 518 unsigned getWavefrontSize(const MCSubtargetInfo *STI) { 519 if (STI->getFeatureBits().test(FeatureWavefrontSize16)) 520 return 16; 521 if (STI->getFeatureBits().test(FeatureWavefrontSize32)) 522 return 32; 523 524 return 64; 525 } 526 527 unsigned getLocalMemorySize(const MCSubtargetInfo *STI) { 528 if (STI->getFeatureBits().test(FeatureLocalMemorySize32768)) 529 return 32768; 530 if (STI->getFeatureBits().test(FeatureLocalMemorySize65536)) 531 return 65536; 532 533 return 0; 534 } 535 536 unsigned getEUsPerCU(const MCSubtargetInfo *STI) { 537 // "Per CU" really means "per whatever functional block the waves of a 538 // workgroup must share". For gfx10 in CU mode this is the CU, which contains 539 // two SIMDs. 540 if (isGFX10Plus(*STI) && STI->getFeatureBits().test(FeatureCuMode)) 541 return 2; 542 // Pre-gfx10 a CU contains four SIMDs. For gfx10 in WGP mode the WGP contains 543 // two CUs, so a total of four SIMDs. 544 return 4; 545 } 546 547 unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo *STI, 548 unsigned FlatWorkGroupSize) { 549 assert(FlatWorkGroupSize != 0); 550 if (STI->getTargetTriple().getArch() != Triple::amdgcn) 551 return 8; 552 unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize); 553 if (N == 1) 554 return 40; 555 N = 40 / N; 556 return std::min(N, 16u); 557 } 558 559 unsigned getMinWavesPerEU(const MCSubtargetInfo *STI) { 560 return 1; 561 } 562 563 unsigned getMaxWavesPerEU(const MCSubtargetInfo *STI) { 564 // FIXME: Need to take scratch memory into account. 565 if (isGFX90A(*STI)) 566 return 8; 567 if (!isGFX10Plus(*STI)) 568 return 10; 569 return hasGFX10_3Insts(*STI) ? 16 : 20; 570 } 571 572 unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo *STI, 573 unsigned FlatWorkGroupSize) { 574 return divideCeil(getWavesPerWorkGroup(STI, FlatWorkGroupSize), 575 getEUsPerCU(STI)); 576 } 577 578 unsigned getMinFlatWorkGroupSize(const MCSubtargetInfo *STI) { 579 return 1; 580 } 581 582 unsigned getMaxFlatWorkGroupSize(const MCSubtargetInfo *STI) { 583 // Some subtargets allow encoding 2048, but this isn't tested or supported. 584 return 1024; 585 } 586 587 unsigned getWavesPerWorkGroup(const MCSubtargetInfo *STI, 588 unsigned FlatWorkGroupSize) { 589 return divideCeil(FlatWorkGroupSize, getWavefrontSize(STI)); 590 } 591 592 unsigned getSGPRAllocGranule(const MCSubtargetInfo *STI) { 593 IsaVersion Version = getIsaVersion(STI->getCPU()); 594 if (Version.Major >= 10) 595 return getAddressableNumSGPRs(STI); 596 if (Version.Major >= 8) 597 return 16; 598 return 8; 599 } 600 601 unsigned getSGPREncodingGranule(const MCSubtargetInfo *STI) { 602 return 8; 603 } 604 605 unsigned getTotalNumSGPRs(const MCSubtargetInfo *STI) { 606 IsaVersion Version = getIsaVersion(STI->getCPU()); 607 if (Version.Major >= 8) 608 return 800; 609 return 512; 610 } 611 612 unsigned getAddressableNumSGPRs(const MCSubtargetInfo *STI) { 613 if (STI->getFeatureBits().test(FeatureSGPRInitBug)) 614 return FIXED_NUM_SGPRS_FOR_INIT_BUG; 615 616 IsaVersion Version = getIsaVersion(STI->getCPU()); 617 if (Version.Major >= 10) 618 return 106; 619 if (Version.Major >= 8) 620 return 102; 621 return 104; 622 } 623 624 unsigned getMinNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 625 assert(WavesPerEU != 0); 626 627 IsaVersion Version = getIsaVersion(STI->getCPU()); 628 if (Version.Major >= 10) 629 return 0; 630 631 if (WavesPerEU >= getMaxWavesPerEU(STI)) 632 return 0; 633 634 unsigned MinNumSGPRs = getTotalNumSGPRs(STI) / (WavesPerEU + 1); 635 if (STI->getFeatureBits().test(FeatureTrapHandler)) 636 MinNumSGPRs -= std::min(MinNumSGPRs, (unsigned)TRAP_NUM_SGPRS); 637 MinNumSGPRs = alignDown(MinNumSGPRs, getSGPRAllocGranule(STI)) + 1; 638 return std::min(MinNumSGPRs, getAddressableNumSGPRs(STI)); 639 } 640 641 unsigned getMaxNumSGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU, 642 bool Addressable) { 643 assert(WavesPerEU != 0); 644 645 unsigned AddressableNumSGPRs = getAddressableNumSGPRs(STI); 646 IsaVersion Version = getIsaVersion(STI->getCPU()); 647 if (Version.Major >= 10) 648 return Addressable ? AddressableNumSGPRs : 108; 649 if (Version.Major >= 8 && !Addressable) 650 AddressableNumSGPRs = 112; 651 unsigned MaxNumSGPRs = getTotalNumSGPRs(STI) / WavesPerEU; 652 if (STI->getFeatureBits().test(FeatureTrapHandler)) 653 MaxNumSGPRs -= std::min(MaxNumSGPRs, (unsigned)TRAP_NUM_SGPRS); 654 MaxNumSGPRs = alignDown(MaxNumSGPRs, getSGPRAllocGranule(STI)); 655 return std::min(MaxNumSGPRs, AddressableNumSGPRs); 656 } 657 658 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed, 659 bool FlatScrUsed, bool XNACKUsed) { 660 unsigned ExtraSGPRs = 0; 661 if (VCCUsed) 662 ExtraSGPRs = 2; 663 664 IsaVersion Version = getIsaVersion(STI->getCPU()); 665 if (Version.Major >= 10) 666 return ExtraSGPRs; 667 668 if (Version.Major < 8) { 669 if (FlatScrUsed) 670 ExtraSGPRs = 4; 671 } else { 672 if (XNACKUsed) 673 ExtraSGPRs = 4; 674 675 if (FlatScrUsed || 676 STI->getFeatureBits().test(AMDGPU::FeatureArchitectedFlatScratch)) 677 ExtraSGPRs = 6; 678 } 679 680 return ExtraSGPRs; 681 } 682 683 unsigned getNumExtraSGPRs(const MCSubtargetInfo *STI, bool VCCUsed, 684 bool FlatScrUsed) { 685 return getNumExtraSGPRs(STI, VCCUsed, FlatScrUsed, 686 STI->getFeatureBits().test(AMDGPU::FeatureXNACK)); 687 } 688 689 unsigned getNumSGPRBlocks(const MCSubtargetInfo *STI, unsigned NumSGPRs) { 690 NumSGPRs = alignTo(std::max(1u, NumSGPRs), getSGPREncodingGranule(STI)); 691 // SGPRBlocks is actual number of SGPR blocks minus 1. 692 return NumSGPRs / getSGPREncodingGranule(STI) - 1; 693 } 694 695 unsigned getVGPRAllocGranule(const MCSubtargetInfo *STI, 696 Optional<bool> EnableWavefrontSize32) { 697 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 698 return 8; 699 700 bool IsWave32 = EnableWavefrontSize32 ? 701 *EnableWavefrontSize32 : 702 STI->getFeatureBits().test(FeatureWavefrontSize32); 703 704 if (hasGFX10_3Insts(*STI)) 705 return IsWave32 ? 16 : 8; 706 707 return IsWave32 ? 8 : 4; 708 } 709 710 unsigned getVGPREncodingGranule(const MCSubtargetInfo *STI, 711 Optional<bool> EnableWavefrontSize32) { 712 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 713 return 8; 714 715 bool IsWave32 = EnableWavefrontSize32 ? 716 *EnableWavefrontSize32 : 717 STI->getFeatureBits().test(FeatureWavefrontSize32); 718 719 return IsWave32 ? 8 : 4; 720 } 721 722 unsigned getTotalNumVGPRs(const MCSubtargetInfo *STI) { 723 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 724 return 512; 725 if (!isGFX10Plus(*STI)) 726 return 256; 727 return STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1024 : 512; 728 } 729 730 unsigned getAddressableNumVGPRs(const MCSubtargetInfo *STI) { 731 if (STI->getFeatureBits().test(FeatureGFX90AInsts)) 732 return 512; 733 return 256; 734 } 735 736 unsigned getMinNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 737 assert(WavesPerEU != 0); 738 739 if (WavesPerEU >= getMaxWavesPerEU(STI)) 740 return 0; 741 unsigned MinNumVGPRs = 742 alignDown(getTotalNumVGPRs(STI) / (WavesPerEU + 1), 743 getVGPRAllocGranule(STI)) + 1; 744 return std::min(MinNumVGPRs, getAddressableNumVGPRs(STI)); 745 } 746 747 unsigned getMaxNumVGPRs(const MCSubtargetInfo *STI, unsigned WavesPerEU) { 748 assert(WavesPerEU != 0); 749 750 unsigned MaxNumVGPRs = alignDown(getTotalNumVGPRs(STI) / WavesPerEU, 751 getVGPRAllocGranule(STI)); 752 unsigned AddressableNumVGPRs = getAddressableNumVGPRs(STI); 753 return std::min(MaxNumVGPRs, AddressableNumVGPRs); 754 } 755 756 unsigned getNumVGPRBlocks(const MCSubtargetInfo *STI, unsigned NumVGPRs, 757 Optional<bool> EnableWavefrontSize32) { 758 NumVGPRs = alignTo(std::max(1u, NumVGPRs), 759 getVGPREncodingGranule(STI, EnableWavefrontSize32)); 760 // VGPRBlocks is actual number of VGPR blocks minus 1. 761 return NumVGPRs / getVGPREncodingGranule(STI, EnableWavefrontSize32) - 1; 762 } 763 764 } // end namespace IsaInfo 765 766 void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header, 767 const MCSubtargetInfo *STI) { 768 IsaVersion Version = getIsaVersion(STI->getCPU()); 769 770 memset(&Header, 0, sizeof(Header)); 771 772 Header.amd_kernel_code_version_major = 1; 773 Header.amd_kernel_code_version_minor = 2; 774 Header.amd_machine_kind = 1; // AMD_MACHINE_KIND_AMDGPU 775 Header.amd_machine_version_major = Version.Major; 776 Header.amd_machine_version_minor = Version.Minor; 777 Header.amd_machine_version_stepping = Version.Stepping; 778 Header.kernel_code_entry_byte_offset = sizeof(Header); 779 Header.wavefront_size = 6; 780 781 // If the code object does not support indirect functions, then the value must 782 // be 0xffffffff. 783 Header.call_convention = -1; 784 785 // These alignment values are specified in powers of two, so alignment = 786 // 2^n. The minimum alignment is 2^4 = 16. 787 Header.kernarg_segment_alignment = 4; 788 Header.group_segment_alignment = 4; 789 Header.private_segment_alignment = 4; 790 791 if (Version.Major >= 10) { 792 if (STI->getFeatureBits().test(FeatureWavefrontSize32)) { 793 Header.wavefront_size = 5; 794 Header.code_properties |= AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32; 795 } 796 Header.compute_pgm_resource_registers |= 797 S_00B848_WGP_MODE(STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1) | 798 S_00B848_MEM_ORDERED(1); 799 } 800 } 801 802 amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor( 803 const MCSubtargetInfo *STI) { 804 IsaVersion Version = getIsaVersion(STI->getCPU()); 805 806 amdhsa::kernel_descriptor_t KD; 807 memset(&KD, 0, sizeof(KD)); 808 809 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 810 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, 811 amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE); 812 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 813 amdhsa::COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP, 1); 814 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 815 amdhsa::COMPUTE_PGM_RSRC1_ENABLE_IEEE_MODE, 1); 816 AMDHSA_BITS_SET(KD.compute_pgm_rsrc2, 817 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, 1); 818 if (Version.Major >= 10) { 819 AMDHSA_BITS_SET(KD.kernel_code_properties, 820 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, 821 STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1 : 0); 822 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 823 amdhsa::COMPUTE_PGM_RSRC1_WGP_MODE, 824 STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1); 825 AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, 826 amdhsa::COMPUTE_PGM_RSRC1_MEM_ORDERED, 1); 827 } 828 if (AMDGPU::isGFX90A(*STI)) { 829 AMDHSA_BITS_SET(KD.compute_pgm_rsrc3, 830 amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, 831 STI->getFeatureBits().test(FeatureTgSplit) ? 1 : 0); 832 } 833 return KD; 834 } 835 836 bool isGroupSegment(const GlobalValue *GV) { 837 return GV->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS; 838 } 839 840 bool isGlobalSegment(const GlobalValue *GV) { 841 return GV->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS; 842 } 843 844 bool isReadOnlySegment(const GlobalValue *GV) { 845 unsigned AS = GV->getAddressSpace(); 846 return AS == AMDGPUAS::CONSTANT_ADDRESS || 847 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT; 848 } 849 850 bool shouldEmitConstantsToTextSection(const Triple &TT) { 851 return TT.getArch() == Triple::r600; 852 } 853 854 int getIntegerAttribute(const Function &F, StringRef Name, int Default) { 855 Attribute A = F.getFnAttribute(Name); 856 int Result = Default; 857 858 if (A.isStringAttribute()) { 859 StringRef Str = A.getValueAsString(); 860 if (Str.getAsInteger(0, Result)) { 861 LLVMContext &Ctx = F.getContext(); 862 Ctx.emitError("can't parse integer attribute " + Name); 863 } 864 } 865 866 return Result; 867 } 868 869 std::pair<int, int> getIntegerPairAttribute(const Function &F, 870 StringRef Name, 871 std::pair<int, int> Default, 872 bool OnlyFirstRequired) { 873 Attribute A = F.getFnAttribute(Name); 874 if (!A.isStringAttribute()) 875 return Default; 876 877 LLVMContext &Ctx = F.getContext(); 878 std::pair<int, int> Ints = Default; 879 std::pair<StringRef, StringRef> Strs = A.getValueAsString().split(','); 880 if (Strs.first.trim().getAsInteger(0, Ints.first)) { 881 Ctx.emitError("can't parse first integer attribute " + Name); 882 return Default; 883 } 884 if (Strs.second.trim().getAsInteger(0, Ints.second)) { 885 if (!OnlyFirstRequired || !Strs.second.trim().empty()) { 886 Ctx.emitError("can't parse second integer attribute " + Name); 887 return Default; 888 } 889 } 890 891 return Ints; 892 } 893 894 unsigned getVmcntBitMask(const IsaVersion &Version) { 895 unsigned VmcntLo = (1 << getVmcntBitWidthLo()) - 1; 896 if (Version.Major < 9) 897 return VmcntLo; 898 899 unsigned VmcntHi = ((1 << getVmcntBitWidthHi()) - 1) << getVmcntBitWidthLo(); 900 return VmcntLo | VmcntHi; 901 } 902 903 unsigned getExpcntBitMask(const IsaVersion &Version) { 904 return (1 << getExpcntBitWidth()) - 1; 905 } 906 907 unsigned getLgkmcntBitMask(const IsaVersion &Version) { 908 return (1 << getLgkmcntBitWidth(Version.Major)) - 1; 909 } 910 911 unsigned getWaitcntBitMask(const IsaVersion &Version) { 912 unsigned VmcntLo = getBitMask(getVmcntBitShiftLo(), getVmcntBitWidthLo()); 913 unsigned Expcnt = getBitMask(getExpcntBitShift(), getExpcntBitWidth()); 914 unsigned Lgkmcnt = getBitMask(getLgkmcntBitShift(), 915 getLgkmcntBitWidth(Version.Major)); 916 unsigned Waitcnt = VmcntLo | Expcnt | Lgkmcnt; 917 if (Version.Major < 9) 918 return Waitcnt; 919 920 unsigned VmcntHi = getBitMask(getVmcntBitShiftHi(), getVmcntBitWidthHi()); 921 return Waitcnt | VmcntHi; 922 } 923 924 unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) { 925 unsigned VmcntLo = 926 unpackBits(Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo()); 927 if (Version.Major < 9) 928 return VmcntLo; 929 930 unsigned VmcntHi = 931 unpackBits(Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi()); 932 VmcntHi <<= getVmcntBitWidthLo(); 933 return VmcntLo | VmcntHi; 934 } 935 936 unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) { 937 return unpackBits(Waitcnt, getExpcntBitShift(), getExpcntBitWidth()); 938 } 939 940 unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) { 941 return unpackBits(Waitcnt, getLgkmcntBitShift(), 942 getLgkmcntBitWidth(Version.Major)); 943 } 944 945 void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, 946 unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt) { 947 Vmcnt = decodeVmcnt(Version, Waitcnt); 948 Expcnt = decodeExpcnt(Version, Waitcnt); 949 Lgkmcnt = decodeLgkmcnt(Version, Waitcnt); 950 } 951 952 Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded) { 953 Waitcnt Decoded; 954 Decoded.VmCnt = decodeVmcnt(Version, Encoded); 955 Decoded.ExpCnt = decodeExpcnt(Version, Encoded); 956 Decoded.LgkmCnt = decodeLgkmcnt(Version, Encoded); 957 return Decoded; 958 } 959 960 unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt, 961 unsigned Vmcnt) { 962 Waitcnt = 963 packBits(Vmcnt, Waitcnt, getVmcntBitShiftLo(), getVmcntBitWidthLo()); 964 if (Version.Major < 9) 965 return Waitcnt; 966 967 Vmcnt >>= getVmcntBitWidthLo(); 968 return packBits(Vmcnt, Waitcnt, getVmcntBitShiftHi(), getVmcntBitWidthHi()); 969 } 970 971 unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt, 972 unsigned Expcnt) { 973 return packBits(Expcnt, Waitcnt, getExpcntBitShift(), getExpcntBitWidth()); 974 } 975 976 unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt, 977 unsigned Lgkmcnt) { 978 return packBits(Lgkmcnt, Waitcnt, getLgkmcntBitShift(), 979 getLgkmcntBitWidth(Version.Major)); 980 } 981 982 unsigned encodeWaitcnt(const IsaVersion &Version, 983 unsigned Vmcnt, unsigned Expcnt, unsigned Lgkmcnt) { 984 unsigned Waitcnt = getWaitcntBitMask(Version); 985 Waitcnt = encodeVmcnt(Version, Waitcnt, Vmcnt); 986 Waitcnt = encodeExpcnt(Version, Waitcnt, Expcnt); 987 Waitcnt = encodeLgkmcnt(Version, Waitcnt, Lgkmcnt); 988 return Waitcnt; 989 } 990 991 unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded) { 992 return encodeWaitcnt(Version, Decoded.VmCnt, Decoded.ExpCnt, Decoded.LgkmCnt); 993 } 994 995 //===----------------------------------------------------------------------===// 996 // hwreg 997 //===----------------------------------------------------------------------===// 998 999 namespace Hwreg { 1000 1001 int64_t getHwregId(const StringRef Name) { 1002 for (int Id = ID_SYMBOLIC_FIRST_; Id < ID_SYMBOLIC_LAST_; ++Id) { 1003 if (IdSymbolic[Id] && Name == IdSymbolic[Id]) 1004 return Id; 1005 } 1006 return ID_UNKNOWN_; 1007 } 1008 1009 static unsigned getLastSymbolicHwreg(const MCSubtargetInfo &STI) { 1010 if (isSI(STI) || isCI(STI) || isVI(STI)) 1011 return ID_SYMBOLIC_FIRST_GFX9_; 1012 else if (isGFX9(STI)) 1013 return ID_SYMBOLIC_FIRST_GFX10_; 1014 else if (isGFX10(STI) && !isGFX10_BEncoding(STI)) 1015 return ID_SYMBOLIC_FIRST_GFX1030_; 1016 else 1017 return ID_SYMBOLIC_LAST_; 1018 } 1019 1020 bool isValidHwreg(int64_t Id, const MCSubtargetInfo &STI) { 1021 return 1022 ID_SYMBOLIC_FIRST_ <= Id && Id < getLastSymbolicHwreg(STI) && 1023 IdSymbolic[Id] && (Id != ID_XNACK_MASK || !AMDGPU::isGFX10_BEncoding(STI)); 1024 } 1025 1026 bool isValidHwreg(int64_t Id) { 1027 return 0 <= Id && isUInt<ID_WIDTH_>(Id); 1028 } 1029 1030 bool isValidHwregOffset(int64_t Offset) { 1031 return 0 <= Offset && isUInt<OFFSET_WIDTH_>(Offset); 1032 } 1033 1034 bool isValidHwregWidth(int64_t Width) { 1035 return 0 <= (Width - 1) && isUInt<WIDTH_M1_WIDTH_>(Width - 1); 1036 } 1037 1038 uint64_t encodeHwreg(uint64_t Id, uint64_t Offset, uint64_t Width) { 1039 return (Id << ID_SHIFT_) | 1040 (Offset << OFFSET_SHIFT_) | 1041 ((Width - 1) << WIDTH_M1_SHIFT_); 1042 } 1043 1044 StringRef getHwreg(unsigned Id, const MCSubtargetInfo &STI) { 1045 return isValidHwreg(Id, STI) ? IdSymbolic[Id] : ""; 1046 } 1047 1048 void decodeHwreg(unsigned Val, unsigned &Id, unsigned &Offset, unsigned &Width) { 1049 Id = (Val & ID_MASK_) >> ID_SHIFT_; 1050 Offset = (Val & OFFSET_MASK_) >> OFFSET_SHIFT_; 1051 Width = ((Val & WIDTH_M1_MASK_) >> WIDTH_M1_SHIFT_) + 1; 1052 } 1053 1054 } // namespace Hwreg 1055 1056 //===----------------------------------------------------------------------===// 1057 // exp tgt 1058 //===----------------------------------------------------------------------===// 1059 1060 namespace Exp { 1061 1062 struct ExpTgt { 1063 StringLiteral Name; 1064 unsigned Tgt; 1065 unsigned MaxIndex; 1066 }; 1067 1068 static constexpr ExpTgt ExpTgtInfo[] = { 1069 {{"null"}, ET_NULL, ET_NULL_MAX_IDX}, 1070 {{"mrtz"}, ET_MRTZ, ET_MRTZ_MAX_IDX}, 1071 {{"prim"}, ET_PRIM, ET_PRIM_MAX_IDX}, 1072 {{"mrt"}, ET_MRT0, ET_MRT_MAX_IDX}, 1073 {{"pos"}, ET_POS0, ET_POS_MAX_IDX}, 1074 {{"param"}, ET_PARAM0, ET_PARAM_MAX_IDX}, 1075 }; 1076 1077 bool getTgtName(unsigned Id, StringRef &Name, int &Index) { 1078 for (const ExpTgt &Val : ExpTgtInfo) { 1079 if (Val.Tgt <= Id && Id <= Val.Tgt + Val.MaxIndex) { 1080 Index = (Val.MaxIndex == 0) ? -1 : (Id - Val.Tgt); 1081 Name = Val.Name; 1082 return true; 1083 } 1084 } 1085 return false; 1086 } 1087 1088 unsigned getTgtId(const StringRef Name) { 1089 1090 for (const ExpTgt &Val : ExpTgtInfo) { 1091 if (Val.MaxIndex == 0 && Name == Val.Name) 1092 return Val.Tgt; 1093 1094 if (Val.MaxIndex > 0 && Name.startswith(Val.Name)) { 1095 StringRef Suffix = Name.drop_front(Val.Name.size()); 1096 1097 unsigned Id; 1098 if (Suffix.getAsInteger(10, Id) || Id > Val.MaxIndex) 1099 return ET_INVALID; 1100 1101 // Disable leading zeroes 1102 if (Suffix.size() > 1 && Suffix[0] == '0') 1103 return ET_INVALID; 1104 1105 return Val.Tgt + Id; 1106 } 1107 } 1108 return ET_INVALID; 1109 } 1110 1111 bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI) { 1112 return (Id != ET_POS4 && Id != ET_PRIM) || isGFX10Plus(STI); 1113 } 1114 1115 } // namespace Exp 1116 1117 //===----------------------------------------------------------------------===// 1118 // MTBUF Format 1119 //===----------------------------------------------------------------------===// 1120 1121 namespace MTBUFFormat { 1122 1123 int64_t getDfmt(const StringRef Name) { 1124 for (int Id = DFMT_MIN; Id <= DFMT_MAX; ++Id) { 1125 if (Name == DfmtSymbolic[Id]) 1126 return Id; 1127 } 1128 return DFMT_UNDEF; 1129 } 1130 1131 StringRef getDfmtName(unsigned Id) { 1132 assert(Id <= DFMT_MAX); 1133 return DfmtSymbolic[Id]; 1134 } 1135 1136 static StringLiteral const *getNfmtLookupTable(const MCSubtargetInfo &STI) { 1137 if (isSI(STI) || isCI(STI)) 1138 return NfmtSymbolicSICI; 1139 if (isVI(STI) || isGFX9(STI)) 1140 return NfmtSymbolicVI; 1141 return NfmtSymbolicGFX10; 1142 } 1143 1144 int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI) { 1145 auto lookupTable = getNfmtLookupTable(STI); 1146 for (int Id = NFMT_MIN; Id <= NFMT_MAX; ++Id) { 1147 if (Name == lookupTable[Id]) 1148 return Id; 1149 } 1150 return NFMT_UNDEF; 1151 } 1152 1153 StringRef getNfmtName(unsigned Id, const MCSubtargetInfo &STI) { 1154 assert(Id <= NFMT_MAX); 1155 return getNfmtLookupTable(STI)[Id]; 1156 } 1157 1158 bool isValidDfmtNfmt(unsigned Id, const MCSubtargetInfo &STI) { 1159 unsigned Dfmt; 1160 unsigned Nfmt; 1161 decodeDfmtNfmt(Id, Dfmt, Nfmt); 1162 return isValidNfmt(Nfmt, STI); 1163 } 1164 1165 bool isValidNfmt(unsigned Id, const MCSubtargetInfo &STI) { 1166 return !getNfmtName(Id, STI).empty(); 1167 } 1168 1169 int64_t encodeDfmtNfmt(unsigned Dfmt, unsigned Nfmt) { 1170 return (Dfmt << DFMT_SHIFT) | (Nfmt << NFMT_SHIFT); 1171 } 1172 1173 void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt) { 1174 Dfmt = (Format >> DFMT_SHIFT) & DFMT_MASK; 1175 Nfmt = (Format >> NFMT_SHIFT) & NFMT_MASK; 1176 } 1177 1178 int64_t getUnifiedFormat(const StringRef Name) { 1179 for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) { 1180 if (Name == UfmtSymbolic[Id]) 1181 return Id; 1182 } 1183 return UFMT_UNDEF; 1184 } 1185 1186 StringRef getUnifiedFormatName(unsigned Id) { 1187 return isValidUnifiedFormat(Id) ? UfmtSymbolic[Id] : ""; 1188 } 1189 1190 bool isValidUnifiedFormat(unsigned Id) { 1191 return Id <= UFMT_LAST; 1192 } 1193 1194 int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt) { 1195 int64_t Fmt = encodeDfmtNfmt(Dfmt, Nfmt); 1196 for (int Id = UFMT_FIRST; Id <= UFMT_LAST; ++Id) { 1197 if (Fmt == DfmtNfmt2UFmt[Id]) 1198 return Id; 1199 } 1200 return UFMT_UNDEF; 1201 } 1202 1203 bool isValidFormatEncoding(unsigned Val, const MCSubtargetInfo &STI) { 1204 return isGFX10Plus(STI) ? (Val <= UFMT_MAX) : (Val <= DFMT_NFMT_MAX); 1205 } 1206 1207 unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI) { 1208 if (isGFX10Plus(STI)) 1209 return UFMT_DEFAULT; 1210 return DFMT_NFMT_DEFAULT; 1211 } 1212 1213 } // namespace MTBUFFormat 1214 1215 //===----------------------------------------------------------------------===// 1216 // SendMsg 1217 //===----------------------------------------------------------------------===// 1218 1219 namespace SendMsg { 1220 1221 int64_t getMsgId(const StringRef Name) { 1222 for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) { 1223 if (IdSymbolic[i] && Name == IdSymbolic[i]) 1224 return i; 1225 } 1226 return ID_UNKNOWN_; 1227 } 1228 1229 bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI, bool Strict) { 1230 if (Strict) { 1231 switch (MsgId) { 1232 case ID_SAVEWAVE: 1233 return isVI(STI) || isGFX9Plus(STI); 1234 case ID_STALL_WAVE_GEN: 1235 case ID_HALT_WAVES: 1236 case ID_ORDERED_PS_DONE: 1237 case ID_GS_ALLOC_REQ: 1238 case ID_GET_DOORBELL: 1239 return isGFX9Plus(STI); 1240 case ID_EARLY_PRIM_DEALLOC: 1241 return isGFX9(STI); 1242 case ID_GET_DDID: 1243 return isGFX10Plus(STI); 1244 default: 1245 return 0 <= MsgId && MsgId < ID_GAPS_LAST_ && IdSymbolic[MsgId]; 1246 } 1247 } else { 1248 return 0 <= MsgId && isUInt<ID_WIDTH_>(MsgId); 1249 } 1250 } 1251 1252 StringRef getMsgName(int64_t MsgId) { 1253 assert(0 <= MsgId && MsgId < ID_GAPS_LAST_); 1254 return IdSymbolic[MsgId]; 1255 } 1256 1257 int64_t getMsgOpId(int64_t MsgId, const StringRef Name) { 1258 const char* const *S = (MsgId == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic; 1259 const int F = (MsgId == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_; 1260 const int L = (MsgId == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_; 1261 for (int i = F; i < L; ++i) { 1262 if (Name == S[i]) { 1263 return i; 1264 } 1265 } 1266 return OP_UNKNOWN_; 1267 } 1268 1269 bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI, 1270 bool Strict) { 1271 assert(isValidMsgId(MsgId, STI, Strict)); 1272 1273 if (!Strict) 1274 return 0 <= OpId && isUInt<OP_WIDTH_>(OpId); 1275 1276 switch(MsgId) 1277 { 1278 case ID_GS: 1279 return (OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_) && OpId != OP_GS_NOP; 1280 case ID_GS_DONE: 1281 return OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_; 1282 case ID_SYSMSG: 1283 return OP_SYS_FIRST_ <= OpId && OpId < OP_SYS_LAST_; 1284 default: 1285 return OpId == OP_NONE_; 1286 } 1287 } 1288 1289 StringRef getMsgOpName(int64_t MsgId, int64_t OpId) { 1290 assert(msgRequiresOp(MsgId)); 1291 return (MsgId == ID_SYSMSG)? OpSysSymbolic[OpId] : OpGsSymbolic[OpId]; 1292 } 1293 1294 bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId, 1295 const MCSubtargetInfo &STI, bool Strict) { 1296 assert(isValidMsgOp(MsgId, OpId, STI, Strict)); 1297 1298 if (!Strict) 1299 return 0 <= StreamId && isUInt<STREAM_ID_WIDTH_>(StreamId); 1300 1301 switch(MsgId) 1302 { 1303 case ID_GS: 1304 return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_; 1305 case ID_GS_DONE: 1306 return (OpId == OP_GS_NOP)? 1307 (StreamId == STREAM_ID_NONE_) : 1308 (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_); 1309 default: 1310 return StreamId == STREAM_ID_NONE_; 1311 } 1312 } 1313 1314 bool msgRequiresOp(int64_t MsgId) { 1315 return MsgId == ID_GS || MsgId == ID_GS_DONE || MsgId == ID_SYSMSG; 1316 } 1317 1318 bool msgSupportsStream(int64_t MsgId, int64_t OpId) { 1319 return (MsgId == ID_GS || MsgId == ID_GS_DONE) && OpId != OP_GS_NOP; 1320 } 1321 1322 void decodeMsg(unsigned Val, 1323 uint16_t &MsgId, 1324 uint16_t &OpId, 1325 uint16_t &StreamId) { 1326 MsgId = Val & ID_MASK_; 1327 OpId = (Val & OP_MASK_) >> OP_SHIFT_; 1328 StreamId = (Val & STREAM_ID_MASK_) >> STREAM_ID_SHIFT_; 1329 } 1330 1331 uint64_t encodeMsg(uint64_t MsgId, 1332 uint64_t OpId, 1333 uint64_t StreamId) { 1334 return (MsgId << ID_SHIFT_) | 1335 (OpId << OP_SHIFT_) | 1336 (StreamId << STREAM_ID_SHIFT_); 1337 } 1338 1339 } // namespace SendMsg 1340 1341 //===----------------------------------------------------------------------===// 1342 // 1343 //===----------------------------------------------------------------------===// 1344 1345 unsigned getInitialPSInputAddr(const Function &F) { 1346 return getIntegerAttribute(F, "InitialPSInputAddr", 0); 1347 } 1348 1349 bool getHasColorExport(const Function &F) { 1350 // As a safe default always respond as if PS has color exports. 1351 return getIntegerAttribute( 1352 F, "amdgpu-color-export", 1353 F.getCallingConv() == CallingConv::AMDGPU_PS ? 1 : 0) != 0; 1354 } 1355 1356 bool getHasDepthExport(const Function &F) { 1357 return getIntegerAttribute(F, "amdgpu-depth-export", 0) != 0; 1358 } 1359 1360 bool isShader(CallingConv::ID cc) { 1361 switch(cc) { 1362 case CallingConv::AMDGPU_VS: 1363 case CallingConv::AMDGPU_LS: 1364 case CallingConv::AMDGPU_HS: 1365 case CallingConv::AMDGPU_ES: 1366 case CallingConv::AMDGPU_GS: 1367 case CallingConv::AMDGPU_PS: 1368 case CallingConv::AMDGPU_CS: 1369 return true; 1370 default: 1371 return false; 1372 } 1373 } 1374 1375 bool isGraphics(CallingConv::ID cc) { 1376 return isShader(cc) || cc == CallingConv::AMDGPU_Gfx; 1377 } 1378 1379 bool isCompute(CallingConv::ID cc) { 1380 return !isGraphics(cc) || cc == CallingConv::AMDGPU_CS; 1381 } 1382 1383 bool isEntryFunctionCC(CallingConv::ID CC) { 1384 switch (CC) { 1385 case CallingConv::AMDGPU_KERNEL: 1386 case CallingConv::SPIR_KERNEL: 1387 case CallingConv::AMDGPU_VS: 1388 case CallingConv::AMDGPU_GS: 1389 case CallingConv::AMDGPU_PS: 1390 case CallingConv::AMDGPU_CS: 1391 case CallingConv::AMDGPU_ES: 1392 case CallingConv::AMDGPU_HS: 1393 case CallingConv::AMDGPU_LS: 1394 return true; 1395 default: 1396 return false; 1397 } 1398 } 1399 1400 bool isModuleEntryFunctionCC(CallingConv::ID CC) { 1401 switch (CC) { 1402 case CallingConv::AMDGPU_Gfx: 1403 return true; 1404 default: 1405 return isEntryFunctionCC(CC); 1406 } 1407 } 1408 1409 bool hasXNACK(const MCSubtargetInfo &STI) { 1410 return STI.getFeatureBits()[AMDGPU::FeatureXNACK]; 1411 } 1412 1413 bool hasSRAMECC(const MCSubtargetInfo &STI) { 1414 return STI.getFeatureBits()[AMDGPU::FeatureSRAMECC]; 1415 } 1416 1417 bool hasMIMG_R128(const MCSubtargetInfo &STI) { 1418 return STI.getFeatureBits()[AMDGPU::FeatureMIMG_R128] && !STI.getFeatureBits()[AMDGPU::FeatureR128A16]; 1419 } 1420 1421 bool hasGFX10A16(const MCSubtargetInfo &STI) { 1422 return STI.getFeatureBits()[AMDGPU::FeatureGFX10A16]; 1423 } 1424 1425 bool hasG16(const MCSubtargetInfo &STI) { 1426 return STI.getFeatureBits()[AMDGPU::FeatureG16]; 1427 } 1428 1429 bool hasPackedD16(const MCSubtargetInfo &STI) { 1430 return !STI.getFeatureBits()[AMDGPU::FeatureUnpackedD16VMem]; 1431 } 1432 1433 bool isSI(const MCSubtargetInfo &STI) { 1434 return STI.getFeatureBits()[AMDGPU::FeatureSouthernIslands]; 1435 } 1436 1437 bool isCI(const MCSubtargetInfo &STI) { 1438 return STI.getFeatureBits()[AMDGPU::FeatureSeaIslands]; 1439 } 1440 1441 bool isVI(const MCSubtargetInfo &STI) { 1442 return STI.getFeatureBits()[AMDGPU::FeatureVolcanicIslands]; 1443 } 1444 1445 bool isGFX9(const MCSubtargetInfo &STI) { 1446 return STI.getFeatureBits()[AMDGPU::FeatureGFX9]; 1447 } 1448 1449 bool isGFX9Plus(const MCSubtargetInfo &STI) { 1450 return isGFX9(STI) || isGFX10Plus(STI); 1451 } 1452 1453 bool isGFX10(const MCSubtargetInfo &STI) { 1454 return STI.getFeatureBits()[AMDGPU::FeatureGFX10]; 1455 } 1456 1457 bool isGFX10Plus(const MCSubtargetInfo &STI) { return isGFX10(STI); } 1458 1459 bool isGCN3Encoding(const MCSubtargetInfo &STI) { 1460 return STI.getFeatureBits()[AMDGPU::FeatureGCN3Encoding]; 1461 } 1462 1463 bool isGFX10_AEncoding(const MCSubtargetInfo &STI) { 1464 return STI.getFeatureBits()[AMDGPU::FeatureGFX10_AEncoding]; 1465 } 1466 1467 bool isGFX10_BEncoding(const MCSubtargetInfo &STI) { 1468 return STI.getFeatureBits()[AMDGPU::FeatureGFX10_BEncoding]; 1469 } 1470 1471 bool hasGFX10_3Insts(const MCSubtargetInfo &STI) { 1472 return STI.getFeatureBits()[AMDGPU::FeatureGFX10_3Insts]; 1473 } 1474 1475 bool isGFX90A(const MCSubtargetInfo &STI) { 1476 return STI.getFeatureBits()[AMDGPU::FeatureGFX90AInsts]; 1477 } 1478 1479 bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI) { 1480 return STI.getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch]; 1481 } 1482 1483 bool isSGPR(unsigned Reg, const MCRegisterInfo* TRI) { 1484 const MCRegisterClass SGPRClass = TRI->getRegClass(AMDGPU::SReg_32RegClassID); 1485 const unsigned FirstSubReg = TRI->getSubReg(Reg, AMDGPU::sub0); 1486 return SGPRClass.contains(FirstSubReg != 0 ? FirstSubReg : Reg) || 1487 Reg == AMDGPU::SCC; 1488 } 1489 1490 bool isRegIntersect(unsigned Reg0, unsigned Reg1, const MCRegisterInfo* TRI) { 1491 for (MCRegAliasIterator R(Reg0, TRI, true); R.isValid(); ++R) { 1492 if (*R == Reg1) return true; 1493 } 1494 return false; 1495 } 1496 1497 #define MAP_REG2REG \ 1498 using namespace AMDGPU; \ 1499 switch(Reg) { \ 1500 default: return Reg; \ 1501 CASE_CI_VI(FLAT_SCR) \ 1502 CASE_CI_VI(FLAT_SCR_LO) \ 1503 CASE_CI_VI(FLAT_SCR_HI) \ 1504 CASE_VI_GFX9PLUS(TTMP0) \ 1505 CASE_VI_GFX9PLUS(TTMP1) \ 1506 CASE_VI_GFX9PLUS(TTMP2) \ 1507 CASE_VI_GFX9PLUS(TTMP3) \ 1508 CASE_VI_GFX9PLUS(TTMP4) \ 1509 CASE_VI_GFX9PLUS(TTMP5) \ 1510 CASE_VI_GFX9PLUS(TTMP6) \ 1511 CASE_VI_GFX9PLUS(TTMP7) \ 1512 CASE_VI_GFX9PLUS(TTMP8) \ 1513 CASE_VI_GFX9PLUS(TTMP9) \ 1514 CASE_VI_GFX9PLUS(TTMP10) \ 1515 CASE_VI_GFX9PLUS(TTMP11) \ 1516 CASE_VI_GFX9PLUS(TTMP12) \ 1517 CASE_VI_GFX9PLUS(TTMP13) \ 1518 CASE_VI_GFX9PLUS(TTMP14) \ 1519 CASE_VI_GFX9PLUS(TTMP15) \ 1520 CASE_VI_GFX9PLUS(TTMP0_TTMP1) \ 1521 CASE_VI_GFX9PLUS(TTMP2_TTMP3) \ 1522 CASE_VI_GFX9PLUS(TTMP4_TTMP5) \ 1523 CASE_VI_GFX9PLUS(TTMP6_TTMP7) \ 1524 CASE_VI_GFX9PLUS(TTMP8_TTMP9) \ 1525 CASE_VI_GFX9PLUS(TTMP10_TTMP11) \ 1526 CASE_VI_GFX9PLUS(TTMP12_TTMP13) \ 1527 CASE_VI_GFX9PLUS(TTMP14_TTMP15) \ 1528 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3) \ 1529 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7) \ 1530 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11) \ 1531 CASE_VI_GFX9PLUS(TTMP12_TTMP13_TTMP14_TTMP15) \ 1532 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7) \ 1533 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11) \ 1534 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \ 1535 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \ 1536 } 1537 1538 #define CASE_CI_VI(node) \ 1539 assert(!isSI(STI)); \ 1540 case node: return isCI(STI) ? node##_ci : node##_vi; 1541 1542 #define CASE_VI_GFX9PLUS(node) \ 1543 case node: return isGFX9Plus(STI) ? node##_gfx9plus : node##_vi; 1544 1545 unsigned getMCReg(unsigned Reg, const MCSubtargetInfo &STI) { 1546 if (STI.getTargetTriple().getArch() == Triple::r600) 1547 return Reg; 1548 MAP_REG2REG 1549 } 1550 1551 #undef CASE_CI_VI 1552 #undef CASE_VI_GFX9PLUS 1553 1554 #define CASE_CI_VI(node) case node##_ci: case node##_vi: return node; 1555 #define CASE_VI_GFX9PLUS(node) case node##_vi: case node##_gfx9plus: return node; 1556 1557 unsigned mc2PseudoReg(unsigned Reg) { 1558 MAP_REG2REG 1559 } 1560 1561 #undef CASE_CI_VI 1562 #undef CASE_VI_GFX9PLUS 1563 #undef MAP_REG2REG 1564 1565 bool isSISrcOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1566 assert(OpNo < Desc.NumOperands); 1567 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1568 return OpType >= AMDGPU::OPERAND_SRC_FIRST && 1569 OpType <= AMDGPU::OPERAND_SRC_LAST; 1570 } 1571 1572 bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1573 assert(OpNo < Desc.NumOperands); 1574 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1575 switch (OpType) { 1576 case AMDGPU::OPERAND_REG_IMM_FP32: 1577 case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED: 1578 case AMDGPU::OPERAND_REG_IMM_FP64: 1579 case AMDGPU::OPERAND_REG_IMM_FP16: 1580 case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED: 1581 case AMDGPU::OPERAND_REG_IMM_V2FP16: 1582 case AMDGPU::OPERAND_REG_IMM_V2INT16: 1583 case AMDGPU::OPERAND_REG_INLINE_C_FP32: 1584 case AMDGPU::OPERAND_REG_INLINE_C_FP64: 1585 case AMDGPU::OPERAND_REG_INLINE_C_FP16: 1586 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: 1587 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: 1588 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: 1589 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: 1590 case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: 1591 case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16: 1592 case AMDGPU::OPERAND_REG_IMM_V2FP32: 1593 case AMDGPU::OPERAND_REG_INLINE_C_V2FP32: 1594 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: 1595 return true; 1596 default: 1597 return false; 1598 } 1599 } 1600 1601 bool isSISrcInlinableOperand(const MCInstrDesc &Desc, unsigned OpNo) { 1602 assert(OpNo < Desc.NumOperands); 1603 unsigned OpType = Desc.OpInfo[OpNo].OperandType; 1604 return OpType >= AMDGPU::OPERAND_REG_INLINE_C_FIRST && 1605 OpType <= AMDGPU::OPERAND_REG_INLINE_C_LAST; 1606 } 1607 1608 // Avoid using MCRegisterClass::getSize, since that function will go away 1609 // (move from MC* level to Target* level). Return size in bits. 1610 unsigned getRegBitWidth(unsigned RCID) { 1611 switch (RCID) { 1612 case AMDGPU::VGPR_LO16RegClassID: 1613 case AMDGPU::VGPR_HI16RegClassID: 1614 case AMDGPU::SGPR_LO16RegClassID: 1615 case AMDGPU::AGPR_LO16RegClassID: 1616 return 16; 1617 case AMDGPU::SGPR_32RegClassID: 1618 case AMDGPU::VGPR_32RegClassID: 1619 case AMDGPU::VRegOrLds_32RegClassID: 1620 case AMDGPU::AGPR_32RegClassID: 1621 case AMDGPU::VS_32RegClassID: 1622 case AMDGPU::AV_32RegClassID: 1623 case AMDGPU::SReg_32RegClassID: 1624 case AMDGPU::SReg_32_XM0RegClassID: 1625 case AMDGPU::SRegOrLds_32RegClassID: 1626 return 32; 1627 case AMDGPU::SGPR_64RegClassID: 1628 case AMDGPU::VS_64RegClassID: 1629 case AMDGPU::SReg_64RegClassID: 1630 case AMDGPU::VReg_64RegClassID: 1631 case AMDGPU::AReg_64RegClassID: 1632 case AMDGPU::SReg_64_XEXECRegClassID: 1633 case AMDGPU::VReg_64_Align2RegClassID: 1634 case AMDGPU::AReg_64_Align2RegClassID: 1635 case AMDGPU::AV_64RegClassID: 1636 case AMDGPU::AV_64_Align2RegClassID: 1637 return 64; 1638 case AMDGPU::SGPR_96RegClassID: 1639 case AMDGPU::SReg_96RegClassID: 1640 case AMDGPU::VReg_96RegClassID: 1641 case AMDGPU::AReg_96RegClassID: 1642 case AMDGPU::VReg_96_Align2RegClassID: 1643 case AMDGPU::AReg_96_Align2RegClassID: 1644 case AMDGPU::AV_96RegClassID: 1645 case AMDGPU::AV_96_Align2RegClassID: 1646 return 96; 1647 case AMDGPU::SGPR_128RegClassID: 1648 case AMDGPU::SReg_128RegClassID: 1649 case AMDGPU::VReg_128RegClassID: 1650 case AMDGPU::AReg_128RegClassID: 1651 case AMDGPU::VReg_128_Align2RegClassID: 1652 case AMDGPU::AReg_128_Align2RegClassID: 1653 case AMDGPU::AV_128RegClassID: 1654 case AMDGPU::AV_128_Align2RegClassID: 1655 return 128; 1656 case AMDGPU::SGPR_160RegClassID: 1657 case AMDGPU::SReg_160RegClassID: 1658 case AMDGPU::VReg_160RegClassID: 1659 case AMDGPU::AReg_160RegClassID: 1660 case AMDGPU::VReg_160_Align2RegClassID: 1661 case AMDGPU::AReg_160_Align2RegClassID: 1662 case AMDGPU::AV_160RegClassID: 1663 case AMDGPU::AV_160_Align2RegClassID: 1664 return 160; 1665 case AMDGPU::SGPR_192RegClassID: 1666 case AMDGPU::SReg_192RegClassID: 1667 case AMDGPU::VReg_192RegClassID: 1668 case AMDGPU::AReg_192RegClassID: 1669 case AMDGPU::VReg_192_Align2RegClassID: 1670 case AMDGPU::AReg_192_Align2RegClassID: 1671 case AMDGPU::AV_192RegClassID: 1672 case AMDGPU::AV_192_Align2RegClassID: 1673 return 192; 1674 case AMDGPU::SGPR_224RegClassID: 1675 case AMDGPU::SReg_224RegClassID: 1676 case AMDGPU::VReg_224RegClassID: 1677 case AMDGPU::AReg_224RegClassID: 1678 case AMDGPU::VReg_224_Align2RegClassID: 1679 case AMDGPU::AReg_224_Align2RegClassID: 1680 case AMDGPU::AV_224RegClassID: 1681 case AMDGPU::AV_224_Align2RegClassID: 1682 return 224; 1683 case AMDGPU::SGPR_256RegClassID: 1684 case AMDGPU::SReg_256RegClassID: 1685 case AMDGPU::VReg_256RegClassID: 1686 case AMDGPU::AReg_256RegClassID: 1687 case AMDGPU::VReg_256_Align2RegClassID: 1688 case AMDGPU::AReg_256_Align2RegClassID: 1689 case AMDGPU::AV_256RegClassID: 1690 case AMDGPU::AV_256_Align2RegClassID: 1691 return 256; 1692 case AMDGPU::SGPR_512RegClassID: 1693 case AMDGPU::SReg_512RegClassID: 1694 case AMDGPU::VReg_512RegClassID: 1695 case AMDGPU::AReg_512RegClassID: 1696 case AMDGPU::VReg_512_Align2RegClassID: 1697 case AMDGPU::AReg_512_Align2RegClassID: 1698 case AMDGPU::AV_512RegClassID: 1699 case AMDGPU::AV_512_Align2RegClassID: 1700 return 512; 1701 case AMDGPU::SGPR_1024RegClassID: 1702 case AMDGPU::SReg_1024RegClassID: 1703 case AMDGPU::VReg_1024RegClassID: 1704 case AMDGPU::AReg_1024RegClassID: 1705 case AMDGPU::VReg_1024_Align2RegClassID: 1706 case AMDGPU::AReg_1024_Align2RegClassID: 1707 case AMDGPU::AV_1024RegClassID: 1708 case AMDGPU::AV_1024_Align2RegClassID: 1709 return 1024; 1710 default: 1711 llvm_unreachable("Unexpected register class"); 1712 } 1713 } 1714 1715 unsigned getRegBitWidth(const MCRegisterClass &RC) { 1716 return getRegBitWidth(RC.getID()); 1717 } 1718 1719 unsigned getRegOperandSize(const MCRegisterInfo *MRI, const MCInstrDesc &Desc, 1720 unsigned OpNo) { 1721 assert(OpNo < Desc.NumOperands); 1722 unsigned RCID = Desc.OpInfo[OpNo].RegClass; 1723 return getRegBitWidth(MRI->getRegClass(RCID)) / 8; 1724 } 1725 1726 bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi) { 1727 if (isInlinableIntLiteral(Literal)) 1728 return true; 1729 1730 uint64_t Val = static_cast<uint64_t>(Literal); 1731 return (Val == DoubleToBits(0.0)) || 1732 (Val == DoubleToBits(1.0)) || 1733 (Val == DoubleToBits(-1.0)) || 1734 (Val == DoubleToBits(0.5)) || 1735 (Val == DoubleToBits(-0.5)) || 1736 (Val == DoubleToBits(2.0)) || 1737 (Val == DoubleToBits(-2.0)) || 1738 (Val == DoubleToBits(4.0)) || 1739 (Val == DoubleToBits(-4.0)) || 1740 (Val == 0x3fc45f306dc9c882 && HasInv2Pi); 1741 } 1742 1743 bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi) { 1744 if (isInlinableIntLiteral(Literal)) 1745 return true; 1746 1747 // The actual type of the operand does not seem to matter as long 1748 // as the bits match one of the inline immediate values. For example: 1749 // 1750 // -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal, 1751 // so it is a legal inline immediate. 1752 // 1753 // 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in 1754 // floating-point, so it is a legal inline immediate. 1755 1756 uint32_t Val = static_cast<uint32_t>(Literal); 1757 return (Val == FloatToBits(0.0f)) || 1758 (Val == FloatToBits(1.0f)) || 1759 (Val == FloatToBits(-1.0f)) || 1760 (Val == FloatToBits(0.5f)) || 1761 (Val == FloatToBits(-0.5f)) || 1762 (Val == FloatToBits(2.0f)) || 1763 (Val == FloatToBits(-2.0f)) || 1764 (Val == FloatToBits(4.0f)) || 1765 (Val == FloatToBits(-4.0f)) || 1766 (Val == 0x3e22f983 && HasInv2Pi); 1767 } 1768 1769 bool isInlinableLiteral16(int16_t Literal, bool HasInv2Pi) { 1770 if (!HasInv2Pi) 1771 return false; 1772 1773 if (isInlinableIntLiteral(Literal)) 1774 return true; 1775 1776 uint16_t Val = static_cast<uint16_t>(Literal); 1777 return Val == 0x3C00 || // 1.0 1778 Val == 0xBC00 || // -1.0 1779 Val == 0x3800 || // 0.5 1780 Val == 0xB800 || // -0.5 1781 Val == 0x4000 || // 2.0 1782 Val == 0xC000 || // -2.0 1783 Val == 0x4400 || // 4.0 1784 Val == 0xC400 || // -4.0 1785 Val == 0x3118; // 1/2pi 1786 } 1787 1788 bool isInlinableLiteralV216(int32_t Literal, bool HasInv2Pi) { 1789 assert(HasInv2Pi); 1790 1791 if (isInt<16>(Literal) || isUInt<16>(Literal)) { 1792 int16_t Trunc = static_cast<int16_t>(Literal); 1793 return AMDGPU::isInlinableLiteral16(Trunc, HasInv2Pi); 1794 } 1795 if (!(Literal & 0xffff)) 1796 return AMDGPU::isInlinableLiteral16(Literal >> 16, HasInv2Pi); 1797 1798 int16_t Lo16 = static_cast<int16_t>(Literal); 1799 int16_t Hi16 = static_cast<int16_t>(Literal >> 16); 1800 return Lo16 == Hi16 && isInlinableLiteral16(Lo16, HasInv2Pi); 1801 } 1802 1803 bool isInlinableIntLiteralV216(int32_t Literal) { 1804 int16_t Lo16 = static_cast<int16_t>(Literal); 1805 if (isInt<16>(Literal) || isUInt<16>(Literal)) 1806 return isInlinableIntLiteral(Lo16); 1807 1808 int16_t Hi16 = static_cast<int16_t>(Literal >> 16); 1809 if (!(Literal & 0xffff)) 1810 return isInlinableIntLiteral(Hi16); 1811 return Lo16 == Hi16 && isInlinableIntLiteral(Lo16); 1812 } 1813 1814 bool isFoldableLiteralV216(int32_t Literal, bool HasInv2Pi) { 1815 assert(HasInv2Pi); 1816 1817 int16_t Lo16 = static_cast<int16_t>(Literal); 1818 if (isInt<16>(Literal) || isUInt<16>(Literal)) 1819 return true; 1820 1821 int16_t Hi16 = static_cast<int16_t>(Literal >> 16); 1822 if (!(Literal & 0xffff)) 1823 return true; 1824 return Lo16 == Hi16; 1825 } 1826 1827 bool isArgPassedInSGPR(const Argument *A) { 1828 const Function *F = A->getParent(); 1829 1830 // Arguments to compute shaders are never a source of divergence. 1831 CallingConv::ID CC = F->getCallingConv(); 1832 switch (CC) { 1833 case CallingConv::AMDGPU_KERNEL: 1834 case CallingConv::SPIR_KERNEL: 1835 return true; 1836 case CallingConv::AMDGPU_VS: 1837 case CallingConv::AMDGPU_LS: 1838 case CallingConv::AMDGPU_HS: 1839 case CallingConv::AMDGPU_ES: 1840 case CallingConv::AMDGPU_GS: 1841 case CallingConv::AMDGPU_PS: 1842 case CallingConv::AMDGPU_CS: 1843 case CallingConv::AMDGPU_Gfx: 1844 // For non-compute shaders, SGPR inputs are marked with either inreg or byval. 1845 // Everything else is in VGPRs. 1846 return F->getAttributes().hasParamAttr(A->getArgNo(), Attribute::InReg) || 1847 F->getAttributes().hasParamAttr(A->getArgNo(), Attribute::ByVal); 1848 default: 1849 // TODO: Should calls support inreg for SGPR inputs? 1850 return false; 1851 } 1852 } 1853 1854 static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) { 1855 return isGCN3Encoding(ST) || isGFX10Plus(ST); 1856 } 1857 1858 static bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST) { 1859 return isGFX9Plus(ST); 1860 } 1861 1862 bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST, 1863 int64_t EncodedOffset) { 1864 return hasSMEMByteOffset(ST) ? isUInt<20>(EncodedOffset) 1865 : isUInt<8>(EncodedOffset); 1866 } 1867 1868 bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST, 1869 int64_t EncodedOffset, 1870 bool IsBuffer) { 1871 return !IsBuffer && 1872 hasSMRDSignedImmOffset(ST) && 1873 isInt<21>(EncodedOffset); 1874 } 1875 1876 static bool isDwordAligned(uint64_t ByteOffset) { 1877 return (ByteOffset & 3) == 0; 1878 } 1879 1880 uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST, 1881 uint64_t ByteOffset) { 1882 if (hasSMEMByteOffset(ST)) 1883 return ByteOffset; 1884 1885 assert(isDwordAligned(ByteOffset)); 1886 return ByteOffset >> 2; 1887 } 1888 1889 Optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST, 1890 int64_t ByteOffset, bool IsBuffer) { 1891 // The signed version is always a byte offset. 1892 if (!IsBuffer && hasSMRDSignedImmOffset(ST)) { 1893 assert(hasSMEMByteOffset(ST)); 1894 return isInt<20>(ByteOffset) ? Optional<int64_t>(ByteOffset) : None; 1895 } 1896 1897 if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST)) 1898 return None; 1899 1900 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset); 1901 return isLegalSMRDEncodedUnsignedOffset(ST, EncodedOffset) 1902 ? Optional<int64_t>(EncodedOffset) 1903 : None; 1904 } 1905 1906 Optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST, 1907 int64_t ByteOffset) { 1908 if (!isCI(ST) || !isDwordAligned(ByteOffset)) 1909 return None; 1910 1911 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset); 1912 return isUInt<32>(EncodedOffset) ? Optional<int64_t>(EncodedOffset) : None; 1913 } 1914 1915 unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST, bool Signed) { 1916 // Address offset is 12-bit signed for GFX10, 13-bit for GFX9. 1917 if (AMDGPU::isGFX10(ST)) 1918 return Signed ? 12 : 11; 1919 1920 return Signed ? 13 : 12; 1921 } 1922 1923 // Given Imm, split it into the values to put into the SOffset and ImmOffset 1924 // fields in an MUBUF instruction. Return false if it is not possible (due to a 1925 // hardware bug needing a workaround). 1926 // 1927 // The required alignment ensures that individual address components remain 1928 // aligned if they are aligned to begin with. It also ensures that additional 1929 // offsets within the given alignment can be added to the resulting ImmOffset. 1930 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset, 1931 const GCNSubtarget *Subtarget, Align Alignment) { 1932 const uint32_t MaxImm = alignDown(4095, Alignment.value()); 1933 uint32_t Overflow = 0; 1934 1935 if (Imm > MaxImm) { 1936 if (Imm <= MaxImm + 64) { 1937 // Use an SOffset inline constant for 4..64 1938 Overflow = Imm - MaxImm; 1939 Imm = MaxImm; 1940 } else { 1941 // Try to keep the same value in SOffset for adjacent loads, so that 1942 // the corresponding register contents can be re-used. 1943 // 1944 // Load values with all low-bits (except for alignment bits) set into 1945 // SOffset, so that a larger range of values can be covered using 1946 // s_movk_i32. 1947 // 1948 // Atomic operations fail to work correctly when individual address 1949 // components are unaligned, even if their sum is aligned. 1950 uint32_t High = (Imm + Alignment.value()) & ~4095; 1951 uint32_t Low = (Imm + Alignment.value()) & 4095; 1952 Imm = Low; 1953 Overflow = High - Alignment.value(); 1954 } 1955 } 1956 1957 // There is a hardware bug in SI and CI which prevents address clamping in 1958 // MUBUF instructions from working correctly with SOffsets. The immediate 1959 // offset is unaffected. 1960 if (Overflow > 0 && 1961 Subtarget->getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS) 1962 return false; 1963 1964 ImmOffset = Imm; 1965 SOffset = Overflow; 1966 return true; 1967 } 1968 1969 SIModeRegisterDefaults::SIModeRegisterDefaults(const Function &F) { 1970 *this = getDefaultForCallingConv(F.getCallingConv()); 1971 1972 StringRef IEEEAttr = F.getFnAttribute("amdgpu-ieee").getValueAsString(); 1973 if (!IEEEAttr.empty()) 1974 IEEE = IEEEAttr == "true"; 1975 1976 StringRef DX10ClampAttr 1977 = F.getFnAttribute("amdgpu-dx10-clamp").getValueAsString(); 1978 if (!DX10ClampAttr.empty()) 1979 DX10Clamp = DX10ClampAttr == "true"; 1980 1981 StringRef DenormF32Attr = F.getFnAttribute("denormal-fp-math-f32").getValueAsString(); 1982 if (!DenormF32Attr.empty()) { 1983 DenormalMode DenormMode = parseDenormalFPAttribute(DenormF32Attr); 1984 FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE; 1985 FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE; 1986 } 1987 1988 StringRef DenormAttr = F.getFnAttribute("denormal-fp-math").getValueAsString(); 1989 if (!DenormAttr.empty()) { 1990 DenormalMode DenormMode = parseDenormalFPAttribute(DenormAttr); 1991 1992 if (DenormF32Attr.empty()) { 1993 FP32InputDenormals = DenormMode.Input == DenormalMode::IEEE; 1994 FP32OutputDenormals = DenormMode.Output == DenormalMode::IEEE; 1995 } 1996 1997 FP64FP16InputDenormals = DenormMode.Input == DenormalMode::IEEE; 1998 FP64FP16OutputDenormals = DenormMode.Output == DenormalMode::IEEE; 1999 } 2000 } 2001 2002 namespace { 2003 2004 struct SourceOfDivergence { 2005 unsigned Intr; 2006 }; 2007 const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr); 2008 2009 #define GET_SourcesOfDivergence_IMPL 2010 #define GET_Gfx9BufferFormat_IMPL 2011 #define GET_Gfx10PlusBufferFormat_IMPL 2012 #include "AMDGPUGenSearchableTables.inc" 2013 2014 } // end anonymous namespace 2015 2016 bool isIntrinsicSourceOfDivergence(unsigned IntrID) { 2017 return lookupSourceOfDivergence(IntrID); 2018 } 2019 2020 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp, 2021 uint8_t NumComponents, 2022 uint8_t NumFormat, 2023 const MCSubtargetInfo &STI) { 2024 return isGFX10Plus(STI) 2025 ? getGfx10PlusBufferFormatInfo(BitsPerComp, NumComponents, 2026 NumFormat) 2027 : getGfx9BufferFormatInfo(BitsPerComp, NumComponents, NumFormat); 2028 } 2029 2030 const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format, 2031 const MCSubtargetInfo &STI) { 2032 return isGFX10Plus(STI) ? getGfx10PlusBufferFormatInfo(Format) 2033 : getGfx9BufferFormatInfo(Format); 2034 } 2035 2036 } // namespace AMDGPU 2037 2038 raw_ostream &operator<<(raw_ostream &OS, 2039 const AMDGPU::IsaInfo::TargetIDSetting S) { 2040 switch (S) { 2041 case (AMDGPU::IsaInfo::TargetIDSetting::Unsupported): 2042 OS << "Unsupported"; 2043 break; 2044 case (AMDGPU::IsaInfo::TargetIDSetting::Any): 2045 OS << "Any"; 2046 break; 2047 case (AMDGPU::IsaInfo::TargetIDSetting::Off): 2048 OS << "Off"; 2049 break; 2050 case (AMDGPU::IsaInfo::TargetIDSetting::On): 2051 OS << "On"; 2052 break; 2053 } 2054 return OS; 2055 } 2056 2057 } // namespace llvm 2058