1 //===-- AMDGPUMCCodeEmitter.cpp - AMDGPU Code Emitter ---------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// The AMDGPU code emitter produces machine code that can be executed 11 /// directly on the GPU device. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/AMDGPUFixupKinds.h" 16 #include "MCTargetDesc/AMDGPUMCExpr.h" 17 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 18 #include "SIDefines.h" 19 #include "Utils/AMDGPUBaseInfo.h" 20 #include "llvm/ADT/APInt.h" 21 #include "llvm/MC/MCCodeEmitter.h" 22 #include "llvm/MC/MCContext.h" 23 #include "llvm/MC/MCExpr.h" 24 #include "llvm/MC/MCInstrInfo.h" 25 #include "llvm/MC/MCRegisterInfo.h" 26 #include "llvm/MC/MCSubtargetInfo.h" 27 #include "llvm/Support/Casting.h" 28 #include "llvm/Support/EndianStream.h" 29 #include <optional> 30 31 using namespace llvm; 32 33 namespace { 34 35 class AMDGPUMCCodeEmitter : public MCCodeEmitter { 36 const MCRegisterInfo &MRI; 37 const MCInstrInfo &MCII; 38 39 public: 40 AMDGPUMCCodeEmitter(const MCInstrInfo &MCII, const MCRegisterInfo &MRI) 41 : MRI(MRI), MCII(MCII) {} 42 43 /// Encode the instruction and write it to the OS. 44 void encodeInstruction(const MCInst &MI, SmallVectorImpl<char> &CB, 45 SmallVectorImpl<MCFixup> &Fixups, 46 const MCSubtargetInfo &STI) const override; 47 48 void getMachineOpValue(const MCInst &MI, const MCOperand &MO, APInt &Op, 49 SmallVectorImpl<MCFixup> &Fixups, 50 const MCSubtargetInfo &STI) const; 51 52 void getMachineOpValueT16(const MCInst &MI, unsigned OpNo, APInt &Op, 53 SmallVectorImpl<MCFixup> &Fixups, 54 const MCSubtargetInfo &STI) const; 55 56 void getMachineOpValueT16Lo128(const MCInst &MI, unsigned OpNo, APInt &Op, 57 SmallVectorImpl<MCFixup> &Fixups, 58 const MCSubtargetInfo &STI) const; 59 60 /// Use a fixup to encode the simm16 field for SOPP branch 61 /// instructions. 62 void getSOPPBrEncoding(const MCInst &MI, unsigned OpNo, APInt &Op, 63 SmallVectorImpl<MCFixup> &Fixups, 64 const MCSubtargetInfo &STI) const; 65 66 void getSMEMOffsetEncoding(const MCInst &MI, unsigned OpNo, APInt &Op, 67 SmallVectorImpl<MCFixup> &Fixups, 68 const MCSubtargetInfo &STI) const; 69 70 void getSDWASrcEncoding(const MCInst &MI, unsigned OpNo, APInt &Op, 71 SmallVectorImpl<MCFixup> &Fixups, 72 const MCSubtargetInfo &STI) const; 73 74 void getSDWAVopcDstEncoding(const MCInst &MI, unsigned OpNo, APInt &Op, 75 SmallVectorImpl<MCFixup> &Fixups, 76 const MCSubtargetInfo &STI) const; 77 78 void getAVOperandEncoding(const MCInst &MI, unsigned OpNo, APInt &Op, 79 SmallVectorImpl<MCFixup> &Fixups, 80 const MCSubtargetInfo &STI) const; 81 82 private: 83 uint64_t getImplicitOpSelHiEncoding(int Opcode) const; 84 void getMachineOpValueCommon(const MCInst &MI, const MCOperand &MO, 85 unsigned OpNo, APInt &Op, 86 SmallVectorImpl<MCFixup> &Fixups, 87 const MCSubtargetInfo &STI) const; 88 89 /// Encode an fp or int literal. 90 std::optional<uint64_t> 91 getLitEncoding(const MCOperand &MO, const MCOperandInfo &OpInfo, 92 const MCSubtargetInfo &STI, 93 bool HasMandatoryLiteral = false) const; 94 95 void getBinaryCodeForInstr(const MCInst &MI, SmallVectorImpl<MCFixup> &Fixups, 96 APInt &Inst, APInt &Scratch, 97 const MCSubtargetInfo &STI) const; 98 }; 99 100 } // end anonymous namespace 101 102 MCCodeEmitter *llvm::createAMDGPUMCCodeEmitter(const MCInstrInfo &MCII, 103 MCContext &Ctx) { 104 return new AMDGPUMCCodeEmitter(MCII, *Ctx.getRegisterInfo()); 105 } 106 107 static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset, 108 const MCExpr *Value, uint16_t Kind, bool PCRel = false) { 109 Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel)); 110 } 111 112 // Returns the encoding value to use if the given integer is an integer inline 113 // immediate value, or 0 if it is not. 114 template <typename IntTy> 115 static uint32_t getIntInlineImmEncoding(IntTy Imm) { 116 if (Imm >= 0 && Imm <= 64) 117 return 128 + Imm; 118 119 if (Imm >= -16 && Imm <= -1) 120 return 192 + std::abs(Imm); 121 122 return 0; 123 } 124 125 static uint32_t getLit16Encoding(uint16_t Val, const MCSubtargetInfo &STI) { 126 uint16_t IntImm = getIntInlineImmEncoding(static_cast<int16_t>(Val)); 127 if (IntImm != 0) 128 return IntImm; 129 130 if (Val == 0x3800) // 0.5 131 return 240; 132 133 if (Val == 0xB800) // -0.5 134 return 241; 135 136 if (Val == 0x3C00) // 1.0 137 return 242; 138 139 if (Val == 0xBC00) // -1.0 140 return 243; 141 142 if (Val == 0x4000) // 2.0 143 return 244; 144 145 if (Val == 0xC000) // -2.0 146 return 245; 147 148 if (Val == 0x4400) // 4.0 149 return 246; 150 151 if (Val == 0xC400) // -4.0 152 return 247; 153 154 if (Val == 0x3118 && // 1.0 / (2.0 * pi) 155 STI.hasFeature(AMDGPU::FeatureInv2PiInlineImm)) 156 return 248; 157 158 return 255; 159 } 160 161 static uint32_t getLitBF16Encoding(uint16_t Val) { 162 uint16_t IntImm = getIntInlineImmEncoding(static_cast<int16_t>(Val)); 163 if (IntImm != 0) 164 return IntImm; 165 166 // clang-format off 167 switch (Val) { 168 case 0x3F00: return 240; // 0.5 169 case 0xBF00: return 241; // -0.5 170 case 0x3F80: return 242; // 1.0 171 case 0xBF80: return 243; // -1.0 172 case 0x4000: return 244; // 2.0 173 case 0xC000: return 245; // -2.0 174 case 0x4080: return 246; // 4.0 175 case 0xC080: return 247; // -4.0 176 case 0x3E22: return 248; // 1.0 / (2.0 * pi) 177 default: return 255; 178 } 179 // clang-format on 180 } 181 182 static uint32_t getLit32Encoding(uint32_t Val, const MCSubtargetInfo &STI) { 183 uint32_t IntImm = getIntInlineImmEncoding(static_cast<int32_t>(Val)); 184 if (IntImm != 0) 185 return IntImm; 186 187 if (Val == llvm::bit_cast<uint32_t>(0.5f)) 188 return 240; 189 190 if (Val == llvm::bit_cast<uint32_t>(-0.5f)) 191 return 241; 192 193 if (Val == llvm::bit_cast<uint32_t>(1.0f)) 194 return 242; 195 196 if (Val == llvm::bit_cast<uint32_t>(-1.0f)) 197 return 243; 198 199 if (Val == llvm::bit_cast<uint32_t>(2.0f)) 200 return 244; 201 202 if (Val == llvm::bit_cast<uint32_t>(-2.0f)) 203 return 245; 204 205 if (Val == llvm::bit_cast<uint32_t>(4.0f)) 206 return 246; 207 208 if (Val == llvm::bit_cast<uint32_t>(-4.0f)) 209 return 247; 210 211 if (Val == 0x3e22f983 && // 1.0 / (2.0 * pi) 212 STI.hasFeature(AMDGPU::FeatureInv2PiInlineImm)) 213 return 248; 214 215 return 255; 216 } 217 218 static uint32_t getLit16IntEncoding(uint32_t Val, const MCSubtargetInfo &STI) { 219 return getLit32Encoding(Val, STI); 220 } 221 222 static uint32_t getLit64Encoding(uint64_t Val, const MCSubtargetInfo &STI, 223 bool IsFP) { 224 uint32_t IntImm = getIntInlineImmEncoding(static_cast<int64_t>(Val)); 225 if (IntImm != 0) 226 return IntImm; 227 228 if (Val == llvm::bit_cast<uint64_t>(0.5)) 229 return 240; 230 231 if (Val == llvm::bit_cast<uint64_t>(-0.5)) 232 return 241; 233 234 if (Val == llvm::bit_cast<uint64_t>(1.0)) 235 return 242; 236 237 if (Val == llvm::bit_cast<uint64_t>(-1.0)) 238 return 243; 239 240 if (Val == llvm::bit_cast<uint64_t>(2.0)) 241 return 244; 242 243 if (Val == llvm::bit_cast<uint64_t>(-2.0)) 244 return 245; 245 246 if (Val == llvm::bit_cast<uint64_t>(4.0)) 247 return 246; 248 249 if (Val == llvm::bit_cast<uint64_t>(-4.0)) 250 return 247; 251 252 if (Val == 0x3fc45f306dc9c882 && // 1.0 / (2.0 * pi) 253 STI.hasFeature(AMDGPU::FeatureInv2PiInlineImm)) 254 return 248; 255 256 // The rest part needs to align with AMDGPUInstPrinter::printImmediate64. 257 258 if (IsFP) { 259 return STI.hasFeature(AMDGPU::Feature64BitLiterals) && Lo_32(Val) ? 254 260 : 255; 261 } 262 263 return STI.hasFeature(AMDGPU::Feature64BitLiterals) && 264 (!isInt<32>(Val) || !isUInt<32>(Val)) 265 ? 254 266 : 255; 267 } 268 269 std::optional<uint64_t> AMDGPUMCCodeEmitter::getLitEncoding( 270 const MCOperand &MO, const MCOperandInfo &OpInfo, 271 const MCSubtargetInfo &STI, bool HasMandatoryLiteral) const { 272 int64_t Imm; 273 if (MO.isExpr()) { 274 if (!MO.getExpr()->evaluateAsAbsolute(Imm)) 275 return (STI.hasFeature(AMDGPU::Feature64BitLiterals) && 276 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_INT64) 277 ? 254 278 : 255; 279 } else { 280 assert(!MO.isDFPImm()); 281 282 if (!MO.isImm()) 283 return {}; 284 285 Imm = MO.getImm(); 286 } 287 288 switch (OpInfo.OperandType) { 289 case AMDGPU::OPERAND_REG_IMM_INT32: 290 case AMDGPU::OPERAND_REG_IMM_FP32: 291 case AMDGPU::OPERAND_REG_INLINE_C_INT32: 292 case AMDGPU::OPERAND_REG_INLINE_C_FP32: 293 case AMDGPU::OPERAND_REG_INLINE_AC_INT32: 294 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: 295 case AMDGPU::OPERAND_REG_IMM_V2INT32: 296 case AMDGPU::OPERAND_REG_IMM_V2FP32: 297 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32: 298 return getLit32Encoding(static_cast<uint32_t>(Imm), STI); 299 300 case AMDGPU::OPERAND_REG_IMM_INT64: 301 case AMDGPU::OPERAND_REG_INLINE_C_INT64: 302 return getLit64Encoding(static_cast<uint64_t>(Imm), STI, false); 303 304 case AMDGPU::OPERAND_REG_INLINE_C_FP64: 305 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: 306 return getLit64Encoding(static_cast<uint64_t>(Imm), STI, true); 307 308 case AMDGPU::OPERAND_REG_IMM_FP64: { 309 auto Enc = getLit64Encoding(static_cast<uint64_t>(Imm), STI, true); 310 return (HasMandatoryLiteral && Enc == 255) ? 254 : Enc; 311 } 312 313 case AMDGPU::OPERAND_REG_IMM_INT16: 314 case AMDGPU::OPERAND_REG_INLINE_C_INT16: 315 return getLit16IntEncoding(static_cast<uint32_t>(Imm), STI); 316 317 case AMDGPU::OPERAND_REG_IMM_FP16: 318 case AMDGPU::OPERAND_REG_INLINE_C_FP16: 319 // FIXME Is this correct? What do inline immediates do on SI for f16 src 320 // which does not have f16 support? 321 return getLit16Encoding(static_cast<uint16_t>(Imm), STI); 322 323 case AMDGPU::OPERAND_REG_IMM_BF16: 324 case AMDGPU::OPERAND_REG_INLINE_C_BF16: 325 // We don't actually need to check Inv2Pi here because BF16 instructions can 326 // only be emitted for targets that already support the feature. 327 return getLitBF16Encoding(static_cast<uint16_t>(Imm)); 328 329 case AMDGPU::OPERAND_REG_IMM_V2INT16: 330 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: 331 return AMDGPU::getInlineEncodingV2I16(static_cast<uint32_t>(Imm)) 332 .value_or(255); 333 334 case AMDGPU::OPERAND_REG_IMM_V2FP16: 335 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: 336 return AMDGPU::getInlineEncodingV2F16(static_cast<uint32_t>(Imm)) 337 .value_or(255); 338 339 case AMDGPU::OPERAND_REG_IMM_V2BF16: 340 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16: 341 return AMDGPU::getInlineEncodingV2BF16(static_cast<uint32_t>(Imm)) 342 .value_or(255); 343 344 case AMDGPU::OPERAND_KIMM32: 345 case AMDGPU::OPERAND_KIMM16: 346 case AMDGPU::OPERAND_KIMM64: 347 return MO.getImm(); 348 default: 349 llvm_unreachable("invalid operand size"); 350 } 351 } 352 353 uint64_t AMDGPUMCCodeEmitter::getImplicitOpSelHiEncoding(int Opcode) const { 354 using namespace AMDGPU::VOP3PEncoding; 355 356 if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::op_sel_hi)) { 357 if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src2)) 358 return 0; 359 if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src1)) 360 return OP_SEL_HI_2; 361 if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src0)) 362 return OP_SEL_HI_1 | OP_SEL_HI_2; 363 } 364 return OP_SEL_HI_0 | OP_SEL_HI_1 | OP_SEL_HI_2; 365 } 366 367 static bool isVCMPX64(const MCInstrDesc &Desc) { 368 return (Desc.TSFlags & SIInstrFlags::VOP3) && 369 Desc.hasImplicitDefOfPhysReg(AMDGPU::EXEC); 370 } 371 372 void AMDGPUMCCodeEmitter::encodeInstruction(const MCInst &MI, 373 SmallVectorImpl<char> &CB, 374 SmallVectorImpl<MCFixup> &Fixups, 375 const MCSubtargetInfo &STI) const { 376 int Opcode = MI.getOpcode(); 377 APInt Encoding, Scratch; 378 getBinaryCodeForInstr(MI, Fixups, Encoding, Scratch, STI); 379 const MCInstrDesc &Desc = MCII.get(MI.getOpcode()); 380 unsigned bytes = Desc.getSize(); 381 382 // Set unused op_sel_hi bits to 1 for VOP3P and MAI instructions. 383 // Note that accvgpr_read/write are MAI, have src0, but do not use op_sel. 384 if (((Desc.TSFlags & SIInstrFlags::VOP3P) || 385 Opcode == AMDGPU::V_ACCVGPR_READ_B32_vi || 386 Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_vi) && 387 // Matrix B reuse operand reuses op_sel_hi. 388 !AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::matrix_b_reuse)) { 389 Encoding |= getImplicitOpSelHiEncoding(Opcode); 390 } 391 392 // GFX10+ v_cmpx opcodes promoted to VOP3 have implied dst=EXEC. 393 // Documentation requires dst to be encoded as EXEC (0x7E), 394 // but it looks like the actual value encoded for dst operand 395 // is ignored by HW. It was decided to define dst as "do not care" 396 // in td files to allow disassembler accept any dst value. 397 // However, dst is encoded as EXEC for compatibility with SP3. 398 if (AMDGPU::isGFX10Plus(STI) && isVCMPX64(Desc)) { 399 assert((Encoding & 0xFF) == 0); 400 Encoding |= MRI.getEncodingValue(AMDGPU::EXEC_LO) & 401 AMDGPU::HWEncoding::REG_IDX_MASK; 402 } 403 404 for (unsigned i = 0; i < bytes; i++) { 405 CB.push_back((uint8_t)Encoding.extractBitsAsZExtValue(8, 8 * i)); 406 } 407 408 // NSA encoding. 409 if (AMDGPU::isGFX10Plus(STI) && Desc.TSFlags & SIInstrFlags::MIMG) { 410 int vaddr0 = AMDGPU::getNamedOperandIdx(MI.getOpcode(), 411 AMDGPU::OpName::vaddr0); 412 int srsrc = AMDGPU::getNamedOperandIdx(MI.getOpcode(), 413 AMDGPU::OpName::srsrc); 414 assert(vaddr0 >= 0 && srsrc > vaddr0); 415 unsigned NumExtraAddrs = srsrc - vaddr0 - 1; 416 unsigned NumPadding = (-NumExtraAddrs) & 3; 417 418 for (unsigned i = 0; i < NumExtraAddrs; ++i) { 419 getMachineOpValue(MI, MI.getOperand(vaddr0 + 1 + i), Encoding, Fixups, 420 STI); 421 CB.push_back((uint8_t)Encoding.getLimitedValue()); 422 } 423 CB.append(NumPadding, 0); 424 } 425 426 if ((bytes > 8 && STI.hasFeature(AMDGPU::FeatureVOP3Literal)) || 427 (bytes > 4 && !STI.hasFeature(AMDGPU::FeatureVOP3Literal))) 428 return; 429 430 // Do not print literals from SISrc Operands for insts with mandatory literals 431 if (AMDGPU::hasNamedOperand(MI.getOpcode(), AMDGPU::OpName::imm)) 432 return; 433 434 // Check for additional literals 435 for (unsigned i = 0, e = Desc.getNumOperands(); i < e; ++i) { 436 437 // Check if this operand should be encoded as [SV]Src 438 if (!AMDGPU::isSISrcOperand(Desc, i)) 439 continue; 440 441 // Is this operand a literal immediate? 442 const MCOperand &Op = MI.getOperand(i); 443 auto Enc = getLitEncoding(Op, Desc.operands()[i], STI); 444 if (!Enc || (*Enc != 255 && *Enc != 254)) 445 continue; 446 447 // Yes! Encode it 448 int64_t Imm = 0; 449 450 if (Op.isImm()) 451 Imm = Op.getImm(); 452 else if (Op.isExpr()) { 453 if (const auto *C = dyn_cast<MCConstantExpr>(Op.getExpr())) 454 Imm = C->getValue(); 455 } else // Exprs will be replaced with a fixup value. 456 llvm_unreachable("Must be immediate or expr"); 457 458 if (*Enc == 254) { 459 assert(STI.hasFeature(AMDGPU::Feature64BitLiterals)); 460 support::endian::write<uint64_t>(CB, Imm, llvm::endianness::little); 461 } else { 462 if (Desc.operands()[i].OperandType == AMDGPU::OPERAND_REG_IMM_FP64) 463 Imm = Hi_32(Imm); 464 support::endian::write<uint32_t>(CB, Imm, llvm::endianness::little); 465 } 466 467 // Only one literal value allowed 468 break; 469 } 470 } 471 472 void AMDGPUMCCodeEmitter::getSOPPBrEncoding(const MCInst &MI, unsigned OpNo, 473 APInt &Op, 474 SmallVectorImpl<MCFixup> &Fixups, 475 const MCSubtargetInfo &STI) const { 476 const MCOperand &MO = MI.getOperand(OpNo); 477 478 if (MO.isExpr()) { 479 const MCExpr *Expr = MO.getExpr(); 480 addFixup(Fixups, 0, Expr, AMDGPU::fixup_si_sopp_br, true); 481 Op = APInt::getZero(96); 482 } else { 483 getMachineOpValue(MI, MO, Op, Fixups, STI); 484 } 485 } 486 487 void AMDGPUMCCodeEmitter::getSMEMOffsetEncoding( 488 const MCInst &MI, unsigned OpNo, APInt &Op, 489 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const { 490 auto Offset = MI.getOperand(OpNo).getImm(); 491 // VI only supports 20-bit unsigned offsets. 492 assert(!AMDGPU::isVI(STI) || isUInt<20>(Offset)); 493 Op = Offset; 494 } 495 496 void AMDGPUMCCodeEmitter::getSDWASrcEncoding(const MCInst &MI, unsigned OpNo, 497 APInt &Op, 498 SmallVectorImpl<MCFixup> &Fixups, 499 const MCSubtargetInfo &STI) const { 500 using namespace AMDGPU::SDWA; 501 502 uint64_t RegEnc = 0; 503 504 const MCOperand &MO = MI.getOperand(OpNo); 505 506 if (MO.isReg()) { 507 MCRegister Reg = MO.getReg(); 508 RegEnc |= MRI.getEncodingValue(Reg); 509 RegEnc &= SDWA9EncValues::SRC_VGPR_MASK; 510 if (AMDGPU::isSGPR(AMDGPU::mc2PseudoReg(Reg), &MRI)) { 511 RegEnc |= SDWA9EncValues::SRC_SGPR_MASK; 512 } 513 Op = RegEnc; 514 return; 515 } else { 516 const MCInstrDesc &Desc = MCII.get(MI.getOpcode()); 517 auto Enc = getLitEncoding(MO, Desc.operands()[OpNo], STI); 518 if (Enc && *Enc != 255) { 519 Op = *Enc | SDWA9EncValues::SRC_SGPR_MASK; 520 return; 521 } 522 } 523 524 llvm_unreachable("Unsupported operand kind"); 525 } 526 527 void AMDGPUMCCodeEmitter::getSDWAVopcDstEncoding( 528 const MCInst &MI, unsigned OpNo, APInt &Op, 529 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const { 530 using namespace AMDGPU::SDWA; 531 532 uint64_t RegEnc = 0; 533 534 const MCOperand &MO = MI.getOperand(OpNo); 535 536 MCRegister Reg = MO.getReg(); 537 if (Reg != AMDGPU::VCC && Reg != AMDGPU::VCC_LO) { 538 RegEnc |= MRI.getEncodingValue(Reg); 539 RegEnc &= SDWA9EncValues::VOPC_DST_SGPR_MASK; 540 RegEnc |= SDWA9EncValues::VOPC_DST_VCC_MASK; 541 } 542 Op = RegEnc; 543 } 544 545 void AMDGPUMCCodeEmitter::getAVOperandEncoding( 546 const MCInst &MI, unsigned OpNo, APInt &Op, 547 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const { 548 MCRegister Reg = MI.getOperand(OpNo).getReg(); 549 unsigned Enc = MRI.getEncodingValue(Reg); 550 unsigned Idx = Enc & AMDGPU::HWEncoding::REG_IDX_MASK; 551 bool IsVGPROrAGPR = 552 Enc & (AMDGPU::HWEncoding::IS_VGPR | AMDGPU::HWEncoding::IS_AGPR); 553 554 // VGPR and AGPR have the same encoding, but SrcA and SrcB operands of mfma 555 // instructions use acc[0:1] modifier bits to distinguish. These bits are 556 // encoded as a virtual 9th bit of the register for these operands. 557 bool IsAGPR = Enc & AMDGPU::HWEncoding::IS_AGPR; 558 559 Op = Idx | (IsVGPROrAGPR << 8) | (IsAGPR << 9); 560 } 561 562 static bool needsPCRel(const MCExpr *Expr) { 563 switch (Expr->getKind()) { 564 case MCExpr::SymbolRef: { 565 auto *SE = cast<MCSymbolRefExpr>(Expr); 566 auto Spec = AMDGPU::getSpecifier(SE); 567 return Spec != AMDGPUMCExpr::S_ABS32_LO && Spec != AMDGPUMCExpr::S_ABS32_HI; 568 } 569 case MCExpr::Binary: { 570 auto *BE = cast<MCBinaryExpr>(Expr); 571 if (BE->getOpcode() == MCBinaryExpr::Sub) 572 return false; 573 return needsPCRel(BE->getLHS()) || needsPCRel(BE->getRHS()); 574 } 575 case MCExpr::Unary: 576 return needsPCRel(cast<MCUnaryExpr>(Expr)->getSubExpr()); 577 case MCExpr::Specifier: 578 case MCExpr::Target: 579 case MCExpr::Constant: 580 return false; 581 } 582 llvm_unreachable("invalid kind"); 583 } 584 585 void AMDGPUMCCodeEmitter::getMachineOpValue(const MCInst &MI, 586 const MCOperand &MO, APInt &Op, 587 SmallVectorImpl<MCFixup> &Fixups, 588 const MCSubtargetInfo &STI) const { 589 if (MO.isReg()){ 590 unsigned Enc = MRI.getEncodingValue(MO.getReg()); 591 unsigned Idx = Enc & AMDGPU::HWEncoding::REG_IDX_MASK; 592 bool IsVGPROrAGPR = 593 Enc & (AMDGPU::HWEncoding::IS_VGPR | AMDGPU::HWEncoding::IS_AGPR); 594 Op = Idx | (IsVGPROrAGPR << 8); 595 return; 596 } 597 unsigned OpNo = &MO - MI.begin(); 598 getMachineOpValueCommon(MI, MO, OpNo, Op, Fixups, STI); 599 } 600 601 void AMDGPUMCCodeEmitter::getMachineOpValueT16( 602 const MCInst &MI, unsigned OpNo, APInt &Op, 603 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const { 604 const MCOperand &MO = MI.getOperand(OpNo); 605 if (MO.isReg()) { 606 unsigned Enc = MRI.getEncodingValue(MO.getReg()); 607 unsigned Idx = Enc & AMDGPU::HWEncoding::REG_IDX_MASK; 608 bool IsVGPR = Enc & AMDGPU::HWEncoding::IS_VGPR; 609 Op = Idx | (IsVGPR << 8); 610 return; 611 } 612 getMachineOpValueCommon(MI, MO, OpNo, Op, Fixups, STI); 613 // VGPRs include the suffix/op_sel bit in the register encoding, but 614 // immediates and SGPRs include it in src_modifiers. Therefore, copy the 615 // op_sel bit from the src operands into src_modifier operands if Op is 616 // src_modifiers and the corresponding src is a VGPR 617 int SrcMOIdx = -1; 618 assert(OpNo < INT_MAX); 619 if ((int)OpNo == AMDGPU::getNamedOperandIdx(MI.getOpcode(), 620 AMDGPU::OpName::src0_modifiers)) { 621 SrcMOIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0); 622 int VDstMOIdx = 623 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdst); 624 if (VDstMOIdx != -1) { 625 auto DstReg = MI.getOperand(VDstMOIdx).getReg(); 626 if (AMDGPU::isHi16Reg(DstReg, MRI)) 627 Op |= SISrcMods::DST_OP_SEL; 628 } 629 } else if ((int)OpNo == AMDGPU::getNamedOperandIdx( 630 MI.getOpcode(), AMDGPU::OpName::src1_modifiers)) 631 SrcMOIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src1); 632 else if ((int)OpNo == AMDGPU::getNamedOperandIdx( 633 MI.getOpcode(), AMDGPU::OpName::src2_modifiers)) 634 SrcMOIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src2); 635 if (SrcMOIdx == -1) 636 return; 637 638 const MCOperand &SrcMO = MI.getOperand(SrcMOIdx); 639 if (!SrcMO.isReg()) 640 return; 641 auto SrcReg = SrcMO.getReg(); 642 if (AMDGPU::isSGPR(SrcReg, &MRI)) 643 return; 644 if (AMDGPU::isHi16Reg(SrcReg, MRI)) 645 Op |= SISrcMods::OP_SEL_0; 646 } 647 648 void AMDGPUMCCodeEmitter::getMachineOpValueT16Lo128( 649 const MCInst &MI, unsigned OpNo, APInt &Op, 650 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const { 651 const MCOperand &MO = MI.getOperand(OpNo); 652 if (MO.isReg()) { 653 uint16_t Encoding = MRI.getEncodingValue(MO.getReg()); 654 unsigned RegIdx = Encoding & AMDGPU::HWEncoding::REG_IDX_MASK; 655 bool IsHi = Encoding & AMDGPU::HWEncoding::IS_HI16; 656 bool IsVGPR = Encoding & AMDGPU::HWEncoding::IS_VGPR; 657 assert((!IsVGPR || isUInt<7>(RegIdx)) && "VGPR0-VGPR127 expected!"); 658 Op = (IsVGPR ? 0x100 : 0) | (IsHi ? 0x80 : 0) | RegIdx; 659 return; 660 } 661 getMachineOpValueCommon(MI, MO, OpNo, Op, Fixups, STI); 662 } 663 664 void AMDGPUMCCodeEmitter::getMachineOpValueCommon( 665 const MCInst &MI, const MCOperand &MO, unsigned OpNo, APInt &Op, 666 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const { 667 bool isLikeImm = false; 668 int64_t Val; 669 670 if (MO.isImm()) { 671 Val = MO.getImm(); 672 isLikeImm = true; 673 } else if (MO.isExpr() && MO.getExpr()->evaluateAsAbsolute(Val)) { 674 isLikeImm = true; 675 } else if (MO.isExpr()) { 676 // FIXME: If this is expression is PCRel or not should not depend on what 677 // the expression looks like. Given that this is just a general expression, 678 // it should probably be FK_Data_4 and whatever is producing 679 // 680 // s_add_u32 s2, s2, (extern_const_addrspace+16 681 // 682 // And expecting a PCRel should instead produce 683 // 684 // .Ltmp1: 685 // s_add_u32 s2, s2, (extern_const_addrspace+16)-.Ltmp1 686 bool PCRel = needsPCRel(MO.getExpr()); 687 const MCInstrDesc &Desc = MCII.get(MI.getOpcode()); 688 uint32_t Offset = Desc.getSize(); 689 assert(Offset == 4 || Offset == 8); 690 addFixup(Fixups, Offset, MO.getExpr(), FK_Data_4, PCRel); 691 } 692 693 const MCInstrDesc &Desc = MCII.get(MI.getOpcode()); 694 if (AMDGPU::isSISrcOperand(Desc, OpNo)) { 695 bool HasMandatoryLiteral = 696 AMDGPU::hasNamedOperand(MI.getOpcode(), AMDGPU::OpName::imm); 697 if (auto Enc = getLitEncoding(MO, Desc.operands()[OpNo], STI, 698 HasMandatoryLiteral)) { 699 Op = *Enc; 700 return; 701 } 702 703 llvm_unreachable("Operand not supported for SISrc"); 704 } 705 706 if (isLikeImm) { 707 Op = Val; 708 return; 709 } 710 711 llvm_unreachable("Encoding of this operand type is not supported yet."); 712 } 713 714 #include "AMDGPUGenMCCodeEmitter.inc" 715