1 //===-- X86MCCodeEmitter.cpp - Convert X86 code to machine code -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the X86MCCodeEmitter class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "MCTargetDesc/X86BaseInfo.h" 14 #include "MCTargetDesc/X86FixupKinds.h" 15 #include "MCTargetDesc/X86MCTargetDesc.h" 16 #include "llvm/ADT/SmallVector.h" 17 #include "llvm/MC/MCCodeEmitter.h" 18 #include "llvm/MC/MCContext.h" 19 #include "llvm/MC/MCExpr.h" 20 #include "llvm/MC/MCFixup.h" 21 #include "llvm/MC/MCInst.h" 22 #include "llvm/MC/MCInstrDesc.h" 23 #include "llvm/MC/MCInstrInfo.h" 24 #include "llvm/MC/MCRegisterInfo.h" 25 #include "llvm/MC/MCSubtargetInfo.h" 26 #include "llvm/MC/MCSymbol.h" 27 #include "llvm/Support/Casting.h" 28 #include "llvm/Support/ErrorHandling.h" 29 #include <cassert> 30 #include <cstdint> 31 #include <cstdlib> 32 33 using namespace llvm; 34 35 #define DEBUG_TYPE "mccodeemitter" 36 37 namespace { 38 39 enum PrefixKind { None, REX, XOP, VEX2, VEX3, EVEX }; 40 41 static void emitByte(uint8_t C, SmallVectorImpl<char> &CB) { CB.push_back(C); } 42 43 class X86OpcodePrefixHelper { 44 // REX (1 byte) 45 // +-----+ +------+ 46 // | 40H | | WRXB | 47 // +-----+ +------+ 48 49 // XOP (3-byte) 50 // +-----+ +--------------+ +-------------------+ 51 // | 8Fh | | RXB | m-mmmm | | W | vvvv | L | pp | 52 // +-----+ +--------------+ +-------------------+ 53 54 // VEX2 (2 bytes) 55 // +-----+ +-------------------+ 56 // | C5h | | R | vvvv | L | pp | 57 // +-----+ +-------------------+ 58 59 // VEX3 (3 bytes) 60 // +-----+ +--------------+ +-------------------+ 61 // | C4h | | RXB | m-mmmm | | W | vvvv | L | pp | 62 // +-----+ +--------------+ +-------------------+ 63 64 // VEX_R: opcode externsion equivalent to REX.R in 65 // 1's complement (inverted) form 66 // 67 // 1: Same as REX_R=0 (must be 1 in 32-bit mode) 68 // 0: Same as REX_R=1 (64 bit mode only) 69 70 // VEX_X: equivalent to REX.X, only used when a 71 // register is used for index in SIB Byte. 72 // 73 // 1: Same as REX.X=0 (must be 1 in 32-bit mode) 74 // 0: Same as REX.X=1 (64-bit mode only) 75 76 // VEX_B: 77 // 1: Same as REX_B=0 (ignored in 32-bit mode) 78 // 0: Same as REX_B=1 (64 bit mode only) 79 80 // VEX_W: opcode specific (use like REX.W, or used for 81 // opcode extension, or ignored, depending on the opcode byte) 82 83 // VEX_5M (VEX m-mmmmm field): 84 // 85 // 0b00000: Reserved for future use 86 // 0b00001: implied 0F leading opcode 87 // 0b00010: implied 0F 38 leading opcode bytes 88 // 0b00011: implied 0F 3A leading opcode bytes 89 // 0b00100: Reserved for future use 90 // 0b00101: VEX MAP5 91 // 0b00110: VEX MAP6 92 // 0b00111-0b11111: Reserved for future use 93 // 0b01000: XOP map select - 08h instructions with imm byte 94 // 0b01001: XOP map select - 09h instructions with no imm byte 95 // 0b01010: XOP map select - 0Ah instructions with imm dword 96 97 // VEX_4V (VEX vvvv field): a register specifier 98 // (in 1's complement form) or 1111 if unused. 99 100 // VEX_PP: opcode extension providing equivalent 101 // functionality of a SIMD prefix 102 // 0b00: None 103 // 0b01: 66 104 // 0b10: F3 105 // 0b11: F2 106 107 // EVEX (4 bytes) 108 // +-----+ +--------------+ +-------------------+ +------------------------+ 109 // | 62h | | RXBR' | 0mmm | | W | vvvv | 1 | pp | | z | L'L | b | v' | aaa | 110 // +-----+ +--------------+ +-------------------+ +------------------------+ 111 112 // EVEX_L2/VEX_L (Vector Length): 113 // L2 L 114 // 0 0: scalar or 128-bit vector 115 // 0 1: 256-bit vector 116 // 1 0: 512-bit vector 117 118 private: 119 unsigned W : 1; 120 unsigned R : 1; 121 unsigned X : 1; 122 unsigned B : 1; 123 unsigned VEX_4V : 4; 124 unsigned VEX_L : 1; 125 unsigned VEX_PP : 2; 126 unsigned VEX_5M : 5; 127 unsigned EVEX_R2 : 1; 128 unsigned EVEX_z : 1; 129 unsigned EVEX_L2 : 1; 130 unsigned EVEX_b : 1; 131 unsigned EVEX_V2 : 1; 132 unsigned EVEX_aaa : 3; 133 PrefixKind Kind = None; 134 const MCRegisterInfo &MRI; 135 136 unsigned getRegEncoding(const MCInst &MI, unsigned OpNum) const { 137 return MRI.getEncodingValue(MI.getOperand(OpNum).getReg()); 138 } 139 140 void setR(unsigned Encoding) { R = Encoding >> 3 & 1; } 141 void setR2(unsigned Encoding) { EVEX_R2 = Encoding >> 4 & 1; } 142 void set4V(unsigned Encoding) { VEX_4V = Encoding & 0xf; } 143 void setV2(unsigned Encoding) { EVEX_V2 = Encoding >> 4 & 1; } 144 145 public: 146 void setW(bool V) { W = V; } 147 void setR(const MCInst &MI, unsigned OpNum) { 148 setR(getRegEncoding(MI, OpNum)); 149 } 150 void setX(const MCInst &MI, unsigned OpNum, unsigned Shift = 3) { 151 X = getRegEncoding(MI, OpNum) >> Shift & 1; 152 } 153 void setB(const MCInst &MI, unsigned OpNum) { 154 B = getRegEncoding(MI, OpNum) >> 3 & 1; 155 } 156 void set4V(const MCInst &MI, unsigned OpNum) { 157 set4V(getRegEncoding(MI, OpNum)); 158 } 159 void setL(bool V) { VEX_L = V; } 160 void setPP(unsigned V) { VEX_PP = V; } 161 void set5M(unsigned V) { VEX_5M = V; } 162 void setR2(const MCInst &MI, unsigned OpNum) { 163 setR2(getRegEncoding(MI, OpNum)); 164 } 165 void setRR2(const MCInst &MI, unsigned OpNum) { 166 unsigned Encoding = getRegEncoding(MI, OpNum); 167 setR(Encoding); 168 setR2(Encoding); 169 } 170 void setZ(bool V) { EVEX_z = V; } 171 void setL2(bool V) { EVEX_L2 = V; } 172 void setEVEX_b(bool V) { EVEX_b = V; } 173 void setV2(const MCInst &MI, unsigned OpNum) { 174 setV2(getRegEncoding(MI, OpNum)); 175 } 176 void set4VV2(const MCInst &MI, unsigned OpNum) { 177 unsigned Encoding = getRegEncoding(MI, OpNum); 178 set4V(Encoding); 179 setV2(Encoding); 180 } 181 void setAAA(const MCInst &MI, unsigned OpNum) { 182 EVEX_aaa = getRegEncoding(MI, OpNum); 183 } 184 185 X86OpcodePrefixHelper(const MCRegisterInfo &MRI) 186 : W(0), R(0), X(0), B(0), VEX_4V(0), VEX_L(0), VEX_PP(0), VEX_5M(0), 187 EVEX_R2(0), EVEX_z(0), EVEX_L2(0), EVEX_b(0), EVEX_V2(0), EVEX_aaa(0), 188 MRI(MRI) {} 189 190 void setLowerBound(PrefixKind K) { Kind = K; } 191 192 PrefixKind determineOptimalKind() { 193 switch (Kind) { 194 case None: 195 Kind = (W | R | X | B) ? REX : None; 196 break; 197 case REX: 198 case XOP: 199 case VEX3: 200 case EVEX: 201 break; 202 case VEX2: 203 Kind = (W | X | B | (VEX_5M != 1)) ? VEX3 : VEX2; 204 break; 205 } 206 return Kind; 207 } 208 209 void emit(SmallVectorImpl<char> &CB) const { 210 uint8_t FirstPayload = 211 ((~R) & 0x1) << 7 | ((~X) & 0x1) << 6 | ((~B) & 0x1) << 5; 212 uint8_t LastPayload = ((~VEX_4V) & 0xf) << 3 | VEX_L << 2 | VEX_PP; 213 switch (Kind) { 214 case None: 215 return; 216 case REX: 217 emitByte(0x40 | W << 3 | R << 2 | X << 1 | B, CB); 218 return; 219 case VEX2: 220 emitByte(0xC5, CB); 221 emitByte(((~R) & 1) << 7 | LastPayload, CB); 222 return; 223 case VEX3: 224 case XOP: 225 emitByte(Kind == VEX3 ? 0xC4 : 0x8F, CB); 226 emitByte(FirstPayload | VEX_5M, CB); 227 emitByte(W << 7 | LastPayload, CB); 228 return; 229 case EVEX: 230 assert(VEX_5M && !(VEX_5M & 0x8) && "invalid mmm fields for EVEX!"); 231 emitByte(0x62, CB); 232 emitByte(FirstPayload | ((~EVEX_R2) & 0x1) << 4 | VEX_5M, CB); 233 emitByte(W << 7 | ((~VEX_4V) & 0xf) << 3 | 1 << 2 | VEX_PP, CB); 234 emitByte(EVEX_z << 7 | EVEX_L2 << 6 | VEX_L << 5 | EVEX_b << 4 | 235 ((~EVEX_V2) & 0x1) << 3 | EVEX_aaa, 236 CB); 237 return; 238 } 239 } 240 }; 241 242 class X86MCCodeEmitter : public MCCodeEmitter { 243 const MCInstrInfo &MCII; 244 MCContext &Ctx; 245 246 public: 247 X86MCCodeEmitter(const MCInstrInfo &mcii, MCContext &ctx) 248 : MCII(mcii), Ctx(ctx) {} 249 X86MCCodeEmitter(const X86MCCodeEmitter &) = delete; 250 X86MCCodeEmitter &operator=(const X86MCCodeEmitter &) = delete; 251 ~X86MCCodeEmitter() override = default; 252 253 void emitPrefix(const MCInst &MI, SmallVectorImpl<char> &CB, 254 const MCSubtargetInfo &STI) const override; 255 256 void encodeInstruction(const MCInst &MI, SmallVectorImpl<char> &CB, 257 SmallVectorImpl<MCFixup> &Fixups, 258 const MCSubtargetInfo &STI) const override; 259 260 private: 261 unsigned getX86RegNum(const MCOperand &MO) const; 262 263 unsigned getX86RegEncoding(const MCInst &MI, unsigned OpNum) const; 264 265 void emitImmediate(const MCOperand &Disp, SMLoc Loc, unsigned ImmSize, 266 MCFixupKind FixupKind, uint64_t StartByte, 267 SmallVectorImpl<char> &CB, 268 SmallVectorImpl<MCFixup> &Fixups, int ImmOffset = 0) const; 269 270 void emitRegModRMByte(const MCOperand &ModRMReg, unsigned RegOpcodeFld, 271 SmallVectorImpl<char> &CB) const; 272 273 void emitSIBByte(unsigned SS, unsigned Index, unsigned Base, 274 SmallVectorImpl<char> &CB) const; 275 276 void emitMemModRMByte(const MCInst &MI, unsigned Op, unsigned RegOpcodeField, 277 uint64_t TSFlags, PrefixKind Kind, uint64_t StartByte, 278 SmallVectorImpl<char> &CB, 279 SmallVectorImpl<MCFixup> &Fixups, 280 const MCSubtargetInfo &STI, 281 bool ForceSIB = false) const; 282 283 PrefixKind emitPrefixImpl(unsigned &CurOp, const MCInst &MI, 284 const MCSubtargetInfo &STI, 285 SmallVectorImpl<char> &CB) const; 286 287 PrefixKind emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, 288 SmallVectorImpl<char> &CB) const; 289 290 void emitSegmentOverridePrefix(unsigned SegOperand, const MCInst &MI, 291 SmallVectorImpl<char> &CB) const; 292 293 PrefixKind emitOpcodePrefix(int MemOperand, const MCInst &MI, 294 const MCSubtargetInfo &STI, 295 SmallVectorImpl<char> &CB) const; 296 297 PrefixKind emitREXPrefix(int MemOperand, const MCInst &MI, 298 const MCSubtargetInfo &STI, 299 SmallVectorImpl<char> &CB) const; 300 }; 301 302 } // end anonymous namespace 303 304 static uint8_t modRMByte(unsigned Mod, unsigned RegOpcode, unsigned RM) { 305 assert(Mod < 4 && RegOpcode < 8 && RM < 8 && "ModRM Fields out of range!"); 306 return RM | (RegOpcode << 3) | (Mod << 6); 307 } 308 309 static void emitConstant(uint64_t Val, unsigned Size, 310 SmallVectorImpl<char> &CB) { 311 // Output the constant in little endian byte order. 312 for (unsigned i = 0; i != Size; ++i) { 313 emitByte(Val & 255, CB); 314 Val >>= 8; 315 } 316 } 317 318 /// Determine if this immediate can fit in a disp8 or a compressed disp8 for 319 /// EVEX instructions. \p will be set to the value to pass to the ImmOffset 320 /// parameter of emitImmediate. 321 static bool isDispOrCDisp8(uint64_t TSFlags, int Value, int &ImmOffset) { 322 bool HasEVEX = (TSFlags & X86II::EncodingMask) == X86II::EVEX; 323 324 unsigned CD8_Scale = 325 (TSFlags & X86II::CD8_Scale_Mask) >> X86II::CD8_Scale_Shift; 326 CD8_Scale = CD8_Scale ? 1U << (CD8_Scale - 1) : 0U; 327 if (!HasEVEX || !CD8_Scale) 328 return isInt<8>(Value); 329 330 assert(isPowerOf2_32(CD8_Scale) && "Unexpected CD8 scale!"); 331 if (Value & (CD8_Scale - 1)) // Unaligned offset 332 return false; 333 334 int CDisp8 = Value / static_cast<int>(CD8_Scale); 335 if (!isInt<8>(CDisp8)) 336 return false; 337 338 // ImmOffset will be added to Value in emitImmediate leaving just CDisp8. 339 ImmOffset = CDisp8 - Value; 340 return true; 341 } 342 343 /// \returns the appropriate fixup kind to use for an immediate in an 344 /// instruction with the specified TSFlags. 345 static MCFixupKind getImmFixupKind(uint64_t TSFlags) { 346 unsigned Size = X86II::getSizeOfImm(TSFlags); 347 bool isPCRel = X86II::isImmPCRel(TSFlags); 348 349 if (X86II::isImmSigned(TSFlags)) { 350 switch (Size) { 351 default: 352 llvm_unreachable("Unsupported signed fixup size!"); 353 case 4: 354 return MCFixupKind(X86::reloc_signed_4byte); 355 } 356 } 357 return MCFixup::getKindForSize(Size, isPCRel); 358 } 359 360 enum GlobalOffsetTableExprKind { GOT_None, GOT_Normal, GOT_SymDiff }; 361 362 /// Check if this expression starts with _GLOBAL_OFFSET_TABLE_ and if it is 363 /// of the form _GLOBAL_OFFSET_TABLE_-symbol. This is needed to support PIC on 364 /// ELF i386 as _GLOBAL_OFFSET_TABLE_ is magical. We check only simple case that 365 /// are know to be used: _GLOBAL_OFFSET_TABLE_ by itself or at the start of a 366 /// binary expression. 367 static GlobalOffsetTableExprKind 368 startsWithGlobalOffsetTable(const MCExpr *Expr) { 369 const MCExpr *RHS = nullptr; 370 if (Expr->getKind() == MCExpr::Binary) { 371 const MCBinaryExpr *BE = static_cast<const MCBinaryExpr *>(Expr); 372 Expr = BE->getLHS(); 373 RHS = BE->getRHS(); 374 } 375 376 if (Expr->getKind() != MCExpr::SymbolRef) 377 return GOT_None; 378 379 const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr); 380 const MCSymbol &S = Ref->getSymbol(); 381 if (S.getName() != "_GLOBAL_OFFSET_TABLE_") 382 return GOT_None; 383 if (RHS && RHS->getKind() == MCExpr::SymbolRef) 384 return GOT_SymDiff; 385 return GOT_Normal; 386 } 387 388 static bool hasSecRelSymbolRef(const MCExpr *Expr) { 389 if (Expr->getKind() == MCExpr::SymbolRef) { 390 const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr); 391 return Ref->getKind() == MCSymbolRefExpr::VK_SECREL; 392 } 393 return false; 394 } 395 396 static bool isPCRel32Branch(const MCInst &MI, const MCInstrInfo &MCII) { 397 unsigned Opcode = MI.getOpcode(); 398 const MCInstrDesc &Desc = MCII.get(Opcode); 399 if ((Opcode != X86::CALL64pcrel32 && Opcode != X86::JMP_4 && 400 Opcode != X86::JCC_4) || 401 getImmFixupKind(Desc.TSFlags) != FK_PCRel_4) 402 return false; 403 404 unsigned CurOp = X86II::getOperandBias(Desc); 405 const MCOperand &Op = MI.getOperand(CurOp); 406 if (!Op.isExpr()) 407 return false; 408 409 const MCSymbolRefExpr *Ref = dyn_cast<MCSymbolRefExpr>(Op.getExpr()); 410 return Ref && Ref->getKind() == MCSymbolRefExpr::VK_None; 411 } 412 413 unsigned X86MCCodeEmitter::getX86RegNum(const MCOperand &MO) const { 414 return Ctx.getRegisterInfo()->getEncodingValue(MO.getReg()) & 0x7; 415 } 416 417 unsigned X86MCCodeEmitter::getX86RegEncoding(const MCInst &MI, 418 unsigned OpNum) const { 419 return Ctx.getRegisterInfo()->getEncodingValue(MI.getOperand(OpNum).getReg()); 420 } 421 422 void X86MCCodeEmitter::emitImmediate(const MCOperand &DispOp, SMLoc Loc, 423 unsigned Size, MCFixupKind FixupKind, 424 uint64_t StartByte, 425 SmallVectorImpl<char> &CB, 426 SmallVectorImpl<MCFixup> &Fixups, 427 int ImmOffset) const { 428 const MCExpr *Expr = nullptr; 429 if (DispOp.isImm()) { 430 // If this is a simple integer displacement that doesn't require a 431 // relocation, emit it now. 432 if (FixupKind != FK_PCRel_1 && FixupKind != FK_PCRel_2 && 433 FixupKind != FK_PCRel_4) { 434 emitConstant(DispOp.getImm() + ImmOffset, Size, CB); 435 return; 436 } 437 Expr = MCConstantExpr::create(DispOp.getImm(), Ctx); 438 } else { 439 Expr = DispOp.getExpr(); 440 } 441 442 // If we have an immoffset, add it to the expression. 443 if ((FixupKind == FK_Data_4 || FixupKind == FK_Data_8 || 444 FixupKind == MCFixupKind(X86::reloc_signed_4byte))) { 445 GlobalOffsetTableExprKind Kind = startsWithGlobalOffsetTable(Expr); 446 if (Kind != GOT_None) { 447 assert(ImmOffset == 0); 448 449 if (Size == 8) { 450 FixupKind = MCFixupKind(X86::reloc_global_offset_table8); 451 } else { 452 assert(Size == 4); 453 FixupKind = MCFixupKind(X86::reloc_global_offset_table); 454 } 455 456 if (Kind == GOT_Normal) 457 ImmOffset = static_cast<int>(CB.size() - StartByte); 458 } else if (Expr->getKind() == MCExpr::SymbolRef) { 459 if (hasSecRelSymbolRef(Expr)) { 460 FixupKind = MCFixupKind(FK_SecRel_4); 461 } 462 } else if (Expr->getKind() == MCExpr::Binary) { 463 const MCBinaryExpr *Bin = static_cast<const MCBinaryExpr *>(Expr); 464 if (hasSecRelSymbolRef(Bin->getLHS()) || 465 hasSecRelSymbolRef(Bin->getRHS())) { 466 FixupKind = MCFixupKind(FK_SecRel_4); 467 } 468 } 469 } 470 471 // If the fixup is pc-relative, we need to bias the value to be relative to 472 // the start of the field, not the end of the field. 473 if (FixupKind == FK_PCRel_4 || 474 FixupKind == MCFixupKind(X86::reloc_riprel_4byte) || 475 FixupKind == MCFixupKind(X86::reloc_riprel_4byte_movq_load) || 476 FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax) || 477 FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax_rex) || 478 FixupKind == MCFixupKind(X86::reloc_branch_4byte_pcrel)) { 479 ImmOffset -= 4; 480 // If this is a pc-relative load off _GLOBAL_OFFSET_TABLE_: 481 // leaq _GLOBAL_OFFSET_TABLE_(%rip), %r15 482 // this needs to be a GOTPC32 relocation. 483 if (startsWithGlobalOffsetTable(Expr) != GOT_None) 484 FixupKind = MCFixupKind(X86::reloc_global_offset_table); 485 } 486 if (FixupKind == FK_PCRel_2) 487 ImmOffset -= 2; 488 if (FixupKind == FK_PCRel_1) 489 ImmOffset -= 1; 490 491 if (ImmOffset) 492 Expr = MCBinaryExpr::createAdd(Expr, MCConstantExpr::create(ImmOffset, Ctx), 493 Ctx); 494 495 // Emit a symbolic constant as a fixup and 4 zeros. 496 Fixups.push_back(MCFixup::create(static_cast<uint32_t>(CB.size() - StartByte), 497 Expr, FixupKind, Loc)); 498 emitConstant(0, Size, CB); 499 } 500 501 void X86MCCodeEmitter::emitRegModRMByte(const MCOperand &ModRMReg, 502 unsigned RegOpcodeFld, 503 SmallVectorImpl<char> &CB) const { 504 emitByte(modRMByte(3, RegOpcodeFld, getX86RegNum(ModRMReg)), CB); 505 } 506 507 void X86MCCodeEmitter::emitSIBByte(unsigned SS, unsigned Index, unsigned Base, 508 SmallVectorImpl<char> &CB) const { 509 // SIB byte is in the same format as the modRMByte. 510 emitByte(modRMByte(SS, Index, Base), CB); 511 } 512 513 void X86MCCodeEmitter::emitMemModRMByte( 514 const MCInst &MI, unsigned Op, unsigned RegOpcodeField, uint64_t TSFlags, 515 PrefixKind Kind, uint64_t StartByte, SmallVectorImpl<char> &CB, 516 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI, 517 bool ForceSIB) const { 518 const MCOperand &Disp = MI.getOperand(Op + X86::AddrDisp); 519 const MCOperand &Base = MI.getOperand(Op + X86::AddrBaseReg); 520 const MCOperand &Scale = MI.getOperand(Op + X86::AddrScaleAmt); 521 const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg); 522 unsigned BaseReg = Base.getReg(); 523 524 // Handle %rip relative addressing. 525 if (BaseReg == X86::RIP || 526 BaseReg == X86::EIP) { // [disp32+rIP] in X86-64 mode 527 assert(STI.hasFeature(X86::Is64Bit) && 528 "Rip-relative addressing requires 64-bit mode"); 529 assert(IndexReg.getReg() == 0 && !ForceSIB && 530 "Invalid rip-relative address"); 531 emitByte(modRMByte(0, RegOpcodeField, 5), CB); 532 533 unsigned Opcode = MI.getOpcode(); 534 unsigned FixupKind = [&]() { 535 // Enable relaxed relocation only for a MCSymbolRefExpr. We cannot use a 536 // relaxed relocation if an offset is present (e.g. x@GOTPCREL+4). 537 if (!(Disp.isExpr() && isa<MCSymbolRefExpr>(Disp.getExpr()))) 538 return X86::reloc_riprel_4byte; 539 540 // Certain loads for GOT references can be relocated against the symbol 541 // directly if the symbol ends up in the same linkage unit. 542 switch (Opcode) { 543 default: 544 return X86::reloc_riprel_4byte; 545 case X86::MOV64rm: 546 // movq loads is a subset of reloc_riprel_4byte_relax_rex. It is a 547 // special case because COFF and Mach-O don't support ELF's more 548 // flexible R_X86_64_REX_GOTPCRELX relaxation. 549 assert(Kind == REX); 550 return X86::reloc_riprel_4byte_movq_load; 551 case X86::ADC32rm: 552 case X86::ADD32rm: 553 case X86::AND32rm: 554 case X86::CMP32rm: 555 case X86::MOV32rm: 556 case X86::OR32rm: 557 case X86::SBB32rm: 558 case X86::SUB32rm: 559 case X86::TEST32mr: 560 case X86::XOR32rm: 561 case X86::CALL64m: 562 case X86::JMP64m: 563 case X86::TAILJMPm64: 564 case X86::TEST64mr: 565 case X86::ADC64rm: 566 case X86::ADD64rm: 567 case X86::AND64rm: 568 case X86::CMP64rm: 569 case X86::OR64rm: 570 case X86::SBB64rm: 571 case X86::SUB64rm: 572 case X86::XOR64rm: 573 return Kind == REX ? X86::reloc_riprel_4byte_relax_rex 574 : X86::reloc_riprel_4byte_relax; 575 } 576 }(); 577 578 // rip-relative addressing is actually relative to the *next* instruction. 579 // Since an immediate can follow the mod/rm byte for an instruction, this 580 // means that we need to bias the displacement field of the instruction with 581 // the size of the immediate field. If we have this case, add it into the 582 // expression to emit. 583 // Note: rip-relative addressing using immediate displacement values should 584 // not be adjusted, assuming it was the user's intent. 585 int ImmSize = !Disp.isImm() && X86II::hasImm(TSFlags) 586 ? X86II::getSizeOfImm(TSFlags) 587 : 0; 588 589 emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), StartByte, CB, 590 Fixups, -ImmSize); 591 return; 592 } 593 594 unsigned BaseRegNo = BaseReg ? getX86RegNum(Base) : -1U; 595 596 // 16-bit addressing forms of the ModR/M byte have a different encoding for 597 // the R/M field and are far more limited in which registers can be used. 598 if (X86_MC::is16BitMemOperand(MI, Op, STI)) { 599 if (BaseReg) { 600 // For 32-bit addressing, the row and column values in Table 2-2 are 601 // basically the same. It's AX/CX/DX/BX/SP/BP/SI/DI in that order, with 602 // some special cases. And getX86RegNum reflects that numbering. 603 // For 16-bit addressing it's more fun, as shown in the SDM Vol 2A, 604 // Table 2-1 "16-Bit Addressing Forms with the ModR/M byte". We can only 605 // use SI/DI/BP/BX, which have "row" values 4-7 in no particular order, 606 // while values 0-3 indicate the allowed combinations (base+index) of 607 // those: 0 for BX+SI, 1 for BX+DI, 2 for BP+SI, 3 for BP+DI. 608 // 609 // R16Table[] is a lookup from the normal RegNo, to the row values from 610 // Table 2-1 for 16-bit addressing modes. Where zero means disallowed. 611 static const unsigned R16Table[] = {0, 0, 0, 7, 0, 6, 4, 5}; 612 unsigned RMfield = R16Table[BaseRegNo]; 613 614 assert(RMfield && "invalid 16-bit base register"); 615 616 if (IndexReg.getReg()) { 617 unsigned IndexReg16 = R16Table[getX86RegNum(IndexReg)]; 618 619 assert(IndexReg16 && "invalid 16-bit index register"); 620 // We must have one of SI/DI (4,5), and one of BP/BX (6,7). 621 assert(((IndexReg16 ^ RMfield) & 2) && 622 "invalid 16-bit base/index register combination"); 623 assert(Scale.getImm() == 1 && 624 "invalid scale for 16-bit memory reference"); 625 626 // Allow base/index to appear in either order (although GAS doesn't). 627 if (IndexReg16 & 2) 628 RMfield = (RMfield & 1) | ((7 - IndexReg16) << 1); 629 else 630 RMfield = (IndexReg16 & 1) | ((7 - RMfield) << 1); 631 } 632 633 if (Disp.isImm() && isInt<8>(Disp.getImm())) { 634 if (Disp.getImm() == 0 && RMfield != 6) { 635 // There is no displacement; just the register. 636 emitByte(modRMByte(0, RegOpcodeField, RMfield), CB); 637 return; 638 } 639 // Use the [REG]+disp8 form, including for [BP] which cannot be encoded. 640 emitByte(modRMByte(1, RegOpcodeField, RMfield), CB); 641 emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, CB, Fixups); 642 return; 643 } 644 // This is the [REG]+disp16 case. 645 emitByte(modRMByte(2, RegOpcodeField, RMfield), CB); 646 } else { 647 assert(IndexReg.getReg() == 0 && "Unexpected index register!"); 648 // There is no BaseReg; this is the plain [disp16] case. 649 emitByte(modRMByte(0, RegOpcodeField, 6), CB); 650 } 651 652 // Emit 16-bit displacement for plain disp16 or [REG]+disp16 cases. 653 emitImmediate(Disp, MI.getLoc(), 2, FK_Data_2, StartByte, CB, Fixups); 654 return; 655 } 656 657 // Check for presence of {disp8} or {disp32} pseudo prefixes. 658 bool UseDisp8 = MI.getFlags() & X86::IP_USE_DISP8; 659 bool UseDisp32 = MI.getFlags() & X86::IP_USE_DISP32; 660 661 // We only allow no displacement if no pseudo prefix is present. 662 bool AllowNoDisp = !UseDisp8 && !UseDisp32; 663 // Disp8 is allowed unless the {disp32} prefix is present. 664 bool AllowDisp8 = !UseDisp32; 665 666 // Determine whether a SIB byte is needed. 667 if (// The SIB byte must be used if there is an index register or the 668 // encoding requires a SIB byte. 669 !ForceSIB && IndexReg.getReg() == 0 && 670 // The SIB byte must be used if the base is ESP/RSP/R12, all of which 671 // encode to an R/M value of 4, which indicates that a SIB byte is 672 // present. 673 BaseRegNo != N86::ESP && 674 // If there is no base register and we're in 64-bit mode, we need a SIB 675 // byte to emit an addr that is just 'disp32' (the non-RIP relative form). 676 (!STI.hasFeature(X86::Is64Bit) || BaseReg != 0)) { 677 678 if (BaseReg == 0) { // [disp32] in X86-32 mode 679 emitByte(modRMByte(0, RegOpcodeField, 5), CB); 680 emitImmediate(Disp, MI.getLoc(), 4, FK_Data_4, StartByte, CB, Fixups); 681 return; 682 } 683 684 // If the base is not EBP/ESP/R12/R13 and there is no displacement, use 685 // simple indirect register encoding, this handles addresses like [EAX]. 686 // The encoding for [EBP] or[R13] with no displacement means [disp32] so we 687 // handle it by emitting a displacement of 0 later. 688 if (BaseRegNo != N86::EBP) { 689 if (Disp.isImm() && Disp.getImm() == 0 && AllowNoDisp) { 690 emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), CB); 691 return; 692 } 693 694 // If the displacement is @tlscall, treat it as a zero. 695 if (Disp.isExpr()) { 696 auto *Sym = dyn_cast<MCSymbolRefExpr>(Disp.getExpr()); 697 if (Sym && Sym->getKind() == MCSymbolRefExpr::VK_TLSCALL) { 698 // This is exclusively used by call *a@tlscall(base). The relocation 699 // (R_386_TLSCALL or R_X86_64_TLSCALL) applies to the beginning. 700 Fixups.push_back(MCFixup::create(0, Sym, FK_NONE, MI.getLoc())); 701 emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), CB); 702 return; 703 } 704 } 705 } 706 707 // Otherwise, if the displacement fits in a byte, encode as [REG+disp8]. 708 // Including a compressed disp8 for EVEX instructions that support it. 709 // This also handles the 0 displacement for [EBP] or [R13]. We can't use 710 // disp8 if the {disp32} pseudo prefix is present. 711 if (Disp.isImm() && AllowDisp8) { 712 int ImmOffset = 0; 713 if (isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) { 714 emitByte(modRMByte(1, RegOpcodeField, BaseRegNo), CB); 715 emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, CB, Fixups, 716 ImmOffset); 717 return; 718 } 719 } 720 721 // Otherwise, emit the most general non-SIB encoding: [REG+disp32]. 722 // Displacement may be 0 for [EBP] or [R13] case if {disp32} pseudo prefix 723 // prevented using disp8 above. 724 emitByte(modRMByte(2, RegOpcodeField, BaseRegNo), CB); 725 unsigned Opcode = MI.getOpcode(); 726 unsigned FixupKind = Opcode == X86::MOV32rm ? X86::reloc_signed_4byte_relax 727 : X86::reloc_signed_4byte; 728 emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), StartByte, CB, 729 Fixups); 730 return; 731 } 732 733 // We need a SIB byte, so start by outputting the ModR/M byte first 734 assert(IndexReg.getReg() != X86::ESP && IndexReg.getReg() != X86::RSP && 735 "Cannot use ESP as index reg!"); 736 737 bool ForceDisp32 = false; 738 bool ForceDisp8 = false; 739 int ImmOffset = 0; 740 if (BaseReg == 0) { 741 // If there is no base register, we emit the special case SIB byte with 742 // MOD=0, BASE=5, to JUST get the index, scale, and displacement. 743 BaseRegNo = 5; 744 emitByte(modRMByte(0, RegOpcodeField, 4), CB); 745 ForceDisp32 = true; 746 } else if (Disp.isImm() && Disp.getImm() == 0 && AllowNoDisp && 747 // Base reg can't be EBP/RBP/R13 as that would end up with '5' as 748 // the base field, but that is the magic [*] nomenclature that 749 // indicates no base when mod=0. For these cases we'll emit a 0 750 // displacement instead. 751 BaseRegNo != N86::EBP) { 752 // Emit no displacement ModR/M byte 753 emitByte(modRMByte(0, RegOpcodeField, 4), CB); 754 } else if (Disp.isImm() && AllowDisp8 && 755 isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) { 756 // Displacement fits in a byte or matches an EVEX compressed disp8, use 757 // disp8 encoding. This also handles EBP/R13 base with 0 displacement unless 758 // {disp32} pseudo prefix was used. 759 emitByte(modRMByte(1, RegOpcodeField, 4), CB); 760 ForceDisp8 = true; 761 } else { 762 // Otherwise, emit the normal disp32 encoding. 763 emitByte(modRMByte(2, RegOpcodeField, 4), CB); 764 ForceDisp32 = true; 765 } 766 767 // Calculate what the SS field value should be... 768 static const unsigned SSTable[] = {~0U, 0, 1, ~0U, 2, ~0U, ~0U, ~0U, 3}; 769 unsigned SS = SSTable[Scale.getImm()]; 770 771 unsigned IndexRegNo = IndexReg.getReg() ? getX86RegNum(IndexReg) : 4; 772 773 emitSIBByte(SS, IndexRegNo, BaseRegNo, CB); 774 775 // Do we need to output a displacement? 776 if (ForceDisp8) 777 emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, CB, Fixups, 778 ImmOffset); 779 else if (ForceDisp32) 780 emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(X86::reloc_signed_4byte), 781 StartByte, CB, Fixups); 782 } 783 784 /// Emit all instruction prefixes. 785 /// 786 /// \returns one of the REX, XOP, VEX2, VEX3, EVEX if any of them is used, 787 /// otherwise returns None. 788 PrefixKind X86MCCodeEmitter::emitPrefixImpl(unsigned &CurOp, const MCInst &MI, 789 const MCSubtargetInfo &STI, 790 SmallVectorImpl<char> &CB) const { 791 uint64_t TSFlags = MCII.get(MI.getOpcode()).TSFlags; 792 // Determine where the memory operand starts, if present. 793 int MemoryOperand = X86II::getMemoryOperandNo(TSFlags); 794 // Emit segment override opcode prefix as needed. 795 if (MemoryOperand != -1) { 796 MemoryOperand += CurOp; 797 emitSegmentOverridePrefix(MemoryOperand + X86::AddrSegmentReg, MI, CB); 798 } 799 800 // Emit the repeat opcode prefix as needed. 801 unsigned Flags = MI.getFlags(); 802 if (TSFlags & X86II::REP || Flags & X86::IP_HAS_REPEAT) 803 emitByte(0xF3, CB); 804 if (Flags & X86::IP_HAS_REPEAT_NE) 805 emitByte(0xF2, CB); 806 807 // Emit the address size opcode prefix as needed. 808 if (X86_MC::needsAddressSizeOverride(MI, STI, MemoryOperand, TSFlags) || 809 Flags & X86::IP_HAS_AD_SIZE) 810 emitByte(0x67, CB); 811 812 uint64_t Form = TSFlags & X86II::FormMask; 813 switch (Form) { 814 default: 815 break; 816 case X86II::RawFrmDstSrc: { 817 // Emit segment override opcode prefix as needed (not for %ds). 818 if (MI.getOperand(2).getReg() != X86::DS) 819 emitSegmentOverridePrefix(2, MI, CB); 820 CurOp += 3; // Consume operands. 821 break; 822 } 823 case X86II::RawFrmSrc: { 824 // Emit segment override opcode prefix as needed (not for %ds). 825 if (MI.getOperand(1).getReg() != X86::DS) 826 emitSegmentOverridePrefix(1, MI, CB); 827 CurOp += 2; // Consume operands. 828 break; 829 } 830 case X86II::RawFrmDst: { 831 ++CurOp; // Consume operand. 832 break; 833 } 834 case X86II::RawFrmMemOffs: { 835 // Emit segment override opcode prefix as needed. 836 emitSegmentOverridePrefix(1, MI, CB); 837 break; 838 } 839 } 840 841 // REX prefix is optional, but if used must be immediately before the opcode 842 // Encoding type for this instruction. 843 return (TSFlags & X86II::EncodingMask) 844 ? emitVEXOpcodePrefix(MemoryOperand, MI, CB) 845 : emitOpcodePrefix(MemoryOperand, MI, STI, CB); 846 } 847 848 // AVX instructions are encoded using an encoding scheme that combines 849 // prefix bytes, opcode extension field, operand encoding fields, and vector 850 // length encoding capability into a new prefix, referred to as VEX. 851 852 // The majority of the AVX-512 family of instructions (operating on 853 // 512/256/128-bit vector register operands) are encoded using a new prefix 854 // (called EVEX). 855 856 // XOP is a revised subset of what was originally intended as SSE5. It was 857 // changed to be similar but not overlapping with AVX. 858 859 /// Emit XOP, VEX2, VEX3 or EVEX prefix. 860 /// \returns the used prefix. 861 PrefixKind 862 X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, 863 SmallVectorImpl<char> &CB) const { 864 const MCInstrDesc &Desc = MCII.get(MI.getOpcode()); 865 uint64_t TSFlags = Desc.TSFlags; 866 867 assert(!(TSFlags & X86II::LOCK) && "Can't have LOCK VEX."); 868 869 X86OpcodePrefixHelper Prefix(*Ctx.getRegisterInfo()); 870 switch (TSFlags & X86II::EncodingMask) { 871 default: 872 break; 873 case X86II::XOP: 874 Prefix.setLowerBound(XOP); 875 break; 876 case X86II::VEX: 877 // VEX can be 2 byte or 3 byte, not determined yet if not explicit 878 Prefix.setLowerBound(MI.getFlags() & X86::IP_USE_VEX3 ? VEX3 : VEX2); 879 break; 880 case X86II::EVEX: 881 Prefix.setLowerBound(EVEX); 882 break; 883 } 884 885 Prefix.setW(TSFlags & X86II::REX_W); 886 887 bool HasEVEX_K = TSFlags & X86II::EVEX_K; 888 bool HasVEX_4V = TSFlags & X86II::VEX_4V; 889 bool HasEVEX_RC = TSFlags & X86II::EVEX_RC; 890 891 switch (TSFlags & X86II::OpMapMask) { 892 default: 893 llvm_unreachable("Invalid prefix!"); 894 case X86II::TB: 895 Prefix.set5M(0x1); // 0F 896 break; 897 case X86II::T8: 898 Prefix.set5M(0x2); // 0F 38 899 break; 900 case X86II::TA: 901 Prefix.set5M(0x3); // 0F 3A 902 break; 903 case X86II::XOP8: 904 Prefix.set5M(0x8); 905 break; 906 case X86II::XOP9: 907 Prefix.set5M(0x9); 908 break; 909 case X86II::XOPA: 910 Prefix.set5M(0xA); 911 break; 912 case X86II::T_MAP5: 913 Prefix.set5M(0x5); 914 break; 915 case X86II::T_MAP6: 916 Prefix.set5M(0x6); 917 break; 918 } 919 920 Prefix.setL(TSFlags & X86II::VEX_L); 921 Prefix.setL2(TSFlags & X86II::EVEX_L2); 922 switch (TSFlags & X86II::OpPrefixMask) { 923 case X86II::PD: 924 Prefix.setPP(0x1); // 66 925 break; 926 case X86II::XS: 927 Prefix.setPP(0x2); // F3 928 break; 929 case X86II::XD: 930 Prefix.setPP(0x3); // F2 931 break; 932 } 933 934 Prefix.setZ(HasEVEX_K && (TSFlags & X86II::EVEX_Z)); 935 Prefix.setEVEX_b(TSFlags & X86II::EVEX_B); 936 937 bool EncodeRC = false; 938 uint8_t EVEX_rc = 0; 939 unsigned CurOp = X86II::getOperandBias(Desc); 940 941 switch (TSFlags & X86II::FormMask) { 942 default: 943 llvm_unreachable("Unexpected form in emitVEXOpcodePrefix!"); 944 case X86II::MRMDestMem4VOp3CC: { 945 // MemAddr, src1(ModR/M), src2(VEX_4V) 946 Prefix.setB(MI, MemOperand + X86::AddrBaseReg); 947 Prefix.setX(MI, MemOperand + X86::AddrIndexReg); 948 CurOp += X86::AddrNumOperands; 949 Prefix.setR(MI, ++CurOp); 950 Prefix.set4V(MI, CurOp++); 951 break; 952 } 953 case X86II::MRM_C0: 954 case X86II::RawFrm: 955 break; 956 case X86II::MRMDestMemFSIB: 957 case X86II::MRMDestMem: { 958 // MRMDestMem instructions forms: 959 // MemAddr, src1(ModR/M) 960 // MemAddr, src1(VEX_4V), src2(ModR/M) 961 // MemAddr, src1(ModR/M), imm8 962 // 963 Prefix.setB(MI, MemOperand + X86::AddrBaseReg); 964 Prefix.setX(MI, MemOperand + X86::AddrIndexReg); 965 if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV. 966 Prefix.setV2(MI, MemOperand + X86::AddrIndexReg); 967 968 CurOp += X86::AddrNumOperands; 969 970 if (HasEVEX_K) 971 Prefix.setAAA(MI, CurOp++); 972 973 if (HasVEX_4V) 974 Prefix.set4VV2(MI, CurOp++); 975 976 Prefix.setRR2(MI, CurOp++); 977 break; 978 } 979 case X86II::MRMSrcMemFSIB: 980 case X86II::MRMSrcMem: { 981 // MRMSrcMem instructions forms: 982 // src1(ModR/M), MemAddr 983 // src1(ModR/M), src2(VEX_4V), MemAddr 984 // src1(ModR/M), MemAddr, imm8 985 // src1(ModR/M), MemAddr, src2(Imm[7:4]) 986 // 987 // FMA4: 988 // dst(ModR/M.reg), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4]) 989 Prefix.setRR2(MI, CurOp++); 990 991 if (HasEVEX_K) 992 Prefix.setAAA(MI, CurOp++); 993 994 if (HasVEX_4V) 995 Prefix.set4VV2(MI, CurOp++); 996 997 Prefix.setB(MI, MemOperand + X86::AddrBaseReg); 998 Prefix.setX(MI, MemOperand + X86::AddrIndexReg); 999 if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV. 1000 Prefix.setV2(MI, MemOperand + X86::AddrIndexReg); 1001 1002 break; 1003 } 1004 case X86II::MRMSrcMem4VOp3: { 1005 // Instruction format for 4VOp3: 1006 // src1(ModR/M), MemAddr, src3(VEX_4V) 1007 Prefix.setR(MI, CurOp++); 1008 Prefix.setB(MI, MemOperand + X86::AddrBaseReg); 1009 Prefix.setX(MI, MemOperand + X86::AddrIndexReg); 1010 Prefix.set4V(MI, CurOp + X86::AddrNumOperands); 1011 break; 1012 } 1013 case X86II::MRMSrcMemOp4: { 1014 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M), 1015 Prefix.setR(MI, CurOp++); 1016 Prefix.set4V(MI, CurOp++); 1017 Prefix.setB(MI, MemOperand + X86::AddrBaseReg); 1018 Prefix.setX(MI, MemOperand + X86::AddrIndexReg); 1019 break; 1020 } 1021 case X86II::MRM0m: 1022 case X86II::MRM1m: 1023 case X86II::MRM2m: 1024 case X86II::MRM3m: 1025 case X86II::MRM4m: 1026 case X86II::MRM5m: 1027 case X86II::MRM6m: 1028 case X86II::MRM7m: { 1029 // MRM[0-9]m instructions forms: 1030 // MemAddr 1031 // src1(VEX_4V), MemAddr 1032 if (HasVEX_4V) 1033 Prefix.set4VV2(MI, CurOp++); 1034 1035 if (HasEVEX_K) 1036 Prefix.setAAA(MI, CurOp++); 1037 1038 Prefix.setB(MI, MemOperand + X86::AddrBaseReg); 1039 Prefix.setX(MI, MemOperand + X86::AddrIndexReg); 1040 if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV. 1041 Prefix.setV2(MI, MemOperand + X86::AddrIndexReg); 1042 1043 break; 1044 } 1045 case X86II::MRMSrcReg: { 1046 // MRMSrcReg instructions forms: 1047 // dst(ModR/M), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4]) 1048 // dst(ModR/M), src1(ModR/M) 1049 // dst(ModR/M), src1(ModR/M), imm8 1050 // 1051 // FMA4: 1052 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M), 1053 Prefix.setRR2(MI, CurOp++); 1054 1055 if (HasEVEX_K) 1056 Prefix.setAAA(MI, CurOp++); 1057 1058 if (HasVEX_4V) 1059 Prefix.set4VV2(MI, CurOp++); 1060 1061 Prefix.setB(MI, CurOp); 1062 Prefix.setX(MI, CurOp, 4); 1063 ++CurOp; 1064 1065 if (TSFlags & X86II::EVEX_B) { 1066 if (HasEVEX_RC) { 1067 unsigned NumOps = Desc.getNumOperands(); 1068 unsigned RcOperand = NumOps - 1; 1069 assert(RcOperand >= CurOp); 1070 EVEX_rc = MI.getOperand(RcOperand).getImm(); 1071 assert(EVEX_rc <= 3 && "Invalid rounding control!"); 1072 } 1073 EncodeRC = true; 1074 } 1075 break; 1076 } 1077 case X86II::MRMSrcReg4VOp3: { 1078 // Instruction format for 4VOp3: 1079 // src1(ModR/M), src2(ModR/M), src3(VEX_4V) 1080 Prefix.setR(MI, CurOp++); 1081 Prefix.setB(MI, CurOp++); 1082 Prefix.set4V(MI, CurOp++); 1083 break; 1084 } 1085 case X86II::MRMSrcRegOp4: { 1086 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M), 1087 Prefix.setR(MI, CurOp++); 1088 Prefix.set4V(MI, CurOp++); 1089 // Skip second register source (encoded in Imm[7:4]) 1090 ++CurOp; 1091 1092 Prefix.setB(MI, CurOp); 1093 Prefix.setX(MI, CurOp, 4); 1094 ++CurOp; 1095 break; 1096 } 1097 case X86II::MRMDestReg: { 1098 // MRMDestReg instructions forms: 1099 // dst(ModR/M), src(ModR/M) 1100 // dst(ModR/M), src(ModR/M), imm8 1101 // dst(ModR/M), src1(VEX_4V), src2(ModR/M) 1102 Prefix.setB(MI, CurOp); 1103 Prefix.setX(MI, CurOp, 4); 1104 ++CurOp; 1105 1106 if (HasEVEX_K) 1107 Prefix.setAAA(MI, CurOp++); 1108 1109 if (HasVEX_4V) 1110 Prefix.set4VV2(MI, CurOp++); 1111 1112 Prefix.setRR2(MI, CurOp++); 1113 if (TSFlags & X86II::EVEX_B) 1114 EncodeRC = true; 1115 break; 1116 } 1117 case X86II::MRMr0: { 1118 // MRMr0 instructions forms: 1119 // 11:rrr:000 1120 // dst(ModR/M) 1121 Prefix.setRR2(MI, CurOp++); 1122 break; 1123 } 1124 case X86II::MRM0r: 1125 case X86II::MRM1r: 1126 case X86II::MRM2r: 1127 case X86II::MRM3r: 1128 case X86II::MRM4r: 1129 case X86II::MRM5r: 1130 case X86II::MRM6r: 1131 case X86II::MRM7r: { 1132 // MRM0r-MRM7r instructions forms: 1133 // dst(VEX_4V), src(ModR/M), imm8 1134 if (HasVEX_4V) 1135 Prefix.set4VV2(MI, CurOp++); 1136 1137 if (HasEVEX_K) 1138 Prefix.setAAA(MI, CurOp++); 1139 1140 Prefix.setB(MI, CurOp); 1141 Prefix.setX(MI, CurOp, 4); 1142 ++CurOp; 1143 break; 1144 } 1145 } 1146 if (EncodeRC) { 1147 Prefix.setL(EVEX_rc & 0x1); 1148 Prefix.setL2(EVEX_rc & 0x2); 1149 } 1150 PrefixKind Kind = Prefix.determineOptimalKind(); 1151 Prefix.emit(CB); 1152 return Kind; 1153 } 1154 1155 /// Emit REX prefix which specifies 1156 /// 1) 64-bit instructions, 1157 /// 2) non-default operand size, and 1158 /// 3) use of X86-64 extended registers. 1159 /// 1160 /// \returns the used prefix (REX or None). 1161 PrefixKind X86MCCodeEmitter::emitREXPrefix(int MemOperand, const MCInst &MI, 1162 const MCSubtargetInfo &STI, 1163 SmallVectorImpl<char> &CB) const { 1164 if (!STI.hasFeature(X86::Is64Bit)) 1165 return None; 1166 X86OpcodePrefixHelper Prefix(*Ctx.getRegisterInfo()); 1167 const MCInstrDesc &Desc = MCII.get(MI.getOpcode()); 1168 uint64_t TSFlags = Desc.TSFlags; 1169 Prefix.setW(TSFlags & X86II::REX_W); 1170 unsigned NumOps = MI.getNumOperands(); 1171 bool UsesHighByteReg = false; 1172 #ifndef NDEBUG 1173 bool HasRegOp = false; 1174 #endif 1175 unsigned CurOp = NumOps ? X86II::getOperandBias(Desc) : 0; 1176 for (unsigned i = CurOp; i != NumOps; ++i) { 1177 const MCOperand &MO = MI.getOperand(i); 1178 if (MO.isReg()) { 1179 #ifndef NDEBUG 1180 HasRegOp = true; 1181 #endif 1182 unsigned Reg = MO.getReg(); 1183 if (Reg == X86::AH || Reg == X86::BH || Reg == X86::CH || Reg == X86::DH) 1184 UsesHighByteReg = true; 1185 // If it accesses SPL, BPL, SIL, or DIL, then it requires a REX prefix. 1186 if (X86II::isX86_64NonExtLowByteReg(Reg)) 1187 Prefix.setLowerBound(REX); 1188 } else if (MO.isExpr() && STI.getTargetTriple().isX32()) { 1189 // GOTTPOFF and TLSDESC relocations require a REX prefix to allow 1190 // linker optimizations: even if the instructions we see may not require 1191 // any prefix, they may be replaced by instructions that do. This is 1192 // handled as a special case here so that it also works for hand-written 1193 // assembly without the user needing to write REX, as with GNU as. 1194 const auto *Ref = dyn_cast<MCSymbolRefExpr>(MO.getExpr()); 1195 if (Ref && (Ref->getKind() == MCSymbolRefExpr::VK_GOTTPOFF || 1196 Ref->getKind() == MCSymbolRefExpr::VK_TLSDESC)) { 1197 Prefix.setLowerBound(REX); 1198 } 1199 } 1200 } 1201 switch (TSFlags & X86II::FormMask) { 1202 default: 1203 assert(!HasRegOp && "Unexpected form in emitREXPrefix!"); 1204 break; 1205 case X86II::RawFrm: 1206 case X86II::RawFrmMemOffs: 1207 case X86II::RawFrmSrc: 1208 case X86II::RawFrmDst: 1209 case X86II::RawFrmDstSrc: 1210 break; 1211 case X86II::AddRegFrm: 1212 Prefix.setB(MI, CurOp++); 1213 break; 1214 case X86II::MRMSrcReg: 1215 case X86II::MRMSrcRegCC: 1216 Prefix.setR(MI, CurOp++); 1217 Prefix.setB(MI, CurOp++); 1218 break; 1219 case X86II::MRMSrcMem: 1220 case X86II::MRMSrcMemCC: 1221 Prefix.setR(MI, CurOp++); 1222 Prefix.setB(MI, MemOperand + X86::AddrBaseReg); 1223 Prefix.setX(MI, MemOperand + X86::AddrIndexReg); 1224 CurOp += X86::AddrNumOperands; 1225 break; 1226 case X86II::MRMDestReg: 1227 Prefix.setB(MI, CurOp++); 1228 Prefix.setR(MI, CurOp++); 1229 break; 1230 case X86II::MRMDestMem: 1231 Prefix.setB(MI, MemOperand + X86::AddrBaseReg); 1232 Prefix.setX(MI, MemOperand + X86::AddrIndexReg); 1233 CurOp += X86::AddrNumOperands; 1234 Prefix.setR(MI, CurOp++); 1235 break; 1236 case X86II::MRMXmCC: 1237 case X86II::MRMXm: 1238 case X86II::MRM0m: 1239 case X86II::MRM1m: 1240 case X86II::MRM2m: 1241 case X86II::MRM3m: 1242 case X86II::MRM4m: 1243 case X86II::MRM5m: 1244 case X86II::MRM6m: 1245 case X86II::MRM7m: 1246 Prefix.setB(MI, MemOperand + X86::AddrBaseReg); 1247 Prefix.setX(MI, MemOperand + X86::AddrIndexReg); 1248 break; 1249 case X86II::MRMXrCC: 1250 case X86II::MRMXr: 1251 case X86II::MRM0r: 1252 case X86II::MRM1r: 1253 case X86II::MRM2r: 1254 case X86II::MRM3r: 1255 case X86II::MRM4r: 1256 case X86II::MRM5r: 1257 case X86II::MRM6r: 1258 case X86II::MRM7r: 1259 Prefix.setB(MI, CurOp++); 1260 break; 1261 } 1262 PrefixKind Kind = Prefix.determineOptimalKind(); 1263 if (Kind && UsesHighByteReg) 1264 report_fatal_error( 1265 "Cannot encode high byte register in REX-prefixed instruction"); 1266 Prefix.emit(CB); 1267 return Kind; 1268 } 1269 1270 /// Emit segment override opcode prefix as needed. 1271 void X86MCCodeEmitter::emitSegmentOverridePrefix( 1272 unsigned SegOperand, const MCInst &MI, SmallVectorImpl<char> &CB) const { 1273 // Check for explicit segment override on memory operand. 1274 if (unsigned Reg = MI.getOperand(SegOperand).getReg()) 1275 emitByte(X86::getSegmentOverridePrefixForReg(Reg), CB); 1276 } 1277 1278 /// Emit all instruction prefixes prior to the opcode. 1279 /// 1280 /// \param MemOperand the operand # of the start of a memory operand if present. 1281 /// If not present, it is -1. 1282 /// 1283 /// \returns the used prefix (REX or None). 1284 PrefixKind X86MCCodeEmitter::emitOpcodePrefix(int MemOperand, const MCInst &MI, 1285 const MCSubtargetInfo &STI, 1286 SmallVectorImpl<char> &CB) const { 1287 const MCInstrDesc &Desc = MCII.get(MI.getOpcode()); 1288 uint64_t TSFlags = Desc.TSFlags; 1289 1290 // Emit the operand size opcode prefix as needed. 1291 if ((TSFlags & X86II::OpSizeMask) == 1292 (STI.hasFeature(X86::Is16Bit) ? X86II::OpSize32 : X86II::OpSize16)) 1293 emitByte(0x66, CB); 1294 1295 // Emit the LOCK opcode prefix. 1296 if (TSFlags & X86II::LOCK || MI.getFlags() & X86::IP_HAS_LOCK) 1297 emitByte(0xF0, CB); 1298 1299 // Emit the NOTRACK opcode prefix. 1300 if (TSFlags & X86II::NOTRACK || MI.getFlags() & X86::IP_HAS_NOTRACK) 1301 emitByte(0x3E, CB); 1302 1303 switch (TSFlags & X86II::OpPrefixMask) { 1304 case X86II::PD: // 66 1305 emitByte(0x66, CB); 1306 break; 1307 case X86II::XS: // F3 1308 emitByte(0xF3, CB); 1309 break; 1310 case X86II::XD: // F2 1311 emitByte(0xF2, CB); 1312 break; 1313 } 1314 1315 // Handle REX prefix. 1316 assert((STI.hasFeature(X86::Is64Bit) || !(TSFlags & X86II::REX_W)) && 1317 "REX.W requires 64bit mode."); 1318 PrefixKind Kind = emitREXPrefix(MemOperand, MI, STI, CB); 1319 1320 // 0x0F escape code must be emitted just before the opcode. 1321 switch (TSFlags & X86II::OpMapMask) { 1322 case X86II::TB: // Two-byte opcode map 1323 case X86II::T8: // 0F 38 1324 case X86II::TA: // 0F 3A 1325 case X86II::ThreeDNow: // 0F 0F, second 0F emitted by caller. 1326 emitByte(0x0F, CB); 1327 break; 1328 } 1329 1330 switch (TSFlags & X86II::OpMapMask) { 1331 case X86II::T8: // 0F 38 1332 emitByte(0x38, CB); 1333 break; 1334 case X86II::TA: // 0F 3A 1335 emitByte(0x3A, CB); 1336 break; 1337 } 1338 1339 return Kind; 1340 } 1341 1342 void X86MCCodeEmitter::emitPrefix(const MCInst &MI, SmallVectorImpl<char> &CB, 1343 const MCSubtargetInfo &STI) const { 1344 unsigned Opcode = MI.getOpcode(); 1345 const MCInstrDesc &Desc = MCII.get(Opcode); 1346 uint64_t TSFlags = Desc.TSFlags; 1347 1348 // Pseudo instructions don't get encoded. 1349 if (X86II::isPseudo(TSFlags)) 1350 return; 1351 1352 unsigned CurOp = X86II::getOperandBias(Desc); 1353 1354 emitPrefixImpl(CurOp, MI, STI, CB); 1355 } 1356 1357 void X86MCCodeEmitter::encodeInstruction(const MCInst &MI, 1358 SmallVectorImpl<char> &CB, 1359 SmallVectorImpl<MCFixup> &Fixups, 1360 const MCSubtargetInfo &STI) const { 1361 unsigned Opcode = MI.getOpcode(); 1362 const MCInstrDesc &Desc = MCII.get(Opcode); 1363 uint64_t TSFlags = Desc.TSFlags; 1364 1365 // Pseudo instructions don't get encoded. 1366 if (X86II::isPseudo(TSFlags)) 1367 return; 1368 1369 unsigned NumOps = Desc.getNumOperands(); 1370 unsigned CurOp = X86II::getOperandBias(Desc); 1371 1372 uint64_t StartByte = CB.size(); 1373 1374 PrefixKind Kind = emitPrefixImpl(CurOp, MI, STI, CB); 1375 1376 // It uses the VEX.VVVV field? 1377 bool HasVEX_4V = TSFlags & X86II::VEX_4V; 1378 bool HasVEX_I8Reg = (TSFlags & X86II::ImmMask) == X86II::Imm8Reg; 1379 1380 // It uses the EVEX.aaa field? 1381 bool HasEVEX_K = TSFlags & X86II::EVEX_K; 1382 bool HasEVEX_RC = TSFlags & X86II::EVEX_RC; 1383 1384 // Used if a register is encoded in 7:4 of immediate. 1385 unsigned I8RegNum = 0; 1386 1387 uint8_t BaseOpcode = X86II::getBaseOpcodeFor(TSFlags); 1388 1389 if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow) 1390 BaseOpcode = 0x0F; // Weird 3DNow! encoding. 1391 1392 unsigned OpcodeOffset = 0; 1393 1394 uint64_t Form = TSFlags & X86II::FormMask; 1395 switch (Form) { 1396 default: 1397 errs() << "FORM: " << Form << "\n"; 1398 llvm_unreachable("Unknown FormMask value in X86MCCodeEmitter!"); 1399 case X86II::Pseudo: 1400 llvm_unreachable("Pseudo instruction shouldn't be emitted"); 1401 case X86II::RawFrmDstSrc: 1402 case X86II::RawFrmSrc: 1403 case X86II::RawFrmDst: 1404 case X86II::PrefixByte: 1405 emitByte(BaseOpcode, CB); 1406 break; 1407 case X86II::AddCCFrm: { 1408 // This will be added to the opcode in the fallthrough. 1409 OpcodeOffset = MI.getOperand(NumOps - 1).getImm(); 1410 assert(OpcodeOffset < 16 && "Unexpected opcode offset!"); 1411 --NumOps; // Drop the operand from the end. 1412 [[fallthrough]]; 1413 case X86II::RawFrm: 1414 emitByte(BaseOpcode + OpcodeOffset, CB); 1415 1416 if (!STI.hasFeature(X86::Is64Bit) || !isPCRel32Branch(MI, MCII)) 1417 break; 1418 1419 const MCOperand &Op = MI.getOperand(CurOp++); 1420 emitImmediate(Op, MI.getLoc(), X86II::getSizeOfImm(TSFlags), 1421 MCFixupKind(X86::reloc_branch_4byte_pcrel), StartByte, CB, 1422 Fixups); 1423 break; 1424 } 1425 case X86II::RawFrmMemOffs: 1426 emitByte(BaseOpcode, CB); 1427 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1428 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags), 1429 StartByte, CB, Fixups); 1430 ++CurOp; // skip segment operand 1431 break; 1432 case X86II::RawFrmImm8: 1433 emitByte(BaseOpcode, CB); 1434 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1435 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags), 1436 StartByte, CB, Fixups); 1437 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1, FK_Data_1, StartByte, 1438 CB, Fixups); 1439 break; 1440 case X86II::RawFrmImm16: 1441 emitByte(BaseOpcode, CB); 1442 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1443 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags), 1444 StartByte, CB, Fixups); 1445 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 2, FK_Data_2, StartByte, 1446 CB, Fixups); 1447 break; 1448 1449 case X86II::AddRegFrm: 1450 emitByte(BaseOpcode + getX86RegNum(MI.getOperand(CurOp++)), CB); 1451 break; 1452 1453 case X86II::MRMDestReg: { 1454 emitByte(BaseOpcode, CB); 1455 unsigned SrcRegNum = CurOp + 1; 1456 1457 if (HasEVEX_K) // Skip writemask 1458 ++SrcRegNum; 1459 1460 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV) 1461 ++SrcRegNum; 1462 1463 emitRegModRMByte(MI.getOperand(CurOp), 1464 getX86RegNum(MI.getOperand(SrcRegNum)), CB); 1465 CurOp = SrcRegNum + 1; 1466 break; 1467 } 1468 case X86II::MRMDestMem4VOp3CC: { 1469 unsigned CC = MI.getOperand(8).getImm(); 1470 emitByte(BaseOpcode + CC, CB); 1471 unsigned SrcRegNum = CurOp + X86::AddrNumOperands; 1472 emitMemModRMByte(MI, CurOp + 1, getX86RegNum(MI.getOperand(0)), TSFlags, 1473 Kind, StartByte, CB, Fixups, STI, false); 1474 CurOp = SrcRegNum + 3; // skip reg, VEX_V4 and CC 1475 break; 1476 } 1477 case X86II::MRMDestMemFSIB: 1478 case X86II::MRMDestMem: { 1479 emitByte(BaseOpcode, CB); 1480 unsigned SrcRegNum = CurOp + X86::AddrNumOperands; 1481 1482 if (HasEVEX_K) // Skip writemask 1483 ++SrcRegNum; 1484 1485 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV) 1486 ++SrcRegNum; 1487 1488 bool ForceSIB = (Form == X86II::MRMDestMemFSIB); 1489 emitMemModRMByte(MI, CurOp, getX86RegNum(MI.getOperand(SrcRegNum)), TSFlags, 1490 Kind, StartByte, CB, Fixups, STI, ForceSIB); 1491 CurOp = SrcRegNum + 1; 1492 break; 1493 } 1494 case X86II::MRMSrcReg: { 1495 emitByte(BaseOpcode, CB); 1496 unsigned SrcRegNum = CurOp + 1; 1497 1498 if (HasEVEX_K) // Skip writemask 1499 ++SrcRegNum; 1500 1501 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV) 1502 ++SrcRegNum; 1503 1504 emitRegModRMByte(MI.getOperand(SrcRegNum), 1505 getX86RegNum(MI.getOperand(CurOp)), CB); 1506 CurOp = SrcRegNum + 1; 1507 if (HasVEX_I8Reg) 1508 I8RegNum = getX86RegEncoding(MI, CurOp++); 1509 // do not count the rounding control operand 1510 if (HasEVEX_RC) 1511 --NumOps; 1512 break; 1513 } 1514 case X86II::MRMSrcReg4VOp3: { 1515 emitByte(BaseOpcode, CB); 1516 unsigned SrcRegNum = CurOp + 1; 1517 1518 emitRegModRMByte(MI.getOperand(SrcRegNum), 1519 getX86RegNum(MI.getOperand(CurOp)), CB); 1520 CurOp = SrcRegNum + 1; 1521 ++CurOp; // Encoded in VEX.VVVV 1522 break; 1523 } 1524 case X86II::MRMSrcRegOp4: { 1525 emitByte(BaseOpcode, CB); 1526 unsigned SrcRegNum = CurOp + 1; 1527 1528 // Skip 1st src (which is encoded in VEX_VVVV) 1529 ++SrcRegNum; 1530 1531 // Capture 2nd src (which is encoded in Imm[7:4]) 1532 assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg"); 1533 I8RegNum = getX86RegEncoding(MI, SrcRegNum++); 1534 1535 emitRegModRMByte(MI.getOperand(SrcRegNum), 1536 getX86RegNum(MI.getOperand(CurOp)), CB); 1537 CurOp = SrcRegNum + 1; 1538 break; 1539 } 1540 case X86II::MRMSrcRegCC: { 1541 unsigned FirstOp = CurOp++; 1542 unsigned SecondOp = CurOp++; 1543 1544 unsigned CC = MI.getOperand(CurOp++).getImm(); 1545 emitByte(BaseOpcode + CC, CB); 1546 1547 emitRegModRMByte(MI.getOperand(SecondOp), 1548 getX86RegNum(MI.getOperand(FirstOp)), CB); 1549 break; 1550 } 1551 case X86II::MRMSrcMemFSIB: 1552 case X86II::MRMSrcMem: { 1553 unsigned FirstMemOp = CurOp + 1; 1554 1555 if (HasEVEX_K) // Skip writemask 1556 ++FirstMemOp; 1557 1558 if (HasVEX_4V) 1559 ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV). 1560 1561 emitByte(BaseOpcode, CB); 1562 1563 bool ForceSIB = (Form == X86II::MRMSrcMemFSIB); 1564 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)), 1565 TSFlags, Kind, StartByte, CB, Fixups, STI, ForceSIB); 1566 CurOp = FirstMemOp + X86::AddrNumOperands; 1567 if (HasVEX_I8Reg) 1568 I8RegNum = getX86RegEncoding(MI, CurOp++); 1569 break; 1570 } 1571 case X86II::MRMSrcMem4VOp3: { 1572 unsigned FirstMemOp = CurOp + 1; 1573 1574 emitByte(BaseOpcode, CB); 1575 1576 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)), 1577 TSFlags, Kind, StartByte, CB, Fixups, STI); 1578 CurOp = FirstMemOp + X86::AddrNumOperands; 1579 ++CurOp; // Encoded in VEX.VVVV. 1580 break; 1581 } 1582 case X86II::MRMSrcMemOp4: { 1583 unsigned FirstMemOp = CurOp + 1; 1584 1585 ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV). 1586 1587 // Capture second register source (encoded in Imm[7:4]) 1588 assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg"); 1589 I8RegNum = getX86RegEncoding(MI, FirstMemOp++); 1590 1591 emitByte(BaseOpcode, CB); 1592 1593 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)), 1594 TSFlags, Kind, StartByte, CB, Fixups, STI); 1595 CurOp = FirstMemOp + X86::AddrNumOperands; 1596 break; 1597 } 1598 case X86II::MRMSrcMemCC: { 1599 unsigned RegOp = CurOp++; 1600 unsigned FirstMemOp = CurOp; 1601 CurOp = FirstMemOp + X86::AddrNumOperands; 1602 1603 unsigned CC = MI.getOperand(CurOp++).getImm(); 1604 emitByte(BaseOpcode + CC, CB); 1605 1606 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(RegOp)), 1607 TSFlags, Kind, StartByte, CB, Fixups, STI); 1608 break; 1609 } 1610 1611 case X86II::MRMXrCC: { 1612 unsigned RegOp = CurOp++; 1613 1614 unsigned CC = MI.getOperand(CurOp++).getImm(); 1615 emitByte(BaseOpcode + CC, CB); 1616 emitRegModRMByte(MI.getOperand(RegOp), 0, CB); 1617 break; 1618 } 1619 1620 case X86II::MRMXr: 1621 case X86II::MRM0r: 1622 case X86II::MRM1r: 1623 case X86II::MRM2r: 1624 case X86II::MRM3r: 1625 case X86II::MRM4r: 1626 case X86II::MRM5r: 1627 case X86II::MRM6r: 1628 case X86II::MRM7r: 1629 if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV). 1630 ++CurOp; 1631 if (HasEVEX_K) // Skip writemask 1632 ++CurOp; 1633 emitByte(BaseOpcode, CB); 1634 emitRegModRMByte(MI.getOperand(CurOp++), 1635 (Form == X86II::MRMXr) ? 0 : Form - X86II::MRM0r, CB); 1636 break; 1637 case X86II::MRMr0: 1638 emitByte(BaseOpcode, CB); 1639 emitByte(modRMByte(3, getX86RegNum(MI.getOperand(CurOp++)), 0), CB); 1640 break; 1641 1642 case X86II::MRMXmCC: { 1643 unsigned FirstMemOp = CurOp; 1644 CurOp = FirstMemOp + X86::AddrNumOperands; 1645 1646 unsigned CC = MI.getOperand(CurOp++).getImm(); 1647 emitByte(BaseOpcode + CC, CB); 1648 1649 emitMemModRMByte(MI, FirstMemOp, 0, TSFlags, Kind, StartByte, CB, Fixups, 1650 STI); 1651 break; 1652 } 1653 1654 case X86II::MRMXm: 1655 case X86II::MRM0m: 1656 case X86II::MRM1m: 1657 case X86II::MRM2m: 1658 case X86II::MRM3m: 1659 case X86II::MRM4m: 1660 case X86II::MRM5m: 1661 case X86II::MRM6m: 1662 case X86II::MRM7m: 1663 if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV). 1664 ++CurOp; 1665 if (HasEVEX_K) // Skip writemask 1666 ++CurOp; 1667 emitByte(BaseOpcode, CB); 1668 emitMemModRMByte(MI, CurOp, 1669 (Form == X86II::MRMXm) ? 0 : Form - X86II::MRM0m, TSFlags, 1670 Kind, StartByte, CB, Fixups, STI); 1671 CurOp += X86::AddrNumOperands; 1672 break; 1673 1674 case X86II::MRM0X: 1675 case X86II::MRM1X: 1676 case X86II::MRM2X: 1677 case X86II::MRM3X: 1678 case X86II::MRM4X: 1679 case X86II::MRM5X: 1680 case X86II::MRM6X: 1681 case X86II::MRM7X: 1682 emitByte(BaseOpcode, CB); 1683 emitByte(0xC0 + ((Form - X86II::MRM0X) << 3), CB); 1684 break; 1685 1686 case X86II::MRM_C0: 1687 case X86II::MRM_C1: 1688 case X86II::MRM_C2: 1689 case X86II::MRM_C3: 1690 case X86II::MRM_C4: 1691 case X86II::MRM_C5: 1692 case X86II::MRM_C6: 1693 case X86II::MRM_C7: 1694 case X86II::MRM_C8: 1695 case X86II::MRM_C9: 1696 case X86II::MRM_CA: 1697 case X86II::MRM_CB: 1698 case X86II::MRM_CC: 1699 case X86II::MRM_CD: 1700 case X86II::MRM_CE: 1701 case X86II::MRM_CF: 1702 case X86II::MRM_D0: 1703 case X86II::MRM_D1: 1704 case X86II::MRM_D2: 1705 case X86II::MRM_D3: 1706 case X86II::MRM_D4: 1707 case X86II::MRM_D5: 1708 case X86II::MRM_D6: 1709 case X86II::MRM_D7: 1710 case X86II::MRM_D8: 1711 case X86II::MRM_D9: 1712 case X86II::MRM_DA: 1713 case X86II::MRM_DB: 1714 case X86II::MRM_DC: 1715 case X86II::MRM_DD: 1716 case X86II::MRM_DE: 1717 case X86II::MRM_DF: 1718 case X86II::MRM_E0: 1719 case X86II::MRM_E1: 1720 case X86II::MRM_E2: 1721 case X86II::MRM_E3: 1722 case X86II::MRM_E4: 1723 case X86II::MRM_E5: 1724 case X86II::MRM_E6: 1725 case X86II::MRM_E7: 1726 case X86II::MRM_E8: 1727 case X86II::MRM_E9: 1728 case X86II::MRM_EA: 1729 case X86II::MRM_EB: 1730 case X86II::MRM_EC: 1731 case X86II::MRM_ED: 1732 case X86II::MRM_EE: 1733 case X86II::MRM_EF: 1734 case X86II::MRM_F0: 1735 case X86II::MRM_F1: 1736 case X86II::MRM_F2: 1737 case X86II::MRM_F3: 1738 case X86II::MRM_F4: 1739 case X86II::MRM_F5: 1740 case X86II::MRM_F6: 1741 case X86II::MRM_F7: 1742 case X86II::MRM_F8: 1743 case X86II::MRM_F9: 1744 case X86II::MRM_FA: 1745 case X86II::MRM_FB: 1746 case X86II::MRM_FC: 1747 case X86II::MRM_FD: 1748 case X86II::MRM_FE: 1749 case X86II::MRM_FF: 1750 emitByte(BaseOpcode, CB); 1751 emitByte(0xC0 + Form - X86II::MRM_C0, CB); 1752 break; 1753 } 1754 1755 if (HasVEX_I8Reg) { 1756 // The last source register of a 4 operand instruction in AVX is encoded 1757 // in bits[7:4] of a immediate byte. 1758 assert(I8RegNum < 16 && "Register encoding out of range"); 1759 I8RegNum <<= 4; 1760 if (CurOp != NumOps) { 1761 unsigned Val = MI.getOperand(CurOp++).getImm(); 1762 assert(Val < 16 && "Immediate operand value out of range"); 1763 I8RegNum |= Val; 1764 } 1765 emitImmediate(MCOperand::createImm(I8RegNum), MI.getLoc(), 1, FK_Data_1, 1766 StartByte, CB, Fixups); 1767 } else { 1768 // If there is a remaining operand, it must be a trailing immediate. Emit it 1769 // according to the right size for the instruction. Some instructions 1770 // (SSE4a extrq and insertq) have two trailing immediates. 1771 while (CurOp != NumOps && NumOps - CurOp <= 2) { 1772 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1773 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags), 1774 StartByte, CB, Fixups); 1775 } 1776 } 1777 1778 if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow) 1779 emitByte(X86II::getBaseOpcodeFor(TSFlags), CB); 1780 1781 assert(CB.size() - StartByte <= 15 && 1782 "The size of instruction must be no longer than 15."); 1783 #ifndef NDEBUG 1784 // FIXME: Verify. 1785 if (/*!Desc.isVariadic() &&*/ CurOp != NumOps) { 1786 errs() << "Cannot encode all operands of: "; 1787 MI.dump(); 1788 errs() << '\n'; 1789 abort(); 1790 } 1791 #endif 1792 } 1793 1794 MCCodeEmitter *llvm::createX86MCCodeEmitter(const MCInstrInfo &MCII, 1795 MCContext &Ctx) { 1796 return new X86MCCodeEmitter(MCII, Ctx); 1797 } 1798