1 //===-- X86MCCodeEmitter.cpp - Convert X86 code to machine code -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the X86MCCodeEmitter class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "MCTargetDesc/X86BaseInfo.h" 14 #include "MCTargetDesc/X86FixupKinds.h" 15 #include "MCTargetDesc/X86MCTargetDesc.h" 16 #include "llvm/ADT/SmallVector.h" 17 #include "llvm/MC/MCCodeEmitter.h" 18 #include "llvm/MC/MCContext.h" 19 #include "llvm/MC/MCExpr.h" 20 #include "llvm/MC/MCFixup.h" 21 #include "llvm/MC/MCInst.h" 22 #include "llvm/MC/MCInstrDesc.h" 23 #include "llvm/MC/MCInstrInfo.h" 24 #include "llvm/MC/MCRegisterInfo.h" 25 #include "llvm/MC/MCSubtargetInfo.h" 26 #include "llvm/MC/MCSymbol.h" 27 #include "llvm/Support/ErrorHandling.h" 28 #include "llvm/Support/raw_ostream.h" 29 #include <cassert> 30 #include <cstdint> 31 #include <cstdlib> 32 33 using namespace llvm; 34 35 #define DEBUG_TYPE "mccodeemitter" 36 37 namespace { 38 39 class X86MCCodeEmitter : public MCCodeEmitter { 40 const MCInstrInfo &MCII; 41 MCContext &Ctx; 42 43 public: 44 X86MCCodeEmitter(const MCInstrInfo &mcii, MCContext &ctx) 45 : MCII(mcii), Ctx(ctx) {} 46 X86MCCodeEmitter(const X86MCCodeEmitter &) = delete; 47 X86MCCodeEmitter &operator=(const X86MCCodeEmitter &) = delete; 48 ~X86MCCodeEmitter() override = default; 49 50 void emitPrefix(const MCInst &MI, raw_ostream &OS, 51 const MCSubtargetInfo &STI) const override; 52 53 void encodeInstruction(const MCInst &MI, raw_ostream &OS, 54 SmallVectorImpl<MCFixup> &Fixups, 55 const MCSubtargetInfo &STI) const override; 56 57 private: 58 unsigned getX86RegNum(const MCOperand &MO) const; 59 60 unsigned getX86RegEncoding(const MCInst &MI, unsigned OpNum) const; 61 62 /// \param MI a single low-level machine instruction. 63 /// \param OpNum the operand #. 64 /// \returns true if the OpNumth operand of MI require a bit to be set in 65 /// REX prefix. 66 bool isREXExtendedReg(const MCInst &MI, unsigned OpNum) const; 67 68 void emitImmediate(const MCOperand &Disp, SMLoc Loc, unsigned ImmSize, 69 MCFixupKind FixupKind, uint64_t StartByte, raw_ostream &OS, 70 SmallVectorImpl<MCFixup> &Fixups, int ImmOffset = 0) const; 71 72 void emitRegModRMByte(const MCOperand &ModRMReg, unsigned RegOpcodeFld, 73 raw_ostream &OS) const; 74 75 void emitSIBByte(unsigned SS, unsigned Index, unsigned Base, 76 raw_ostream &OS) const; 77 78 void emitMemModRMByte(const MCInst &MI, unsigned Op, unsigned RegOpcodeField, 79 uint64_t TSFlags, bool HasREX, uint64_t StartByte, 80 raw_ostream &OS, SmallVectorImpl<MCFixup> &Fixups, 81 const MCSubtargetInfo &STI, 82 bool ForceSIB = false) const; 83 84 bool emitPrefixImpl(unsigned &CurOp, const MCInst &MI, 85 const MCSubtargetInfo &STI, raw_ostream &OS) const; 86 87 void emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, 88 raw_ostream &OS) const; 89 90 void emitSegmentOverridePrefix(unsigned SegOperand, const MCInst &MI, 91 raw_ostream &OS) const; 92 93 bool emitOpcodePrefix(int MemOperand, const MCInst &MI, 94 const MCSubtargetInfo &STI, raw_ostream &OS) const; 95 96 bool emitREXPrefix(int MemOperand, const MCInst &MI, 97 const MCSubtargetInfo &STI, raw_ostream &OS) const; 98 }; 99 100 } // end anonymous namespace 101 102 static uint8_t modRMByte(unsigned Mod, unsigned RegOpcode, unsigned RM) { 103 assert(Mod < 4 && RegOpcode < 8 && RM < 8 && "ModRM Fields out of range!"); 104 return RM | (RegOpcode << 3) | (Mod << 6); 105 } 106 107 static void emitByte(uint8_t C, raw_ostream &OS) { OS << static_cast<char>(C); } 108 109 static void emitConstant(uint64_t Val, unsigned Size, raw_ostream &OS) { 110 // Output the constant in little endian byte order. 111 for (unsigned i = 0; i != Size; ++i) { 112 emitByte(Val & 255, OS); 113 Val >>= 8; 114 } 115 } 116 117 /// Determine if this immediate can fit in a disp8 or a compressed disp8 for 118 /// EVEX instructions. \p will be set to the value to pass to the ImmOffset 119 /// parameter of emitImmediate. 120 static bool isDispOrCDisp8(uint64_t TSFlags, int Value, int &ImmOffset) { 121 bool HasEVEX = (TSFlags & X86II::EncodingMask) == X86II::EVEX; 122 123 int CD8_Scale = 124 (TSFlags & X86II::CD8_Scale_Mask) >> X86II::CD8_Scale_Shift; 125 if (!HasEVEX || CD8_Scale == 0) 126 return isInt<8>(Value); 127 128 assert(isPowerOf2_32(CD8_Scale) && "Unexpected CD8 scale!"); 129 if (Value & (CD8_Scale - 1)) // Unaligned offset 130 return false; 131 132 int CDisp8 = Value / CD8_Scale; 133 if (!isInt<8>(CDisp8)) 134 return false; 135 136 // ImmOffset will be added to Value in emitImmediate leaving just CDisp8. 137 ImmOffset = CDisp8 - Value; 138 return true; 139 } 140 141 /// \returns the appropriate fixup kind to use for an immediate in an 142 /// instruction with the specified TSFlags. 143 static MCFixupKind getImmFixupKind(uint64_t TSFlags) { 144 unsigned Size = X86II::getSizeOfImm(TSFlags); 145 bool isPCRel = X86II::isImmPCRel(TSFlags); 146 147 if (X86II::isImmSigned(TSFlags)) { 148 switch (Size) { 149 default: 150 llvm_unreachable("Unsupported signed fixup size!"); 151 case 4: 152 return MCFixupKind(X86::reloc_signed_4byte); 153 } 154 } 155 return MCFixup::getKindForSize(Size, isPCRel); 156 } 157 158 /// \param Op operand # of the memory operand. 159 /// 160 /// \returns true if the specified instruction has a 16-bit memory operand. 161 static bool is16BitMemOperand(const MCInst &MI, unsigned Op, 162 const MCSubtargetInfo &STI) { 163 const MCOperand &Base = MI.getOperand(Op + X86::AddrBaseReg); 164 const MCOperand &Index = MI.getOperand(Op + X86::AddrIndexReg); 165 166 unsigned BaseReg = Base.getReg(); 167 unsigned IndexReg = Index.getReg(); 168 169 if (STI.hasFeature(X86::Mode16Bit) && BaseReg == 0 && IndexReg == 0) 170 return true; 171 if ((BaseReg != 0 && 172 X86MCRegisterClasses[X86::GR16RegClassID].contains(BaseReg)) || 173 (IndexReg != 0 && 174 X86MCRegisterClasses[X86::GR16RegClassID].contains(IndexReg))) 175 return true; 176 return false; 177 } 178 179 /// \param Op operand # of the memory operand. 180 /// 181 /// \returns true if the specified instruction has a 32-bit memory operand. 182 static bool is32BitMemOperand(const MCInst &MI, unsigned Op) { 183 const MCOperand &BaseReg = MI.getOperand(Op + X86::AddrBaseReg); 184 const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg); 185 186 if ((BaseReg.getReg() != 0 && 187 X86MCRegisterClasses[X86::GR32RegClassID].contains(BaseReg.getReg())) || 188 (IndexReg.getReg() != 0 && 189 X86MCRegisterClasses[X86::GR32RegClassID].contains(IndexReg.getReg()))) 190 return true; 191 if (BaseReg.getReg() == X86::EIP) { 192 assert(IndexReg.getReg() == 0 && "Invalid eip-based address."); 193 return true; 194 } 195 if (IndexReg.getReg() == X86::EIZ) 196 return true; 197 return false; 198 } 199 200 /// \param Op operand # of the memory operand. 201 /// 202 /// \returns true if the specified instruction has a 64-bit memory operand. 203 #ifndef NDEBUG 204 static bool is64BitMemOperand(const MCInst &MI, unsigned Op) { 205 const MCOperand &BaseReg = MI.getOperand(Op + X86::AddrBaseReg); 206 const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg); 207 208 if ((BaseReg.getReg() != 0 && 209 X86MCRegisterClasses[X86::GR64RegClassID].contains(BaseReg.getReg())) || 210 (IndexReg.getReg() != 0 && 211 X86MCRegisterClasses[X86::GR64RegClassID].contains(IndexReg.getReg()))) 212 return true; 213 return false; 214 } 215 #endif 216 217 enum GlobalOffsetTableExprKind { GOT_None, GOT_Normal, GOT_SymDiff }; 218 219 /// Check if this expression starts with _GLOBAL_OFFSET_TABLE_ and if it is 220 /// of the form _GLOBAL_OFFSET_TABLE_-symbol. This is needed to support PIC on 221 /// ELF i386 as _GLOBAL_OFFSET_TABLE_ is magical. We check only simple case that 222 /// are know to be used: _GLOBAL_OFFSET_TABLE_ by itself or at the start of a 223 /// binary expression. 224 static GlobalOffsetTableExprKind 225 startsWithGlobalOffsetTable(const MCExpr *Expr) { 226 const MCExpr *RHS = nullptr; 227 if (Expr->getKind() == MCExpr::Binary) { 228 const MCBinaryExpr *BE = static_cast<const MCBinaryExpr *>(Expr); 229 Expr = BE->getLHS(); 230 RHS = BE->getRHS(); 231 } 232 233 if (Expr->getKind() != MCExpr::SymbolRef) 234 return GOT_None; 235 236 const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr); 237 const MCSymbol &S = Ref->getSymbol(); 238 if (S.getName() != "_GLOBAL_OFFSET_TABLE_") 239 return GOT_None; 240 if (RHS && RHS->getKind() == MCExpr::SymbolRef) 241 return GOT_SymDiff; 242 return GOT_Normal; 243 } 244 245 static bool hasSecRelSymbolRef(const MCExpr *Expr) { 246 if (Expr->getKind() == MCExpr::SymbolRef) { 247 const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr); 248 return Ref->getKind() == MCSymbolRefExpr::VK_SECREL; 249 } 250 return false; 251 } 252 253 static bool isPCRel32Branch(const MCInst &MI, const MCInstrInfo &MCII) { 254 unsigned Opcode = MI.getOpcode(); 255 const MCInstrDesc &Desc = MCII.get(Opcode); 256 if ((Opcode != X86::CALL64pcrel32 && Opcode != X86::JMP_4 && 257 Opcode != X86::JCC_4) || 258 getImmFixupKind(Desc.TSFlags) != FK_PCRel_4) 259 return false; 260 261 unsigned CurOp = X86II::getOperandBias(Desc); 262 const MCOperand &Op = MI.getOperand(CurOp); 263 if (!Op.isExpr()) 264 return false; 265 266 const MCSymbolRefExpr *Ref = dyn_cast<MCSymbolRefExpr>(Op.getExpr()); 267 return Ref && Ref->getKind() == MCSymbolRefExpr::VK_None; 268 } 269 270 unsigned X86MCCodeEmitter::getX86RegNum(const MCOperand &MO) const { 271 return Ctx.getRegisterInfo()->getEncodingValue(MO.getReg()) & 0x7; 272 } 273 274 unsigned X86MCCodeEmitter::getX86RegEncoding(const MCInst &MI, 275 unsigned OpNum) const { 276 return Ctx.getRegisterInfo()->getEncodingValue(MI.getOperand(OpNum).getReg()); 277 } 278 279 /// \param MI a single low-level machine instruction. 280 /// \param OpNum the operand #. 281 /// \returns true if the OpNumth operand of MI require a bit to be set in 282 /// REX prefix. 283 bool X86MCCodeEmitter::isREXExtendedReg(const MCInst &MI, 284 unsigned OpNum) const { 285 return (getX86RegEncoding(MI, OpNum) >> 3) & 1; 286 } 287 288 void X86MCCodeEmitter::emitImmediate(const MCOperand &DispOp, SMLoc Loc, 289 unsigned Size, MCFixupKind FixupKind, 290 uint64_t StartByte, raw_ostream &OS, 291 SmallVectorImpl<MCFixup> &Fixups, 292 int ImmOffset) const { 293 const MCExpr *Expr = nullptr; 294 if (DispOp.isImm()) { 295 // If this is a simple integer displacement that doesn't require a 296 // relocation, emit it now. 297 if (FixupKind != FK_PCRel_1 && FixupKind != FK_PCRel_2 && 298 FixupKind != FK_PCRel_4) { 299 emitConstant(DispOp.getImm() + ImmOffset, Size, OS); 300 return; 301 } 302 Expr = MCConstantExpr::create(DispOp.getImm(), Ctx); 303 } else { 304 Expr = DispOp.getExpr(); 305 } 306 307 // If we have an immoffset, add it to the expression. 308 if ((FixupKind == FK_Data_4 || FixupKind == FK_Data_8 || 309 FixupKind == MCFixupKind(X86::reloc_signed_4byte))) { 310 GlobalOffsetTableExprKind Kind = startsWithGlobalOffsetTable(Expr); 311 if (Kind != GOT_None) { 312 assert(ImmOffset == 0); 313 314 if (Size == 8) { 315 FixupKind = MCFixupKind(X86::reloc_global_offset_table8); 316 } else { 317 assert(Size == 4); 318 FixupKind = MCFixupKind(X86::reloc_global_offset_table); 319 } 320 321 if (Kind == GOT_Normal) 322 ImmOffset = static_cast<int>(OS.tell() - StartByte); 323 } else if (Expr->getKind() == MCExpr::SymbolRef) { 324 if (hasSecRelSymbolRef(Expr)) { 325 FixupKind = MCFixupKind(FK_SecRel_4); 326 } 327 } else if (Expr->getKind() == MCExpr::Binary) { 328 const MCBinaryExpr *Bin = static_cast<const MCBinaryExpr *>(Expr); 329 if (hasSecRelSymbolRef(Bin->getLHS()) || 330 hasSecRelSymbolRef(Bin->getRHS())) { 331 FixupKind = MCFixupKind(FK_SecRel_4); 332 } 333 } 334 } 335 336 // If the fixup is pc-relative, we need to bias the value to be relative to 337 // the start of the field, not the end of the field. 338 if (FixupKind == FK_PCRel_4 || 339 FixupKind == MCFixupKind(X86::reloc_riprel_4byte) || 340 FixupKind == MCFixupKind(X86::reloc_riprel_4byte_movq_load) || 341 FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax) || 342 FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax_rex) || 343 FixupKind == MCFixupKind(X86::reloc_branch_4byte_pcrel)) { 344 ImmOffset -= 4; 345 // If this is a pc-relative load off _GLOBAL_OFFSET_TABLE_: 346 // leaq _GLOBAL_OFFSET_TABLE_(%rip), %r15 347 // this needs to be a GOTPC32 relocation. 348 if (startsWithGlobalOffsetTable(Expr) != GOT_None) 349 FixupKind = MCFixupKind(X86::reloc_global_offset_table); 350 } 351 if (FixupKind == FK_PCRel_2) 352 ImmOffset -= 2; 353 if (FixupKind == FK_PCRel_1) 354 ImmOffset -= 1; 355 356 if (ImmOffset) 357 Expr = MCBinaryExpr::createAdd(Expr, MCConstantExpr::create(ImmOffset, Ctx), 358 Ctx); 359 360 // Emit a symbolic constant as a fixup and 4 zeros. 361 Fixups.push_back(MCFixup::create(static_cast<uint32_t>(OS.tell() - StartByte), 362 Expr, FixupKind, Loc)); 363 emitConstant(0, Size, OS); 364 } 365 366 void X86MCCodeEmitter::emitRegModRMByte(const MCOperand &ModRMReg, 367 unsigned RegOpcodeFld, 368 raw_ostream &OS) const { 369 emitByte(modRMByte(3, RegOpcodeFld, getX86RegNum(ModRMReg)), OS); 370 } 371 372 void X86MCCodeEmitter::emitSIBByte(unsigned SS, unsigned Index, unsigned Base, 373 raw_ostream &OS) const { 374 // SIB byte is in the same format as the modRMByte. 375 emitByte(modRMByte(SS, Index, Base), OS); 376 } 377 378 void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op, 379 unsigned RegOpcodeField, 380 uint64_t TSFlags, bool HasREX, 381 uint64_t StartByte, raw_ostream &OS, 382 SmallVectorImpl<MCFixup> &Fixups, 383 const MCSubtargetInfo &STI, 384 bool ForceSIB) const { 385 const MCOperand &Disp = MI.getOperand(Op + X86::AddrDisp); 386 const MCOperand &Base = MI.getOperand(Op + X86::AddrBaseReg); 387 const MCOperand &Scale = MI.getOperand(Op + X86::AddrScaleAmt); 388 const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg); 389 unsigned BaseReg = Base.getReg(); 390 391 // Handle %rip relative addressing. 392 if (BaseReg == X86::RIP || 393 BaseReg == X86::EIP) { // [disp32+rIP] in X86-64 mode 394 assert(STI.hasFeature(X86::Mode64Bit) && 395 "Rip-relative addressing requires 64-bit mode"); 396 assert(IndexReg.getReg() == 0 && !ForceSIB && 397 "Invalid rip-relative address"); 398 emitByte(modRMByte(0, RegOpcodeField, 5), OS); 399 400 unsigned Opcode = MI.getOpcode(); 401 unsigned FixupKind = [&]() { 402 // Enable relaxed relocation only for a MCSymbolRefExpr. We cannot use a 403 // relaxed relocation if an offset is present (e.g. x@GOTPCREL+4). 404 if (!(Disp.isExpr() && isa<MCSymbolRefExpr>(Disp.getExpr()))) 405 return X86::reloc_riprel_4byte; 406 407 // Certain loads for GOT references can be relocated against the symbol 408 // directly if the symbol ends up in the same linkage unit. 409 switch (Opcode) { 410 default: 411 return X86::reloc_riprel_4byte; 412 case X86::MOV64rm: 413 // movq loads is a subset of reloc_riprel_4byte_relax_rex. It is a 414 // special case because COFF and Mach-O don't support ELF's more 415 // flexible R_X86_64_REX_GOTPCRELX relaxation. 416 assert(HasREX); 417 return X86::reloc_riprel_4byte_movq_load; 418 case X86::ADC32rm: 419 case X86::ADD32rm: 420 case X86::AND32rm: 421 case X86::CMP32rm: 422 case X86::MOV32rm: 423 case X86::OR32rm: 424 case X86::SBB32rm: 425 case X86::SUB32rm: 426 case X86::TEST32mr: 427 case X86::XOR32rm: 428 case X86::CALL64m: 429 case X86::JMP64m: 430 case X86::TAILJMPm64: 431 case X86::TEST64mr: 432 case X86::ADC64rm: 433 case X86::ADD64rm: 434 case X86::AND64rm: 435 case X86::CMP64rm: 436 case X86::OR64rm: 437 case X86::SBB64rm: 438 case X86::SUB64rm: 439 case X86::XOR64rm: 440 return HasREX ? X86::reloc_riprel_4byte_relax_rex 441 : X86::reloc_riprel_4byte_relax; 442 } 443 }(); 444 445 // rip-relative addressing is actually relative to the *next* instruction. 446 // Since an immediate can follow the mod/rm byte for an instruction, this 447 // means that we need to bias the displacement field of the instruction with 448 // the size of the immediate field. If we have this case, add it into the 449 // expression to emit. 450 // Note: rip-relative addressing using immediate displacement values should 451 // not be adjusted, assuming it was the user's intent. 452 int ImmSize = !Disp.isImm() && X86II::hasImm(TSFlags) 453 ? X86II::getSizeOfImm(TSFlags) 454 : 0; 455 456 emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), StartByte, OS, 457 Fixups, -ImmSize); 458 return; 459 } 460 461 unsigned BaseRegNo = BaseReg ? getX86RegNum(Base) : -1U; 462 463 // 16-bit addressing forms of the ModR/M byte have a different encoding for 464 // the R/M field and are far more limited in which registers can be used. 465 if (is16BitMemOperand(MI, Op, STI)) { 466 if (BaseReg) { 467 // For 32-bit addressing, the row and column values in Table 2-2 are 468 // basically the same. It's AX/CX/DX/BX/SP/BP/SI/DI in that order, with 469 // some special cases. And getX86RegNum reflects that numbering. 470 // For 16-bit addressing it's more fun, as shown in the SDM Vol 2A, 471 // Table 2-1 "16-Bit Addressing Forms with the ModR/M byte". We can only 472 // use SI/DI/BP/BX, which have "row" values 4-7 in no particular order, 473 // while values 0-3 indicate the allowed combinations (base+index) of 474 // those: 0 for BX+SI, 1 for BX+DI, 2 for BP+SI, 3 for BP+DI. 475 // 476 // R16Table[] is a lookup from the normal RegNo, to the row values from 477 // Table 2-1 for 16-bit addressing modes. Where zero means disallowed. 478 static const unsigned R16Table[] = {0, 0, 0, 7, 0, 6, 4, 5}; 479 unsigned RMfield = R16Table[BaseRegNo]; 480 481 assert(RMfield && "invalid 16-bit base register"); 482 483 if (IndexReg.getReg()) { 484 unsigned IndexReg16 = R16Table[getX86RegNum(IndexReg)]; 485 486 assert(IndexReg16 && "invalid 16-bit index register"); 487 // We must have one of SI/DI (4,5), and one of BP/BX (6,7). 488 assert(((IndexReg16 ^ RMfield) & 2) && 489 "invalid 16-bit base/index register combination"); 490 assert(Scale.getImm() == 1 && 491 "invalid scale for 16-bit memory reference"); 492 493 // Allow base/index to appear in either order (although GAS doesn't). 494 if (IndexReg16 & 2) 495 RMfield = (RMfield & 1) | ((7 - IndexReg16) << 1); 496 else 497 RMfield = (IndexReg16 & 1) | ((7 - RMfield) << 1); 498 } 499 500 if (Disp.isImm() && isInt<8>(Disp.getImm())) { 501 if (Disp.getImm() == 0 && RMfield != 6) { 502 // There is no displacement; just the register. 503 emitByte(modRMByte(0, RegOpcodeField, RMfield), OS); 504 return; 505 } 506 // Use the [REG]+disp8 form, including for [BP] which cannot be encoded. 507 emitByte(modRMByte(1, RegOpcodeField, RMfield), OS); 508 emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups); 509 return; 510 } 511 // This is the [REG]+disp16 case. 512 emitByte(modRMByte(2, RegOpcodeField, RMfield), OS); 513 } else { 514 assert(IndexReg.getReg() == 0 && "Unexpected index register!"); 515 // There is no BaseReg; this is the plain [disp16] case. 516 emitByte(modRMByte(0, RegOpcodeField, 6), OS); 517 } 518 519 // Emit 16-bit displacement for plain disp16 or [REG]+disp16 cases. 520 emitImmediate(Disp, MI.getLoc(), 2, FK_Data_2, StartByte, OS, Fixups); 521 return; 522 } 523 524 // Check for presence of {disp8} or {disp32} pseudo prefixes. 525 bool UseDisp8 = MI.getFlags() & X86::IP_USE_DISP8; 526 bool UseDisp32 = MI.getFlags() & X86::IP_USE_DISP32; 527 528 // We only allow no displacement if no pseudo prefix is present. 529 bool AllowNoDisp = !UseDisp8 && !UseDisp32; 530 // Disp8 is allowed unless the {disp32} prefix is present. 531 bool AllowDisp8 = !UseDisp32; 532 533 // Determine whether a SIB byte is needed. 534 if (// The SIB byte must be used if there is an index register or the 535 // encoding requires a SIB byte. 536 !ForceSIB && IndexReg.getReg() == 0 && 537 // The SIB byte must be used if the base is ESP/RSP/R12, all of which 538 // encode to an R/M value of 4, which indicates that a SIB byte is 539 // present. 540 BaseRegNo != N86::ESP && 541 // If there is no base register and we're in 64-bit mode, we need a SIB 542 // byte to emit an addr that is just 'disp32' (the non-RIP relative form). 543 (!STI.hasFeature(X86::Mode64Bit) || BaseReg != 0)) { 544 545 if (BaseReg == 0) { // [disp32] in X86-32 mode 546 emitByte(modRMByte(0, RegOpcodeField, 5), OS); 547 emitImmediate(Disp, MI.getLoc(), 4, FK_Data_4, StartByte, OS, Fixups); 548 return; 549 } 550 551 // If the base is not EBP/ESP/R12/R13 and there is no displacement, use 552 // simple indirect register encoding, this handles addresses like [EAX]. 553 // The encoding for [EBP] or[R13] with no displacement means [disp32] so we 554 // handle it by emitting a displacement of 0 later. 555 if (BaseRegNo != N86::EBP) { 556 if (Disp.isImm() && Disp.getImm() == 0 && AllowNoDisp) { 557 emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), OS); 558 return; 559 } 560 561 // If the displacement is @tlscall, treat it as a zero. 562 if (Disp.isExpr()) { 563 auto *Sym = dyn_cast<MCSymbolRefExpr>(Disp.getExpr()); 564 if (Sym && Sym->getKind() == MCSymbolRefExpr::VK_TLSCALL) { 565 // This is exclusively used by call *a@tlscall(base). The relocation 566 // (R_386_TLSCALL or R_X86_64_TLSCALL) applies to the beginning. 567 Fixups.push_back(MCFixup::create(0, Sym, FK_NONE, MI.getLoc())); 568 emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), OS); 569 return; 570 } 571 } 572 } 573 574 // Otherwise, if the displacement fits in a byte, encode as [REG+disp8]. 575 // Including a compressed disp8 for EVEX instructions that support it. 576 // This also handles the 0 displacement for [EBP] or [R13]. We can't use 577 // disp8 if the {disp32} pseudo prefix is present. 578 if (Disp.isImm() && AllowDisp8) { 579 int ImmOffset = 0; 580 if (isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) { 581 emitByte(modRMByte(1, RegOpcodeField, BaseRegNo), OS); 582 emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups, 583 ImmOffset); 584 return; 585 } 586 } 587 588 // Otherwise, emit the most general non-SIB encoding: [REG+disp32]. 589 // Displacement may be 0 for [EBP] or [R13] case if {disp32} pseudo prefix 590 // prevented using disp8 above. 591 emitByte(modRMByte(2, RegOpcodeField, BaseRegNo), OS); 592 unsigned Opcode = MI.getOpcode(); 593 unsigned FixupKind = Opcode == X86::MOV32rm ? X86::reloc_signed_4byte_relax 594 : X86::reloc_signed_4byte; 595 emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), StartByte, OS, 596 Fixups); 597 return; 598 } 599 600 // We need a SIB byte, so start by outputting the ModR/M byte first 601 assert(IndexReg.getReg() != X86::ESP && IndexReg.getReg() != X86::RSP && 602 "Cannot use ESP as index reg!"); 603 604 bool ForceDisp32 = false; 605 bool ForceDisp8 = false; 606 int ImmOffset = 0; 607 if (BaseReg == 0) { 608 // If there is no base register, we emit the special case SIB byte with 609 // MOD=0, BASE=5, to JUST get the index, scale, and displacement. 610 BaseRegNo = 5; 611 emitByte(modRMByte(0, RegOpcodeField, 4), OS); 612 ForceDisp32 = true; 613 } else if (Disp.isImm() && Disp.getImm() == 0 && AllowNoDisp && 614 // Base reg can't be EBP/RBP/R13 as that would end up with '5' as 615 // the base field, but that is the magic [*] nomenclature that 616 // indicates no base when mod=0. For these cases we'll emit a 0 617 // displacement instead. 618 BaseRegNo != N86::EBP) { 619 // Emit no displacement ModR/M byte 620 emitByte(modRMByte(0, RegOpcodeField, 4), OS); 621 } else if (Disp.isImm() && AllowDisp8 && 622 isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) { 623 // Displacement fits in a byte or matches an EVEX compressed disp8, use 624 // disp8 encoding. This also handles EBP/R13 base with 0 displacement unless 625 // {disp32} pseudo prefix was used. 626 emitByte(modRMByte(1, RegOpcodeField, 4), OS); 627 ForceDisp8 = true; 628 } else { 629 // Otherwise, emit the normal disp32 encoding. 630 emitByte(modRMByte(2, RegOpcodeField, 4), OS); 631 ForceDisp32 = true; 632 } 633 634 // Calculate what the SS field value should be... 635 static const unsigned SSTable[] = {~0U, 0, 1, ~0U, 2, ~0U, ~0U, ~0U, 3}; 636 unsigned SS = SSTable[Scale.getImm()]; 637 638 unsigned IndexRegNo = IndexReg.getReg() ? getX86RegNum(IndexReg) : 4; 639 640 emitSIBByte(SS, IndexRegNo, BaseRegNo, OS); 641 642 // Do we need to output a displacement? 643 if (ForceDisp8) 644 emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups, 645 ImmOffset); 646 else if (ForceDisp32) 647 emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(X86::reloc_signed_4byte), 648 StartByte, OS, Fixups); 649 } 650 651 /// Emit all instruction prefixes. 652 /// 653 /// \returns true if REX prefix is used, otherwise returns false. 654 bool X86MCCodeEmitter::emitPrefixImpl(unsigned &CurOp, const MCInst &MI, 655 const MCSubtargetInfo &STI, 656 raw_ostream &OS) const { 657 uint64_t TSFlags = MCII.get(MI.getOpcode()).TSFlags; 658 // Determine where the memory operand starts, if present. 659 int MemoryOperand = X86II::getMemoryOperandNo(TSFlags); 660 // Emit segment override opcode prefix as needed. 661 if (MemoryOperand != -1) { 662 MemoryOperand += CurOp; 663 emitSegmentOverridePrefix(MemoryOperand + X86::AddrSegmentReg, MI, OS); 664 } 665 666 // Emit the repeat opcode prefix as needed. 667 unsigned Flags = MI.getFlags(); 668 if (TSFlags & X86II::REP || Flags & X86::IP_HAS_REPEAT) 669 emitByte(0xF3, OS); 670 if (Flags & X86::IP_HAS_REPEAT_NE) 671 emitByte(0xF2, OS); 672 673 // Emit the address size opcode prefix as needed. 674 bool NeedAddressOverride; 675 uint64_t AdSize = TSFlags & X86II::AdSizeMask; 676 if ((STI.hasFeature(X86::Mode16Bit) && AdSize == X86II::AdSize32) || 677 (STI.hasFeature(X86::Mode32Bit) && AdSize == X86II::AdSize16) || 678 (STI.hasFeature(X86::Mode64Bit) && AdSize == X86II::AdSize32)) { 679 NeedAddressOverride = true; 680 } else if (MemoryOperand < 0) { 681 NeedAddressOverride = false; 682 } else if (STI.hasFeature(X86::Mode64Bit)) { 683 assert(!is16BitMemOperand(MI, MemoryOperand, STI)); 684 NeedAddressOverride = is32BitMemOperand(MI, MemoryOperand); 685 } else if (STI.hasFeature(X86::Mode32Bit)) { 686 assert(!is64BitMemOperand(MI, MemoryOperand)); 687 NeedAddressOverride = is16BitMemOperand(MI, MemoryOperand, STI); 688 } else { 689 assert(STI.hasFeature(X86::Mode16Bit)); 690 assert(!is64BitMemOperand(MI, MemoryOperand)); 691 NeedAddressOverride = !is16BitMemOperand(MI, MemoryOperand, STI); 692 } 693 694 if (NeedAddressOverride) 695 emitByte(0x67, OS); 696 697 // Encoding type for this instruction. 698 uint64_t Encoding = TSFlags & X86II::EncodingMask; 699 bool HasREX = false; 700 if (Encoding) 701 emitVEXOpcodePrefix(MemoryOperand, MI, OS); 702 else 703 HasREX = emitOpcodePrefix(MemoryOperand, MI, STI, OS); 704 705 uint64_t Form = TSFlags & X86II::FormMask; 706 switch (Form) { 707 default: 708 break; 709 case X86II::RawFrmDstSrc: { 710 unsigned siReg = MI.getOperand(1).getReg(); 711 assert(((siReg == X86::SI && MI.getOperand(0).getReg() == X86::DI) || 712 (siReg == X86::ESI && MI.getOperand(0).getReg() == X86::EDI) || 713 (siReg == X86::RSI && MI.getOperand(0).getReg() == X86::RDI)) && 714 "SI and DI register sizes do not match"); 715 // Emit segment override opcode prefix as needed (not for %ds). 716 if (MI.getOperand(2).getReg() != X86::DS) 717 emitSegmentOverridePrefix(2, MI, OS); 718 // Emit AdSize prefix as needed. 719 if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::ESI) || 720 (STI.hasFeature(X86::Mode32Bit) && siReg == X86::SI)) 721 emitByte(0x67, OS); 722 CurOp += 3; // Consume operands. 723 break; 724 } 725 case X86II::RawFrmSrc: { 726 unsigned siReg = MI.getOperand(0).getReg(); 727 // Emit segment override opcode prefix as needed (not for %ds). 728 if (MI.getOperand(1).getReg() != X86::DS) 729 emitSegmentOverridePrefix(1, MI, OS); 730 // Emit AdSize prefix as needed. 731 if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::ESI) || 732 (STI.hasFeature(X86::Mode32Bit) && siReg == X86::SI)) 733 emitByte(0x67, OS); 734 CurOp += 2; // Consume operands. 735 break; 736 } 737 case X86II::RawFrmDst: { 738 unsigned siReg = MI.getOperand(0).getReg(); 739 // Emit AdSize prefix as needed. 740 if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::EDI) || 741 (STI.hasFeature(X86::Mode32Bit) && siReg == X86::DI)) 742 emitByte(0x67, OS); 743 ++CurOp; // Consume operand. 744 break; 745 } 746 case X86II::RawFrmMemOffs: { 747 // Emit segment override opcode prefix as needed. 748 emitSegmentOverridePrefix(1, MI, OS); 749 break; 750 } 751 } 752 753 return HasREX; 754 } 755 756 /// AVX instructions are encoded using a opcode prefix called VEX. 757 void X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI, 758 raw_ostream &OS) const { 759 const MCInstrDesc &Desc = MCII.get(MI.getOpcode()); 760 uint64_t TSFlags = Desc.TSFlags; 761 762 assert(!(TSFlags & X86II::LOCK) && "Can't have LOCK VEX."); 763 764 uint64_t Encoding = TSFlags & X86II::EncodingMask; 765 bool HasEVEX_K = TSFlags & X86II::EVEX_K; 766 bool HasVEX_4V = TSFlags & X86II::VEX_4V; 767 bool HasEVEX_RC = TSFlags & X86II::EVEX_RC; 768 769 // VEX_R: opcode externsion equivalent to REX.R in 770 // 1's complement (inverted) form 771 // 772 // 1: Same as REX_R=0 (must be 1 in 32-bit mode) 773 // 0: Same as REX_R=1 (64 bit mode only) 774 // 775 uint8_t VEX_R = 0x1; 776 uint8_t EVEX_R2 = 0x1; 777 778 // VEX_X: equivalent to REX.X, only used when a 779 // register is used for index in SIB Byte. 780 // 781 // 1: Same as REX.X=0 (must be 1 in 32-bit mode) 782 // 0: Same as REX.X=1 (64-bit mode only) 783 uint8_t VEX_X = 0x1; 784 785 // VEX_B: 786 // 787 // 1: Same as REX_B=0 (ignored in 32-bit mode) 788 // 0: Same as REX_B=1 (64 bit mode only) 789 // 790 uint8_t VEX_B = 0x1; 791 792 // VEX_W: opcode specific (use like REX.W, or used for 793 // opcode extension, or ignored, depending on the opcode byte) 794 uint8_t VEX_W = (TSFlags & X86II::VEX_W) ? 1 : 0; 795 796 // VEX_5M (VEX m-mmmmm field): 797 // 798 // 0b00000: Reserved for future use 799 // 0b00001: implied 0F leading opcode 800 // 0b00010: implied 0F 38 leading opcode bytes 801 // 0b00011: implied 0F 3A leading opcode bytes 802 // 0b00100: Reserved for future use 803 // 0b00101: VEX MAP5 804 // 0b00110: VEX MAP6 805 // 0b00111-0b11111: Reserved for future use 806 // 0b01000: XOP map select - 08h instructions with imm byte 807 // 0b01001: XOP map select - 09h instructions with no imm byte 808 // 0b01010: XOP map select - 0Ah instructions with imm dword 809 uint8_t VEX_5M; 810 switch (TSFlags & X86II::OpMapMask) { 811 default: 812 llvm_unreachable("Invalid prefix!"); 813 case X86II::TB: 814 VEX_5M = 0x1; 815 break; // 0F 816 case X86II::T8: 817 VEX_5M = 0x2; 818 break; // 0F 38 819 case X86II::TA: 820 VEX_5M = 0x3; 821 break; // 0F 3A 822 case X86II::XOP8: 823 VEX_5M = 0x8; 824 break; 825 case X86II::XOP9: 826 VEX_5M = 0x9; 827 break; 828 case X86II::XOPA: 829 VEX_5M = 0xA; 830 break; 831 case X86II::T_MAP5: 832 VEX_5M = 0x5; 833 break; 834 case X86II::T_MAP6: 835 VEX_5M = 0x6; 836 break; 837 } 838 839 // VEX_4V (VEX vvvv field): a register specifier 840 // (in 1's complement form) or 1111 if unused. 841 uint8_t VEX_4V = 0xf; 842 uint8_t EVEX_V2 = 0x1; 843 844 // EVEX_L2/VEX_L (Vector Length): 845 // 846 // L2 L 847 // 0 0: scalar or 128-bit vector 848 // 0 1: 256-bit vector 849 // 1 0: 512-bit vector 850 // 851 uint8_t VEX_L = (TSFlags & X86II::VEX_L) ? 1 : 0; 852 uint8_t EVEX_L2 = (TSFlags & X86II::EVEX_L2) ? 1 : 0; 853 854 // VEX_PP: opcode extension providing equivalent 855 // functionality of a SIMD prefix 856 // 857 // 0b00: None 858 // 0b01: 66 859 // 0b10: F3 860 // 0b11: F2 861 // 862 uint8_t VEX_PP = 0; 863 switch (TSFlags & X86II::OpPrefixMask) { 864 case X86II::PD: 865 VEX_PP = 0x1; 866 break; // 66 867 case X86II::XS: 868 VEX_PP = 0x2; 869 break; // F3 870 case X86II::XD: 871 VEX_PP = 0x3; 872 break; // F2 873 } 874 875 // EVEX_U 876 uint8_t EVEX_U = 1; // Always '1' so far 877 878 // EVEX_z 879 uint8_t EVEX_z = (HasEVEX_K && (TSFlags & X86II::EVEX_Z)) ? 1 : 0; 880 881 // EVEX_b 882 uint8_t EVEX_b = (TSFlags & X86II::EVEX_B) ? 1 : 0; 883 884 // EVEX_rc 885 uint8_t EVEX_rc = 0; 886 887 // EVEX_aaa 888 uint8_t EVEX_aaa = 0; 889 890 bool EncodeRC = false; 891 892 // Classify VEX_B, VEX_4V, VEX_R, VEX_X 893 unsigned NumOps = Desc.getNumOperands(); 894 unsigned CurOp = X86II::getOperandBias(Desc); 895 896 switch (TSFlags & X86II::FormMask) { 897 default: 898 llvm_unreachable("Unexpected form in emitVEXOpcodePrefix!"); 899 case X86II::MRM_C0: 900 case X86II::RawFrm: 901 case X86II::PrefixByte: 902 break; 903 case X86II::MRMDestMemFSIB: 904 case X86II::MRMDestMem: { 905 // MRMDestMem instructions forms: 906 // MemAddr, src1(ModR/M) 907 // MemAddr, src1(VEX_4V), src2(ModR/M) 908 // MemAddr, src1(ModR/M), imm8 909 // 910 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg); 911 VEX_B = ~(BaseRegEnc >> 3) & 1; 912 unsigned IndexRegEnc = 913 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg); 914 VEX_X = ~(IndexRegEnc >> 3) & 1; 915 if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV. 916 EVEX_V2 = ~(IndexRegEnc >> 4) & 1; 917 918 CurOp += X86::AddrNumOperands; 919 920 if (HasEVEX_K) 921 EVEX_aaa = getX86RegEncoding(MI, CurOp++); 922 923 if (HasVEX_4V) { 924 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++); 925 VEX_4V = ~VRegEnc & 0xf; 926 EVEX_V2 = ~(VRegEnc >> 4) & 1; 927 } 928 929 unsigned RegEnc = getX86RegEncoding(MI, CurOp++); 930 VEX_R = ~(RegEnc >> 3) & 1; 931 EVEX_R2 = ~(RegEnc >> 4) & 1; 932 break; 933 } 934 case X86II::MRMSrcMemFSIB: 935 case X86II::MRMSrcMem: { 936 // MRMSrcMem instructions forms: 937 // src1(ModR/M), MemAddr 938 // src1(ModR/M), src2(VEX_4V), MemAddr 939 // src1(ModR/M), MemAddr, imm8 940 // src1(ModR/M), MemAddr, src2(Imm[7:4]) 941 // 942 // FMA4: 943 // dst(ModR/M.reg), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4]) 944 unsigned RegEnc = getX86RegEncoding(MI, CurOp++); 945 VEX_R = ~(RegEnc >> 3) & 1; 946 EVEX_R2 = ~(RegEnc >> 4) & 1; 947 948 if (HasEVEX_K) 949 EVEX_aaa = getX86RegEncoding(MI, CurOp++); 950 951 if (HasVEX_4V) { 952 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++); 953 VEX_4V = ~VRegEnc & 0xf; 954 EVEX_V2 = ~(VRegEnc >> 4) & 1; 955 } 956 957 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg); 958 VEX_B = ~(BaseRegEnc >> 3) & 1; 959 unsigned IndexRegEnc = 960 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg); 961 VEX_X = ~(IndexRegEnc >> 3) & 1; 962 if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV. 963 EVEX_V2 = ~(IndexRegEnc >> 4) & 1; 964 965 break; 966 } 967 case X86II::MRMSrcMem4VOp3: { 968 // Instruction format for 4VOp3: 969 // src1(ModR/M), MemAddr, src3(VEX_4V) 970 unsigned RegEnc = getX86RegEncoding(MI, CurOp++); 971 VEX_R = ~(RegEnc >> 3) & 1; 972 973 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg); 974 VEX_B = ~(BaseRegEnc >> 3) & 1; 975 unsigned IndexRegEnc = 976 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg); 977 VEX_X = ~(IndexRegEnc >> 3) & 1; 978 979 VEX_4V = ~getX86RegEncoding(MI, CurOp + X86::AddrNumOperands) & 0xf; 980 break; 981 } 982 case X86II::MRMSrcMemOp4: { 983 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M), 984 unsigned RegEnc = getX86RegEncoding(MI, CurOp++); 985 VEX_R = ~(RegEnc >> 3) & 1; 986 987 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++); 988 VEX_4V = ~VRegEnc & 0xf; 989 990 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg); 991 VEX_B = ~(BaseRegEnc >> 3) & 1; 992 unsigned IndexRegEnc = 993 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg); 994 VEX_X = ~(IndexRegEnc >> 3) & 1; 995 break; 996 } 997 case X86II::MRM0m: 998 case X86II::MRM1m: 999 case X86II::MRM2m: 1000 case X86II::MRM3m: 1001 case X86II::MRM4m: 1002 case X86II::MRM5m: 1003 case X86II::MRM6m: 1004 case X86II::MRM7m: { 1005 // MRM[0-9]m instructions forms: 1006 // MemAddr 1007 // src1(VEX_4V), MemAddr 1008 if (HasVEX_4V) { 1009 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++); 1010 VEX_4V = ~VRegEnc & 0xf; 1011 EVEX_V2 = ~(VRegEnc >> 4) & 1; 1012 } 1013 1014 if (HasEVEX_K) 1015 EVEX_aaa = getX86RegEncoding(MI, CurOp++); 1016 1017 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg); 1018 VEX_B = ~(BaseRegEnc >> 3) & 1; 1019 unsigned IndexRegEnc = 1020 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg); 1021 VEX_X = ~(IndexRegEnc >> 3) & 1; 1022 if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV. 1023 EVEX_V2 = ~(IndexRegEnc >> 4) & 1; 1024 1025 break; 1026 } 1027 case X86II::MRMSrcReg: { 1028 // MRMSrcReg instructions forms: 1029 // dst(ModR/M), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4]) 1030 // dst(ModR/M), src1(ModR/M) 1031 // dst(ModR/M), src1(ModR/M), imm8 1032 // 1033 // FMA4: 1034 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M), 1035 unsigned RegEnc = getX86RegEncoding(MI, CurOp++); 1036 VEX_R = ~(RegEnc >> 3) & 1; 1037 EVEX_R2 = ~(RegEnc >> 4) & 1; 1038 1039 if (HasEVEX_K) 1040 EVEX_aaa = getX86RegEncoding(MI, CurOp++); 1041 1042 if (HasVEX_4V) { 1043 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++); 1044 VEX_4V = ~VRegEnc & 0xf; 1045 EVEX_V2 = ~(VRegEnc >> 4) & 1; 1046 } 1047 1048 RegEnc = getX86RegEncoding(MI, CurOp++); 1049 VEX_B = ~(RegEnc >> 3) & 1; 1050 VEX_X = ~(RegEnc >> 4) & 1; 1051 1052 if (EVEX_b) { 1053 if (HasEVEX_RC) { 1054 unsigned RcOperand = NumOps - 1; 1055 assert(RcOperand >= CurOp); 1056 EVEX_rc = MI.getOperand(RcOperand).getImm(); 1057 assert(EVEX_rc <= 3 && "Invalid rounding control!"); 1058 } 1059 EncodeRC = true; 1060 } 1061 break; 1062 } 1063 case X86II::MRMSrcReg4VOp3: { 1064 // Instruction format for 4VOp3: 1065 // src1(ModR/M), src2(ModR/M), src3(VEX_4V) 1066 unsigned RegEnc = getX86RegEncoding(MI, CurOp++); 1067 VEX_R = ~(RegEnc >> 3) & 1; 1068 1069 RegEnc = getX86RegEncoding(MI, CurOp++); 1070 VEX_B = ~(RegEnc >> 3) & 1; 1071 1072 VEX_4V = ~getX86RegEncoding(MI, CurOp++) & 0xf; 1073 break; 1074 } 1075 case X86II::MRMSrcRegOp4: { 1076 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M), 1077 unsigned RegEnc = getX86RegEncoding(MI, CurOp++); 1078 VEX_R = ~(RegEnc >> 3) & 1; 1079 1080 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++); 1081 VEX_4V = ~VRegEnc & 0xf; 1082 1083 // Skip second register source (encoded in Imm[7:4]) 1084 ++CurOp; 1085 1086 RegEnc = getX86RegEncoding(MI, CurOp++); 1087 VEX_B = ~(RegEnc >> 3) & 1; 1088 VEX_X = ~(RegEnc >> 4) & 1; 1089 break; 1090 } 1091 case X86II::MRMDestReg: { 1092 // MRMDestReg instructions forms: 1093 // dst(ModR/M), src(ModR/M) 1094 // dst(ModR/M), src(ModR/M), imm8 1095 // dst(ModR/M), src1(VEX_4V), src2(ModR/M) 1096 unsigned RegEnc = getX86RegEncoding(MI, CurOp++); 1097 VEX_B = ~(RegEnc >> 3) & 1; 1098 VEX_X = ~(RegEnc >> 4) & 1; 1099 1100 if (HasEVEX_K) 1101 EVEX_aaa = getX86RegEncoding(MI, CurOp++); 1102 1103 if (HasVEX_4V) { 1104 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++); 1105 VEX_4V = ~VRegEnc & 0xf; 1106 EVEX_V2 = ~(VRegEnc >> 4) & 1; 1107 } 1108 1109 RegEnc = getX86RegEncoding(MI, CurOp++); 1110 VEX_R = ~(RegEnc >> 3) & 1; 1111 EVEX_R2 = ~(RegEnc >> 4) & 1; 1112 if (EVEX_b) 1113 EncodeRC = true; 1114 break; 1115 } 1116 case X86II::MRMr0: { 1117 // MRMr0 instructions forms: 1118 // 11:rrr:000 1119 // dst(ModR/M) 1120 unsigned RegEnc = getX86RegEncoding(MI, CurOp++); 1121 VEX_R = ~(RegEnc >> 3) & 1; 1122 EVEX_R2 = ~(RegEnc >> 4) & 1; 1123 break; 1124 } 1125 case X86II::MRM0r: 1126 case X86II::MRM1r: 1127 case X86II::MRM2r: 1128 case X86II::MRM3r: 1129 case X86II::MRM4r: 1130 case X86II::MRM5r: 1131 case X86II::MRM6r: 1132 case X86II::MRM7r: { 1133 // MRM0r-MRM7r instructions forms: 1134 // dst(VEX_4V), src(ModR/M), imm8 1135 if (HasVEX_4V) { 1136 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++); 1137 VEX_4V = ~VRegEnc & 0xf; 1138 EVEX_V2 = ~(VRegEnc >> 4) & 1; 1139 } 1140 if (HasEVEX_K) 1141 EVEX_aaa = getX86RegEncoding(MI, CurOp++); 1142 1143 unsigned RegEnc = getX86RegEncoding(MI, CurOp++); 1144 VEX_B = ~(RegEnc >> 3) & 1; 1145 VEX_X = ~(RegEnc >> 4) & 1; 1146 break; 1147 } 1148 } 1149 1150 if (Encoding == X86II::VEX || Encoding == X86II::XOP) { 1151 // VEX opcode prefix can have 2 or 3 bytes 1152 // 1153 // 3 bytes: 1154 // +-----+ +--------------+ +-------------------+ 1155 // | C4h | | RXB | m-mmmm | | W | vvvv | L | pp | 1156 // +-----+ +--------------+ +-------------------+ 1157 // 2 bytes: 1158 // +-----+ +-------------------+ 1159 // | C5h | | R | vvvv | L | pp | 1160 // +-----+ +-------------------+ 1161 // 1162 // XOP uses a similar prefix: 1163 // +-----+ +--------------+ +-------------------+ 1164 // | 8Fh | | RXB | m-mmmm | | W | vvvv | L | pp | 1165 // +-----+ +--------------+ +-------------------+ 1166 uint8_t LastByte = VEX_PP | (VEX_L << 2) | (VEX_4V << 3); 1167 1168 // Can we use the 2 byte VEX prefix? 1169 if (!(MI.getFlags() & X86::IP_USE_VEX3) && Encoding == X86II::VEX && 1170 VEX_B && VEX_X && !VEX_W && (VEX_5M == 1)) { 1171 emitByte(0xC5, OS); 1172 emitByte(LastByte | (VEX_R << 7), OS); 1173 return; 1174 } 1175 1176 // 3 byte VEX prefix 1177 emitByte(Encoding == X86II::XOP ? 0x8F : 0xC4, OS); 1178 emitByte(VEX_R << 7 | VEX_X << 6 | VEX_B << 5 | VEX_5M, OS); 1179 emitByte(LastByte | (VEX_W << 7), OS); 1180 } else { 1181 assert(Encoding == X86II::EVEX && "unknown encoding!"); 1182 // EVEX opcode prefix can have 4 bytes 1183 // 1184 // +-----+ +--------------+ +-------------------+ +------------------------+ 1185 // | 62h | | RXBR' | 0mmm | | W | vvvv | U | pp | | z | L'L | b | v' | aaa | 1186 // +-----+ +--------------+ +-------------------+ +------------------------+ 1187 assert((VEX_5M & 0x7) == VEX_5M && 1188 "More than 3 significant bits in VEX.m-mmmm fields for EVEX!"); 1189 1190 emitByte(0x62, OS); 1191 emitByte((VEX_R << 7) | (VEX_X << 6) | (VEX_B << 5) | (EVEX_R2 << 4) | 1192 VEX_5M, 1193 OS); 1194 emitByte((VEX_W << 7) | (VEX_4V << 3) | (EVEX_U << 2) | VEX_PP, OS); 1195 if (EncodeRC) 1196 emitByte((EVEX_z << 7) | (EVEX_rc << 5) | (EVEX_b << 4) | (EVEX_V2 << 3) | 1197 EVEX_aaa, 1198 OS); 1199 else 1200 emitByte((EVEX_z << 7) | (EVEX_L2 << 6) | (VEX_L << 5) | (EVEX_b << 4) | 1201 (EVEX_V2 << 3) | EVEX_aaa, 1202 OS); 1203 } 1204 } 1205 1206 /// Emit REX prefix which specifies 1207 /// 1) 64-bit instructions, 1208 /// 2) non-default operand size, and 1209 /// 3) use of X86-64 extended registers. 1210 /// 1211 /// \returns true if REX prefix is used, otherwise returns false. 1212 bool X86MCCodeEmitter::emitREXPrefix(int MemOperand, const MCInst &MI, 1213 const MCSubtargetInfo &STI, 1214 raw_ostream &OS) const { 1215 uint8_t REX = [&, MemOperand]() { 1216 uint8_t REX = 0; 1217 bool UsesHighByteReg = false; 1218 1219 const MCInstrDesc &Desc = MCII.get(MI.getOpcode()); 1220 uint64_t TSFlags = Desc.TSFlags; 1221 1222 if (TSFlags & X86II::REX_W) 1223 REX |= 1 << 3; // set REX.W 1224 1225 if (MI.getNumOperands() == 0) 1226 return REX; 1227 1228 unsigned NumOps = MI.getNumOperands(); 1229 unsigned CurOp = X86II::getOperandBias(Desc); 1230 1231 // If it accesses SPL, BPL, SIL, or DIL, then it requires a 0x40 REX prefix. 1232 for (unsigned i = CurOp; i != NumOps; ++i) { 1233 const MCOperand &MO = MI.getOperand(i); 1234 if (MO.isReg()) { 1235 unsigned Reg = MO.getReg(); 1236 if (Reg == X86::AH || Reg == X86::BH || Reg == X86::CH || 1237 Reg == X86::DH) 1238 UsesHighByteReg = true; 1239 if (X86II::isX86_64NonExtLowByteReg(Reg)) 1240 // FIXME: The caller of determineREXPrefix slaps this prefix onto 1241 // anything that returns non-zero. 1242 REX |= 0x40; // REX fixed encoding prefix 1243 } else if (MO.isExpr() && STI.getTargetTriple().isX32()) { 1244 // GOTTPOFF and TLSDESC relocations require a REX prefix to allow 1245 // linker optimizations: even if the instructions we see may not require 1246 // any prefix, they may be replaced by instructions that do. This is 1247 // handled as a special case here so that it also works for hand-written 1248 // assembly without the user needing to write REX, as with GNU as. 1249 const auto *Ref = dyn_cast<MCSymbolRefExpr>(MO.getExpr()); 1250 if (Ref && (Ref->getKind() == MCSymbolRefExpr::VK_GOTTPOFF || 1251 Ref->getKind() == MCSymbolRefExpr::VK_TLSDESC)) { 1252 REX |= 0x40; // REX fixed encoding prefix 1253 } 1254 } 1255 } 1256 1257 switch (TSFlags & X86II::FormMask) { 1258 case X86II::AddRegFrm: 1259 REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B 1260 break; 1261 case X86II::MRMSrcReg: 1262 case X86II::MRMSrcRegCC: 1263 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R 1264 REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B 1265 break; 1266 case X86II::MRMSrcMem: 1267 case X86II::MRMSrcMemCC: 1268 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R 1269 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0; // REX.B 1270 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X 1271 CurOp += X86::AddrNumOperands; 1272 break; 1273 case X86II::MRMDestReg: 1274 REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B 1275 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R 1276 break; 1277 case X86II::MRMDestMem: 1278 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0; // REX.B 1279 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X 1280 CurOp += X86::AddrNumOperands; 1281 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R 1282 break; 1283 case X86II::MRMXmCC: 1284 case X86II::MRMXm: 1285 case X86II::MRM0m: 1286 case X86II::MRM1m: 1287 case X86II::MRM2m: 1288 case X86II::MRM3m: 1289 case X86II::MRM4m: 1290 case X86II::MRM5m: 1291 case X86II::MRM6m: 1292 case X86II::MRM7m: 1293 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0; // REX.B 1294 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X 1295 break; 1296 case X86II::MRMXrCC: 1297 case X86II::MRMXr: 1298 case X86II::MRM0r: 1299 case X86II::MRM1r: 1300 case X86II::MRM2r: 1301 case X86II::MRM3r: 1302 case X86II::MRM4r: 1303 case X86II::MRM5r: 1304 case X86II::MRM6r: 1305 case X86II::MRM7r: 1306 REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B 1307 break; 1308 case X86II::MRMr0: 1309 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R 1310 break; 1311 case X86II::MRMDestMemFSIB: 1312 llvm_unreachable("FSIB format never need REX prefix!"); 1313 } 1314 if (REX && UsesHighByteReg) 1315 report_fatal_error( 1316 "Cannot encode high byte register in REX-prefixed instruction"); 1317 return REX; 1318 }(); 1319 1320 if (!REX) 1321 return false; 1322 1323 emitByte(0x40 | REX, OS); 1324 return true; 1325 } 1326 1327 /// Emit segment override opcode prefix as needed. 1328 void X86MCCodeEmitter::emitSegmentOverridePrefix(unsigned SegOperand, 1329 const MCInst &MI, 1330 raw_ostream &OS) const { 1331 // Check for explicit segment override on memory operand. 1332 if (unsigned Reg = MI.getOperand(SegOperand).getReg()) 1333 emitByte(X86::getSegmentOverridePrefixForReg(Reg), OS); 1334 } 1335 1336 /// Emit all instruction prefixes prior to the opcode. 1337 /// 1338 /// \param MemOperand the operand # of the start of a memory operand if present. 1339 /// If not present, it is -1. 1340 /// 1341 /// \returns true if REX prefix is used, otherwise returns false. 1342 bool X86MCCodeEmitter::emitOpcodePrefix(int MemOperand, const MCInst &MI, 1343 const MCSubtargetInfo &STI, 1344 raw_ostream &OS) const { 1345 const MCInstrDesc &Desc = MCII.get(MI.getOpcode()); 1346 uint64_t TSFlags = Desc.TSFlags; 1347 1348 // Emit the operand size opcode prefix as needed. 1349 if ((TSFlags & X86II::OpSizeMask) == 1350 (STI.hasFeature(X86::Mode16Bit) ? X86II::OpSize32 : X86II::OpSize16)) 1351 emitByte(0x66, OS); 1352 1353 // Emit the LOCK opcode prefix. 1354 if (TSFlags & X86II::LOCK || MI.getFlags() & X86::IP_HAS_LOCK) 1355 emitByte(0xF0, OS); 1356 1357 // Emit the NOTRACK opcode prefix. 1358 if (TSFlags & X86II::NOTRACK || MI.getFlags() & X86::IP_HAS_NOTRACK) 1359 emitByte(0x3E, OS); 1360 1361 switch (TSFlags & X86II::OpPrefixMask) { 1362 case X86II::PD: // 66 1363 emitByte(0x66, OS); 1364 break; 1365 case X86II::XS: // F3 1366 emitByte(0xF3, OS); 1367 break; 1368 case X86II::XD: // F2 1369 emitByte(0xF2, OS); 1370 break; 1371 } 1372 1373 // Handle REX prefix. 1374 assert((STI.hasFeature(X86::Mode64Bit) || !(TSFlags & X86II::REX_W)) && 1375 "REX.W requires 64bit mode."); 1376 bool HasREX = STI.hasFeature(X86::Mode64Bit) 1377 ? emitREXPrefix(MemOperand, MI, STI, OS) 1378 : false; 1379 1380 // 0x0F escape code must be emitted just before the opcode. 1381 switch (TSFlags & X86II::OpMapMask) { 1382 case X86II::TB: // Two-byte opcode map 1383 case X86II::T8: // 0F 38 1384 case X86II::TA: // 0F 3A 1385 case X86II::ThreeDNow: // 0F 0F, second 0F emitted by caller. 1386 emitByte(0x0F, OS); 1387 break; 1388 } 1389 1390 switch (TSFlags & X86II::OpMapMask) { 1391 case X86II::T8: // 0F 38 1392 emitByte(0x38, OS); 1393 break; 1394 case X86II::TA: // 0F 3A 1395 emitByte(0x3A, OS); 1396 break; 1397 } 1398 1399 return HasREX; 1400 } 1401 1402 void X86MCCodeEmitter::emitPrefix(const MCInst &MI, raw_ostream &OS, 1403 const MCSubtargetInfo &STI) const { 1404 unsigned Opcode = MI.getOpcode(); 1405 const MCInstrDesc &Desc = MCII.get(Opcode); 1406 uint64_t TSFlags = Desc.TSFlags; 1407 1408 // Pseudo instructions don't get encoded. 1409 if (X86II::isPseudo(TSFlags)) 1410 return; 1411 1412 unsigned CurOp = X86II::getOperandBias(Desc); 1413 1414 emitPrefixImpl(CurOp, MI, STI, OS); 1415 } 1416 1417 void X86MCCodeEmitter::encodeInstruction(const MCInst &MI, raw_ostream &OS, 1418 SmallVectorImpl<MCFixup> &Fixups, 1419 const MCSubtargetInfo &STI) const { 1420 unsigned Opcode = MI.getOpcode(); 1421 const MCInstrDesc &Desc = MCII.get(Opcode); 1422 uint64_t TSFlags = Desc.TSFlags; 1423 1424 // Pseudo instructions don't get encoded. 1425 if (X86II::isPseudo(TSFlags)) 1426 return; 1427 1428 unsigned NumOps = Desc.getNumOperands(); 1429 unsigned CurOp = X86II::getOperandBias(Desc); 1430 1431 uint64_t StartByte = OS.tell(); 1432 1433 bool HasREX = emitPrefixImpl(CurOp, MI, STI, OS); 1434 1435 // It uses the VEX.VVVV field? 1436 bool HasVEX_4V = TSFlags & X86II::VEX_4V; 1437 bool HasVEX_I8Reg = (TSFlags & X86II::ImmMask) == X86II::Imm8Reg; 1438 1439 // It uses the EVEX.aaa field? 1440 bool HasEVEX_K = TSFlags & X86II::EVEX_K; 1441 bool HasEVEX_RC = TSFlags & X86II::EVEX_RC; 1442 1443 // Used if a register is encoded in 7:4 of immediate. 1444 unsigned I8RegNum = 0; 1445 1446 uint8_t BaseOpcode = X86II::getBaseOpcodeFor(TSFlags); 1447 1448 if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow) 1449 BaseOpcode = 0x0F; // Weird 3DNow! encoding. 1450 1451 unsigned OpcodeOffset = 0; 1452 1453 uint64_t Form = TSFlags & X86II::FormMask; 1454 switch (Form) { 1455 default: 1456 errs() << "FORM: " << Form << "\n"; 1457 llvm_unreachable("Unknown FormMask value in X86MCCodeEmitter!"); 1458 case X86II::Pseudo: 1459 llvm_unreachable("Pseudo instruction shouldn't be emitted"); 1460 case X86II::RawFrmDstSrc: 1461 case X86II::RawFrmSrc: 1462 case X86II::RawFrmDst: 1463 case X86II::PrefixByte: 1464 emitByte(BaseOpcode, OS); 1465 break; 1466 case X86II::AddCCFrm: { 1467 // This will be added to the opcode in the fallthrough. 1468 OpcodeOffset = MI.getOperand(NumOps - 1).getImm(); 1469 assert(OpcodeOffset < 16 && "Unexpected opcode offset!"); 1470 --NumOps; // Drop the operand from the end. 1471 LLVM_FALLTHROUGH; 1472 case X86II::RawFrm: 1473 emitByte(BaseOpcode + OpcodeOffset, OS); 1474 1475 if (!STI.hasFeature(X86::Mode64Bit) || !isPCRel32Branch(MI, MCII)) 1476 break; 1477 1478 const MCOperand &Op = MI.getOperand(CurOp++); 1479 emitImmediate(Op, MI.getLoc(), X86II::getSizeOfImm(TSFlags), 1480 MCFixupKind(X86::reloc_branch_4byte_pcrel), StartByte, OS, 1481 Fixups); 1482 break; 1483 } 1484 case X86II::RawFrmMemOffs: 1485 emitByte(BaseOpcode, OS); 1486 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1487 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags), 1488 StartByte, OS, Fixups); 1489 ++CurOp; // skip segment operand 1490 break; 1491 case X86II::RawFrmImm8: 1492 emitByte(BaseOpcode, OS); 1493 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1494 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags), 1495 StartByte, OS, Fixups); 1496 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1, FK_Data_1, StartByte, 1497 OS, Fixups); 1498 break; 1499 case X86II::RawFrmImm16: 1500 emitByte(BaseOpcode, OS); 1501 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1502 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags), 1503 StartByte, OS, Fixups); 1504 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 2, FK_Data_2, StartByte, 1505 OS, Fixups); 1506 break; 1507 1508 case X86II::AddRegFrm: 1509 emitByte(BaseOpcode + getX86RegNum(MI.getOperand(CurOp++)), OS); 1510 break; 1511 1512 case X86II::MRMDestReg: { 1513 emitByte(BaseOpcode, OS); 1514 unsigned SrcRegNum = CurOp + 1; 1515 1516 if (HasEVEX_K) // Skip writemask 1517 ++SrcRegNum; 1518 1519 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV) 1520 ++SrcRegNum; 1521 1522 emitRegModRMByte(MI.getOperand(CurOp), 1523 getX86RegNum(MI.getOperand(SrcRegNum)), OS); 1524 CurOp = SrcRegNum + 1; 1525 break; 1526 } 1527 case X86II::MRMDestMemFSIB: 1528 case X86II::MRMDestMem: { 1529 emitByte(BaseOpcode, OS); 1530 unsigned SrcRegNum = CurOp + X86::AddrNumOperands; 1531 1532 if (HasEVEX_K) // Skip writemask 1533 ++SrcRegNum; 1534 1535 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV) 1536 ++SrcRegNum; 1537 1538 bool ForceSIB = (Form == X86II::MRMDestMemFSIB); 1539 emitMemModRMByte(MI, CurOp, getX86RegNum(MI.getOperand(SrcRegNum)), TSFlags, 1540 HasREX, StartByte, OS, Fixups, STI, ForceSIB); 1541 CurOp = SrcRegNum + 1; 1542 break; 1543 } 1544 case X86II::MRMSrcReg: { 1545 emitByte(BaseOpcode, OS); 1546 unsigned SrcRegNum = CurOp + 1; 1547 1548 if (HasEVEX_K) // Skip writemask 1549 ++SrcRegNum; 1550 1551 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV) 1552 ++SrcRegNum; 1553 1554 emitRegModRMByte(MI.getOperand(SrcRegNum), 1555 getX86RegNum(MI.getOperand(CurOp)), OS); 1556 CurOp = SrcRegNum + 1; 1557 if (HasVEX_I8Reg) 1558 I8RegNum = getX86RegEncoding(MI, CurOp++); 1559 // do not count the rounding control operand 1560 if (HasEVEX_RC) 1561 --NumOps; 1562 break; 1563 } 1564 case X86II::MRMSrcReg4VOp3: { 1565 emitByte(BaseOpcode, OS); 1566 unsigned SrcRegNum = CurOp + 1; 1567 1568 emitRegModRMByte(MI.getOperand(SrcRegNum), 1569 getX86RegNum(MI.getOperand(CurOp)), OS); 1570 CurOp = SrcRegNum + 1; 1571 ++CurOp; // Encoded in VEX.VVVV 1572 break; 1573 } 1574 case X86II::MRMSrcRegOp4: { 1575 emitByte(BaseOpcode, OS); 1576 unsigned SrcRegNum = CurOp + 1; 1577 1578 // Skip 1st src (which is encoded in VEX_VVVV) 1579 ++SrcRegNum; 1580 1581 // Capture 2nd src (which is encoded in Imm[7:4]) 1582 assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg"); 1583 I8RegNum = getX86RegEncoding(MI, SrcRegNum++); 1584 1585 emitRegModRMByte(MI.getOperand(SrcRegNum), 1586 getX86RegNum(MI.getOperand(CurOp)), OS); 1587 CurOp = SrcRegNum + 1; 1588 break; 1589 } 1590 case X86II::MRMSrcRegCC: { 1591 unsigned FirstOp = CurOp++; 1592 unsigned SecondOp = CurOp++; 1593 1594 unsigned CC = MI.getOperand(CurOp++).getImm(); 1595 emitByte(BaseOpcode + CC, OS); 1596 1597 emitRegModRMByte(MI.getOperand(SecondOp), 1598 getX86RegNum(MI.getOperand(FirstOp)), OS); 1599 break; 1600 } 1601 case X86II::MRMSrcMemFSIB: 1602 case X86II::MRMSrcMem: { 1603 unsigned FirstMemOp = CurOp + 1; 1604 1605 if (HasEVEX_K) // Skip writemask 1606 ++FirstMemOp; 1607 1608 if (HasVEX_4V) 1609 ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV). 1610 1611 emitByte(BaseOpcode, OS); 1612 1613 bool ForceSIB = (Form == X86II::MRMSrcMemFSIB); 1614 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)), 1615 TSFlags, HasREX, StartByte, OS, Fixups, STI, ForceSIB); 1616 CurOp = FirstMemOp + X86::AddrNumOperands; 1617 if (HasVEX_I8Reg) 1618 I8RegNum = getX86RegEncoding(MI, CurOp++); 1619 break; 1620 } 1621 case X86II::MRMSrcMem4VOp3: { 1622 unsigned FirstMemOp = CurOp + 1; 1623 1624 emitByte(BaseOpcode, OS); 1625 1626 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)), 1627 TSFlags, HasREX, StartByte, OS, Fixups, STI); 1628 CurOp = FirstMemOp + X86::AddrNumOperands; 1629 ++CurOp; // Encoded in VEX.VVVV. 1630 break; 1631 } 1632 case X86II::MRMSrcMemOp4: { 1633 unsigned FirstMemOp = CurOp + 1; 1634 1635 ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV). 1636 1637 // Capture second register source (encoded in Imm[7:4]) 1638 assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg"); 1639 I8RegNum = getX86RegEncoding(MI, FirstMemOp++); 1640 1641 emitByte(BaseOpcode, OS); 1642 1643 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)), 1644 TSFlags, HasREX, StartByte, OS, Fixups, STI); 1645 CurOp = FirstMemOp + X86::AddrNumOperands; 1646 break; 1647 } 1648 case X86II::MRMSrcMemCC: { 1649 unsigned RegOp = CurOp++; 1650 unsigned FirstMemOp = CurOp; 1651 CurOp = FirstMemOp + X86::AddrNumOperands; 1652 1653 unsigned CC = MI.getOperand(CurOp++).getImm(); 1654 emitByte(BaseOpcode + CC, OS); 1655 1656 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(RegOp)), 1657 TSFlags, HasREX, StartByte, OS, Fixups, STI); 1658 break; 1659 } 1660 1661 case X86II::MRMXrCC: { 1662 unsigned RegOp = CurOp++; 1663 1664 unsigned CC = MI.getOperand(CurOp++).getImm(); 1665 emitByte(BaseOpcode + CC, OS); 1666 emitRegModRMByte(MI.getOperand(RegOp), 0, OS); 1667 break; 1668 } 1669 1670 case X86II::MRMXr: 1671 case X86II::MRM0r: 1672 case X86II::MRM1r: 1673 case X86II::MRM2r: 1674 case X86II::MRM3r: 1675 case X86II::MRM4r: 1676 case X86II::MRM5r: 1677 case X86II::MRM6r: 1678 case X86II::MRM7r: 1679 if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV). 1680 ++CurOp; 1681 if (HasEVEX_K) // Skip writemask 1682 ++CurOp; 1683 emitByte(BaseOpcode, OS); 1684 emitRegModRMByte(MI.getOperand(CurOp++), 1685 (Form == X86II::MRMXr) ? 0 : Form - X86II::MRM0r, OS); 1686 break; 1687 case X86II::MRMr0: 1688 emitByte(BaseOpcode, OS); 1689 emitByte(modRMByte(3, getX86RegNum(MI.getOperand(CurOp++)),0), OS); 1690 break; 1691 1692 case X86II::MRMXmCC: { 1693 unsigned FirstMemOp = CurOp; 1694 CurOp = FirstMemOp + X86::AddrNumOperands; 1695 1696 unsigned CC = MI.getOperand(CurOp++).getImm(); 1697 emitByte(BaseOpcode + CC, OS); 1698 1699 emitMemModRMByte(MI, FirstMemOp, 0, TSFlags, HasREX, StartByte, OS, Fixups, 1700 STI); 1701 break; 1702 } 1703 1704 case X86II::MRMXm: 1705 case X86II::MRM0m: 1706 case X86II::MRM1m: 1707 case X86II::MRM2m: 1708 case X86II::MRM3m: 1709 case X86II::MRM4m: 1710 case X86II::MRM5m: 1711 case X86II::MRM6m: 1712 case X86II::MRM7m: 1713 if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV). 1714 ++CurOp; 1715 if (HasEVEX_K) // Skip writemask 1716 ++CurOp; 1717 emitByte(BaseOpcode, OS); 1718 emitMemModRMByte(MI, CurOp, 1719 (Form == X86II::MRMXm) ? 0 : Form - X86II::MRM0m, TSFlags, 1720 HasREX, StartByte, OS, Fixups, STI); 1721 CurOp += X86::AddrNumOperands; 1722 break; 1723 1724 case X86II::MRM0X: 1725 case X86II::MRM1X: 1726 case X86II::MRM2X: 1727 case X86II::MRM3X: 1728 case X86II::MRM4X: 1729 case X86II::MRM5X: 1730 case X86II::MRM6X: 1731 case X86II::MRM7X: 1732 emitByte(BaseOpcode, OS); 1733 emitByte(0xC0 + ((Form - X86II::MRM0X) << 3), OS); 1734 break; 1735 1736 case X86II::MRM_C0: 1737 case X86II::MRM_C1: 1738 case X86II::MRM_C2: 1739 case X86II::MRM_C3: 1740 case X86II::MRM_C4: 1741 case X86II::MRM_C5: 1742 case X86II::MRM_C6: 1743 case X86II::MRM_C7: 1744 case X86II::MRM_C8: 1745 case X86II::MRM_C9: 1746 case X86II::MRM_CA: 1747 case X86II::MRM_CB: 1748 case X86II::MRM_CC: 1749 case X86II::MRM_CD: 1750 case X86II::MRM_CE: 1751 case X86II::MRM_CF: 1752 case X86II::MRM_D0: 1753 case X86II::MRM_D1: 1754 case X86II::MRM_D2: 1755 case X86II::MRM_D3: 1756 case X86II::MRM_D4: 1757 case X86II::MRM_D5: 1758 case X86II::MRM_D6: 1759 case X86II::MRM_D7: 1760 case X86II::MRM_D8: 1761 case X86II::MRM_D9: 1762 case X86II::MRM_DA: 1763 case X86II::MRM_DB: 1764 case X86II::MRM_DC: 1765 case X86II::MRM_DD: 1766 case X86II::MRM_DE: 1767 case X86II::MRM_DF: 1768 case X86II::MRM_E0: 1769 case X86II::MRM_E1: 1770 case X86II::MRM_E2: 1771 case X86II::MRM_E3: 1772 case X86II::MRM_E4: 1773 case X86II::MRM_E5: 1774 case X86II::MRM_E6: 1775 case X86II::MRM_E7: 1776 case X86II::MRM_E8: 1777 case X86II::MRM_E9: 1778 case X86II::MRM_EA: 1779 case X86II::MRM_EB: 1780 case X86II::MRM_EC: 1781 case X86II::MRM_ED: 1782 case X86II::MRM_EE: 1783 case X86II::MRM_EF: 1784 case X86II::MRM_F0: 1785 case X86II::MRM_F1: 1786 case X86II::MRM_F2: 1787 case X86II::MRM_F3: 1788 case X86II::MRM_F4: 1789 case X86II::MRM_F5: 1790 case X86II::MRM_F6: 1791 case X86II::MRM_F7: 1792 case X86II::MRM_F8: 1793 case X86II::MRM_F9: 1794 case X86II::MRM_FA: 1795 case X86II::MRM_FB: 1796 case X86II::MRM_FC: 1797 case X86II::MRM_FD: 1798 case X86II::MRM_FE: 1799 case X86II::MRM_FF: 1800 emitByte(BaseOpcode, OS); 1801 emitByte(0xC0 + Form - X86II::MRM_C0, OS); 1802 break; 1803 } 1804 1805 if (HasVEX_I8Reg) { 1806 // The last source register of a 4 operand instruction in AVX is encoded 1807 // in bits[7:4] of a immediate byte. 1808 assert(I8RegNum < 16 && "Register encoding out of range"); 1809 I8RegNum <<= 4; 1810 if (CurOp != NumOps) { 1811 unsigned Val = MI.getOperand(CurOp++).getImm(); 1812 assert(Val < 16 && "Immediate operand value out of range"); 1813 I8RegNum |= Val; 1814 } 1815 emitImmediate(MCOperand::createImm(I8RegNum), MI.getLoc(), 1, FK_Data_1, 1816 StartByte, OS, Fixups); 1817 } else { 1818 // If there is a remaining operand, it must be a trailing immediate. Emit it 1819 // according to the right size for the instruction. Some instructions 1820 // (SSE4a extrq and insertq) have two trailing immediates. 1821 while (CurOp != NumOps && NumOps - CurOp <= 2) { 1822 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1823 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags), 1824 StartByte, OS, Fixups); 1825 } 1826 } 1827 1828 if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow) 1829 emitByte(X86II::getBaseOpcodeFor(TSFlags), OS); 1830 1831 assert(OS.tell() - StartByte <= 15 && 1832 "The size of instruction must be no longer than 15."); 1833 #ifndef NDEBUG 1834 // FIXME: Verify. 1835 if (/*!Desc.isVariadic() &&*/ CurOp != NumOps) { 1836 errs() << "Cannot encode all operands of: "; 1837 MI.dump(); 1838 errs() << '\n'; 1839 abort(); 1840 } 1841 #endif 1842 } 1843 1844 MCCodeEmitter *llvm::createX86MCCodeEmitter(const MCInstrInfo &MCII, 1845 const MCRegisterInfo &MRI, 1846 MCContext &Ctx) { 1847 return new X86MCCodeEmitter(MCII, Ctx); 1848 } 1849