Lines Matching +full:bp +full:- +full:disabled +full:- +full:ports
1 // SPDX-License-Identifier: GPL-2.0-only
5 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
9 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
18 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
27 #include <asm/nospec-branch.h>
29 #include <asm/text-patching.h>
45 #define OpMem64 6ull /* Memory, 64-bit */
46 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
49 #define OpImmByte 10ull /* 8-bit sign extended immediate */
51 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
52 #define OpMem16 13ull /* Memory operand (16-bit). */
53 #define OpMem32 14ull /* Memory operand (32-bit). */
65 #define OpMem8 26ull /* 8-bit zero extended memory operand */
66 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
67 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
69 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
72 #define OpMask ((1ull << OpBits) - 1)
75 * Opcode effective-address decode tables.
83 /* Operand sizes: 8-bit operands or specified/overridden size. */
84 #define ByteOp (1<<0) /* 8-bit operands. */
85 #define DstShift 1 /* Destination operand type at bits 1-5 */
97 #define SrcShift 6 /* Source operand type at bits 6-10 */
123 #define GroupMask (7<<15) /* Group mechanisms, at bits 15-17 */
134 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
141 #define No64 (1<<28) /* Instruction generates #UD in 64-bit mode */
145 #define Src2Shift (32) /* Source 2 operand type at bits 32-36 */
159 /* free: 37-39 */
161 #define AlignMask ((u64)3 << 41) /* Memory alignment requirement at bits 41-42 */
165 /* free: 43-44 */
251 unsigned long dirty = ctxt->regs_dirty; in writeback_registers()
255 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]); in writeback_registers()
260 ctxt->regs_dirty = 0; in invalidate_registers()
261 ctxt->regs_valid = 0; in invalidate_registers()
280 unsigned long flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF; \
282 if (!(ctxt->d & ByteOp)) \
283 bytes = ctxt->dst.bytes; \
291 : "+a" (ctxt->dst.val), \
292 "+d" (ctxt->src.val), \
295 : "c" (ctxt->src2.val))
312 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK); \
316 /* 1-operand, using "a" (dst) */
325 /* 1-operand, using "c" (src2) */
334 /* 1-operand, using "c" (src2) with exception */
343 /* 2-operand, using "a" (dst), "d" (src) */
352 /* 2-operand, reversed */
361 /* 2-operand, word only (no byte op) */
370 /* 2-operand, using "a" (dst) and CL (src2) */
379 /* 3-operand, using "a" (dst), "d" (src) and CL (src2) */
393 ctxt->dst.val = 0xFF * !!(ctxt->eflags & X86_EFLAGS_CF); in em_salc()
419 .rep_prefix = ctxt->rep_prefix, in emulator_check_intercept()
420 .modrm_mod = ctxt->modrm_mod, in emulator_check_intercept()
421 .modrm_reg = ctxt->modrm_reg, in emulator_check_intercept()
422 .modrm_rm = ctxt->modrm_rm, in emulator_check_intercept()
423 .src_val = ctxt->src.val64, in emulator_check_intercept()
424 .dst_val = ctxt->dst.val64, in emulator_check_intercept()
425 .src_bytes = ctxt->src.bytes, in emulator_check_intercept()
426 .dst_bytes = ctxt->dst.bytes, in emulator_check_intercept()
427 .src_type = ctxt->src.type, in emulator_check_intercept()
428 .dst_type = ctxt->dst.type, in emulator_check_intercept()
429 .ad_bytes = ctxt->ad_bytes, in emulator_check_intercept()
430 .rip = ctxt->eip, in emulator_check_intercept()
431 .next_rip = ctxt->_eip, in emulator_check_intercept()
434 return ctxt->ops->intercept(ctxt, &info, stage); in emulator_check_intercept()
444 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */ in assign_register()
454 break; /* 64b: zero-extend */ in assign_register()
463 return (1UL << (ctxt->ad_bytes << 3)) - 1; in ad_mask()
471 if (ctxt->mode == X86EMUL_MODE_PROT64) in stack_mask()
473 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS); in stack_mask()
486 if (ctxt->ad_bytes == sizeof(unsigned long)) in address_mask()
508 assign_register(preg, *preg + inc, ctxt->ad_bytes); in register_address_increment()
520 return desc->g ? (limit << 12) | 0xfff : limit; in desc_limit_scaled()
525 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS) in seg_base()
528 return ctxt->ops->get_cached_segment_base(ctxt, seg); in seg_base()
537 ctxt->exception.vector = vec; in emulate_exception()
538 ctxt->exception.error_code = error; in emulate_exception()
539 ctxt->exception.error_code_valid = valid; in emulate_exception()
583 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg); in get_segment_selector()
594 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg); in set_segment_selector()
595 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg); in set_segment_selector()
600 return (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_LA57) ? 57 : 48; in ctxt_virt_addr_bits()
607 return !ctxt->ops->is_canonical_addr(ctxt, la, flags); in emul_is_noncanonical_address()
621 u64 alignment = ctxt->d & AlignMask; in insn_alignment()
654 *linear = la = ctxt->ops->get_untagged_addr(ctxt, la, flags); in __linearize()
659 *max_size = min_t(u64, ~0u, (1ull << va_bits) - la); in __linearize()
665 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL, in __linearize()
669 /* code segment in protected mode or read-only data segment */ in __linearize()
670 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8)) || !(desc.type & 2)) && in __linearize()
678 /* expand-down segment */ in __linearize()
688 *max_size = (u64)lim + 1 - addr.ea; in __linearize()
694 if (la & (insn_alignment(ctxt, size) - 1)) in __linearize()
710 return __linearize(ctxt, addr, &max_size, size, ctxt->mode, linear, in linearize()
722 if (ctxt->op_bytes != sizeof(unsigned long)) in assign_eip()
723 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1); in assign_eip()
724 rc = __linearize(ctxt, addr, &max_size, 1, ctxt->mode, &linear, in assign_eip()
727 ctxt->_eip = addr.ea; in assign_eip()
738 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer); in emulator_recalc_and_set_mode()
740 if (!(ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PE)) { in emulator_recalc_and_set_mode()
744 ctxt->mode = X86EMUL_MODE_REAL; in emulator_recalc_and_set_mode()
748 if (ctxt->eflags & X86_EFLAGS_VM) { in emulator_recalc_and_set_mode()
752 ctxt->mode = X86EMUL_MODE_VM86; in emulator_recalc_and_set_mode()
756 if (!ctxt->ops->get_segment(ctxt, &selector, &cs, &base3, VCPU_SREG_CS)) in emulator_recalc_and_set_mode()
762 ctxt->mode = X86EMUL_MODE_PROT64; in emulator_recalc_and_set_mode()
765 ctxt->mode = X86EMUL_MODE_PROT32; in emulator_recalc_and_set_mode()
767 ctxt->mode = X86EMUL_MODE_PROT16; in emulator_recalc_and_set_mode()
771 ctxt->mode = cs.d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16; in emulator_recalc_and_set_mode()
794 return assign_eip_near(ctxt, ctxt->_eip + rel); in jmp_rel()
800 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true); in linear_read_system()
807 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true); in linear_write_system()
821 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false); in segmented_read_std()
835 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false); in segmented_write_std()
847 int cur_size = ctxt->fetch.end - ctxt->fetch.data; in __do_insn_fetch_bytes()
849 .ea = ctxt->eip + cur_size }; in __do_insn_fetch_bytes()
861 rc = __linearize(ctxt, addr, &max_size, 0, ctxt->mode, &linear, in __do_insn_fetch_bytes()
867 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear)); in __do_insn_fetch_bytes()
873 * still, we must have hit the 15-byte boundary. in __do_insn_fetch_bytes()
878 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end, in __do_insn_fetch_bytes()
879 size, &ctxt->exception); in __do_insn_fetch_bytes()
882 ctxt->fetch.end += size; in __do_insn_fetch_bytes()
889 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr; in do_insn_fetch_bytes()
892 return __do_insn_fetch_bytes(ctxt, size - done_size); in do_insn_fetch_bytes()
904 ctxt->_eip += sizeof(_type); \
905 memcpy(&_x, ctxt->fetch.ptr, sizeof(_type)); \
906 ctxt->fetch.ptr += sizeof(_type); \
915 ctxt->_eip += (_size); \
916 memcpy(_arr, ctxt->fetch.ptr, _size); \
917 ctxt->fetch.ptr += (_size); \
929 int highbyte_regs = (ctxt->rex_prefix == REX_NONE) && byteop; in decode_register()
1001 if (ctxt->src.val == 0) in em_bsf_c()
1002 ctxt->dst.type = OP_NONE; in em_bsf_c()
1009 if (ctxt->src.val == 0) in em_bsr_c()
1010 ctxt->dst.type = OP_NONE; in em_bsr_c()
1021 switch (op->bytes) { in fetch_register_operand()
1023 op->val = *(u8 *)op->addr.reg; in fetch_register_operand()
1026 op->val = *(u16 *)op->addr.reg; in fetch_register_operand()
1029 op->val = *(u32 *)op->addr.reg; in fetch_register_operand()
1032 op->val = *(u64 *)op->addr.reg; in fetch_register_operand()
1035 op->orig_val = op->val; in fetch_register_operand()
1040 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM)) in em_fninit()
1053 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM)) in em_fnstcw()
1060 ctxt->dst.val = fcw; in em_fnstcw()
1069 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM)) in em_fnstsw()
1076 ctxt->dst.val = fsw; in em_fnstsw()
1084 if ((ctxt->d & Avx) && ctxt->op_bytes == 32) { in __decode_register_operand()
1085 op->type = OP_YMM; in __decode_register_operand()
1086 op->bytes = 32; in __decode_register_operand()
1087 op->addr.xmm = reg; in __decode_register_operand()
1088 kvm_read_avx_reg(reg, &op->vec_val2); in __decode_register_operand()
1091 if (ctxt->d & (Avx|Sse)) { in __decode_register_operand()
1092 op->type = OP_XMM; in __decode_register_operand()
1093 op->bytes = 16; in __decode_register_operand()
1094 op->addr.xmm = reg; in __decode_register_operand()
1095 kvm_read_sse_reg(reg, &op->vec_val); in __decode_register_operand()
1098 if (ctxt->d & Mmx) { in __decode_register_operand()
1100 op->type = OP_MM; in __decode_register_operand()
1101 op->bytes = 8; in __decode_register_operand()
1102 op->addr.mm = reg; in __decode_register_operand()
1106 op->type = OP_REG; in __decode_register_operand()
1107 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; in __decode_register_operand()
1108 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp); in __decode_register_operand()
1117 if (ctxt->d & ModRM) in decode_register_operand()
1118 reg = ctxt->modrm_reg; in decode_register_operand()
1120 reg = (ctxt->b & 7) | (ctxt->rex_bits & REX_B ? 8 : 0); in decode_register_operand()
1128 ctxt->modrm_seg = VCPU_SREG_SS; in adjust_modrm_seg()
1139 ctxt->modrm_reg = (ctxt->rex_bits & REX_R ? 8 : 0); in decode_modrm()
1140 index_reg = (ctxt->rex_bits & REX_X ? 8 : 0); in decode_modrm()
1141 base_reg = (ctxt->rex_bits & REX_B ? 8 : 0); in decode_modrm()
1143 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6; in decode_modrm()
1144 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3; in decode_modrm()
1145 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07); in decode_modrm()
1146 ctxt->modrm_seg = VCPU_SREG_DS; in decode_modrm()
1148 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) { in decode_modrm()
1149 __decode_register_operand(ctxt, op, ctxt->modrm_rm); in decode_modrm()
1153 op->type = OP_MEM; in decode_modrm()
1155 if (ctxt->ad_bytes == 2) { in decode_modrm()
1157 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP); in decode_modrm() local
1161 /* 16-bit ModR/M decode. */ in decode_modrm()
1162 switch (ctxt->modrm_mod) { in decode_modrm()
1164 if (ctxt->modrm_rm == 6) in decode_modrm()
1174 switch (ctxt->modrm_rm) { in decode_modrm()
1182 modrm_ea += bp + si; in decode_modrm()
1185 modrm_ea += bp + di; in decode_modrm()
1194 if (ctxt->modrm_mod != 0) in decode_modrm()
1195 modrm_ea += bp; in decode_modrm()
1201 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 || in decode_modrm()
1202 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0)) in decode_modrm()
1203 ctxt->modrm_seg = VCPU_SREG_SS; in decode_modrm()
1206 /* 32/64-bit ModR/M decode. */ in decode_modrm()
1207 if ((ctxt->modrm_rm & 7) == 4) { in decode_modrm()
1213 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0) in decode_modrm()
1219 if ((ctxt->d & IncSP) && in decode_modrm()
1221 modrm_ea += ctxt->op_bytes; in decode_modrm()
1225 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) { in decode_modrm()
1227 if (ctxt->mode == X86EMUL_MODE_PROT64) in decode_modrm()
1228 ctxt->rip_relative = 1; in decode_modrm()
1230 base_reg = ctxt->modrm_rm; in decode_modrm()
1234 switch (ctxt->modrm_mod) { in decode_modrm()
1243 op->addr.mem.ea = modrm_ea; in decode_modrm()
1244 if (ctxt->ad_bytes != 8) in decode_modrm()
1245 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea; in decode_modrm()
1256 op->type = OP_MEM; in decode_abs()
1257 switch (ctxt->ad_bytes) { in decode_abs()
1259 op->addr.mem.ea = insn_fetch(u16, ctxt); in decode_abs()
1262 op->addr.mem.ea = insn_fetch(u32, ctxt); in decode_abs()
1265 op->addr.mem.ea = insn_fetch(u64, ctxt); in decode_abs()
1276 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) { in fetch_bit_operand()
1277 mask = ~((long)ctxt->dst.bytes * 8 - 1); in fetch_bit_operand()
1279 if (ctxt->src.bytes == 2) in fetch_bit_operand()
1280 sv = (s16)ctxt->src.val & (s16)mask; in fetch_bit_operand()
1281 else if (ctxt->src.bytes == 4) in fetch_bit_operand()
1282 sv = (s32)ctxt->src.val & (s32)mask; in fetch_bit_operand()
1284 sv = (s64)ctxt->src.val & (s64)mask; in fetch_bit_operand()
1286 ctxt->dst.addr.mem.ea = address_mask(ctxt, in fetch_bit_operand()
1287 ctxt->dst.addr.mem.ea + (sv >> 3)); in fetch_bit_operand()
1291 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1; in fetch_bit_operand()
1298 struct read_cache *mc = &ctxt->mem_read; in read_emulated()
1300 if (mc->pos < mc->end) in read_emulated()
1303 if (KVM_EMULATOR_BUG_ON((mc->end + size) >= sizeof(mc->data), ctxt)) in read_emulated()
1306 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size, in read_emulated()
1307 &ctxt->exception); in read_emulated()
1311 mc->end += size; in read_emulated()
1314 memcpy(dest, mc->data + mc->pos, size); in read_emulated()
1315 mc->pos += size; in read_emulated()
1344 return ctxt->ops->write_emulated(ctxt, linear, data, size, in segmented_write()
1345 &ctxt->exception); in segmented_write()
1359 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data, in segmented_cmpxchg()
1360 size, &ctxt->exception); in segmented_cmpxchg()
1367 struct read_cache *rc = &ctxt->io_read; in pio_in_emulated()
1369 if (rc->pos == rc->end) { /* refill pio read ahead */ in pio_in_emulated()
1371 unsigned int count = ctxt->rep_prefix ? in pio_in_emulated()
1373 in_page = (ctxt->eflags & X86_EFLAGS_DF) ? in pio_in_emulated()
1375 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)); in pio_in_emulated()
1376 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count); in pio_in_emulated()
1379 rc->pos = rc->end = 0; in pio_in_emulated()
1380 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n)) in pio_in_emulated()
1382 rc->end = n * size; in pio_in_emulated()
1385 if (ctxt->rep_prefix && (ctxt->d & String) && in pio_in_emulated()
1386 !(ctxt->eflags & X86_EFLAGS_DF)) { in pio_in_emulated()
1387 ctxt->dst.data = rc->data + rc->pos; in pio_in_emulated()
1388 ctxt->dst.type = OP_MEM_STR; in pio_in_emulated()
1389 ctxt->dst.count = (rc->end - rc->pos) / size; in pio_in_emulated()
1390 rc->pos = rc->end; in pio_in_emulated()
1392 memcpy(dest, rc->data + rc->pos, size); in pio_in_emulated()
1393 rc->pos += size; in pio_in_emulated()
1404 ctxt->ops->get_idt(ctxt, &dt); in read_interrupt_descriptor()
1416 const struct x86_emulate_ops *ops = ctxt->ops; in get_descriptor_table_ptr()
1424 if (!ops->get_segment(ctxt, &sel, &desc, &base3, in get_descriptor_table_ptr()
1428 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */ in get_descriptor_table_ptr()
1429 dt->address = get_desc_base(&desc) | ((u64)base3 << 32); in get_descriptor_table_ptr()
1431 ops->get_gdt(ctxt, dt); in get_descriptor_table_ptr()
1452 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer); in get_descriptor_ptr()
1454 addr &= (u32)-1; in get_descriptor_ptr()
1495 if (!(ctxt->ops->get_cr(ctxt, 4) & X86_CR4_CET)) in emulator_is_ssp_invalid()
1498 if (ctxt->ops->get_msr(ctxt, MSR_EFER, &efer)) in emulator_is_ssp_invalid()
1501 /* SSP is guaranteed to be valid if the vCPU was already in 32-bit mode. */ in emulator_is_ssp_invalid()
1505 if (ctxt->ops->get_msr(ctxt, MSR_IA32_X_CET, &cet)) in emulator_is_ssp_invalid()
1511 if (ctxt->ops->get_msr(ctxt, MSR_KVM_INTERNAL_GUEST_SSP, &ssp)) in emulator_is_ssp_invalid()
1515 * On transfer from 64-bit mode to compatibility mode, SSP[63:32] must in emulator_is_ssp_invalid()
1516 * be 0, i.e. SSP must be a 32-bit value outside of 64-bit mode. in emulator_is_ssp_invalid()
1530 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */ in __load_segment_descriptor()
1538 if (ctxt->mode == X86EMUL_MODE_REAL) { in __load_segment_descriptor()
1541 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg); in __load_segment_descriptor()
1544 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) { in __load_segment_descriptor()
1567 if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl) in __load_segment_descriptor()
1571 * ctxt->ops->set_segment expects the CPL to be in in __load_segment_descriptor()
1572 * SS.DPL, so fake an expand-up 32-bit data segment. in __load_segment_descriptor()
1628 /* Outer-privilege level return is not implemented */ in __load_segment_descriptor()
1653 /* in long-mode d/b must be clear if l is set */ in __load_segment_descriptor()
1657 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer); in __load_segment_descriptor()
1666 /* CS(RPL) <- CPL */ in __load_segment_descriptor()
1704 } else if (ctxt->mode == X86EMUL_MODE_PROT64) { in __load_segment_descriptor()
1717 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc, in __load_segment_descriptor()
1718 sizeof(seg_desc), &ctxt->exception); in __load_segment_descriptor()
1723 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg); in __load_segment_descriptor()
1734 u8 cpl = ctxt->ops->cpl(ctxt); in load_segment_descriptor()
1747 ctxt->mode == X86EMUL_MODE_PROT64) in load_segment_descriptor()
1756 return assign_register(op->addr.reg, op->val, op->bytes); in write_register_operand()
1761 switch (op->type) { in writeback()
1766 if (ctxt->lock_prefix) in writeback()
1768 op->addr.mem, in writeback()
1769 &op->orig_val, in writeback()
1770 &op->val, in writeback()
1771 op->bytes); in writeback()
1774 op->addr.mem, in writeback()
1775 &op->val, in writeback()
1776 op->bytes); in writeback()
1779 op->addr.mem, in writeback()
1780 op->data, in writeback()
1781 op->bytes * op->count); in writeback()
1783 if (!(ctxt->d & Avx)) { in writeback()
1784 kvm_write_sse_reg(op->addr.xmm, &op->vec_val); in writeback()
1788 memset(op->valptr + 16, 0, 16); in writeback()
1791 kvm_write_avx_reg(op->addr.xmm, &op->vec_val2); in writeback()
1794 kvm_write_mmx_reg(op->addr.mm, &op->mm_val); in writeback()
1809 rsp_increment(ctxt, -len); in emulate_push()
1819 ctxt->dst.type = OP_NONE; in em_push()
1820 return emulate_push(ctxt, &ctxt->src.val, ctxt->op_bytes); in em_push()
1841 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes); in em_pop()
1850 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT; in emulate_popf()
1851 int cpl = ctxt->ops->cpl(ctxt); in emulate_popf()
1862 switch(ctxt->mode) { in emulate_popf()
1882 (ctxt->eflags & ~change_mask) | (val & change_mask); in emulate_popf()
1889 ctxt->dst.type = OP_REG; in em_popf()
1890 ctxt->dst.addr.reg = &ctxt->eflags; in em_popf()
1891 ctxt->dst.bytes = ctxt->op_bytes; in em_popf()
1892 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes); in em_popf()
1898 unsigned frame_size = ctxt->src.val; in em_enter()
1899 unsigned nesting_level = ctxt->src2.val & 31; in em_enter()
1912 reg_read(ctxt, VCPU_REGS_RSP) - frame_size, in em_enter()
1921 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes); in em_leave()
1926 int seg = ctxt->src2.val; in em_push_sreg()
1928 ctxt->src.val = get_segment_selector(ctxt, seg); in em_push_sreg()
1929 if (ctxt->op_bytes == 4) { in em_push_sreg()
1930 rsp_increment(ctxt, -2); in em_push_sreg()
1931 ctxt->op_bytes = 2; in em_push_sreg()
1939 int seg = ctxt->src2.val; in em_pop_sreg()
1948 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS; in em_pop_sreg()
1949 if (ctxt->op_bytes > 2) in em_pop_sreg()
1950 rsp_increment(ctxt, ctxt->op_bytes - 2); in em_pop_sreg()
1964 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg)); in em_pusha()
1978 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM; in em_pushf()
1990 rsp_increment(ctxt, ctxt->op_bytes); in em_popa()
1991 --reg; in em_popa()
1994 rc = emulate_pop(ctxt, &val, ctxt->op_bytes); in em_popa()
1997 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes); in em_popa()
1998 --reg; in em_popa()
2005 const struct x86_emulate_ops *ops = ctxt->ops; in __emulate_int_real()
2013 ctxt->src.val = ctxt->eflags; in __emulate_int_real()
2018 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC); in __emulate_int_real()
2020 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS); in __emulate_int_real()
2025 ctxt->src.val = ctxt->_eip; in __emulate_int_real()
2030 ops->get_idt(ctxt, &dt); in __emulate_int_real()
2047 ctxt->_eip = eip; in __emulate_int_real()
2065 switch(ctxt->mode) { in emulate_int()
2095 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes); in emulate_iret_real()
2103 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes); in emulate_iret_real()
2108 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes); in emulate_iret_real()
2118 ctxt->_eip = temp_eip; in emulate_iret_real()
2120 if (ctxt->op_bytes == 4) in emulate_iret_real()
2121 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask)); in emulate_iret_real()
2122 else if (ctxt->op_bytes == 2) { in emulate_iret_real()
2123 ctxt->eflags &= ~0xffff; in emulate_iret_real()
2124 ctxt->eflags |= temp_eflags; in emulate_iret_real()
2127 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */ in emulate_iret_real()
2128 ctxt->eflags |= X86_EFLAGS_FIXED; in emulate_iret_real()
2129 ctxt->ops->set_nmi_mask(ctxt, false); in emulate_iret_real()
2136 switch(ctxt->mode) { in em_iret()
2154 u8 cpl = ctxt->ops->cpl(ctxt); in em_jmp_far()
2156 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2); in em_jmp_far()
2164 rc = assign_eip_far(ctxt, ctxt->src.val); in em_jmp_far()
2174 return assign_eip_near(ctxt, ctxt->src.val); in em_jmp_abs()
2182 old_eip = ctxt->_eip; in em_call_near_abs()
2183 rc = assign_eip_near(ctxt, ctxt->src.val); in em_call_near_abs()
2186 ctxt->src.val = old_eip; in em_call_near_abs()
2193 u64 old = ctxt->dst.orig_val64; in em_cmpxchg8b()
2195 if (ctxt->dst.bytes == 16) in em_cmpxchg8b()
2202 ctxt->eflags &= ~X86_EFLAGS_ZF; in em_cmpxchg8b()
2204 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) | in em_cmpxchg8b()
2207 ctxt->eflags |= X86_EFLAGS_ZF; in em_cmpxchg8b()
2217 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes); in em_ret()
2229 int cpl = ctxt->ops->cpl(ctxt); in em_ret_far()
2232 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes); in em_ret_far()
2235 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes); in em_ret_far()
2258 rsp_increment(ctxt, ctxt->src.val); in em_ret_far_imm()
2265 ctxt->dst.orig_val = ctxt->dst.val; in em_cmpxchg()
2266 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX); in em_cmpxchg()
2267 ctxt->src.orig_val = ctxt->src.val; in em_cmpxchg()
2268 ctxt->src.val = ctxt->dst.orig_val; in em_cmpxchg()
2271 if (ctxt->eflags & X86_EFLAGS_ZF) { in em_cmpxchg()
2273 ctxt->src.type = OP_NONE; in em_cmpxchg()
2274 ctxt->dst.val = ctxt->src.orig_val; in em_cmpxchg()
2277 ctxt->src.type = OP_REG; in em_cmpxchg()
2278 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX); in em_cmpxchg()
2279 ctxt->src.val = ctxt->dst.orig_val; in em_cmpxchg()
2280 /* Create write-cycle to dest by writing the same value */ in em_cmpxchg()
2281 ctxt->dst.val = ctxt->dst.orig_val; in em_cmpxchg()
2288 int seg = ctxt->src2.val; in em_lseg()
2292 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2); in em_lseg()
2298 ctxt->dst.val = ctxt->src.val; in em_lseg()
2304 if (!ctxt->ops->is_smm(ctxt)) in em_rsm()
2307 if (ctxt->ops->leave_smm(ctxt)) in em_rsm()
2308 ctxt->ops->triple_fault(ctxt); in em_rsm()
2316 cs->l = 0; /* will be adjusted later */ in setup_syscalls_segments()
2318 cs->g = 1; /* 4kb granularity */ in setup_syscalls_segments()
2320 cs->type = 0x0b; /* Read, Execute, Accessed */ in setup_syscalls_segments()
2321 cs->s = 1; in setup_syscalls_segments()
2322 cs->dpl = 0; /* will be adjusted later */ in setup_syscalls_segments()
2323 cs->p = 1; in setup_syscalls_segments()
2324 cs->d = 1; in setup_syscalls_segments()
2325 cs->avl = 0; in setup_syscalls_segments()
2329 ss->g = 1; /* 4kb granularity */ in setup_syscalls_segments()
2330 ss->s = 1; in setup_syscalls_segments()
2331 ss->type = 0x03; /* Read/Write, Accessed */ in setup_syscalls_segments()
2332 ss->d = 1; /* 32bit stack segment */ in setup_syscalls_segments()
2333 ss->dpl = 0; in setup_syscalls_segments()
2334 ss->p = 1; in setup_syscalls_segments()
2335 ss->l = 0; in setup_syscalls_segments()
2336 ss->avl = 0; in setup_syscalls_segments()
2341 const struct x86_emulate_ops *ops = ctxt->ops; in em_syscall()
2348 if (ctxt->mode == X86EMUL_MODE_REAL || in em_syscall()
2349 ctxt->mode == X86EMUL_MODE_VM86) in em_syscall()
2353 * Intel compatible CPUs only support SYSCALL in 64-bit mode, whereas in em_syscall()
2357 * for KVM to trap-and-emulate, unlike emulating AMD on Intel. in em_syscall()
2359 if (ctxt->mode != X86EMUL_MODE_PROT64 && in em_syscall()
2360 ctxt->ops->guest_cpuid_is_intel_compatible(ctxt)) in em_syscall()
2363 ops->get_msr(ctxt, MSR_EFER, &efer); in em_syscall()
2368 ops->get_msr(ctxt, MSR_STAR, &msr_data); in em_syscall()
2377 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS); in em_syscall()
2378 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS); in em_syscall()
2380 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip; in em_syscall()
2383 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags; in em_syscall()
2385 ops->get_msr(ctxt, in em_syscall()
2386 ctxt->mode == X86EMUL_MODE_PROT64 ? in em_syscall()
2388 ctxt->_eip = msr_data; in em_syscall()
2390 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data); in em_syscall()
2391 ctxt->eflags &= ~msr_data; in em_syscall()
2392 ctxt->eflags |= X86_EFLAGS_FIXED; in em_syscall()
2396 ops->get_msr(ctxt, MSR_STAR, &msr_data); in em_syscall()
2397 ctxt->_eip = (u32)msr_data; in em_syscall()
2399 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF); in em_syscall()
2402 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0; in em_syscall()
2408 const struct x86_emulate_ops *ops = ctxt->ops; in em_sysenter()
2414 ops->get_msr(ctxt, MSR_EFER, &efer); in em_sysenter()
2416 if (ctxt->mode == X86EMUL_MODE_REAL) in em_sysenter()
2423 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA) && in em_sysenter()
2424 !ctxt->ops->guest_cpuid_is_intel_compatible(ctxt)) in em_sysenter()
2428 if (ctxt->mode == X86EMUL_MODE_PROT64) in em_sysenter()
2431 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data); in em_sysenter()
2436 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF); in em_sysenter()
2444 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS); in em_sysenter()
2445 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS); in em_sysenter()
2447 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data); in em_sysenter()
2448 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data; in em_sysenter()
2450 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data); in em_sysenter()
2454 ctxt->mode = X86EMUL_MODE_PROT64; in em_sysenter()
2461 const struct x86_emulate_ops *ops = ctxt->ops; in em_sysexit()
2468 if (ctxt->mode == X86EMUL_MODE_REAL || in em_sysexit()
2469 ctxt->mode == X86EMUL_MODE_VM86) in em_sysexit()
2474 if (ctxt->rex_bits & REX_W) in em_sysexit()
2484 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data); in em_sysexit()
2509 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS); in em_sysexit()
2510 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS); in em_sysexit()
2512 ctxt->_eip = rdx; in em_sysexit()
2513 ctxt->mode = usermode; in em_sysexit()
2522 if (ctxt->mode == X86EMUL_MODE_REAL) in emulator_bad_iopl()
2524 if (ctxt->mode == X86EMUL_MODE_VM86) in emulator_bad_iopl()
2526 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT; in emulator_bad_iopl()
2527 return ctxt->ops->cpl(ctxt) > iopl; in emulator_bad_iopl()
2536 const struct x86_emulate_ops *ops = ctxt->ops; in emulator_io_port_access_allowed()
2541 unsigned mask = (1 << len) - 1; in emulator_io_port_access_allowed()
2545 * VMware allows access to these ports even if denied in emulator_io_port_access_allowed()
2552 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR); in emulator_io_port_access_allowed()
2561 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true); in emulator_io_port_access_allowed()
2566 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true); in emulator_io_port_access_allowed()
2577 if (ctxt->perm_ok) in emulator_io_permitted()
2584 ctxt->perm_ok = true; in emulator_io_permitted()
2593 * manner when ECX is zero due to REP-string optimizations. in string_registers_quirk()
2598 if (ctxt->ad_bytes != 4) in string_registers_quirk()
2602 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true); in string_registers_quirk()
2608 switch (ctxt->b) { in string_registers_quirk()
2611 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1; in string_registers_quirk()
2615 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1; in string_registers_quirk()
2623 tss->ip = ctxt->_eip; in save_state_to_tss16()
2624 tss->flag = ctxt->eflags; in save_state_to_tss16()
2625 tss->ax = reg_read(ctxt, VCPU_REGS_RAX); in save_state_to_tss16()
2626 tss->cx = reg_read(ctxt, VCPU_REGS_RCX); in save_state_to_tss16()
2627 tss->dx = reg_read(ctxt, VCPU_REGS_RDX); in save_state_to_tss16()
2628 tss->bx = reg_read(ctxt, VCPU_REGS_RBX); in save_state_to_tss16()
2629 tss->sp = reg_read(ctxt, VCPU_REGS_RSP); in save_state_to_tss16()
2630 tss->bp = reg_read(ctxt, VCPU_REGS_RBP); in save_state_to_tss16()
2631 tss->si = reg_read(ctxt, VCPU_REGS_RSI); in save_state_to_tss16()
2632 tss->di = reg_read(ctxt, VCPU_REGS_RDI); in save_state_to_tss16()
2634 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES); in save_state_to_tss16()
2635 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS); in save_state_to_tss16()
2636 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS); in save_state_to_tss16()
2637 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS); in save_state_to_tss16()
2638 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR); in save_state_to_tss16()
2647 ctxt->_eip = tss->ip; in load_state_from_tss16()
2648 ctxt->eflags = tss->flag | 2; in load_state_from_tss16()
2649 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax; in load_state_from_tss16()
2650 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx; in load_state_from_tss16()
2651 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx; in load_state_from_tss16()
2652 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx; in load_state_from_tss16()
2653 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp; in load_state_from_tss16()
2654 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp; in load_state_from_tss16()
2655 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si; in load_state_from_tss16()
2656 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di; in load_state_from_tss16()
2662 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR); in load_state_from_tss16()
2663 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES); in load_state_from_tss16()
2664 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS); in load_state_from_tss16()
2665 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS); in load_state_from_tss16()
2666 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS); in load_state_from_tss16()
2668 cpl = tss->cs & 3; in load_state_from_tss16()
2674 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl, in load_state_from_tss16()
2678 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, in load_state_from_tss16()
2682 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, in load_state_from_tss16()
2686 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, in load_state_from_tss16()
2690 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, in load_state_from_tss16()
2736 tss->eip = ctxt->_eip; in save_state_to_tss32()
2737 tss->eflags = ctxt->eflags; in save_state_to_tss32()
2738 tss->eax = reg_read(ctxt, VCPU_REGS_RAX); in save_state_to_tss32()
2739 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX); in save_state_to_tss32()
2740 tss->edx = reg_read(ctxt, VCPU_REGS_RDX); in save_state_to_tss32()
2741 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX); in save_state_to_tss32()
2742 tss->esp = reg_read(ctxt, VCPU_REGS_RSP); in save_state_to_tss32()
2743 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP); in save_state_to_tss32()
2744 tss->esi = reg_read(ctxt, VCPU_REGS_RSI); in save_state_to_tss32()
2745 tss->edi = reg_read(ctxt, VCPU_REGS_RDI); in save_state_to_tss32()
2747 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES); in save_state_to_tss32()
2748 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS); in save_state_to_tss32()
2749 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS); in save_state_to_tss32()
2750 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS); in save_state_to_tss32()
2751 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS); in save_state_to_tss32()
2752 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS); in save_state_to_tss32()
2761 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3)) in load_state_from_tss32()
2763 ctxt->_eip = tss->eip; in load_state_from_tss32()
2764 ctxt->eflags = tss->eflags | 2; in load_state_from_tss32()
2767 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax; in load_state_from_tss32()
2768 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx; in load_state_from_tss32()
2769 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx; in load_state_from_tss32()
2770 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx; in load_state_from_tss32()
2771 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp; in load_state_from_tss32()
2772 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp; in load_state_from_tss32()
2773 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi; in load_state_from_tss32()
2774 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi; in load_state_from_tss32()
2781 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR); in load_state_from_tss32()
2782 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES); in load_state_from_tss32()
2783 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS); in load_state_from_tss32()
2784 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS); in load_state_from_tss32()
2785 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS); in load_state_from_tss32()
2786 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS); in load_state_from_tss32()
2787 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS); in load_state_from_tss32()
2794 if (ctxt->eflags & X86_EFLAGS_VM) { in load_state_from_tss32()
2795 ctxt->mode = X86EMUL_MODE_VM86; in load_state_from_tss32()
2798 ctxt->mode = X86EMUL_MODE_PROT32; in load_state_from_tss32()
2799 cpl = tss->cs & 3; in load_state_from_tss32()
2806 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR, in load_state_from_tss32()
2810 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, in load_state_from_tss32()
2814 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, in load_state_from_tss32()
2818 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, in load_state_from_tss32()
2822 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, in load_state_from_tss32()
2826 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl, in load_state_from_tss32()
2830 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl, in load_state_from_tss32()
2853 ldt_sel_offset - eip_offset); in task_switch_32()
2878 const struct x86_emulate_ops *ops = ctxt->ops; in emulator_do_task_switch()
2883 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR); in emulator_do_task_switch()
2903 * 3. jmp/call to TSS/task-gate: No check is performed since the in emulator_do_task_switch()
2907 if (idt_index != -1) { in emulator_do_task_switch()
2918 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl) in emulator_do_task_switch()
2936 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT; in emulator_do_task_switch()
2952 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT; in emulator_do_task_switch()
2959 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS); in emulator_do_task_switch()
2960 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR); in emulator_do_task_switch()
2963 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2; in emulator_do_task_switch()
2964 ctxt->lock_prefix = 0; in emulator_do_task_switch()
2965 ctxt->src.val = (unsigned long) error_code; in emulator_do_task_switch()
2969 dr7 = ops->get_dr(ctxt, 7); in emulator_do_task_switch()
2970 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN)); in emulator_do_task_switch()
2982 ctxt->_eip = ctxt->eip; in emulator_task_switch()
2983 ctxt->dst.type = OP_NONE; in emulator_task_switch()
2989 ctxt->eip = ctxt->_eip; in emulator_task_switch()
2999 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count; in string_addr_inc()
3001 register_address_increment(ctxt, reg, df * op->bytes); in string_addr_inc()
3002 op->addr.mem.ea = register_address(ctxt, reg); in string_addr_inc()
3010 cf = ctxt->eflags & X86_EFLAGS_CF; in em_das()
3011 al = ctxt->dst.val; in em_das()
3016 af = ctxt->eflags & X86_EFLAGS_AF; in em_das()
3018 al -= 6; in em_das()
3025 al -= 0x60; in em_das()
3029 ctxt->dst.val = al; in em_das()
3031 ctxt->src.type = OP_IMM; in em_das()
3032 ctxt->src.val = 0; in em_das()
3033 ctxt->src.bytes = 1; in em_das()
3035 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF); in em_das()
3037 ctxt->eflags |= X86_EFLAGS_CF; in em_das()
3039 ctxt->eflags |= X86_EFLAGS_AF; in em_das()
3047 if (ctxt->src.val == 0) in em_aam()
3050 al = ctxt->dst.val & 0xff; in em_aam()
3051 ah = al / ctxt->src.val; in em_aam()
3052 al %= ctxt->src.val; in em_aam()
3054 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8); in em_aam()
3057 ctxt->src.type = OP_IMM; in em_aam()
3058 ctxt->src.val = 0; in em_aam()
3059 ctxt->src.bytes = 1; in em_aam()
3067 u8 al = ctxt->dst.val & 0xff; in em_aad()
3068 u8 ah = (ctxt->dst.val >> 8) & 0xff; in em_aad()
3070 al = (al + (ah * ctxt->src.val)) & 0xff; in em_aad()
3072 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al; in em_aad()
3075 ctxt->src.type = OP_IMM; in em_aad()
3076 ctxt->src.val = 0; in em_aad()
3077 ctxt->src.bytes = 1; in em_aad()
3086 long rel = ctxt->src.val; in em_call()
3088 ctxt->src.val = (unsigned long)ctxt->_eip; in em_call()
3101 const struct x86_emulate_ops *ops = ctxt->ops; in em_call_far()
3102 int cpl = ctxt->ops->cpl(ctxt); in em_call_far()
3103 enum x86emul_mode prev_mode = ctxt->mode; in em_call_far()
3105 old_eip = ctxt->_eip; in em_call_far()
3106 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS); in em_call_far()
3108 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2); in em_call_far()
3114 rc = assign_eip_far(ctxt, ctxt->src.val); in em_call_far()
3118 ctxt->src.val = old_cs; in em_call_far()
3123 ctxt->src.val = old_eip; in em_call_far()
3133 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS); in em_call_far()
3134 ctxt->mode = prev_mode; in em_call_far()
3144 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes); in em_ret_near_imm()
3150 rsp_increment(ctxt, ctxt->src.val); in em_ret_near_imm()
3157 ctxt->src.val = ctxt->dst.val; in em_xchg()
3158 write_register_operand(&ctxt->src); in em_xchg()
3161 ctxt->dst.val = ctxt->src.orig_val; in em_xchg()
3162 ctxt->lock_prefix = 1; in em_xchg()
3168 ctxt->dst.val = ctxt->src2.val; in em_imul_3op()
3174 ctxt->dst.type = OP_REG; in em_cwd()
3175 ctxt->dst.bytes = ctxt->src.bytes; in em_cwd()
3176 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX); in em_cwd()
3177 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1); in em_cwd()
3186 if (!ctxt->ops->guest_has_rdpid(ctxt)) in em_rdpid()
3189 ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux); in em_rdpid()
3190 ctxt->dst.val = tsc_aux; in em_rdpid()
3198 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc); in em_rdtsc()
3208 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc)) in em_rdpmc()
3217 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr)); in em_mov()
3225 if (!ctxt->ops->guest_has_movbe(ctxt)) in em_movbe()
3228 switch (ctxt->op_bytes) { in em_movbe()
3235 * Both casting ->valptr and ->val to u16 breaks strict aliasing in em_movbe()
3238 tmp = (u16)ctxt->src.val; in em_movbe()
3239 ctxt->dst.val &= ~0xffffUL; in em_movbe()
3240 ctxt->dst.val |= (unsigned long)swab16(tmp); in em_movbe()
3243 ctxt->dst.val = swab32((u32)ctxt->src.val); in em_movbe()
3246 ctxt->dst.val = swab64(ctxt->src.val); in em_movbe()
3256 int cr_num = ctxt->modrm_reg; in em_cr_write()
3259 if (ctxt->ops->set_cr(ctxt, cr_num, ctxt->src.val)) in em_cr_write()
3263 ctxt->dst.type = OP_NONE; in em_cr_write()
3282 if (ctxt->mode == X86EMUL_MODE_PROT64) in em_dr_write()
3283 val = ctxt->src.val & ~0ULL; in em_dr_write()
3285 val = ctxt->src.val & ~0U; in em_dr_write()
3288 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0) in em_dr_write()
3292 ctxt->dst.type = OP_NONE; in em_dr_write()
3304 r = ctxt->ops->set_msr_with_filter(ctxt, msr_index, msr_data); in em_wrmsr()
3318 r = ctxt->ops->get_msr_with_filter(ctxt, msr_index, &msr_data); in em_rdmsr()
3333 (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) && in em_store_sreg()
3334 ctxt->ops->cpl(ctxt) > 0) in em_store_sreg()
3337 ctxt->dst.val = get_segment_selector(ctxt, segment); in em_store_sreg()
3338 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM) in em_store_sreg()
3339 ctxt->dst.bytes = 2; in em_store_sreg()
3345 if (ctxt->modrm_reg > VCPU_SREG_GS) in em_mov_rm_sreg()
3348 return em_store_sreg(ctxt, ctxt->modrm_reg); in em_mov_rm_sreg()
3353 u16 sel = ctxt->src.val; in em_mov_sreg_rm()
3355 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS) in em_mov_sreg_rm()
3358 if (ctxt->modrm_reg == VCPU_SREG_SS) in em_mov_sreg_rm()
3359 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS; in em_mov_sreg_rm()
3362 ctxt->dst.type = OP_NONE; in em_mov_sreg_rm()
3363 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg); in em_mov_sreg_rm()
3373 u16 sel = ctxt->src.val; in em_lldt()
3376 ctxt->dst.type = OP_NONE; in em_lldt()
3387 u16 sel = ctxt->src.val; in em_ltr()
3390 ctxt->dst.type = OP_NONE; in em_ltr()
3400 rc = __linearize(ctxt, ctxt->src.addr.mem, &max_size, 1, ctxt->mode, in em_invlpg()
3403 ctxt->ops->invlpg(ctxt, linear); in em_invlpg()
3405 ctxt->dst.type = OP_NONE; in em_invlpg()
3413 cr0 = ctxt->ops->get_cr(ctxt, 0); in em_clts()
3415 ctxt->ops->set_cr(ctxt, 0, cr0); in em_clts()
3421 int rc = ctxt->ops->fix_hypercall(ctxt); in em_hypercall()
3426 /* Let the processor re-execute the fixed hypercall */ in em_hypercall()
3427 ctxt->_eip = ctxt->eip; in em_hypercall()
3429 ctxt->dst.type = OP_NONE; in em_hypercall()
3439 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) && in emulate_store_desc_ptr()
3440 ctxt->ops->cpl(ctxt) > 0) in emulate_store_desc_ptr()
3443 if (ctxt->mode == X86EMUL_MODE_PROT64) in emulate_store_desc_ptr()
3444 ctxt->op_bytes = 8; in emulate_store_desc_ptr()
3446 if (ctxt->op_bytes == 2) { in emulate_store_desc_ptr()
3447 ctxt->op_bytes = 4; in emulate_store_desc_ptr()
3451 ctxt->dst.type = OP_NONE; in emulate_store_desc_ptr()
3452 return segmented_write_std(ctxt, ctxt->dst.addr.mem, in emulate_store_desc_ptr()
3453 &desc_ptr, 2 + ctxt->op_bytes); in emulate_store_desc_ptr()
3458 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt); in em_sgdt()
3463 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt); in em_sidt()
3471 if (ctxt->mode == X86EMUL_MODE_PROT64) in em_lgdt_lidt()
3472 ctxt->op_bytes = 8; in em_lgdt_lidt()
3473 rc = read_descriptor(ctxt, ctxt->src.addr.mem, in em_lgdt_lidt()
3475 ctxt->op_bytes); in em_lgdt_lidt()
3478 if (ctxt->mode == X86EMUL_MODE_PROT64 && in em_lgdt_lidt()
3483 ctxt->ops->set_gdt(ctxt, &desc_ptr); in em_lgdt_lidt()
3485 ctxt->ops->set_idt(ctxt, &desc_ptr); in em_lgdt_lidt()
3487 ctxt->dst.type = OP_NONE; in em_lgdt_lidt()
3503 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) && in em_smsw()
3504 ctxt->ops->cpl(ctxt) > 0) in em_smsw()
3507 if (ctxt->dst.type == OP_MEM) in em_smsw()
3508 ctxt->dst.bytes = 2; in em_smsw()
3509 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0); in em_smsw()
3515 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul) in em_lmsw()
3516 | (ctxt->src.val & 0x0f)); in em_lmsw()
3517 ctxt->dst.type = OP_NONE; in em_lmsw()
3525 register_address_increment(ctxt, VCPU_REGS_RCX, -1); in em_loop()
3527 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags))) in em_loop()
3528 rc = jmp_rel(ctxt, ctxt->src.val); in em_loop()
3538 rc = jmp_rel(ctxt, ctxt->src.val); in em_jcxz()
3545 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val, in em_in()
3546 &ctxt->dst.val)) in em_in()
3554 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val, in em_out()
3555 &ctxt->src.val, 1); in em_out()
3557 ctxt->dst.type = OP_NONE; in em_out()
3566 ctxt->eflags &= ~X86_EFLAGS_IF; in em_cli()
3575 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI; in em_sti()
3576 ctxt->eflags |= X86_EFLAGS_IF; in em_sti()
3585 ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr); in em_cpuid()
3587 ctxt->ops->cpl(ctxt)) { in em_cpuid()
3593 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false); in em_cpuid()
3609 ctxt->eflags &= ~0xffUL; in em_sahf()
3610 ctxt->eflags |= flags | X86_EFLAGS_FIXED; in em_sahf()
3617 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8; in em_lahf()
3623 switch (ctxt->op_bytes) { in em_bswap()
3626 asm("bswap %0" : "+r"(ctxt->dst.val)); in em_bswap()
3630 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val)); in em_bswap()
3650 ctxt->dst.val = (s32) ctxt->src.val; in em_movsxd()
3656 if (!ctxt->ops->guest_has_fxsr(ctxt)) in check_fxsr()
3659 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM)) in check_fxsr()
3666 if (ctxt->mode >= X86EMUL_MODE_PROT64) in check_fxsr()
3673 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but does save
3684 if (ctxt->mode == X86EMUL_MODE_PROT64) in fxstate_size()
3687 cr4_osfxsr = ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR; in fxstate_size()
3695 * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs
3698 * 3) 64-bit mode with REX.W prefix
3699 * - like (2), but XMM 8-15 are being saved and restored
3700 * 4) 64-bit mode without REX.W prefix
3701 * - like (3), but FIP and FDP are 64 bit
3703 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
3727 return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state, in em_fxsave()
3734 * (preemption has to stay disabled until FXRSTOR).
3746 __fxstate_size(16) - used_size); in fxregs_fixup()
3762 rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, size); in em_fxrstor()
3792 if (!(ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSXSAVE)) in em_xsetbv()
3799 if (ctxt->ops->set_xcr(ctxt, ecx, ((u64)edx << 32) | eax)) in em_xsetbv()
3819 if (!valid_cr(ctxt->modrm_reg)) in check_cr_access()
3827 int dr = ctxt->modrm_reg; in check_dr_read()
3833 cr4 = ctxt->ops->get_cr(ctxt, 4); in check_dr_read()
3837 if (ctxt->ops->get_dr(ctxt, 7) & DR7_GD) { in check_dr_read()
3840 dr6 = ctxt->ops->get_dr(ctxt, 6); in check_dr_read()
3843 ctxt->ops->set_dr(ctxt, 6, dr6); in check_dr_read()
3852 u64 new_val = ctxt->src.val64; in check_dr_write()
3853 int dr = ctxt->modrm_reg; in check_dr_write()
3865 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer); in check_svme()
3886 u64 cr4 = ctxt->ops->get_cr(ctxt, 4); in check_rdtsc()
3888 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt)) in check_rdtsc()
3896 u64 cr4 = ctxt->ops->get_cr(ctxt, 4); in check_rdpmc()
3900 * VMware allows access to these Pseduo-PMCs even when read via RDPMC in check_rdpmc()
3911 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) || in check_rdpmc()
3912 ctxt->ops->check_rdpmc_early(ctxt, rcx)) in check_rdpmc()
3920 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u); in check_perm_in()
3921 if (!emulator_io_permitted(ctxt, ctxt->src.val, ctxt->dst.bytes)) in check_perm_in()
3929 ctxt->src.bytes = min(ctxt->src.bytes, 4u); in check_perm_out()
3930 if (!emulator_io_permitted(ctxt, ctxt->dst.val, ctxt->src.bytes)) in check_perm_out()
4150 /* 0xC0 - 0xC7 */
4152 /* 0xC8 - 0xCF */
4154 /* 0xD0 - 0xC7 */
4156 /* 0xD8 - 0xDF */
4158 /* 0xE0 - 0xE7 */
4160 /* 0xE8 - 0xEF */
4162 /* 0xF0 - 0xF7 */
4164 /* 0xF8 - 0xFF */
4171 /* 0xC0 - 0xC7 */
4173 /* 0xC8 - 0xCF */
4175 /* 0xD0 - 0xC7 */
4177 /* 0xD8 - 0xDF */
4179 /* 0xE0 - 0xE7 */
4181 /* 0xE8 - 0xEF */
4183 /* 0xF0 - 0xF7 */
4185 /* 0xF8 - 0xFF */
4192 /* 0xC0 - 0xC7 */
4194 /* 0xC8 - 0xCF */
4196 /* 0xD0 - 0xC7 */
4198 /* 0xD8 - 0xDF */
4200 /* 0xE0 - 0xE7 */
4202 /* 0xE8 - 0xEF */
4204 /* 0xF0 - 0xF7 */
4206 /* 0xF8 - 0xFF */
4223 /* 0x00 - 0x07 */
4227 /* 0x08 - 0x0F */
4231 /* 0x10 - 0x17 */
4235 /* 0x18 - 0x1F */
4239 /* 0x20 - 0x27 */
4241 /* 0x28 - 0x2F */
4243 /* 0x30 - 0x37 */
4245 /* 0x38 - 0x3F */
4247 /* 0x40 - 0x4F */
4249 /* 0x50 - 0x57 */
4251 /* 0x58 - 0x5F */
4253 /* 0x60 - 0x67 */
4258 /* 0x68 - 0x6F */
4265 /* 0x70 - 0x7F */
4267 /* 0x80 - 0x87 */
4274 /* 0x88 - 0x8F */
4281 /* 0x90 - 0x97 */
4283 /* 0x98 - 0x9F */
4289 /* 0xA0 - 0xA7 */
4294 /* 0xA8 - 0xAF */
4299 /* 0xB0 - 0xB7 */
4301 /* 0xB8 - 0xBF */
4303 /* 0xC0 - 0xC7 */
4310 /* 0xC8 - 0xCF */
4318 /* 0xD0 - 0xD7 */
4325 /* 0xD8 - 0xDF */
4327 /* 0xE0 - 0xE7 */
4332 /* 0xE8 - 0xEF */
4339 /* 0xF0 - 0xF7 */
4343 /* 0xF8 - 0xFF */
4350 /* 0x00 - 0x0F */
4356 /* 0x10 - 0x1F */
4366 /* 0x20 - 0x2F */
4378 /* 0x30 - 0x3F */
4387 /* 0x40 - 0x4F */
4389 /* 0x50 - 0x5F */
4391 /* 0x60 - 0x6F */
4396 /* 0x70 - 0x7F */
4401 /* 0x80 - 0x8F */
4403 /* 0x90 - 0x9F */
4405 /* 0xA0 - 0xA7 */
4411 /* 0xA8 - 0xAF */
4418 /* 0xB0 - 0xB7 */
4425 /* 0xB8 - 0xBF */
4432 /* 0xC0 - 0xC7 */
4436 /* 0xC8 - 0xCF */
4438 /* 0xD0 - 0xDF */
4440 /* 0xE0 - 0xEF */
4443 /* 0xF0 - 0xFF */
4468 /* 0x00 - 0x1f */
4470 /* 0x20 - 0x2f */
4473 /* 0x30 - 0x7f */
4475 /* 0x80 - 0xef */
4477 /* 0xf0 - 0xf1 */
4480 /* 0xf2 - 0xff */
4502 return ctxt->d & ShadowStack; in is_shstk_instruction()
4507 u64 flags = ctxt->d; in is_ibt_instruction()
4516 * state. IRET #GPs when returning to virtual-8086 and IBT or SHSTK is in is_ibt_instruction()
4521 return ctxt->execute != em_iret && in is_ibt_instruction()
4522 ctxt->execute != em_ret_far && in is_ibt_instruction()
4523 ctxt->execute != em_ret_far_imm && in is_ibt_instruction()
4524 ctxt->execute != em_sysexit; in is_ibt_instruction()
4558 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; in imm_size()
4569 op->type = OP_IMM; in decode_imm()
4570 op->bytes = size; in decode_imm()
4571 op->addr.mem.ea = ctxt->_eip; in decode_imm()
4572 /* NB. Immediates are sign-extended as necessary. */ in decode_imm()
4573 switch (op->bytes) { in decode_imm()
4575 op->val = insn_fetch(s8, ctxt); in decode_imm()
4578 op->val = insn_fetch(s16, ctxt); in decode_imm()
4581 op->val = insn_fetch(s32, ctxt); in decode_imm()
4584 op->val = insn_fetch(s64, ctxt); in decode_imm()
4588 switch (op->bytes) { in decode_imm()
4590 op->val &= 0xff; in decode_imm()
4593 op->val &= 0xffff; in decode_imm()
4596 op->val &= 0xffffffff; in decode_imm()
4617 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; in decode_operand()
4619 *op = ctxt->memop; in decode_operand()
4620 ctxt->memopp = op; in decode_operand()
4621 if (ctxt->d & BitOp) in decode_operand()
4623 op->orig_val = op->val; in decode_operand()
4626 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8; in decode_operand()
4629 op->type = OP_REG; in decode_operand()
4630 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; in decode_operand()
4631 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX); in decode_operand()
4635 op->type = OP_REG; in decode_operand()
4636 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes; in decode_operand()
4637 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX); in decode_operand()
4641 if (ctxt->d & ByteOp) { in decode_operand()
4642 op->type = OP_NONE; in decode_operand()
4645 op->type = OP_REG; in decode_operand()
4646 op->bytes = ctxt->op_bytes; in decode_operand()
4647 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX); in decode_operand()
4651 op->type = OP_MEM; in decode_operand()
4652 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; in decode_operand()
4653 op->addr.mem.ea = in decode_operand()
4655 op->addr.mem.seg = VCPU_SREG_ES; in decode_operand()
4656 op->val = 0; in decode_operand()
4657 op->count = 1; in decode_operand()
4660 op->type = OP_REG; in decode_operand()
4661 op->bytes = 2; in decode_operand()
4662 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX); in decode_operand()
4666 op->type = OP_IMM; in decode_operand()
4667 op->bytes = 1; in decode_operand()
4668 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff; in decode_operand()
4674 op->type = OP_IMM; in decode_operand()
4675 op->bytes = 1; in decode_operand()
4676 op->val = 1; in decode_operand()
4682 rc = decode_imm(ctxt, op, ctxt->op_bytes, true); in decode_operand()
4685 ctxt->memop.bytes = 1; in decode_operand()
4686 if (ctxt->memop.type == OP_REG) { in decode_operand()
4687 ctxt->memop.addr.reg = decode_register(ctxt, in decode_operand()
4688 ctxt->modrm_rm, true); in decode_operand()
4689 fetch_register_operand(&ctxt->memop); in decode_operand()
4693 ctxt->memop.bytes = 2; in decode_operand()
4696 ctxt->memop.bytes = 4; in decode_operand()
4705 op->type = OP_MEM; in decode_operand()
4706 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; in decode_operand()
4707 op->addr.mem.ea = in decode_operand()
4709 op->addr.mem.seg = ctxt->seg_override; in decode_operand()
4710 op->val = 0; in decode_operand()
4711 op->count = 1; in decode_operand()
4714 op->type = OP_MEM; in decode_operand()
4715 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; in decode_operand()
4716 op->addr.mem.ea = in decode_operand()
4720 op->addr.mem.seg = ctxt->seg_override; in decode_operand()
4721 op->val = 0; in decode_operand()
4724 op->type = OP_IMM; in decode_operand()
4725 op->addr.mem.ea = ctxt->_eip; in decode_operand()
4726 op->bytes = ctxt->op_bytes + 2; in decode_operand()
4727 insn_fetch_arr(op->valptr, op->bytes, ctxt); in decode_operand()
4730 ctxt->memop.bytes = ctxt->op_bytes + 2; in decode_operand()
4733 op->type = OP_IMM; in decode_operand()
4734 op->val = VCPU_SREG_ES; in decode_operand()
4737 op->type = OP_IMM; in decode_operand()
4738 op->val = VCPU_SREG_CS; in decode_operand()
4741 op->type = OP_IMM; in decode_operand()
4742 op->val = VCPU_SREG_SS; in decode_operand()
4745 op->type = OP_IMM; in decode_operand()
4746 op->val = VCPU_SREG_DS; in decode_operand()
4749 op->type = OP_IMM; in decode_operand()
4750 op->val = VCPU_SREG_FS; in decode_operand()
4753 op->type = OP_IMM; in decode_operand()
4754 op->val = VCPU_SREG_GS; in decode_operand()
4759 op->type = OP_NONE; /* Disable writeback. */ in decode_operand()
4773 if (ctxt->rep_prefix || ctxt->op_prefix || ctxt->rex_prefix) in x86_decode_avx()
4788 ctxt->rex_prefix = REX_PREFIX; in x86_decode_avx()
4789 ctxt->rex_bits = (vex_2nd & 0xE0) >> 5; /* RXB */ in x86_decode_avx()
4790 ctxt->rex_bits |= (vex_3rd & 0x80) >> 4; /* w */ in x86_decode_avx()
4791 if (ctxt->rex_bits && ctxt->mode != X86EMUL_MODE_PROT64) in x86_decode_avx()
4799 ctxt->b = insn_fetch(u8, ctxt); in x86_decode_avx()
4802 ctxt->opcode_len = 2; in x86_decode_avx()
4803 *opcode = twobyte_table[ctxt->b]; in x86_decode_avx()
4806 ctxt->opcode_len = 3; in x86_decode_avx()
4807 *opcode = opcode_map_0f_38[ctxt->b]; in x86_decode_avx()
4824 ctxt->op_bytes = 32; in x86_decode_avx()
4826 ctxt->op_bytes = 16; in x86_decode_avx()
4830 case 1: ctxt->op_prefix = true; break; in x86_decode_avx()
4831 case 2: ctxt->rep_prefix = 0xf3; break; in x86_decode_avx()
4832 case 3: ctxt->rep_prefix = 0xf2; break; in x86_decode_avx()
4845 int mode = ctxt->mode; in x86_decode_insn()
4853 ctxt->memop.type = OP_NONE; in x86_decode_insn()
4854 ctxt->memopp = NULL; in x86_decode_insn()
4855 ctxt->_eip = ctxt->eip; in x86_decode_insn()
4856 ctxt->fetch.ptr = ctxt->fetch.data; in x86_decode_insn()
4857 ctxt->fetch.end = ctxt->fetch.data + insn_len; in x86_decode_insn()
4858 ctxt->opcode_len = 1; in x86_decode_insn()
4859 ctxt->intercept = x86_intercept_none; in x86_decode_insn()
4861 memcpy(ctxt->fetch.data, insn, insn_len); in x86_decode_insn()
4872 ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS); in x86_decode_insn()
4892 ctxt->op_bytes = def_op_bytes; in x86_decode_insn()
4893 ctxt->ad_bytes = def_ad_bytes; in x86_decode_insn()
4897 switch (ctxt->b = insn_fetch(u8, ctxt)) { in x86_decode_insn()
4898 case 0x66: /* operand-size override */ in x86_decode_insn()
4899 ctxt->op_prefix = true; in x86_decode_insn()
4901 ctxt->op_bytes = def_op_bytes ^ 6; in x86_decode_insn()
4903 case 0x67: /* address-size override */ in x86_decode_insn()
4906 ctxt->ad_bytes = def_ad_bytes ^ 12; in x86_decode_insn()
4909 ctxt->ad_bytes = def_ad_bytes ^ 6; in x86_decode_insn()
4913 ctxt->seg_override = VCPU_SREG_ES; in x86_decode_insn()
4917 ctxt->seg_override = VCPU_SREG_CS; in x86_decode_insn()
4921 ctxt->seg_override = VCPU_SREG_SS; in x86_decode_insn()
4925 ctxt->seg_override = VCPU_SREG_DS; in x86_decode_insn()
4929 ctxt->seg_override = VCPU_SREG_FS; in x86_decode_insn()
4933 ctxt->seg_override = VCPU_SREG_GS; in x86_decode_insn()
4938 ctxt->rex_prefix = REX_PREFIX; in x86_decode_insn()
4939 ctxt->rex_bits = ctxt->b & 0xf; in x86_decode_insn()
4942 ctxt->lock_prefix = 1; in x86_decode_insn()
4946 ctxt->rep_prefix = ctxt->b; in x86_decode_insn()
4953 ctxt->rex_prefix = REX_NONE; in x86_decode_insn()
4954 ctxt->rex_bits = 0; in x86_decode_insn()
4960 if (ctxt->rex_bits & REX_W) in x86_decode_insn()
4961 ctxt->op_bytes = 8; in x86_decode_insn()
4964 if (ctxt->b == 0xc4 || ctxt->b == 0xc5) { in x86_decode_insn()
4968 opcode = opcode_table[ctxt->b]; in x86_decode_insn()
4969 ctxt->modrm = vex_2nd; in x86_decode_insn()
4975 rc = x86_decode_avx(ctxt, ctxt->b, vex_2nd, &opcode); in x86_decode_insn()
4978 } else if (ctxt->b == 0x0f) { in x86_decode_insn()
4979 /* Two- or three-byte opcode */ in x86_decode_insn()
4980 ctxt->opcode_len = 2; in x86_decode_insn()
4981 ctxt->b = insn_fetch(u8, ctxt); in x86_decode_insn()
4982 opcode = twobyte_table[ctxt->b]; in x86_decode_insn()
4985 if (ctxt->b == 0x38) { in x86_decode_insn()
4986 ctxt->opcode_len = 3; in x86_decode_insn()
4987 ctxt->b = insn_fetch(u8, ctxt); in x86_decode_insn()
4988 opcode = opcode_map_0f_38[ctxt->b]; in x86_decode_insn()
4992 opcode = opcode_table[ctxt->b]; in x86_decode_insn()
4996 ctxt->modrm = insn_fetch(u8, ctxt); in x86_decode_insn()
4999 ctxt->d = opcode.flags; in x86_decode_insn()
5000 while (ctxt->d & GroupMask) { in x86_decode_insn()
5001 switch (ctxt->d & GroupMask) { in x86_decode_insn()
5003 goffset = (ctxt->modrm >> 3) & 7; in x86_decode_insn()
5007 goffset = (ctxt->modrm >> 3) & 7; in x86_decode_insn()
5008 if ((ctxt->modrm >> 6) == 3) in x86_decode_insn()
5009 opcode = opcode.u.gdual->mod3[goffset]; in x86_decode_insn()
5011 opcode = opcode.u.gdual->mod012[goffset]; in x86_decode_insn()
5014 goffset = ctxt->modrm & 7; in x86_decode_insn()
5018 if (ctxt->rep_prefix && ctxt->op_prefix) in x86_decode_insn()
5020 simd_prefix = ctxt->op_prefix ? 0x66 : ctxt->rep_prefix; in x86_decode_insn()
5022 case 0x00: opcode = opcode.u.gprefix->pfx_no; break; in x86_decode_insn()
5023 case 0x66: opcode = opcode.u.gprefix->pfx_66; break; in x86_decode_insn()
5024 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break; in x86_decode_insn()
5025 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break; in x86_decode_insn()
5029 if (ctxt->modrm > 0xbf) { in x86_decode_insn()
5030 size_t size = ARRAY_SIZE(opcode.u.esc->high); in x86_decode_insn()
5032 ctxt->modrm - 0xc0, size); in x86_decode_insn()
5034 opcode = opcode.u.esc->high[index]; in x86_decode_insn()
5036 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7]; in x86_decode_insn()
5040 if ((ctxt->modrm >> 6) == 3) in x86_decode_insn()
5041 opcode = opcode.u.idual->mod3; in x86_decode_insn()
5043 opcode = opcode.u.idual->mod012; in x86_decode_insn()
5046 if (ctxt->mode == X86EMUL_MODE_PROT64) in x86_decode_insn()
5047 opcode = opcode.u.mdual->mode64; in x86_decode_insn()
5049 opcode = opcode.u.mdual->mode32; in x86_decode_insn()
5055 ctxt->d &= ~(u64)GroupMask; in x86_decode_insn()
5056 ctxt->d |= opcode.flags; in x86_decode_insn()
5059 ctxt->is_branch = opcode.flags & IsBranch; in x86_decode_insn()
5062 if (ctxt->d == 0) in x86_decode_insn()
5071 if (!(ctxt->d & Avx)) in x86_decode_insn()
5074 if (!(ctxt->d & AlignMask)) in x86_decode_insn()
5075 ctxt->d |= Unaligned; in x86_decode_insn()
5078 ctxt->execute = opcode.u.execute; in x86_decode_insn()
5086 ctxt->ops->get_cr(ctxt, 4) & X86_CR4_CET) { in x86_decode_insn()
5090 * Check both User and Supervisor on far transfers as inter- in x86_decode_insn()
5094 * any CET-affected instructions at any privilege level. in x86_decode_insn()
5096 if (!(ctxt->d & NearBranch)) in x86_decode_insn()
5098 else if (ctxt->ops->cpl(ctxt) == 3) in x86_decode_insn()
5103 if ((u_cet && ctxt->ops->get_msr(ctxt, MSR_IA32_U_CET, &u_cet)) || in x86_decode_insn()
5104 (s_cet && ctxt->ops->get_msr(ctxt, MSR_IA32_S_CET, &s_cet))) in x86_decode_insn()
5115 likely(!(ctxt->d & EmulateOnUD))) in x86_decode_insn()
5118 if (unlikely(ctxt->d & in x86_decode_insn()
5125 ctxt->check_perm = opcode.check_perm; in x86_decode_insn()
5126 ctxt->intercept = opcode.intercept; in x86_decode_insn()
5128 if (ctxt->d & NotImpl) in x86_decode_insn()
5132 if (ctxt->op_bytes == 4 && (ctxt->d & Stack)) in x86_decode_insn()
5133 ctxt->op_bytes = 8; in x86_decode_insn()
5134 else if (ctxt->d & NearBranch) in x86_decode_insn()
5135 ctxt->op_bytes = 8; in x86_decode_insn()
5138 if (ctxt->d & Op3264) { in x86_decode_insn()
5140 ctxt->op_bytes = 8; in x86_decode_insn()
5142 ctxt->op_bytes = 4; in x86_decode_insn()
5145 if ((ctxt->d & No16) && ctxt->op_bytes == 2) in x86_decode_insn()
5146 ctxt->op_bytes = 4; in x86_decode_insn()
5150 else if (ctxt->d & Sse) in x86_decode_insn()
5151 ctxt->op_bytes = 16, ctxt->d &= ~Avx; in x86_decode_insn()
5152 else if (ctxt->d & Mmx) in x86_decode_insn()
5153 ctxt->op_bytes = 8; in x86_decode_insn()
5157 if (ctxt->d & ModRM) { in x86_decode_insn()
5158 rc = decode_modrm(ctxt, &ctxt->memop); in x86_decode_insn()
5161 ctxt->seg_override = ctxt->modrm_seg; in x86_decode_insn()
5163 } else if (ctxt->d & MemAbs) in x86_decode_insn()
5164 rc = decode_abs(ctxt, &ctxt->memop); in x86_decode_insn()
5169 ctxt->seg_override = VCPU_SREG_DS; in x86_decode_insn()
5171 ctxt->memop.addr.mem.seg = ctxt->seg_override; in x86_decode_insn()
5177 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask); in x86_decode_insn()
5185 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask); in x86_decode_insn()
5190 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask); in x86_decode_insn()
5192 if (ctxt->rip_relative && likely(ctxt->memopp)) in x86_decode_insn()
5193 ctxt->memopp->addr.mem.ea = address_mask(ctxt, in x86_decode_insn()
5194 ctxt->memopp->addr.mem.ea + ctxt->_eip); in x86_decode_insn()
5198 ctxt->have_exception = true; in x86_decode_insn()
5204 return ctxt->d & PageTable; in x86_page_table_writing_insn()
5213 * - if REPE/REPZ and ZF = 0 then done in string_insn_completed()
5214 * - if REPNE/REPNZ and ZF = 1 then done in string_insn_completed()
5216 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) || in string_insn_completed()
5217 (ctxt->b == 0xae) || (ctxt->b == 0xaf)) in string_insn_completed()
5218 && (((ctxt->rep_prefix == REPE_PREFIX) && in string_insn_completed()
5219 ((ctxt->eflags & X86_EFLAGS_ZF) == 0)) in string_insn_completed()
5220 || ((ctxt->rep_prefix == REPNE_PREFIX) && in string_insn_completed()
5221 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF)))) in string_insn_completed()
5243 if (op->type == OP_MM) in fetch_possible_mmx_operand()
5244 kvm_read_mmx_reg(op->addr.mm, &op->mm_val); in fetch_possible_mmx_operand()
5250 ctxt->rip_relative = false; in init_decode_cache()
5251 ctxt->rex_prefix = REX_NONE; in init_decode_cache()
5252 ctxt->rex_bits = 0; in init_decode_cache()
5253 ctxt->lock_prefix = 0; in init_decode_cache()
5254 ctxt->op_prefix = false; in init_decode_cache()
5255 ctxt->rep_prefix = 0; in init_decode_cache()
5256 ctxt->regs_valid = 0; in init_decode_cache()
5257 ctxt->regs_dirty = 0; in init_decode_cache()
5259 ctxt->io_read.pos = 0; in init_decode_cache()
5260 ctxt->io_read.end = 0; in init_decode_cache()
5261 ctxt->mem_read.end = 0; in init_decode_cache()
5266 const struct x86_emulate_ops *ops = ctxt->ops; in x86_emulate_insn()
5268 int saved_dst_type = ctxt->dst.type; in x86_emulate_insn()
5270 ctxt->mem_read.pos = 0; in x86_emulate_insn()
5273 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) { in x86_emulate_insn()
5278 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) { in x86_emulate_insn()
5283 if (unlikely(ctxt->d & in x86_emulate_insn()
5285 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) || in x86_emulate_insn()
5286 (ctxt->d & Undefined)) { in x86_emulate_insn()
5291 if ((ctxt->d & (Avx|Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM))) { in x86_emulate_insn()
5296 if (ctxt->d & Avx) { in x86_emulate_insn()
5298 if (!(ops->get_cr(ctxt, 4) & X86_CR4_OSXSAVE) in x86_emulate_insn()
5299 || ops->get_xcr(ctxt, 0, &xcr) in x86_emulate_insn()
5304 } else if (ctxt->d & Sse) { in x86_emulate_insn()
5305 if (!(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR)) { in x86_emulate_insn()
5311 if ((ctxt->d & (Avx|Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) { in x86_emulate_insn()
5316 if (ctxt->d & Mmx) { in x86_emulate_insn()
5324 fetch_possible_mmx_operand(&ctxt->src); in x86_emulate_insn()
5325 fetch_possible_mmx_operand(&ctxt->src2); in x86_emulate_insn()
5326 if (!(ctxt->d & Mov)) in x86_emulate_insn()
5327 fetch_possible_mmx_operand(&ctxt->dst); in x86_emulate_insn()
5330 if (unlikely(check_intercepts) && ctxt->intercept) { in x86_emulate_insn()
5331 rc = emulator_check_intercept(ctxt, ctxt->intercept, in x86_emulate_insn()
5338 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) { in x86_emulate_insn()
5344 if ((ctxt->d & Priv) && ops->cpl(ctxt)) { in x86_emulate_insn()
5345 if (ctxt->d & PrivUD) in x86_emulate_insn()
5353 if (ctxt->d & CheckPerm) { in x86_emulate_insn()
5354 rc = ctxt->check_perm(ctxt); in x86_emulate_insn()
5359 if (unlikely(check_intercepts) && (ctxt->d & Intercept)) { in x86_emulate_insn()
5360 rc = emulator_check_intercept(ctxt, ctxt->intercept, in x86_emulate_insn()
5366 if (ctxt->rep_prefix && (ctxt->d & String)) { in x86_emulate_insn()
5370 ctxt->eip = ctxt->_eip; in x86_emulate_insn()
5371 ctxt->eflags &= ~X86_EFLAGS_RF; in x86_emulate_insn()
5377 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) { in x86_emulate_insn()
5378 rc = segmented_read(ctxt, ctxt->src.addr.mem, in x86_emulate_insn()
5379 ctxt->src.valptr, ctxt->src.bytes); in x86_emulate_insn()
5382 ctxt->src.orig_val64 = ctxt->src.val64; in x86_emulate_insn()
5385 if (ctxt->src2.type == OP_MEM) { in x86_emulate_insn()
5386 rc = segmented_read(ctxt, ctxt->src2.addr.mem, in x86_emulate_insn()
5387 &ctxt->src2.val, ctxt->src2.bytes); in x86_emulate_insn()
5392 if ((ctxt->d & DstMask) == ImplicitOps) in x86_emulate_insn()
5396 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) { in x86_emulate_insn()
5397 /* optimisation - avoid slow emulated read if Mov */ in x86_emulate_insn()
5398 rc = segmented_read(ctxt, ctxt->dst.addr.mem, in x86_emulate_insn()
5399 &ctxt->dst.val, ctxt->dst.bytes); in x86_emulate_insn()
5401 if (!(ctxt->d & NoWrite) && in x86_emulate_insn()
5403 ctxt->exception.vector == PF_VECTOR) in x86_emulate_insn()
5404 ctxt->exception.error_code |= PFERR_WRITE_MASK; in x86_emulate_insn()
5408 /* Copy full 64-bit value for CMPXCHG8B. */ in x86_emulate_insn()
5409 ctxt->dst.orig_val64 = ctxt->dst.val64; in x86_emulate_insn()
5413 if (unlikely(check_intercepts) && (ctxt->d & Intercept)) { in x86_emulate_insn()
5414 rc = emulator_check_intercept(ctxt, ctxt->intercept, in x86_emulate_insn()
5420 if (ctxt->rep_prefix && (ctxt->d & String)) in x86_emulate_insn()
5421 ctxt->eflags |= X86_EFLAGS_RF; in x86_emulate_insn()
5423 ctxt->eflags &= ~X86_EFLAGS_RF; in x86_emulate_insn()
5425 if (ctxt->execute) { in x86_emulate_insn()
5426 rc = ctxt->execute(ctxt); in x86_emulate_insn()
5432 if (ctxt->opcode_len == 2) in x86_emulate_insn()
5434 else if (ctxt->opcode_len == 3) in x86_emulate_insn()
5437 switch (ctxt->b) { in x86_emulate_insn()
5439 if (test_cc(ctxt->b, ctxt->eflags)) in x86_emulate_insn()
5440 rc = jmp_rel(ctxt, ctxt->src.val); in x86_emulate_insn()
5443 ctxt->dst.val = ctxt->src.addr.mem.ea; in x86_emulate_insn()
5446 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX)) in x86_emulate_insn()
5447 ctxt->dst.type = OP_NONE; in x86_emulate_insn()
5452 switch (ctxt->op_bytes) { in x86_emulate_insn()
5453 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break; in x86_emulate_insn()
5454 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break; in x86_emulate_insn()
5455 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break; in x86_emulate_insn()
5462 rc = emulate_int(ctxt, ctxt->src.val); in x86_emulate_insn()
5465 if (ctxt->eflags & X86_EFLAGS_OF) in x86_emulate_insn()
5470 rc = jmp_rel(ctxt, ctxt->src.val); in x86_emulate_insn()
5471 ctxt->dst.type = OP_NONE; /* Disable writeback. */ in x86_emulate_insn()
5474 ctxt->ops->halt(ctxt); in x86_emulate_insn()
5478 ctxt->eflags ^= X86_EFLAGS_CF; in x86_emulate_insn()
5481 ctxt->eflags &= ~X86_EFLAGS_CF; in x86_emulate_insn()
5484 ctxt->eflags |= X86_EFLAGS_CF; in x86_emulate_insn()
5487 ctxt->eflags &= ~X86_EFLAGS_DF; in x86_emulate_insn()
5490 ctxt->eflags |= X86_EFLAGS_DF; in x86_emulate_insn()
5500 if (ctxt->d & SrcWrite) { in x86_emulate_insn()
5501 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR); in x86_emulate_insn()
5502 rc = writeback(ctxt, &ctxt->src); in x86_emulate_insn()
5506 if (!(ctxt->d & NoWrite)) { in x86_emulate_insn()
5507 rc = writeback(ctxt, &ctxt->dst); in x86_emulate_insn()
5516 ctxt->dst.type = saved_dst_type; in x86_emulate_insn()
5518 if ((ctxt->d & SrcMask) == SrcSI) in x86_emulate_insn()
5519 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src); in x86_emulate_insn()
5521 if ((ctxt->d & DstMask) == DstDI) in x86_emulate_insn()
5522 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst); in x86_emulate_insn()
5524 if (ctxt->rep_prefix && (ctxt->d & String)) { in x86_emulate_insn()
5526 struct read_cache *r = &ctxt->io_read; in x86_emulate_insn()
5527 if ((ctxt->d & SrcMask) == SrcSI) in x86_emulate_insn()
5528 count = ctxt->src.count; in x86_emulate_insn()
5530 count = ctxt->dst.count; in x86_emulate_insn()
5531 register_address_increment(ctxt, VCPU_REGS_RCX, -count); in x86_emulate_insn()
5535 * Re-enter guest when pio read ahead buffer is empty in x86_emulate_insn()
5538 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) && in x86_emulate_insn()
5539 (r->end == 0 || r->end != r->pos)) { in x86_emulate_insn()
5545 ctxt->mem_read.end = 0; in x86_emulate_insn()
5551 ctxt->eflags &= ~X86_EFLAGS_RF; in x86_emulate_insn()
5554 ctxt->eip = ctxt->_eip; in x86_emulate_insn()
5555 if (ctxt->mode != X86EMUL_MODE_PROT64) in x86_emulate_insn()
5556 ctxt->eip = (u32)ctxt->_eip; in x86_emulate_insn()
5560 if (KVM_EMULATOR_BUG_ON(ctxt->exception.vector > 0x1f, ctxt)) in x86_emulate_insn()
5562 ctxt->have_exception = true; in x86_emulate_insn()
5573 switch (ctxt->b) { in x86_emulate_insn()
5575 (ctxt->ops->wbinvd)(ctxt); in x86_emulate_insn()
5583 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg); in x86_emulate_insn()
5586 ctxt->dst.val = ops->get_dr(ctxt, ctxt->modrm_reg); in x86_emulate_insn()
5589 if (test_cc(ctxt->b, ctxt->eflags)) in x86_emulate_insn()
5590 ctxt->dst.val = ctxt->src.val; in x86_emulate_insn()
5591 else if (ctxt->op_bytes != 4) in x86_emulate_insn()
5592 ctxt->dst.type = OP_NONE; /* no writeback */ in x86_emulate_insn()
5595 if (test_cc(ctxt->b, ctxt->eflags)) in x86_emulate_insn()
5596 rc = jmp_rel(ctxt, ctxt->src.val); in x86_emulate_insn()
5599 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags); in x86_emulate_insn()
5602 ctxt->dst.bytes = ctxt->op_bytes; in x86_emulate_insn()
5603 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val in x86_emulate_insn()
5604 : (u16) ctxt->src.val; in x86_emulate_insn()
5607 ctxt->dst.bytes = ctxt->op_bytes; in x86_emulate_insn()
5608 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val : in x86_emulate_insn()
5609 (s16) ctxt->src.val; in x86_emulate_insn()
5638 if (ctxt->rep_prefix && (ctxt->d & String)) in emulator_can_use_gpa()
5641 if (ctxt->d & TwoMemOp) in emulator_can_use_gpa()