1 // SPDX-License-Identifier: GPL-2.0-only
2 /******************************************************************************
5 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
7 * Copyright (c) 2005 Keir Fraser
9 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
10 * privileged instructions:
12 * Copyright (C) 2006 Qumranet
13 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
15 * Avi Kivity <avi@qumranet.com>
16 * Yaniv Kamay <yaniv@qumranet.com>
18 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
21 #include <linux/kvm_host.h>
22 #include "kvm_cache_regs.h"
23 #include <asm/kvm_emulate.h>
24 #include <linux/stringify.h>
25 #include <asm/debugreg.h>
26 #include <asm/nospec-branch.h>
37 #define OpImplicit 1ull /* No generic decode */
38 #define OpReg 2ull /* Register */
39 #define OpMem 3ull /* Memory */
40 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
41 #define OpDI 5ull /* ES:DI/EDI/RDI */
42 #define OpMem64 6ull /* Memory, 64-bit */
43 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
44 #define OpDX 8ull /* DX register */
45 #define OpCL 9ull /* CL register (for shifts) */
46 #define OpImmByte 10ull /* 8-bit sign extended immediate */
47 #define OpOne 11ull /* Implied 1 */
48 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
49 #define OpMem16 13ull /* Memory operand (16-bit). */
50 #define OpMem32 14ull /* Memory operand (32-bit). */
51 #define OpImmU 15ull /* Immediate operand, zero extended */
52 #define OpSI 16ull /* SI/ESI/RSI */
53 #define OpImmFAddr 17ull /* Immediate far address */
54 #define OpMemFAddr 18ull /* Far address in memory */
55 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
56 #define OpES 20ull /* ES */
57 #define OpCS 21ull /* CS */
58 #define OpSS 22ull /* SS */
59 #define OpDS 23ull /* DS */
60 #define OpFS 24ull /* FS */
61 #define OpGS 25ull /* GS */
62 #define OpMem8 26ull /* 8-bit zero extended memory operand */
63 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
64 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
65 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
66 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
68 #define OpBits 5 /* Width of operand field */
69 #define OpMask ((1ull << OpBits) - 1)
72 * Opcode effective-address decode tables.
73 * Note that we only emulate instructions that have at least one memory
74 * operand (excluding implicit stack references). We assume that stack
75 * references and instruction fetches will never occur in special memory
76 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
80 /* Operand sizes: 8-bit operands or specified/overridden size. */
81 #define ByteOp (1<<0) /* 8-bit operands. */
82 /* Destination operand type. */
84 #define ImplicitOps (OpImplicit << DstShift)
85 #define DstReg (OpReg << DstShift)
86 #define DstMem (OpMem << DstShift)
87 #define DstAcc (OpAcc << DstShift)
88 #define DstDI (OpDI << DstShift)
89 #define DstMem64 (OpMem64 << DstShift)
90 #define DstMem16 (OpMem16 << DstShift)
91 #define DstImmUByte (OpImmUByte << DstShift)
92 #define DstDX (OpDX << DstShift)
93 #define DstAccLo (OpAccLo << DstShift)
94 #define DstMask (OpMask << DstShift)
95 /* Source operand type. */
97 #define SrcNone (OpNone << SrcShift)
98 #define SrcReg (OpReg << SrcShift)
99 #define SrcMem (OpMem << SrcShift)
100 #define SrcMem16 (OpMem16 << SrcShift)
101 #define SrcMem32 (OpMem32 << SrcShift)
102 #define SrcImm (OpImm << SrcShift)
103 #define SrcImmByte (OpImmByte << SrcShift)
104 #define SrcOne (OpOne << SrcShift)
105 #define SrcImmUByte (OpImmUByte << SrcShift)
106 #define SrcImmU (OpImmU << SrcShift)
107 #define SrcSI (OpSI << SrcShift)
108 #define SrcXLat (OpXLat << SrcShift)
109 #define SrcImmFAddr (OpImmFAddr << SrcShift)
110 #define SrcMemFAddr (OpMemFAddr << SrcShift)
111 #define SrcAcc (OpAcc << SrcShift)
112 #define SrcImmU16 (OpImmU16 << SrcShift)
113 #define SrcImm64 (OpImm64 << SrcShift)
114 #define SrcDX (OpDX << SrcShift)
115 #define SrcMem8 (OpMem8 << SrcShift)
116 #define SrcAccHi (OpAccHi << SrcShift)
117 #define SrcMask (OpMask << SrcShift)
118 #define BitOp (1<<11)
119 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
120 #define String (1<<13) /* String instruction (rep capable) */
121 #define Stack (1<<14) /* Stack instruction (push/pop) */
122 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
123 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
124 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
125 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
126 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
127 #define Escape (5<<15) /* Escape to coprocessor instruction */
128 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
129 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
130 #define Sse (1<<18) /* SSE Vector instruction */
131 /* Generic ModRM decode. */
132 #define ModRM (1<<19)
133 /* Destination is only written; never read. */
136 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
137 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
138 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
139 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
140 #define Undefined (1<<25) /* No Such Instruction */
141 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
142 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
144 #define PageTable (1 << 29) /* instruction used to write page table */
145 #define NotImpl (1 << 30) /* instruction is not implemented */
146 /* Source 2 operand type */
147 #define Src2Shift (31)
148 #define Src2None (OpNone << Src2Shift)
149 #define Src2Mem (OpMem << Src2Shift)
150 #define Src2CL (OpCL << Src2Shift)
151 #define Src2ImmByte (OpImmByte << Src2Shift)
152 #define Src2One (OpOne << Src2Shift)
153 #define Src2Imm (OpImm << Src2Shift)
154 #define Src2ES (OpES << Src2Shift)
155 #define Src2CS (OpCS << Src2Shift)
156 #define Src2SS (OpSS << Src2Shift)
157 #define Src2DS (OpDS << Src2Shift)
158 #define Src2FS (OpFS << Src2Shift)
159 #define Src2GS (OpGS << Src2Shift)
160 #define Src2Mask (OpMask << Src2Shift)
161 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
162 #define AlignMask ((u64)7 << 41)
163 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
164 #define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */
165 #define Avx ((u64)3 << 41) /* Advanced Vector Extensions */
166 #define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
167 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
168 #define NoWrite ((u64)1 << 45) /* No writeback */
169 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
170 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
171 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
172 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
173 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
174 #define NearBranch ((u64)1 << 52) /* Near branches */
175 #define No16 ((u64)1 << 53) /* No 16 bit operand */
176 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
177 #define TwoMemOp ((u64)1 << 55) /* Instruction has two memory operand */
179 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
181 #define X2(x...) x, x
182 #define X3(x...) X2(x), x
183 #define X4(x...) X2(x), X2(x)
184 #define X5(x...) X4(x), x
185 #define X6(x...) X4(x), X2(x)
186 #define X7(x...) X4(x), X3(x)
187 #define X8(x...) X4(x), X4(x)
188 #define X16(x...) X8(x), X8(x)
190 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
191 #define FASTOP_SIZE 8
194 * fastop functions have a special calling convention:
199 * flags: rflags (in/out)
200 * ex: rsi (in:fastop pointer, out:zero if exception)
202 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
203 * different operand sizes can be reached by calculation, rather than a jump
204 * table (which would be bigger than the code).
206 * fastop functions are declared as taking a never-defined fastop parameter,
207 * so they can't be called from C directly.
216 int (*execute)(struct x86_emulate_ctxt *ctxt);
217 const struct opcode *group;
218 const struct group_dual *gdual;
219 const struct gprefix *gprefix;
220 const struct escape *esc;
221 const struct instr_dual *idual;
222 const struct mode_dual *mdual;
223 void (*fastop)(struct fastop *fake);
225 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
229 struct opcode mod012[8];
230 struct opcode mod3[8];
234 struct opcode pfx_no;
235 struct opcode pfx_66;
236 struct opcode pfx_f2;
237 struct opcode pfx_f3;
242 struct opcode high[64];
246 struct opcode mod012;
251 struct opcode mode32;
252 struct opcode mode64;
255 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
257 enum x86_transfer_type {
259 X86_TRANSFER_CALL_JMP,
261 X86_TRANSFER_TASK_SWITCH,
264 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
266 if (!(ctxt->regs_valid & (1 << nr))) {
267 ctxt->regs_valid |= 1 << nr;
268 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
270 return ctxt->_regs[nr];
273 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
275 ctxt->regs_valid |= 1 << nr;
276 ctxt->regs_dirty |= 1 << nr;
277 return &ctxt->_regs[nr];
280 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
283 return reg_write(ctxt, nr);
286 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
290 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
291 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
294 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
296 ctxt->regs_dirty = 0;
297 ctxt->regs_valid = 0;
301 * These EFLAGS bits are restored from saved value during emulation, and
302 * any changes are written back to the saved value after emulation.
304 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
305 X86_EFLAGS_PF|X86_EFLAGS_CF)
313 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
315 #define __FOP_FUNC(name) \
316 ".align " __stringify(FASTOP_SIZE) " \n\t" \
317 ".type " name ", @function \n\t" \
320 #define FOP_FUNC(name) \
323 #define __FOP_RET(name) \
325 ".size " name ", .-" name "\n\t"
327 #define FOP_RET(name) \
330 #define FOP_START(op) \
331 extern void em_##op(struct fastop *fake); \
332 asm(".pushsection .text, \"ax\" \n\t" \
333 ".global em_" #op " \n\t" \
334 ".align " __stringify(FASTOP_SIZE) " \n\t" \
340 #define __FOPNOP(name) \
345 __FOPNOP(__stringify(__UNIQUE_ID(nop)))
347 #define FOP1E(op, dst) \
348 __FOP_FUNC(#op "_" #dst) \
349 "10: " #op " %" #dst " \n\t" \
350 __FOP_RET(#op "_" #dst)
352 #define FOP1EEX(op, dst) \
353 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
355 #define FASTOP1(op) \
360 ON64(FOP1E(op##q, rax)) \
363 /* 1-operand, using src2 (for MUL/DIV r/m) */
364 #define FASTOP1SRC2(op, name) \
369 ON64(FOP1E(op, rcx)) \
372 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
373 #define FASTOP1SRC2EX(op, name) \
378 ON64(FOP1EEX(op, rcx)) \
381 #define FOP2E(op, dst, src) \
382 __FOP_FUNC(#op "_" #dst "_" #src) \
383 #op " %" #src ", %" #dst " \n\t" \
384 __FOP_RET(#op "_" #dst "_" #src)
386 #define FASTOP2(op) \
388 FOP2E(op##b, al, dl) \
389 FOP2E(op##w, ax, dx) \
390 FOP2E(op##l, eax, edx) \
391 ON64(FOP2E(op##q, rax, rdx)) \
394 /* 2 operand, word only */
395 #define FASTOP2W(op) \
398 FOP2E(op##w, ax, dx) \
399 FOP2E(op##l, eax, edx) \
400 ON64(FOP2E(op##q, rax, rdx)) \
403 /* 2 operand, src is CL */
404 #define FASTOP2CL(op) \
406 FOP2E(op##b, al, cl) \
407 FOP2E(op##w, ax, cl) \
408 FOP2E(op##l, eax, cl) \
409 ON64(FOP2E(op##q, rax, cl)) \
412 /* 2 operand, src and dest are reversed */
413 #define FASTOP2R(op, name) \
415 FOP2E(op##b, dl, al) \
416 FOP2E(op##w, dx, ax) \
417 FOP2E(op##l, edx, eax) \
418 ON64(FOP2E(op##q, rdx, rax)) \
421 #define FOP3E(op, dst, src, src2) \
422 __FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
423 #op " %" #src2 ", %" #src ", %" #dst " \n\t"\
424 __FOP_RET(#op "_" #dst "_" #src "_" #src2)
426 /* 3-operand, word-only, src2=cl */
427 #define FASTOP3WCL(op) \
430 FOP3E(op##w, ax, dx, cl) \
431 FOP3E(op##l, eax, edx, cl) \
432 ON64(FOP3E(op##q, rax, rdx, cl)) \
435 /* Special case for SETcc - 1 instruction per cc */
436 #define FOP_SETCC(op) \
438 ".type " #op ", @function \n\t" \
443 asm(".pushsection .fixup, \"ax\"\n"
444 ".global kvm_fastop_exception \n"
445 "kvm_fastop_exception: xor %esi, %esi; ret\n"
469 "pushf; sbb %al, %al; popf \n\t"
474 * XXX: inoutclob user must know where the argument is being expanded.
475 * Relying on CONFIG_CC_HAS_ASM_GOTO would allow us to remove _fault.
477 #define asm_safe(insn, inoutclob...) \
481 asm volatile("1:" insn "\n" \
483 ".pushsection .fixup, \"ax\"\n" \
484 "3: movl $1, %[_fault]\n" \
487 _ASM_EXTABLE(1b, 3b) \
488 : [_fault] "+qm"(_fault) inoutclob ); \
490 _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
493 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
494 enum x86_intercept intercept,
495 enum x86_intercept_stage stage)
497 struct x86_instruction_info info = {
498 .intercept = intercept,
499 .rep_prefix = ctxt->rep_prefix,
500 .modrm_mod = ctxt->modrm_mod,
501 .modrm_reg = ctxt->modrm_reg,
502 .modrm_rm = ctxt->modrm_rm,
503 .src_val = ctxt->src.val64,
504 .dst_val = ctxt->dst.val64,
505 .src_bytes = ctxt->src.bytes,
506 .dst_bytes = ctxt->dst.bytes,
507 .ad_bytes = ctxt->ad_bytes,
508 .next_rip = ctxt->eip,
511 return ctxt->ops->intercept(ctxt, &info, stage);
514 static void assign_masked(ulong *dest, ulong src, ulong mask)
516 *dest = (*dest & ~mask) | (src & mask);
519 static void assign_register(unsigned long *reg, u64 val, int bytes)
521 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
524 *(u8 *)reg = (u8)val;
527 *(u16 *)reg = (u16)val;
531 break; /* 64b: zero-extend */
538 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
540 return (1UL << (ctxt->ad_bytes << 3)) - 1;
543 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
546 struct desc_struct ss;
548 if (ctxt->mode == X86EMUL_MODE_PROT64)
550 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
551 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
554 static int stack_size(struct x86_emulate_ctxt *ctxt)
556 return (__fls(stack_mask(ctxt)) + 1) >> 3;
559 /* Access/update address held in a register, based on addressing mode. */
560 static inline unsigned long
561 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
563 if (ctxt->ad_bytes == sizeof(unsigned long))
566 return reg & ad_mask(ctxt);
569 static inline unsigned long
570 register_address(struct x86_emulate_ctxt *ctxt, int reg)
572 return address_mask(ctxt, reg_read(ctxt, reg));
575 static void masked_increment(ulong *reg, ulong mask, int inc)
577 assign_masked(reg, *reg + inc, mask);
581 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
583 ulong *preg = reg_rmw(ctxt, reg);
585 assign_register(preg, *preg + inc, ctxt->ad_bytes);
588 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
590 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
593 static u32 desc_limit_scaled(struct desc_struct *desc)
595 u32 limit = get_desc_limit(desc);
597 return desc->g ? (limit << 12) | 0xfff : limit;
600 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
602 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
605 return ctxt->ops->get_cached_segment_base(ctxt, seg);
608 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
609 u32 error, bool valid)
612 ctxt->exception.vector = vec;
613 ctxt->exception.error_code = error;
614 ctxt->exception.error_code_valid = valid;
615 return X86EMUL_PROPAGATE_FAULT;
618 static int emulate_db(struct x86_emulate_ctxt *ctxt)
620 return emulate_exception(ctxt, DB_VECTOR, 0, false);
623 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
625 return emulate_exception(ctxt, GP_VECTOR, err, true);
628 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
630 return emulate_exception(ctxt, SS_VECTOR, err, true);
633 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
635 return emulate_exception(ctxt, UD_VECTOR, 0, false);
638 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
640 return emulate_exception(ctxt, TS_VECTOR, err, true);
643 static int emulate_de(struct x86_emulate_ctxt *ctxt)
645 return emulate_exception(ctxt, DE_VECTOR, 0, false);
648 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
650 return emulate_exception(ctxt, NM_VECTOR, 0, false);
653 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
656 struct desc_struct desc;
658 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
662 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
667 struct desc_struct desc;
669 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
670 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
674 * x86 defines three classes of vector instructions: explicitly
675 * aligned, explicitly unaligned, and the rest, which change behaviour
676 * depending on whether they're AVX encoded or not.
678 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
679 * subject to the same check. FXSAVE and FXRSTOR are checked here too as their
680 * 512 bytes of data must be aligned to a 16 byte boundary.
682 static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
684 u64 alignment = ctxt->d & AlignMask;
686 if (likely(size < 16))
701 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
702 struct segmented_address addr,
703 unsigned *max_size, unsigned size,
704 bool write, bool fetch,
705 enum x86emul_mode mode, ulong *linear)
707 struct desc_struct desc;
714 la = seg_base(ctxt, addr.seg) + addr.ea;
717 case X86EMUL_MODE_PROT64:
719 va_bits = ctxt_virt_addr_bits(ctxt);
720 if (get_canonical(la, va_bits) != la)
723 *max_size = min_t(u64, ~0u, (1ull << va_bits) - la);
724 if (size > *max_size)
728 *linear = la = (u32)la;
729 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
733 /* code segment in protected mode or read-only data segment */
734 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
735 || !(desc.type & 2)) && write)
737 /* unreadable code segment */
738 if (!fetch && (desc.type & 8) && !(desc.type & 2))
740 lim = desc_limit_scaled(&desc);
741 if (!(desc.type & 8) && (desc.type & 4)) {
742 /* expand-down segment */
745 lim = desc.d ? 0xffffffff : 0xffff;
749 if (lim == 0xffffffff)
752 *max_size = (u64)lim + 1 - addr.ea;
753 if (size > *max_size)
758 if (la & (insn_alignment(ctxt, size) - 1))
759 return emulate_gp(ctxt, 0);
760 return X86EMUL_CONTINUE;
762 if (addr.seg == VCPU_SREG_SS)
763 return emulate_ss(ctxt, 0);
765 return emulate_gp(ctxt, 0);
768 static int linearize(struct x86_emulate_ctxt *ctxt,
769 struct segmented_address addr,
770 unsigned size, bool write,
774 return __linearize(ctxt, addr, &max_size, size, write, false,
778 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
779 enum x86emul_mode mode)
784 struct segmented_address addr = { .seg = VCPU_SREG_CS,
787 if (ctxt->op_bytes != sizeof(unsigned long))
788 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
789 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
790 if (rc == X86EMUL_CONTINUE)
791 ctxt->_eip = addr.ea;
795 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
797 return assign_eip(ctxt, dst, ctxt->mode);
800 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
801 const struct desc_struct *cs_desc)
803 enum x86emul_mode mode = ctxt->mode;
807 if (ctxt->mode >= X86EMUL_MODE_PROT16) {
811 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
813 mode = X86EMUL_MODE_PROT64;
815 mode = X86EMUL_MODE_PROT32; /* temporary value */
818 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
819 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
820 rc = assign_eip(ctxt, dst, mode);
821 if (rc == X86EMUL_CONTINUE)
826 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
828 return assign_eip_near(ctxt, ctxt->_eip + rel);
831 static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,
832 void *data, unsigned size)
834 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);
837 static int linear_write_system(struct x86_emulate_ctxt *ctxt,
838 ulong linear, void *data,
841 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);
844 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
845 struct segmented_address addr,
852 rc = linearize(ctxt, addr, size, false, &linear);
853 if (rc != X86EMUL_CONTINUE)
855 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);
858 static int segmented_write_std(struct x86_emulate_ctxt *ctxt,
859 struct segmented_address addr,
866 rc = linearize(ctxt, addr, size, true, &linear);
867 if (rc != X86EMUL_CONTINUE)
869 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);
873 * Prefetch the remaining bytes of the instruction without crossing page
874 * boundary if they are not in fetch_cache yet.
876 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
879 unsigned size, max_size;
880 unsigned long linear;
881 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
882 struct segmented_address addr = { .seg = VCPU_SREG_CS,
883 .ea = ctxt->eip + cur_size };
886 * We do not know exactly how many bytes will be needed, and
887 * __linearize is expensive, so fetch as much as possible. We
888 * just have to avoid going beyond the 15 byte limit, the end
889 * of the segment, or the end of the page.
891 * __linearize is called with size 0 so that it does not do any
892 * boundary check itself. Instead, we use max_size to check
895 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
897 if (unlikely(rc != X86EMUL_CONTINUE))
900 size = min_t(unsigned, 15UL ^ cur_size, max_size);
901 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
904 * One instruction can only straddle two pages,
905 * and one has been loaded at the beginning of
906 * x86_decode_insn. So, if not enough bytes
907 * still, we must have hit the 15-byte boundary.
909 if (unlikely(size < op_size))
910 return emulate_gp(ctxt, 0);
912 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
913 size, &ctxt->exception);
914 if (unlikely(rc != X86EMUL_CONTINUE))
916 ctxt->fetch.end += size;
917 return X86EMUL_CONTINUE;
920 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
923 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
925 if (unlikely(done_size < size))
926 return __do_insn_fetch_bytes(ctxt, size - done_size);
928 return X86EMUL_CONTINUE;
931 /* Fetch next part of the instruction being emulated. */
932 #define insn_fetch(_type, _ctxt) \
935 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
936 if (rc != X86EMUL_CONTINUE) \
938 ctxt->_eip += sizeof(_type); \
939 memcpy(&_x, ctxt->fetch.ptr, sizeof(_type)); \
940 ctxt->fetch.ptr += sizeof(_type); \
944 #define insn_fetch_arr(_arr, _size, _ctxt) \
946 rc = do_insn_fetch_bytes(_ctxt, _size); \
947 if (rc != X86EMUL_CONTINUE) \
949 ctxt->_eip += (_size); \
950 memcpy(_arr, ctxt->fetch.ptr, _size); \
951 ctxt->fetch.ptr += (_size); \
955 * Given the 'reg' portion of a ModRM byte, and a register block, return a
956 * pointer into the block that addresses the relevant register.
957 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
959 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
963 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
965 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
966 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
968 p = reg_rmw(ctxt, modrm_reg);
972 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
973 struct segmented_address addr,
974 u16 *size, unsigned long *address, int op_bytes)
981 rc = segmented_read_std(ctxt, addr, size, 2);
982 if (rc != X86EMUL_CONTINUE)
985 rc = segmented_read_std(ctxt, addr, address, op_bytes);
999 FASTOP1SRC2(mul, mul_ex);
1000 FASTOP1SRC2(imul, imul_ex);
1001 FASTOP1SRC2EX(div, div_ex);
1002 FASTOP1SRC2EX(idiv, idiv_ex);
1031 FASTOP2R(cmp, cmp_r);
1033 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
1035 /* If src is zero, do not writeback, but update flags */
1036 if (ctxt->src.val == 0)
1037 ctxt->dst.type = OP_NONE;
1038 return fastop(ctxt, em_bsf);
1041 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
1043 /* If src is zero, do not writeback, but update flags */
1044 if (ctxt->src.val == 0)
1045 ctxt->dst.type = OP_NONE;
1046 return fastop(ctxt, em_bsr);
1049 static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
1052 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
1054 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1055 asm("push %[flags]; popf; " CALL_NOSPEC
1056 : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags));
1060 static void fetch_register_operand(struct operand *op)
1062 switch (op->bytes) {
1064 op->val = *(u8 *)op->addr.reg;
1067 op->val = *(u16 *)op->addr.reg;
1070 op->val = *(u32 *)op->addr.reg;
1073 op->val = *(u64 *)op->addr.reg;
1078 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
1081 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
1082 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
1083 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
1084 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
1085 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
1086 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
1087 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
1088 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1089 #ifdef CONFIG_X86_64
1090 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1091 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1092 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1093 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1094 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1095 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1096 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1097 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1103 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1107 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1108 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1109 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1110 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1111 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1112 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1113 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1114 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1115 #ifdef CONFIG_X86_64
1116 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1117 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1118 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1119 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1120 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1121 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1122 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1123 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1129 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1132 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1133 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1134 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1135 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1136 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1137 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1138 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1139 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1144 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1147 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1148 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1149 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1150 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1151 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1152 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1153 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1154 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1159 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1161 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1162 return emulate_nm(ctxt);
1164 asm volatile("fninit");
1165 return X86EMUL_CONTINUE;
1168 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1172 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1173 return emulate_nm(ctxt);
1175 asm volatile("fnstcw %0": "+m"(fcw));
1177 ctxt->dst.val = fcw;
1179 return X86EMUL_CONTINUE;
1182 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1186 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1187 return emulate_nm(ctxt);
1189 asm volatile("fnstsw %0": "+m"(fsw));
1191 ctxt->dst.val = fsw;
1193 return X86EMUL_CONTINUE;
1196 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1199 unsigned reg = ctxt->modrm_reg;
1201 if (!(ctxt->d & ModRM))
1202 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1204 if (ctxt->d & Sse) {
1208 read_sse_reg(ctxt, &op->vec_val, reg);
1211 if (ctxt->d & Mmx) {
1220 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1221 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1223 fetch_register_operand(op);
1224 op->orig_val = op->val;
1227 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1229 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1230 ctxt->modrm_seg = VCPU_SREG_SS;
1233 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1237 int index_reg, base_reg, scale;
1238 int rc = X86EMUL_CONTINUE;
1241 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1242 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1243 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1245 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1246 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1247 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1248 ctxt->modrm_seg = VCPU_SREG_DS;
1250 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1252 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1253 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1255 if (ctxt->d & Sse) {
1258 op->addr.xmm = ctxt->modrm_rm;
1259 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1262 if (ctxt->d & Mmx) {
1265 op->addr.mm = ctxt->modrm_rm & 7;
1268 fetch_register_operand(op);
1274 if (ctxt->ad_bytes == 2) {
1275 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1276 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1277 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1278 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1280 /* 16-bit ModR/M decode. */
1281 switch (ctxt->modrm_mod) {
1283 if (ctxt->modrm_rm == 6)
1284 modrm_ea += insn_fetch(u16, ctxt);
1287 modrm_ea += insn_fetch(s8, ctxt);
1290 modrm_ea += insn_fetch(u16, ctxt);
1293 switch (ctxt->modrm_rm) {
1295 modrm_ea += bx + si;
1298 modrm_ea += bx + di;
1301 modrm_ea += bp + si;
1304 modrm_ea += bp + di;
1313 if (ctxt->modrm_mod != 0)
1320 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1321 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1322 ctxt->modrm_seg = VCPU_SREG_SS;
1323 modrm_ea = (u16)modrm_ea;
1325 /* 32/64-bit ModR/M decode. */
1326 if ((ctxt->modrm_rm & 7) == 4) {
1327 sib = insn_fetch(u8, ctxt);
1328 index_reg |= (sib >> 3) & 7;
1329 base_reg |= sib & 7;
1332 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1333 modrm_ea += insn_fetch(s32, ctxt);
1335 modrm_ea += reg_read(ctxt, base_reg);
1336 adjust_modrm_seg(ctxt, base_reg);
1337 /* Increment ESP on POP [ESP] */
1338 if ((ctxt->d & IncSP) &&
1339 base_reg == VCPU_REGS_RSP)
1340 modrm_ea += ctxt->op_bytes;
1343 modrm_ea += reg_read(ctxt, index_reg) << scale;
1344 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1345 modrm_ea += insn_fetch(s32, ctxt);
1346 if (ctxt->mode == X86EMUL_MODE_PROT64)
1347 ctxt->rip_relative = 1;
1349 base_reg = ctxt->modrm_rm;
1350 modrm_ea += reg_read(ctxt, base_reg);
1351 adjust_modrm_seg(ctxt, base_reg);
1353 switch (ctxt->modrm_mod) {
1355 modrm_ea += insn_fetch(s8, ctxt);
1358 modrm_ea += insn_fetch(s32, ctxt);
1362 op->addr.mem.ea = modrm_ea;
1363 if (ctxt->ad_bytes != 8)
1364 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1370 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1373 int rc = X86EMUL_CONTINUE;
1376 switch (ctxt->ad_bytes) {
1378 op->addr.mem.ea = insn_fetch(u16, ctxt);
1381 op->addr.mem.ea = insn_fetch(u32, ctxt);
1384 op->addr.mem.ea = insn_fetch(u64, ctxt);
1391 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1395 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1396 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1398 if (ctxt->src.bytes == 2)
1399 sv = (s16)ctxt->src.val & (s16)mask;
1400 else if (ctxt->src.bytes == 4)
1401 sv = (s32)ctxt->src.val & (s32)mask;
1403 sv = (s64)ctxt->src.val & (s64)mask;
1405 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1406 ctxt->dst.addr.mem.ea + (sv >> 3));
1409 /* only subword offset */
1410 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1413 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1414 unsigned long addr, void *dest, unsigned size)
1417 struct read_cache *mc = &ctxt->mem_read;
1419 if (mc->pos < mc->end)
1422 WARN_ON((mc->end + size) >= sizeof(mc->data));
1424 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1426 if (rc != X86EMUL_CONTINUE)
1432 memcpy(dest, mc->data + mc->pos, size);
1434 return X86EMUL_CONTINUE;
1437 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1438 struct segmented_address addr,
1445 rc = linearize(ctxt, addr, size, false, &linear);
1446 if (rc != X86EMUL_CONTINUE)
1448 return read_emulated(ctxt, linear, data, size);
1451 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1452 struct segmented_address addr,
1459 rc = linearize(ctxt, addr, size, true, &linear);
1460 if (rc != X86EMUL_CONTINUE)
1462 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1466 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1467 struct segmented_address addr,
1468 const void *orig_data, const void *data,
1474 rc = linearize(ctxt, addr, size, true, &linear);
1475 if (rc != X86EMUL_CONTINUE)
1477 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1478 size, &ctxt->exception);
1481 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1482 unsigned int size, unsigned short port,
1485 struct read_cache *rc = &ctxt->io_read;
1487 if (rc->pos == rc->end) { /* refill pio read ahead */
1488 unsigned int in_page, n;
1489 unsigned int count = ctxt->rep_prefix ?
1490 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1491 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1492 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1493 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1494 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1497 rc->pos = rc->end = 0;
1498 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1503 if (ctxt->rep_prefix && (ctxt->d & String) &&
1504 !(ctxt->eflags & X86_EFLAGS_DF)) {
1505 ctxt->dst.data = rc->data + rc->pos;
1506 ctxt->dst.type = OP_MEM_STR;
1507 ctxt->dst.count = (rc->end - rc->pos) / size;
1510 memcpy(dest, rc->data + rc->pos, size);
1516 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1517 u16 index, struct desc_struct *desc)
1522 ctxt->ops->get_idt(ctxt, &dt);
1524 if (dt.size < index * 8 + 7)
1525 return emulate_gp(ctxt, index << 3 | 0x2);
1527 addr = dt.address + index * 8;
1528 return linear_read_system(ctxt, addr, desc, sizeof(*desc));
1531 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1532 u16 selector, struct desc_ptr *dt)
1534 const struct x86_emulate_ops *ops = ctxt->ops;
1537 if (selector & 1 << 2) {
1538 struct desc_struct desc;
1541 memset(dt, 0, sizeof(*dt));
1542 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1546 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1547 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1549 ops->get_gdt(ctxt, dt);
1552 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1553 u16 selector, ulong *desc_addr_p)
1556 u16 index = selector >> 3;
1559 get_descriptor_table_ptr(ctxt, selector, &dt);
1561 if (dt.size < index * 8 + 7)
1562 return emulate_gp(ctxt, selector & 0xfffc);
1564 addr = dt.address + index * 8;
1566 #ifdef CONFIG_X86_64
1567 if (addr >> 32 != 0) {
1570 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1571 if (!(efer & EFER_LMA))
1576 *desc_addr_p = addr;
1577 return X86EMUL_CONTINUE;
1580 /* allowed just for 8 bytes segments */
1581 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1582 u16 selector, struct desc_struct *desc,
1587 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1588 if (rc != X86EMUL_CONTINUE)
1591 return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));
1594 /* allowed just for 8 bytes segments */
1595 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1596 u16 selector, struct desc_struct *desc)
1601 rc = get_descriptor_ptr(ctxt, selector, &addr);
1602 if (rc != X86EMUL_CONTINUE)
1605 return linear_write_system(ctxt, addr, desc, sizeof(*desc));
1608 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1609 u16 selector, int seg, u8 cpl,
1610 enum x86_transfer_type transfer,
1611 struct desc_struct *desc)
1613 struct desc_struct seg_desc, old_desc;
1615 unsigned err_vec = GP_VECTOR;
1617 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1623 memset(&seg_desc, 0, sizeof(seg_desc));
1625 if (ctxt->mode == X86EMUL_MODE_REAL) {
1626 /* set real mode segment descriptor (keep limit etc. for
1628 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1629 set_desc_base(&seg_desc, selector << 4);
1631 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1632 /* VM86 needs a clean new segment descriptor */
1633 set_desc_base(&seg_desc, selector << 4);
1634 set_desc_limit(&seg_desc, 0xffff);
1644 /* TR should be in GDT only */
1645 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1648 /* NULL selector is not valid for TR, CS and (except for long mode) SS */
1649 if (null_selector) {
1650 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR)
1653 if (seg == VCPU_SREG_SS) {
1654 if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)
1658 * ctxt->ops->set_segment expects the CPL to be in
1659 * SS.DPL, so fake an expand-up 32-bit data segment.
1669 /* Skip all following checks */
1673 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1674 if (ret != X86EMUL_CONTINUE)
1677 err_code = selector & 0xfffc;
1678 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1681 /* can't load system descriptor into segment selector */
1682 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1683 if (transfer == X86_TRANSFER_CALL_JMP)
1684 return X86EMUL_UNHANDLEABLE;
1689 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1698 * segment is not a writable data segment or segment
1699 * selector's RPL != CPL or segment selector's RPL != CPL
1701 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1705 if (!(seg_desc.type & 8))
1708 if (seg_desc.type & 4) {
1714 if (rpl > cpl || dpl != cpl)
1717 /* in long-mode d/b must be clear if l is set */
1718 if (seg_desc.d && seg_desc.l) {
1721 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1722 if (efer & EFER_LMA)
1726 /* CS(RPL) <- CPL */
1727 selector = (selector & 0xfffc) | cpl;
1730 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1732 old_desc = seg_desc;
1733 seg_desc.type |= 2; /* busy */
1734 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1735 sizeof(seg_desc), &ctxt->exception);
1736 if (ret != X86EMUL_CONTINUE)
1739 case VCPU_SREG_LDTR:
1740 if (seg_desc.s || seg_desc.type != 2)
1743 default: /* DS, ES, FS, or GS */
1745 * segment is not a data or readable code segment or
1746 * ((segment is a data or nonconforming code segment)
1747 * and (both RPL and CPL > DPL))
1749 if ((seg_desc.type & 0xa) == 0x8 ||
1750 (((seg_desc.type & 0xc) != 0xc) &&
1751 (rpl > dpl && cpl > dpl)))
1757 /* mark segment as accessed */
1758 if (!(seg_desc.type & 1)) {
1760 ret = write_segment_descriptor(ctxt, selector,
1762 if (ret != X86EMUL_CONTINUE)
1765 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1766 ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));
1767 if (ret != X86EMUL_CONTINUE)
1769 if (emul_is_noncanonical_address(get_desc_base(&seg_desc) |
1770 ((u64)base3 << 32), ctxt))
1771 return emulate_gp(ctxt, 0);
1774 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1777 return X86EMUL_CONTINUE;
1779 return emulate_exception(ctxt, err_vec, err_code, true);
1782 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1783 u16 selector, int seg)
1785 u8 cpl = ctxt->ops->cpl(ctxt);
1788 * None of MOV, POP and LSS can load a NULL selector in CPL=3, but
1789 * they can load it at CPL<3 (Intel's manual says only LSS can,
1792 * However, the Intel manual says that putting IST=1/DPL=3 in
1793 * an interrupt gate will result in SS=3 (the AMD manual instead
1794 * says it doesn't), so allow SS=3 in __load_segment_descriptor
1795 * and only forbid it here.
1797 if (seg == VCPU_SREG_SS && selector == 3 &&
1798 ctxt->mode == X86EMUL_MODE_PROT64)
1799 return emulate_exception(ctxt, GP_VECTOR, 0, true);
1801 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1802 X86_TRANSFER_NONE, NULL);
1805 static void write_register_operand(struct operand *op)
1807 return assign_register(op->addr.reg, op->val, op->bytes);
1810 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1814 write_register_operand(op);
1817 if (ctxt->lock_prefix)
1818 return segmented_cmpxchg(ctxt,
1824 return segmented_write(ctxt,
1830 return segmented_write(ctxt,
1833 op->bytes * op->count);
1836 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1839 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1847 return X86EMUL_CONTINUE;
1850 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1852 struct segmented_address addr;
1854 rsp_increment(ctxt, -bytes);
1855 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1856 addr.seg = VCPU_SREG_SS;
1858 return segmented_write(ctxt, addr, data, bytes);
1861 static int em_push(struct x86_emulate_ctxt *ctxt)
1863 /* Disable writeback. */
1864 ctxt->dst.type = OP_NONE;
1865 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1868 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1869 void *dest, int len)
1872 struct segmented_address addr;
1874 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1875 addr.seg = VCPU_SREG_SS;
1876 rc = segmented_read(ctxt, addr, dest, len);
1877 if (rc != X86EMUL_CONTINUE)
1880 rsp_increment(ctxt, len);
1884 static int em_pop(struct x86_emulate_ctxt *ctxt)
1886 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1889 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1890 void *dest, int len)
1893 unsigned long val, change_mask;
1894 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1895 int cpl = ctxt->ops->cpl(ctxt);
1897 rc = emulate_pop(ctxt, &val, len);
1898 if (rc != X86EMUL_CONTINUE)
1901 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1902 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1903 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1904 X86_EFLAGS_AC | X86_EFLAGS_ID;
1906 switch(ctxt->mode) {
1907 case X86EMUL_MODE_PROT64:
1908 case X86EMUL_MODE_PROT32:
1909 case X86EMUL_MODE_PROT16:
1911 change_mask |= X86_EFLAGS_IOPL;
1913 change_mask |= X86_EFLAGS_IF;
1915 case X86EMUL_MODE_VM86:
1917 return emulate_gp(ctxt, 0);
1918 change_mask |= X86_EFLAGS_IF;
1920 default: /* real mode */
1921 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1925 *(unsigned long *)dest =
1926 (ctxt->eflags & ~change_mask) | (val & change_mask);
1931 static int em_popf(struct x86_emulate_ctxt *ctxt)
1933 ctxt->dst.type = OP_REG;
1934 ctxt->dst.addr.reg = &ctxt->eflags;
1935 ctxt->dst.bytes = ctxt->op_bytes;
1936 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1939 static int em_enter(struct x86_emulate_ctxt *ctxt)
1942 unsigned frame_size = ctxt->src.val;
1943 unsigned nesting_level = ctxt->src2.val & 31;
1947 return X86EMUL_UNHANDLEABLE;
1949 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1950 rc = push(ctxt, &rbp, stack_size(ctxt));
1951 if (rc != X86EMUL_CONTINUE)
1953 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1955 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1956 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1958 return X86EMUL_CONTINUE;
1961 static int em_leave(struct x86_emulate_ctxt *ctxt)
1963 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1965 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1968 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1970 int seg = ctxt->src2.val;
1972 ctxt->src.val = get_segment_selector(ctxt, seg);
1973 if (ctxt->op_bytes == 4) {
1974 rsp_increment(ctxt, -2);
1978 return em_push(ctxt);
1981 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1983 int seg = ctxt->src2.val;
1984 unsigned long selector;
1987 rc = emulate_pop(ctxt, &selector, 2);
1988 if (rc != X86EMUL_CONTINUE)
1991 if (ctxt->modrm_reg == VCPU_SREG_SS)
1992 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1993 if (ctxt->op_bytes > 2)
1994 rsp_increment(ctxt, ctxt->op_bytes - 2);
1996 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
2000 static int em_pusha(struct x86_emulate_ctxt *ctxt)
2002 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
2003 int rc = X86EMUL_CONTINUE;
2004 int reg = VCPU_REGS_RAX;
2006 while (reg <= VCPU_REGS_RDI) {
2007 (reg == VCPU_REGS_RSP) ?
2008 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
2011 if (rc != X86EMUL_CONTINUE)
2020 static int em_pushf(struct x86_emulate_ctxt *ctxt)
2022 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
2023 return em_push(ctxt);
2026 static int em_popa(struct x86_emulate_ctxt *ctxt)
2028 int rc = X86EMUL_CONTINUE;
2029 int reg = VCPU_REGS_RDI;
2032 while (reg >= VCPU_REGS_RAX) {
2033 if (reg == VCPU_REGS_RSP) {
2034 rsp_increment(ctxt, ctxt->op_bytes);
2038 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
2039 if (rc != X86EMUL_CONTINUE)
2041 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
2047 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2049 const struct x86_emulate_ops *ops = ctxt->ops;
2056 /* TODO: Add limit checks */
2057 ctxt->src.val = ctxt->eflags;
2059 if (rc != X86EMUL_CONTINUE)
2062 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
2064 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
2066 if (rc != X86EMUL_CONTINUE)
2069 ctxt->src.val = ctxt->_eip;
2071 if (rc != X86EMUL_CONTINUE)
2074 ops->get_idt(ctxt, &dt);
2076 eip_addr = dt.address + (irq << 2);
2077 cs_addr = dt.address + (irq << 2) + 2;
2079 rc = linear_read_system(ctxt, cs_addr, &cs, 2);
2080 if (rc != X86EMUL_CONTINUE)
2083 rc = linear_read_system(ctxt, eip_addr, &eip, 2);
2084 if (rc != X86EMUL_CONTINUE)
2087 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2088 if (rc != X86EMUL_CONTINUE)
2096 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2100 invalidate_registers(ctxt);
2101 rc = __emulate_int_real(ctxt, irq);
2102 if (rc == X86EMUL_CONTINUE)
2103 writeback_registers(ctxt);
2107 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2109 switch(ctxt->mode) {
2110 case X86EMUL_MODE_REAL:
2111 return __emulate_int_real(ctxt, irq);
2112 case X86EMUL_MODE_VM86:
2113 case X86EMUL_MODE_PROT16:
2114 case X86EMUL_MODE_PROT32:
2115 case X86EMUL_MODE_PROT64:
2117 /* Protected mode interrupts unimplemented yet */
2118 return X86EMUL_UNHANDLEABLE;
2122 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2124 int rc = X86EMUL_CONTINUE;
2125 unsigned long temp_eip = 0;
2126 unsigned long temp_eflags = 0;
2127 unsigned long cs = 0;
2128 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2129 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2130 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2131 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2132 X86_EFLAGS_AC | X86_EFLAGS_ID |
2134 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2137 /* TODO: Add stack limit check */
2139 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2141 if (rc != X86EMUL_CONTINUE)
2144 if (temp_eip & ~0xffff)
2145 return emulate_gp(ctxt, 0);
2147 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2149 if (rc != X86EMUL_CONTINUE)
2152 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2154 if (rc != X86EMUL_CONTINUE)
2157 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2159 if (rc != X86EMUL_CONTINUE)
2162 ctxt->_eip = temp_eip;
2164 if (ctxt->op_bytes == 4)
2165 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2166 else if (ctxt->op_bytes == 2) {
2167 ctxt->eflags &= ~0xffff;
2168 ctxt->eflags |= temp_eflags;
2171 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2172 ctxt->eflags |= X86_EFLAGS_FIXED;
2173 ctxt->ops->set_nmi_mask(ctxt, false);
2178 static int em_iret(struct x86_emulate_ctxt *ctxt)
2180 switch(ctxt->mode) {
2181 case X86EMUL_MODE_REAL:
2182 return emulate_iret_real(ctxt);
2183 case X86EMUL_MODE_VM86:
2184 case X86EMUL_MODE_PROT16:
2185 case X86EMUL_MODE_PROT32:
2186 case X86EMUL_MODE_PROT64:
2188 /* iret from protected mode unimplemented yet */
2189 return X86EMUL_UNHANDLEABLE;
2193 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2197 struct desc_struct new_desc;
2198 u8 cpl = ctxt->ops->cpl(ctxt);
2200 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2202 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2203 X86_TRANSFER_CALL_JMP,
2205 if (rc != X86EMUL_CONTINUE)
2208 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2209 /* Error handling is not implemented. */
2210 if (rc != X86EMUL_CONTINUE)
2211 return X86EMUL_UNHANDLEABLE;
2216 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2218 return assign_eip_near(ctxt, ctxt->src.val);
2221 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2226 old_eip = ctxt->_eip;
2227 rc = assign_eip_near(ctxt, ctxt->src.val);
2228 if (rc != X86EMUL_CONTINUE)
2230 ctxt->src.val = old_eip;
2235 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2237 u64 old = ctxt->dst.orig_val64;
2239 if (ctxt->dst.bytes == 16)
2240 return X86EMUL_UNHANDLEABLE;
2242 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2243 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2244 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2245 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2246 ctxt->eflags &= ~X86_EFLAGS_ZF;
2248 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2249 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2251 ctxt->eflags |= X86_EFLAGS_ZF;
2253 return X86EMUL_CONTINUE;
2256 static int em_ret(struct x86_emulate_ctxt *ctxt)
2261 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2262 if (rc != X86EMUL_CONTINUE)
2265 return assign_eip_near(ctxt, eip);
2268 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2271 unsigned long eip, cs;
2272 int cpl = ctxt->ops->cpl(ctxt);
2273 struct desc_struct new_desc;
2275 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2276 if (rc != X86EMUL_CONTINUE)
2278 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2279 if (rc != X86EMUL_CONTINUE)
2281 /* Outer-privilege level return is not implemented */
2282 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2283 return X86EMUL_UNHANDLEABLE;
2284 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2287 if (rc != X86EMUL_CONTINUE)
2289 rc = assign_eip_far(ctxt, eip, &new_desc);
2290 /* Error handling is not implemented. */
2291 if (rc != X86EMUL_CONTINUE)
2292 return X86EMUL_UNHANDLEABLE;
2297 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2301 rc = em_ret_far(ctxt);
2302 if (rc != X86EMUL_CONTINUE)
2304 rsp_increment(ctxt, ctxt->src.val);
2305 return X86EMUL_CONTINUE;
2308 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2310 /* Save real source value, then compare EAX against destination. */
2311 ctxt->dst.orig_val = ctxt->dst.val;
2312 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2313 ctxt->src.orig_val = ctxt->src.val;
2314 ctxt->src.val = ctxt->dst.orig_val;
2315 fastop(ctxt, em_cmp);
2317 if (ctxt->eflags & X86_EFLAGS_ZF) {
2318 /* Success: write back to memory; no update of EAX */
2319 ctxt->src.type = OP_NONE;
2320 ctxt->dst.val = ctxt->src.orig_val;
2322 /* Failure: write the value we saw to EAX. */
2323 ctxt->src.type = OP_REG;
2324 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2325 ctxt->src.val = ctxt->dst.orig_val;
2326 /* Create write-cycle to dest by writing the same value */
2327 ctxt->dst.val = ctxt->dst.orig_val;
2329 return X86EMUL_CONTINUE;
2332 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2334 int seg = ctxt->src2.val;
2338 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2340 rc = load_segment_descriptor(ctxt, sel, seg);
2341 if (rc != X86EMUL_CONTINUE)
2344 ctxt->dst.val = ctxt->src.val;
2348 static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
2350 #ifdef CONFIG_X86_64
2351 u32 eax, ebx, ecx, edx;
2355 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2356 return edx & bit(X86_FEATURE_LM);
2362 static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
2364 desc->g = (flags >> 23) & 1;
2365 desc->d = (flags >> 22) & 1;
2366 desc->l = (flags >> 21) & 1;
2367 desc->avl = (flags >> 20) & 1;
2368 desc->p = (flags >> 15) & 1;
2369 desc->dpl = (flags >> 13) & 3;
2370 desc->s = (flags >> 12) & 1;
2371 desc->type = (flags >> 8) & 15;
2374 static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, const char *smstate,
2377 struct desc_struct desc;
2381 selector = GET_SMSTATE(u32, smstate, 0x7fa8 + n * 4);
2384 offset = 0x7f84 + n * 12;
2386 offset = 0x7f2c + (n - 3) * 12;
2388 set_desc_base(&desc, GET_SMSTATE(u32, smstate, offset + 8));
2389 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, offset + 4));
2390 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, offset));
2391 ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
2392 return X86EMUL_CONTINUE;
2395 #ifdef CONFIG_X86_64
2396 static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, const char *smstate,
2399 struct desc_struct desc;
2404 offset = 0x7e00 + n * 16;
2406 selector = GET_SMSTATE(u16, smstate, offset);
2407 rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smstate, offset + 2) << 8);
2408 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, offset + 4));
2409 set_desc_base(&desc, GET_SMSTATE(u32, smstate, offset + 8));
2410 base3 = GET_SMSTATE(u32, smstate, offset + 12);
2412 ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
2413 return X86EMUL_CONTINUE;
2417 static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,
2418 u64 cr0, u64 cr3, u64 cr4)
2423 /* In order to later set CR4.PCIDE, CR3[11:0] must be zero. */
2425 if (cr4 & X86_CR4_PCIDE) {
2430 bad = ctxt->ops->set_cr(ctxt, 3, cr3);
2432 return X86EMUL_UNHANDLEABLE;
2435 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
2436 * Then enable protected mode. However, PCID cannot be enabled
2437 * if EFER.LMA=0, so set it separately.
2439 bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2441 return X86EMUL_UNHANDLEABLE;
2443 bad = ctxt->ops->set_cr(ctxt, 0, cr0);
2445 return X86EMUL_UNHANDLEABLE;
2447 if (cr4 & X86_CR4_PCIDE) {
2448 bad = ctxt->ops->set_cr(ctxt, 4, cr4);
2450 return X86EMUL_UNHANDLEABLE;
2452 bad = ctxt->ops->set_cr(ctxt, 3, cr3 | pcid);
2454 return X86EMUL_UNHANDLEABLE;
2459 return X86EMUL_CONTINUE;
2462 static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt,
2463 const char *smstate)
2465 struct desc_struct desc;
2468 u32 val, cr0, cr3, cr4;
2471 cr0 = GET_SMSTATE(u32, smstate, 0x7ffc);
2472 cr3 = GET_SMSTATE(u32, smstate, 0x7ff8);
2473 ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7ff4) | X86_EFLAGS_FIXED;
2474 ctxt->_eip = GET_SMSTATE(u32, smstate, 0x7ff0);
2476 for (i = 0; i < 8; i++)
2477 *reg_write(ctxt, i) = GET_SMSTATE(u32, smstate, 0x7fd0 + i * 4);
2479 val = GET_SMSTATE(u32, smstate, 0x7fcc);
2480 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2481 val = GET_SMSTATE(u32, smstate, 0x7fc8);
2482 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2484 selector = GET_SMSTATE(u32, smstate, 0x7fc4);
2485 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7f64));
2486 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7f60));
2487 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7f5c));
2488 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);
2490 selector = GET_SMSTATE(u32, smstate, 0x7fc0);
2491 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7f80));
2492 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7f7c));
2493 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7f78));
2494 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);
2496 dt.address = GET_SMSTATE(u32, smstate, 0x7f74);
2497 dt.size = GET_SMSTATE(u32, smstate, 0x7f70);
2498 ctxt->ops->set_gdt(ctxt, &dt);
2500 dt.address = GET_SMSTATE(u32, smstate, 0x7f58);
2501 dt.size = GET_SMSTATE(u32, smstate, 0x7f54);
2502 ctxt->ops->set_idt(ctxt, &dt);
2504 for (i = 0; i < 6; i++) {
2505 int r = rsm_load_seg_32(ctxt, smstate, i);
2506 if (r != X86EMUL_CONTINUE)
2510 cr4 = GET_SMSTATE(u32, smstate, 0x7f14);
2512 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7ef8));
2514 return rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2517 #ifdef CONFIG_X86_64
2518 static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt,
2519 const char *smstate)
2521 struct desc_struct desc;
2523 u64 val, cr0, cr3, cr4;
2528 for (i = 0; i < 16; i++)
2529 *reg_write(ctxt, i) = GET_SMSTATE(u64, smstate, 0x7ff8 - i * 8);
2531 ctxt->_eip = GET_SMSTATE(u64, smstate, 0x7f78);
2532 ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7f70) | X86_EFLAGS_FIXED;
2534 val = GET_SMSTATE(u32, smstate, 0x7f68);
2535 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2536 val = GET_SMSTATE(u32, smstate, 0x7f60);
2537 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2539 cr0 = GET_SMSTATE(u64, smstate, 0x7f58);
2540 cr3 = GET_SMSTATE(u64, smstate, 0x7f50);
2541 cr4 = GET_SMSTATE(u64, smstate, 0x7f48);
2542 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7f00));
2543 val = GET_SMSTATE(u64, smstate, 0x7ed0);
2544 ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA);
2546 selector = GET_SMSTATE(u32, smstate, 0x7e90);
2547 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7e92) << 8);
2548 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7e94));
2549 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7e98));
2550 base3 = GET_SMSTATE(u32, smstate, 0x7e9c);
2551 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);
2553 dt.size = GET_SMSTATE(u32, smstate, 0x7e84);
2554 dt.address = GET_SMSTATE(u64, smstate, 0x7e88);
2555 ctxt->ops->set_idt(ctxt, &dt);
2557 selector = GET_SMSTATE(u32, smstate, 0x7e70);
2558 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7e72) << 8);
2559 set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7e74));
2560 set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7e78));
2561 base3 = GET_SMSTATE(u32, smstate, 0x7e7c);
2562 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);
2564 dt.size = GET_SMSTATE(u32, smstate, 0x7e64);
2565 dt.address = GET_SMSTATE(u64, smstate, 0x7e68);
2566 ctxt->ops->set_gdt(ctxt, &dt);
2568 r = rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2569 if (r != X86EMUL_CONTINUE)
2572 for (i = 0; i < 6; i++) {
2573 r = rsm_load_seg_64(ctxt, smstate, i);
2574 if (r != X86EMUL_CONTINUE)
2578 return X86EMUL_CONTINUE;
2582 static int em_rsm(struct x86_emulate_ctxt *ctxt)
2584 unsigned long cr0, cr4, efer;
2589 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0)
2590 return emulate_ud(ctxt);
2592 smbase = ctxt->ops->get_smbase(ctxt);
2594 ret = ctxt->ops->read_phys(ctxt, smbase + 0xfe00, buf, sizeof(buf));
2595 if (ret != X86EMUL_CONTINUE)
2596 return X86EMUL_UNHANDLEABLE;
2598 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
2599 ctxt->ops->set_nmi_mask(ctxt, false);
2601 ctxt->ops->set_hflags(ctxt, ctxt->ops->get_hflags(ctxt) &
2602 ~(X86EMUL_SMM_INSIDE_NMI_MASK | X86EMUL_SMM_MASK));
2605 * Get back to real mode, to prepare a safe state in which to load
2606 * CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU
2607 * supports long mode.
2609 if (emulator_has_longmode(ctxt)) {
2610 struct desc_struct cs_desc;
2612 /* Zero CR4.PCIDE before CR0.PG. */
2613 cr4 = ctxt->ops->get_cr(ctxt, 4);
2614 if (cr4 & X86_CR4_PCIDE)
2615 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2617 /* A 32-bit code segment is required to clear EFER.LMA. */
2618 memset(&cs_desc, 0, sizeof(cs_desc));
2620 cs_desc.s = cs_desc.g = cs_desc.p = 1;
2621 ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);
2624 /* For the 64-bit case, this will clear EFER.LMA. */
2625 cr0 = ctxt->ops->get_cr(ctxt, 0);
2626 if (cr0 & X86_CR0_PE)
2627 ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
2629 if (emulator_has_longmode(ctxt)) {
2630 /* Clear CR4.PAE before clearing EFER.LME. */
2631 cr4 = ctxt->ops->get_cr(ctxt, 4);
2632 if (cr4 & X86_CR4_PAE)
2633 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);
2635 /* And finally go back to 32-bit mode. */
2637 ctxt->ops->set_msr(ctxt, MSR_EFER, efer);
2641 * Give pre_leave_smm() a chance to make ISA-specific changes to the
2642 * vCPU state (e.g. enter guest mode) before loading state from the SMM
2645 if (ctxt->ops->pre_leave_smm(ctxt, buf))
2646 return X86EMUL_UNHANDLEABLE;
2648 #ifdef CONFIG_X86_64
2649 if (emulator_has_longmode(ctxt))
2650 ret = rsm_load_state_64(ctxt, buf);
2653 ret = rsm_load_state_32(ctxt, buf);
2655 if (ret != X86EMUL_CONTINUE) {
2656 /* FIXME: should triple fault */
2657 return X86EMUL_UNHANDLEABLE;
2660 ctxt->ops->post_leave_smm(ctxt);
2662 return X86EMUL_CONTINUE;
2666 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2667 struct desc_struct *cs, struct desc_struct *ss)
2669 cs->l = 0; /* will be adjusted later */
2670 set_desc_base(cs, 0); /* flat segment */
2671 cs->g = 1; /* 4kb granularity */
2672 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2673 cs->type = 0x0b; /* Read, Execute, Accessed */
2675 cs->dpl = 0; /* will be adjusted later */
2680 set_desc_base(ss, 0); /* flat segment */
2681 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2682 ss->g = 1; /* 4kb granularity */
2684 ss->type = 0x03; /* Read/Write, Accessed */
2685 ss->d = 1; /* 32bit stack segment */
2692 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2694 u32 eax, ebx, ecx, edx;
2697 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2698 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2699 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2700 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2703 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2705 const struct x86_emulate_ops *ops = ctxt->ops;
2706 u32 eax, ebx, ecx, edx;
2709 * syscall should always be enabled in longmode - so only become
2710 * vendor specific (cpuid) if other modes are active...
2712 if (ctxt->mode == X86EMUL_MODE_PROT64)
2717 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2719 * Intel ("GenuineIntel")
2720 * remark: Intel CPUs only support "syscall" in 64bit
2721 * longmode. Also an 64bit guest with a
2722 * 32bit compat-app running will #UD !! While this
2723 * behaviour can be fixed (by emulating) into AMD
2724 * response - CPUs of AMD can't behave like Intel.
2726 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2727 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2728 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2731 /* AMD ("AuthenticAMD") */
2732 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2733 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2734 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2737 /* AMD ("AMDisbetter!") */
2738 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2739 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2740 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2743 /* Hygon ("HygonGenuine") */
2744 if (ebx == X86EMUL_CPUID_VENDOR_HygonGenuine_ebx &&
2745 ecx == X86EMUL_CPUID_VENDOR_HygonGenuine_ecx &&
2746 edx == X86EMUL_CPUID_VENDOR_HygonGenuine_edx)
2750 * default: (not Intel, not AMD, not Hygon), apply Intel's
2756 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2758 const struct x86_emulate_ops *ops = ctxt->ops;
2759 struct desc_struct cs, ss;
2764 /* syscall is not available in real mode */
2765 if (ctxt->mode == X86EMUL_MODE_REAL ||
2766 ctxt->mode == X86EMUL_MODE_VM86)
2767 return emulate_ud(ctxt);
2769 if (!(em_syscall_is_enabled(ctxt)))
2770 return emulate_ud(ctxt);
2772 ops->get_msr(ctxt, MSR_EFER, &efer);
2773 if (!(efer & EFER_SCE))
2774 return emulate_ud(ctxt);
2776 setup_syscalls_segments(ctxt, &cs, &ss);
2777 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2779 cs_sel = (u16)(msr_data & 0xfffc);
2780 ss_sel = (u16)(msr_data + 8);
2782 if (efer & EFER_LMA) {
2786 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2787 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2789 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2790 if (efer & EFER_LMA) {
2791 #ifdef CONFIG_X86_64
2792 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2795 ctxt->mode == X86EMUL_MODE_PROT64 ?
2796 MSR_LSTAR : MSR_CSTAR, &msr_data);
2797 ctxt->_eip = msr_data;
2799 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2800 ctxt->eflags &= ~msr_data;
2801 ctxt->eflags |= X86_EFLAGS_FIXED;
2805 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2806 ctxt->_eip = (u32)msr_data;
2808 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2811 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
2812 return X86EMUL_CONTINUE;
2815 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2817 const struct x86_emulate_ops *ops = ctxt->ops;
2818 struct desc_struct cs, ss;
2823 ops->get_msr(ctxt, MSR_EFER, &efer);
2824 /* inject #GP if in real mode */
2825 if (ctxt->mode == X86EMUL_MODE_REAL)
2826 return emulate_gp(ctxt, 0);
2829 * Not recognized on AMD in compat mode (but is recognized in legacy
2832 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2833 && !vendor_intel(ctxt))
2834 return emulate_ud(ctxt);
2836 /* sysenter/sysexit have not been tested in 64bit mode. */
2837 if (ctxt->mode == X86EMUL_MODE_PROT64)
2838 return X86EMUL_UNHANDLEABLE;
2840 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2841 if ((msr_data & 0xfffc) == 0x0)
2842 return emulate_gp(ctxt, 0);
2844 setup_syscalls_segments(ctxt, &cs, &ss);
2845 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2846 cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2847 ss_sel = cs_sel + 8;
2848 if (efer & EFER_LMA) {
2853 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2854 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2856 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2857 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2859 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2860 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2863 return X86EMUL_CONTINUE;
2866 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2868 const struct x86_emulate_ops *ops = ctxt->ops;
2869 struct desc_struct cs, ss;
2870 u64 msr_data, rcx, rdx;
2872 u16 cs_sel = 0, ss_sel = 0;
2874 /* inject #GP if in real mode or Virtual 8086 mode */
2875 if (ctxt->mode == X86EMUL_MODE_REAL ||
2876 ctxt->mode == X86EMUL_MODE_VM86)
2877 return emulate_gp(ctxt, 0);
2879 setup_syscalls_segments(ctxt, &cs, &ss);
2881 if ((ctxt->rex_prefix & 0x8) != 0x0)
2882 usermode = X86EMUL_MODE_PROT64;
2884 usermode = X86EMUL_MODE_PROT32;
2886 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2887 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2891 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2893 case X86EMUL_MODE_PROT32:
2894 cs_sel = (u16)(msr_data + 16);
2895 if ((msr_data & 0xfffc) == 0x0)
2896 return emulate_gp(ctxt, 0);
2897 ss_sel = (u16)(msr_data + 24);
2901 case X86EMUL_MODE_PROT64:
2902 cs_sel = (u16)(msr_data + 32);
2903 if (msr_data == 0x0)
2904 return emulate_gp(ctxt, 0);
2905 ss_sel = cs_sel + 8;
2908 if (emul_is_noncanonical_address(rcx, ctxt) ||
2909 emul_is_noncanonical_address(rdx, ctxt))
2910 return emulate_gp(ctxt, 0);
2913 cs_sel |= SEGMENT_RPL_MASK;
2914 ss_sel |= SEGMENT_RPL_MASK;
2916 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2917 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2920 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2922 return X86EMUL_CONTINUE;
2925 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2928 if (ctxt->mode == X86EMUL_MODE_REAL)
2930 if (ctxt->mode == X86EMUL_MODE_VM86)
2932 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2933 return ctxt->ops->cpl(ctxt) > iopl;
2936 #define VMWARE_PORT_VMPORT (0x5658)
2937 #define VMWARE_PORT_VMRPC (0x5659)
2939 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2942 const struct x86_emulate_ops *ops = ctxt->ops;
2943 struct desc_struct tr_seg;
2946 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2947 unsigned mask = (1 << len) - 1;
2951 * VMware allows access to these ports even if denied
2952 * by TSS I/O permission bitmap. Mimic behavior.
2954 if (enable_vmware_backdoor &&
2955 ((port == VMWARE_PORT_VMPORT) || (port == VMWARE_PORT_VMRPC)))
2958 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2961 if (desc_limit_scaled(&tr_seg) < 103)
2963 base = get_desc_base(&tr_seg);
2964 #ifdef CONFIG_X86_64
2965 base |= ((u64)base3) << 32;
2967 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true);
2968 if (r != X86EMUL_CONTINUE)
2970 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2972 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true);
2973 if (r != X86EMUL_CONTINUE)
2975 if ((perm >> bit_idx) & mask)
2980 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2986 if (emulator_bad_iopl(ctxt))
2987 if (!emulator_io_port_access_allowed(ctxt, port, len))
2990 ctxt->perm_ok = true;
2995 static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
2998 * Intel CPUs mask the counter and pointers in quite strange
2999 * manner when ECX is zero due to REP-string optimizations.
3001 #ifdef CONFIG_X86_64
3002 if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
3005 *reg_write(ctxt, VCPU_REGS_RCX) = 0;
3008 case 0xa4: /* movsb */
3009 case 0xa5: /* movsd/w */
3010 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
3012 case 0xaa: /* stosb */
3013 case 0xab: /* stosd/w */
3014 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
3019 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
3020 struct tss_segment_16 *tss)
3022 tss->ip = ctxt->_eip;
3023 tss->flag = ctxt->eflags;
3024 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
3025 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
3026 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
3027 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
3028 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
3029 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
3030 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
3031 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
3033 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3034 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3035 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3036 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3037 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
3040 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
3041 struct tss_segment_16 *tss)
3046 ctxt->_eip = tss->ip;
3047 ctxt->eflags = tss->flag | 2;
3048 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
3049 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
3050 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
3051 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
3052 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
3053 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
3054 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
3055 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
3058 * SDM says that segment selectors are loaded before segment
3061 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
3062 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3063 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3064 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3065 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3070 * Now load segment descriptors. If fault happens at this stage
3071 * it is handled in a context of new task
3073 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
3074 X86_TRANSFER_TASK_SWITCH, NULL);
3075 if (ret != X86EMUL_CONTINUE)
3077 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3078 X86_TRANSFER_TASK_SWITCH, NULL);
3079 if (ret != X86EMUL_CONTINUE)
3081 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3082 X86_TRANSFER_TASK_SWITCH, NULL);
3083 if (ret != X86EMUL_CONTINUE)
3085 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3086 X86_TRANSFER_TASK_SWITCH, NULL);
3087 if (ret != X86EMUL_CONTINUE)
3089 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3090 X86_TRANSFER_TASK_SWITCH, NULL);
3091 if (ret != X86EMUL_CONTINUE)
3094 return X86EMUL_CONTINUE;
3097 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
3098 u16 tss_selector, u16 old_tss_sel,
3099 ulong old_tss_base, struct desc_struct *new_desc)
3101 struct tss_segment_16 tss_seg;
3103 u32 new_tss_base = get_desc_base(new_desc);
3105 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3106 if (ret != X86EMUL_CONTINUE)
3109 save_state_to_tss16(ctxt, &tss_seg);
3111 ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3112 if (ret != X86EMUL_CONTINUE)
3115 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));
3116 if (ret != X86EMUL_CONTINUE)
3119 if (old_tss_sel != 0xffff) {
3120 tss_seg.prev_task_link = old_tss_sel;
3122 ret = linear_write_system(ctxt, new_tss_base,
3123 &tss_seg.prev_task_link,
3124 sizeof(tss_seg.prev_task_link));
3125 if (ret != X86EMUL_CONTINUE)
3129 return load_state_from_tss16(ctxt, &tss_seg);
3132 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
3133 struct tss_segment_32 *tss)
3135 /* CR3 and ldt selector are not saved intentionally */
3136 tss->eip = ctxt->_eip;
3137 tss->eflags = ctxt->eflags;
3138 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
3139 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
3140 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
3141 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
3142 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
3143 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
3144 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
3145 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
3147 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3148 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3149 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3150 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3151 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
3152 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
3155 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
3156 struct tss_segment_32 *tss)
3161 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
3162 return emulate_gp(ctxt, 0);
3163 ctxt->_eip = tss->eip;
3164 ctxt->eflags = tss->eflags | 2;
3166 /* General purpose registers */
3167 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
3168 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
3169 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
3170 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
3171 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
3172 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
3173 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
3174 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
3177 * SDM says that segment selectors are loaded before segment
3178 * descriptors. This is important because CPL checks will
3181 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
3182 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3183 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3184 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3185 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3186 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
3187 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
3190 * If we're switching between Protected Mode and VM86, we need to make
3191 * sure to update the mode before loading the segment descriptors so
3192 * that the selectors are interpreted correctly.
3194 if (ctxt->eflags & X86_EFLAGS_VM) {
3195 ctxt->mode = X86EMUL_MODE_VM86;
3198 ctxt->mode = X86EMUL_MODE_PROT32;
3203 * Now load segment descriptors. If fault happenes at this stage
3204 * it is handled in a context of new task
3206 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
3207 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
3208 if (ret != X86EMUL_CONTINUE)
3210 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3211 X86_TRANSFER_TASK_SWITCH, NULL);
3212 if (ret != X86EMUL_CONTINUE)
3214 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3215 X86_TRANSFER_TASK_SWITCH, NULL);
3216 if (ret != X86EMUL_CONTINUE)
3218 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3219 X86_TRANSFER_TASK_SWITCH, NULL);
3220 if (ret != X86EMUL_CONTINUE)
3222 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3223 X86_TRANSFER_TASK_SWITCH, NULL);
3224 if (ret != X86EMUL_CONTINUE)
3226 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
3227 X86_TRANSFER_TASK_SWITCH, NULL);
3228 if (ret != X86EMUL_CONTINUE)
3230 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
3231 X86_TRANSFER_TASK_SWITCH, NULL);
3236 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
3237 u16 tss_selector, u16 old_tss_sel,
3238 ulong old_tss_base, struct desc_struct *new_desc)
3240 struct tss_segment_32 tss_seg;
3242 u32 new_tss_base = get_desc_base(new_desc);
3243 u32 eip_offset = offsetof(struct tss_segment_32, eip);
3244 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3246 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof(tss_seg));
3247 if (ret != X86EMUL_CONTINUE)
3250 save_state_to_tss32(ctxt, &tss_seg);
3252 /* Only GP registers and segment selectors are saved */
3253 ret = linear_write_system(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
3254 ldt_sel_offset - eip_offset);
3255 if (ret != X86EMUL_CONTINUE)
3258 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof(tss_seg));
3259 if (ret != X86EMUL_CONTINUE)
3262 if (old_tss_sel != 0xffff) {
3263 tss_seg.prev_task_link = old_tss_sel;
3265 ret = linear_write_system(ctxt, new_tss_base,
3266 &tss_seg.prev_task_link,
3267 sizeof(tss_seg.prev_task_link));
3268 if (ret != X86EMUL_CONTINUE)
3272 return load_state_from_tss32(ctxt, &tss_seg);
3275 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
3276 u16 tss_selector, int idt_index, int reason,
3277 bool has_error_code, u32 error_code)
3279 const struct x86_emulate_ops *ops = ctxt->ops;
3280 struct desc_struct curr_tss_desc, next_tss_desc;
3282 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
3283 ulong old_tss_base =
3284 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
3286 ulong desc_addr, dr7;
3288 /* FIXME: old_tss_base == ~0 ? */
3290 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
3291 if (ret != X86EMUL_CONTINUE)
3293 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
3294 if (ret != X86EMUL_CONTINUE)
3297 /* FIXME: check that next_tss_desc is tss */
3300 * Check privileges. The three cases are task switch caused by...
3302 * 1. jmp/call/int to task gate: Check against DPL of the task gate
3303 * 2. Exception/IRQ/iret: No check is performed
3304 * 3. jmp/call to TSS/task-gate: No check is performed since the
3305 * hardware checks it before exiting.
3307 if (reason == TASK_SWITCH_GATE) {
3308 if (idt_index != -1) {
3309 /* Software interrupts */
3310 struct desc_struct task_gate_desc;
3313 ret = read_interrupt_descriptor(ctxt, idt_index,
3315 if (ret != X86EMUL_CONTINUE)
3318 dpl = task_gate_desc.dpl;
3319 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
3320 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
3324 desc_limit = desc_limit_scaled(&next_tss_desc);
3325 if (!next_tss_desc.p ||
3326 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
3327 desc_limit < 0x2b)) {
3328 return emulate_ts(ctxt, tss_selector & 0xfffc);
3331 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
3332 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
3333 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
3336 if (reason == TASK_SWITCH_IRET)
3337 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
3339 /* set back link to prev task only if NT bit is set in eflags
3340 note that old_tss_sel is not used after this point */
3341 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
3342 old_tss_sel = 0xffff;
3344 if (next_tss_desc.type & 8)
3345 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
3346 old_tss_base, &next_tss_desc);
3348 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
3349 old_tss_base, &next_tss_desc);
3350 if (ret != X86EMUL_CONTINUE)
3353 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
3354 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
3356 if (reason != TASK_SWITCH_IRET) {
3357 next_tss_desc.type |= (1 << 1); /* set busy flag */
3358 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
3361 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
3362 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
3364 if (has_error_code) {
3365 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
3366 ctxt->lock_prefix = 0;
3367 ctxt->src.val = (unsigned long) error_code;
3368 ret = em_push(ctxt);
3371 ops->get_dr(ctxt, 7, &dr7);
3372 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));
3377 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
3378 u16 tss_selector, int idt_index, int reason,
3379 bool has_error_code, u32 error_code)
3383 invalidate_registers(ctxt);
3384 ctxt->_eip = ctxt->eip;
3385 ctxt->dst.type = OP_NONE;
3387 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
3388 has_error_code, error_code);
3390 if (rc == X86EMUL_CONTINUE) {
3391 ctxt->eip = ctxt->_eip;
3392 writeback_registers(ctxt);
3395 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3398 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
3401 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
3403 register_address_increment(ctxt, reg, df * op->bytes);
3404 op->addr.mem.ea = register_address(ctxt, reg);
3407 static int em_das(struct x86_emulate_ctxt *ctxt)
3410 bool af, cf, old_cf;
3412 cf = ctxt->eflags & X86_EFLAGS_CF;
3418 af = ctxt->eflags & X86_EFLAGS_AF;
3419 if ((al & 0x0f) > 9 || af) {
3421 cf = old_cf | (al >= 250);
3426 if (old_al > 0x99 || old_cf) {
3432 /* Set PF, ZF, SF */
3433 ctxt->src.type = OP_IMM;
3435 ctxt->src.bytes = 1;
3436 fastop(ctxt, em_or);
3437 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3439 ctxt->eflags |= X86_EFLAGS_CF;
3441 ctxt->eflags |= X86_EFLAGS_AF;
3442 return X86EMUL_CONTINUE;
3445 static int em_aam(struct x86_emulate_ctxt *ctxt)
3449 if (ctxt->src.val == 0)
3450 return emulate_de(ctxt);
3452 al = ctxt->dst.val & 0xff;
3453 ah = al / ctxt->src.val;
3454 al %= ctxt->src.val;
3456 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3458 /* Set PF, ZF, SF */
3459 ctxt->src.type = OP_IMM;
3461 ctxt->src.bytes = 1;
3462 fastop(ctxt, em_or);
3464 return X86EMUL_CONTINUE;
3467 static int em_aad(struct x86_emulate_ctxt *ctxt)
3469 u8 al = ctxt->dst.val & 0xff;
3470 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3472 al = (al + (ah * ctxt->src.val)) & 0xff;
3474 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3476 /* Set PF, ZF, SF */
3477 ctxt->src.type = OP_IMM;
3479 ctxt->src.bytes = 1;
3480 fastop(ctxt, em_or);
3482 return X86EMUL_CONTINUE;
3485 static int em_call(struct x86_emulate_ctxt *ctxt)
3488 long rel = ctxt->src.val;
3490 ctxt->src.val = (unsigned long)ctxt->_eip;
3491 rc = jmp_rel(ctxt, rel);
3492 if (rc != X86EMUL_CONTINUE)
3494 return em_push(ctxt);
3497 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3502 struct desc_struct old_desc, new_desc;
3503 const struct x86_emulate_ops *ops = ctxt->ops;
3504 int cpl = ctxt->ops->cpl(ctxt);
3505 enum x86emul_mode prev_mode = ctxt->mode;
3507 old_eip = ctxt->_eip;
3508 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3510 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3511 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3512 X86_TRANSFER_CALL_JMP, &new_desc);
3513 if (rc != X86EMUL_CONTINUE)
3516 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3517 if (rc != X86EMUL_CONTINUE)
3520 ctxt->src.val = old_cs;
3522 if (rc != X86EMUL_CONTINUE)
3525 ctxt->src.val = old_eip;
3527 /* If we failed, we tainted the memory, but the very least we should
3529 if (rc != X86EMUL_CONTINUE) {
3530 pr_warn_once("faulting far call emulation tainted memory\n");
3535 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3536 ctxt->mode = prev_mode;
3541 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3546 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3547 if (rc != X86EMUL_CONTINUE)
3549 rc = assign_eip_near(ctxt, eip);
3550 if (rc != X86EMUL_CONTINUE)
3552 rsp_increment(ctxt, ctxt->src.val);
3553 return X86EMUL_CONTINUE;
3556 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3558 /* Write back the register source. */
3559 ctxt->src.val = ctxt->dst.val;
3560 write_register_operand(&ctxt->src);
3562 /* Write back the memory destination with implicit LOCK prefix. */
3563 ctxt->dst.val = ctxt->src.orig_val;
3564 ctxt->lock_prefix = 1;
3565 return X86EMUL_CONTINUE;
3568 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3570 ctxt->dst.val = ctxt->src2.val;
3571 return fastop(ctxt, em_imul);
3574 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3576 ctxt->dst.type = OP_REG;
3577 ctxt->dst.bytes = ctxt->src.bytes;
3578 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3579 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3581 return X86EMUL_CONTINUE;
3584 static int em_rdpid(struct x86_emulate_ctxt *ctxt)
3588 if (ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux))
3589 return emulate_gp(ctxt, 0);
3590 ctxt->dst.val = tsc_aux;
3591 return X86EMUL_CONTINUE;
3594 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3598 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3599 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3600 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3601 return X86EMUL_CONTINUE;
3604 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3608 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3609 return emulate_gp(ctxt, 0);
3610 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3611 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3612 return X86EMUL_CONTINUE;
3615 static int em_mov(struct x86_emulate_ctxt *ctxt)
3617 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3618 return X86EMUL_CONTINUE;
3621 #define FFL(x) bit(X86_FEATURE_##x)
3623 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3625 u32 ebx, ecx, edx, eax = 1;
3629 * Check MOVBE is set in the guest-visible CPUID leaf.
3631 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
3632 if (!(ecx & FFL(MOVBE)))
3633 return emulate_ud(ctxt);
3635 switch (ctxt->op_bytes) {
3638 * From MOVBE definition: "...When the operand size is 16 bits,
3639 * the upper word of the destination register remains unchanged
3642 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3643 * rules so we have to do the operation almost per hand.
3645 tmp = (u16)ctxt->src.val;
3646 ctxt->dst.val &= ~0xffffUL;
3647 ctxt->dst.val |= (unsigned long)swab16(tmp);
3650 ctxt->dst.val = swab32((u32)ctxt->src.val);
3653 ctxt->dst.val = swab64(ctxt->src.val);
3658 return X86EMUL_CONTINUE;
3661 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3663 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3664 return emulate_gp(ctxt, 0);
3666 /* Disable writeback. */
3667 ctxt->dst.type = OP_NONE;
3668 return X86EMUL_CONTINUE;
3671 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3675 if (ctxt->mode == X86EMUL_MODE_PROT64)
3676 val = ctxt->src.val & ~0ULL;
3678 val = ctxt->src.val & ~0U;
3680 /* #UD condition is already handled. */
3681 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3682 return emulate_gp(ctxt, 0);
3684 /* Disable writeback. */
3685 ctxt->dst.type = OP_NONE;
3686 return X86EMUL_CONTINUE;
3689 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3693 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3694 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3695 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3696 return emulate_gp(ctxt, 0);
3698 return X86EMUL_CONTINUE;
3701 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3705 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3706 return emulate_gp(ctxt, 0);
3708 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3709 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3710 return X86EMUL_CONTINUE;
3713 static int em_store_sreg(struct x86_emulate_ctxt *ctxt, int segment)
3715 if (segment > VCPU_SREG_GS &&
3716 (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3717 ctxt->ops->cpl(ctxt) > 0)
3718 return emulate_gp(ctxt, 0);
3720 ctxt->dst.val = get_segment_selector(ctxt, segment);
3721 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3722 ctxt->dst.bytes = 2;
3723 return X86EMUL_CONTINUE;
3726 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3728 if (ctxt->modrm_reg > VCPU_SREG_GS)
3729 return emulate_ud(ctxt);
3731 return em_store_sreg(ctxt, ctxt->modrm_reg);
3734 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3736 u16 sel = ctxt->src.val;
3738 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3739 return emulate_ud(ctxt);
3741 if (ctxt->modrm_reg == VCPU_SREG_SS)
3742 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3744 /* Disable writeback. */
3745 ctxt->dst.type = OP_NONE;
3746 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3749 static int em_sldt(struct x86_emulate_ctxt *ctxt)
3751 return em_store_sreg(ctxt, VCPU_SREG_LDTR);
3754 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3756 u16 sel = ctxt->src.val;
3758 /* Disable writeback. */
3759 ctxt->dst.type = OP_NONE;
3760 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3763 static int em_str(struct x86_emulate_ctxt *ctxt)
3765 return em_store_sreg(ctxt, VCPU_SREG_TR);
3768 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3770 u16 sel = ctxt->src.val;
3772 /* Disable writeback. */
3773 ctxt->dst.type = OP_NONE;
3774 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3777 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3782 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3783 if (rc == X86EMUL_CONTINUE)
3784 ctxt->ops->invlpg(ctxt, linear);
3785 /* Disable writeback. */
3786 ctxt->dst.type = OP_NONE;
3787 return X86EMUL_CONTINUE;
3790 static int em_clts(struct x86_emulate_ctxt *ctxt)
3794 cr0 = ctxt->ops->get_cr(ctxt, 0);
3796 ctxt->ops->set_cr(ctxt, 0, cr0);
3797 return X86EMUL_CONTINUE;
3800 static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3802 int rc = ctxt->ops->fix_hypercall(ctxt);
3804 if (rc != X86EMUL_CONTINUE)
3807 /* Let the processor re-execute the fixed hypercall */
3808 ctxt->_eip = ctxt->eip;
3809 /* Disable writeback. */
3810 ctxt->dst.type = OP_NONE;
3811 return X86EMUL_CONTINUE;
3814 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3815 void (*get)(struct x86_emulate_ctxt *ctxt,
3816 struct desc_ptr *ptr))
3818 struct desc_ptr desc_ptr;
3820 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3821 ctxt->ops->cpl(ctxt) > 0)
3822 return emulate_gp(ctxt, 0);
3824 if (ctxt->mode == X86EMUL_MODE_PROT64)
3826 get(ctxt, &desc_ptr);
3827 if (ctxt->op_bytes == 2) {
3829 desc_ptr.address &= 0x00ffffff;
3831 /* Disable writeback. */
3832 ctxt->dst.type = OP_NONE;
3833 return segmented_write_std(ctxt, ctxt->dst.addr.mem,
3834 &desc_ptr, 2 + ctxt->op_bytes);
3837 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3839 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3842 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3844 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3847 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3849 struct desc_ptr desc_ptr;
3852 if (ctxt->mode == X86EMUL_MODE_PROT64)
3854 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3855 &desc_ptr.size, &desc_ptr.address,
3857 if (rc != X86EMUL_CONTINUE)
3859 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3860 emul_is_noncanonical_address(desc_ptr.address, ctxt))
3861 return emulate_gp(ctxt, 0);
3863 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3865 ctxt->ops->set_idt(ctxt, &desc_ptr);
3866 /* Disable writeback. */
3867 ctxt->dst.type = OP_NONE;
3868 return X86EMUL_CONTINUE;
3871 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3873 return em_lgdt_lidt(ctxt, true);
3876 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3878 return em_lgdt_lidt(ctxt, false);
3881 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3883 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3884 ctxt->ops->cpl(ctxt) > 0)
3885 return emulate_gp(ctxt, 0);
3887 if (ctxt->dst.type == OP_MEM)
3888 ctxt->dst.bytes = 2;
3889 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3890 return X86EMUL_CONTINUE;
3893 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3895 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3896 | (ctxt->src.val & 0x0f));
3897 ctxt->dst.type = OP_NONE;
3898 return X86EMUL_CONTINUE;
3901 static int em_loop(struct x86_emulate_ctxt *ctxt)
3903 int rc = X86EMUL_CONTINUE;
3905 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3906 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3907 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3908 rc = jmp_rel(ctxt, ctxt->src.val);
3913 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3915 int rc = X86EMUL_CONTINUE;
3917 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3918 rc = jmp_rel(ctxt, ctxt->src.val);
3923 static int em_in(struct x86_emulate_ctxt *ctxt)
3925 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3927 return X86EMUL_IO_NEEDED;
3929 return X86EMUL_CONTINUE;
3932 static int em_out(struct x86_emulate_ctxt *ctxt)
3934 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3936 /* Disable writeback. */
3937 ctxt->dst.type = OP_NONE;
3938 return X86EMUL_CONTINUE;
3941 static int em_cli(struct x86_emulate_ctxt *ctxt)
3943 if (emulator_bad_iopl(ctxt))
3944 return emulate_gp(ctxt, 0);
3946 ctxt->eflags &= ~X86_EFLAGS_IF;
3947 return X86EMUL_CONTINUE;
3950 static int em_sti(struct x86_emulate_ctxt *ctxt)
3952 if (emulator_bad_iopl(ctxt))
3953 return emulate_gp(ctxt, 0);
3955 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3956 ctxt->eflags |= X86_EFLAGS_IF;
3957 return X86EMUL_CONTINUE;
3960 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3962 u32 eax, ebx, ecx, edx;
3965 ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr);
3966 if (msr & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
3967 ctxt->ops->cpl(ctxt)) {
3968 return emulate_gp(ctxt, 0);
3971 eax = reg_read(ctxt, VCPU_REGS_RAX);
3972 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3973 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
3974 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3975 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3976 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3977 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3978 return X86EMUL_CONTINUE;
3981 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3985 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
3987 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3989 ctxt->eflags &= ~0xffUL;
3990 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3991 return X86EMUL_CONTINUE;
3994 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3996 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3997 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3998 return X86EMUL_CONTINUE;
4001 static int em_bswap(struct x86_emulate_ctxt *ctxt)
4003 switch (ctxt->op_bytes) {
4004 #ifdef CONFIG_X86_64
4006 asm("bswap %0" : "+r"(ctxt->dst.val));
4010 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
4013 return X86EMUL_CONTINUE;
4016 static int em_clflush(struct x86_emulate_ctxt *ctxt)
4018 /* emulating clflush regardless of cpuid */
4019 return X86EMUL_CONTINUE;
4022 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
4024 ctxt->dst.val = (s32) ctxt->src.val;
4025 return X86EMUL_CONTINUE;
4028 static int check_fxsr(struct x86_emulate_ctxt *ctxt)
4030 u32 eax = 1, ebx, ecx = 0, edx;
4032 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
4033 if (!(edx & FFL(FXSR)))
4034 return emulate_ud(ctxt);
4036 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
4037 return emulate_nm(ctxt);
4040 * Don't emulate a case that should never be hit, instead of working
4041 * around a lack of fxsave64/fxrstor64 on old compilers.
4043 if (ctxt->mode >= X86EMUL_MODE_PROT64)
4044 return X86EMUL_UNHANDLEABLE;
4046 return X86EMUL_CONTINUE;
4050 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but does save
4051 * and restore MXCSR.
4053 static size_t __fxstate_size(int nregs)
4055 return offsetof(struct fxregs_state, xmm_space[0]) + nregs * 16;
4058 static inline size_t fxstate_size(struct x86_emulate_ctxt *ctxt)
4061 if (ctxt->mode == X86EMUL_MODE_PROT64)
4062 return __fxstate_size(16);
4064 cr4_osfxsr = ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR;
4065 return __fxstate_size(cr4_osfxsr ? 8 : 0);
4069 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
4072 * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs
4073 * preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
4075 * 3) 64-bit mode with REX.W prefix
4076 * - like (2), but XMM 8-15 are being saved and restored
4077 * 4) 64-bit mode without REX.W prefix
4078 * - like (3), but FIP and FDP are 64 bit
4080 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
4081 * desired result. (4) is not emulated.
4083 * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
4084 * and FPU DS) should match.
4086 static int em_fxsave(struct x86_emulate_ctxt *ctxt)
4088 struct fxregs_state fx_state;
4091 rc = check_fxsr(ctxt);
4092 if (rc != X86EMUL_CONTINUE)
4095 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));
4097 if (rc != X86EMUL_CONTINUE)
4100 return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state,
4101 fxstate_size(ctxt));
4105 * FXRSTOR might restore XMM registers not provided by the guest. Fill
4106 * in the host registers (via FXSAVE) instead, so they won't be modified.
4107 * (preemption has to stay disabled until FXRSTOR).
4109 * Use noinline to keep the stack for other functions called by callers small.
4111 static noinline int fxregs_fixup(struct fxregs_state *fx_state,
4112 const size_t used_size)
4114 struct fxregs_state fx_tmp;
4117 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_tmp));
4118 memcpy((void *)fx_state + used_size, (void *)&fx_tmp + used_size,
4119 __fxstate_size(16) - used_size);
4124 static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
4126 struct fxregs_state fx_state;
4130 rc = check_fxsr(ctxt);
4131 if (rc != X86EMUL_CONTINUE)
4134 size = fxstate_size(ctxt);
4135 rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, size);
4136 if (rc != X86EMUL_CONTINUE)
4139 if (size < __fxstate_size(16)) {
4140 rc = fxregs_fixup(&fx_state, size);
4141 if (rc != X86EMUL_CONTINUE)
4145 if (fx_state.mxcsr >> 16) {
4146 rc = emulate_gp(ctxt, 0);
4150 if (rc == X86EMUL_CONTINUE)
4151 rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));
4157 static int em_xsetbv(struct x86_emulate_ctxt *ctxt)
4161 eax = reg_read(ctxt, VCPU_REGS_RAX);
4162 edx = reg_read(ctxt, VCPU_REGS_RDX);
4163 ecx = reg_read(ctxt, VCPU_REGS_RCX);
4165 if (ctxt->ops->set_xcr(ctxt, ecx, ((u64)edx << 32) | eax))
4166 return emulate_gp(ctxt, 0);
4168 return X86EMUL_CONTINUE;
4171 static bool valid_cr(int nr)
4183 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
4185 if (!valid_cr(ctxt->modrm_reg))
4186 return emulate_ud(ctxt);
4188 return X86EMUL_CONTINUE;
4191 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
4193 u64 new_val = ctxt->src.val64;
4194 int cr = ctxt->modrm_reg;
4197 static u64 cr_reserved_bits[] = {
4198 0xffffffff00000000ULL,
4199 0, 0, 0, /* CR3 checked later */
4206 return emulate_ud(ctxt);
4208 if (new_val & cr_reserved_bits[cr])
4209 return emulate_gp(ctxt, 0);
4214 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
4215 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
4216 return emulate_gp(ctxt, 0);
4218 cr4 = ctxt->ops->get_cr(ctxt, 4);
4219 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4221 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
4222 !(cr4 & X86_CR4_PAE))
4223 return emulate_gp(ctxt, 0);
4230 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4231 if (efer & EFER_LMA) {
4233 u32 eax, ebx, ecx, edx;
4237 if (ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx,
4239 maxphyaddr = eax & 0xff;
4242 rsvd = rsvd_bits(maxphyaddr, 63);
4243 if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PCIDE)
4244 rsvd &= ~X86_CR3_PCID_NOFLUSH;
4248 return emulate_gp(ctxt, 0);
4253 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4255 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
4256 return emulate_gp(ctxt, 0);
4262 return X86EMUL_CONTINUE;
4265 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
4269 ctxt->ops->get_dr(ctxt, 7, &dr7);
4271 /* Check if DR7.Global_Enable is set */
4272 return dr7 & (1 << 13);
4275 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
4277 int dr = ctxt->modrm_reg;
4281 return emulate_ud(ctxt);
4283 cr4 = ctxt->ops->get_cr(ctxt, 4);
4284 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
4285 return emulate_ud(ctxt);
4287 if (check_dr7_gd(ctxt)) {
4290 ctxt->ops->get_dr(ctxt, 6, &dr6);
4291 dr6 &= ~DR_TRAP_BITS;
4292 dr6 |= DR6_BD | DR6_RTM;
4293 ctxt->ops->set_dr(ctxt, 6, dr6);
4294 return emulate_db(ctxt);
4297 return X86EMUL_CONTINUE;
4300 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
4302 u64 new_val = ctxt->src.val64;
4303 int dr = ctxt->modrm_reg;
4305 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
4306 return emulate_gp(ctxt, 0);
4308 return check_dr_read(ctxt);
4311 static int check_svme(struct x86_emulate_ctxt *ctxt)
4315 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4317 if (!(efer & EFER_SVME))
4318 return emulate_ud(ctxt);
4320 return X86EMUL_CONTINUE;
4323 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
4325 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
4327 /* Valid physical address? */
4328 if (rax & 0xffff000000000000ULL)
4329 return emulate_gp(ctxt, 0);
4331 return check_svme(ctxt);
4334 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
4336 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4338 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
4339 return emulate_ud(ctxt);
4341 return X86EMUL_CONTINUE;
4344 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
4346 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4347 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
4350 * VMware allows access to these Pseduo-PMCs even when read via RDPMC
4351 * in Ring3 when CR4.PCE=0.
4353 if (enable_vmware_backdoor && is_vmware_backdoor_pmc(rcx))
4354 return X86EMUL_CONTINUE;
4356 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
4357 ctxt->ops->check_pmc(ctxt, rcx))
4358 return emulate_gp(ctxt, 0);
4360 return X86EMUL_CONTINUE;
4363 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
4365 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
4366 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
4367 return emulate_gp(ctxt, 0);
4369 return X86EMUL_CONTINUE;
4372 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
4374 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
4375 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
4376 return emulate_gp(ctxt, 0);
4378 return X86EMUL_CONTINUE;
4381 #define D(_y) { .flags = (_y) }
4382 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4383 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4384 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4385 #define N D(NotImpl)
4386 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4387 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4388 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4389 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4390 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4391 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4392 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4393 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4394 #define II(_f, _e, _i) \
4395 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4396 #define IIP(_f, _e, _i, _p) \
4397 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4398 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4399 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4401 #define D2bv(_f) D((_f) | ByteOp), D(_f)
4402 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4403 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4404 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4405 #define I2bvIP(_f, _e, _i, _p) \
4406 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4408 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4409 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4410 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4412 static const struct opcode group7_rm0[] = {
4414 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
4418 static const struct opcode group7_rm1[] = {
4419 DI(SrcNone | Priv, monitor),
4420 DI(SrcNone | Priv, mwait),
4424 static const struct opcode group7_rm2[] = {
4426 II(ImplicitOps | Priv, em_xsetbv, xsetbv),
4430 static const struct opcode group7_rm3[] = {
4431 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
4432 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
4433 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
4434 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
4435 DIP(SrcNone | Prot | Priv, stgi, check_svme),
4436 DIP(SrcNone | Prot | Priv, clgi, check_svme),
4437 DIP(SrcNone | Prot | Priv, skinit, check_svme),
4438 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
4441 static const struct opcode group7_rm7[] = {
4443 DIP(SrcNone, rdtscp, check_rdtsc),
4447 static const struct opcode group1[] = {
4449 F(Lock | PageTable, em_or),
4452 F(Lock | PageTable, em_and),
4458 static const struct opcode group1A[] = {
4459 I(DstMem | SrcNone | Mov | Stack | IncSP | TwoMemOp, em_pop), N, N, N, N, N, N, N,
4462 static const struct opcode group2[] = {
4463 F(DstMem | ModRM, em_rol),
4464 F(DstMem | ModRM, em_ror),
4465 F(DstMem | ModRM, em_rcl),
4466 F(DstMem | ModRM, em_rcr),
4467 F(DstMem | ModRM, em_shl),
4468 F(DstMem | ModRM, em_shr),
4469 F(DstMem | ModRM, em_shl),
4470 F(DstMem | ModRM, em_sar),
4473 static const struct opcode group3[] = {
4474 F(DstMem | SrcImm | NoWrite, em_test),
4475 F(DstMem | SrcImm | NoWrite, em_test),
4476 F(DstMem | SrcNone | Lock, em_not),
4477 F(DstMem | SrcNone | Lock, em_neg),
4478 F(DstXacc | Src2Mem, em_mul_ex),
4479 F(DstXacc | Src2Mem, em_imul_ex),
4480 F(DstXacc | Src2Mem, em_div_ex),
4481 F(DstXacc | Src2Mem, em_idiv_ex),
4484 static const struct opcode group4[] = {
4485 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
4486 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
4490 static const struct opcode group5[] = {
4491 F(DstMem | SrcNone | Lock, em_inc),
4492 F(DstMem | SrcNone | Lock, em_dec),
4493 I(SrcMem | NearBranch, em_call_near_abs),
4494 I(SrcMemFAddr | ImplicitOps, em_call_far),
4495 I(SrcMem | NearBranch, em_jmp_abs),
4496 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
4497 I(SrcMem | Stack | TwoMemOp, em_push), D(Undefined),
4500 static const struct opcode group6[] = {
4501 II(Prot | DstMem, em_sldt, sldt),
4502 II(Prot | DstMem, em_str, str),
4503 II(Prot | Priv | SrcMem16, em_lldt, lldt),
4504 II(Prot | Priv | SrcMem16, em_ltr, ltr),
4508 static const struct group_dual group7 = { {
4509 II(Mov | DstMem, em_sgdt, sgdt),
4510 II(Mov | DstMem, em_sidt, sidt),
4511 II(SrcMem | Priv, em_lgdt, lgdt),
4512 II(SrcMem | Priv, em_lidt, lidt),
4513 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4514 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4515 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
4521 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4522 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4526 static const struct opcode group8[] = {
4528 F(DstMem | SrcImmByte | NoWrite, em_bt),
4529 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
4530 F(DstMem | SrcImmByte | Lock, em_btr),
4531 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
4535 * The "memory" destination is actually always a register, since we come
4536 * from the register case of group9.
4538 static const struct gprefix pfx_0f_c7_7 = {
4539 N, N, N, II(DstMem | ModRM | Op3264 | EmulateOnUD, em_rdpid, rdtscp),
4543 static const struct group_dual group9 = { {
4544 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
4546 N, N, N, N, N, N, N,
4547 GP(0, &pfx_0f_c7_7),
4550 static const struct opcode group11[] = {
4551 I(DstMem | SrcImm | Mov | PageTable, em_mov),
4555 static const struct gprefix pfx_0f_ae_7 = {
4556 I(SrcMem | ByteOp, em_clflush), N, N, N,
4559 static const struct group_dual group15 = { {
4560 I(ModRM | Aligned16, em_fxsave),
4561 I(ModRM | Aligned16, em_fxrstor),
4562 N, N, N, N, N, GP(0, &pfx_0f_ae_7),
4564 N, N, N, N, N, N, N, N,
4567 static const struct gprefix pfx_0f_6f_0f_7f = {
4568 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
4571 static const struct instr_dual instr_dual_0f_2b = {
4575 static const struct gprefix pfx_0f_2b = {
4576 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
4579 static const struct gprefix pfx_0f_10_0f_11 = {
4580 I(Unaligned, em_mov), I(Unaligned, em_mov), N, N,
4583 static const struct gprefix pfx_0f_28_0f_29 = {
4584 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4587 static const struct gprefix pfx_0f_e7 = {
4588 N, I(Sse, em_mov), N, N,
4591 static const struct escape escape_d9 = { {
4592 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
4595 N, N, N, N, N, N, N, N,
4597 N, N, N, N, N, N, N, N,
4599 N, N, N, N, N, N, N, N,
4601 N, N, N, N, N, N, N, N,
4603 N, N, N, N, N, N, N, N,
4605 N, N, N, N, N, N, N, N,
4607 N, N, N, N, N, N, N, N,
4609 N, N, N, N, N, N, N, N,
4612 static const struct escape escape_db = { {
4613 N, N, N, N, N, N, N, N,
4616 N, N, N, N, N, N, N, N,
4618 N, N, N, N, N, N, N, N,
4620 N, N, N, N, N, N, N, N,
4622 N, N, N, N, N, N, N, N,
4624 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
4626 N, N, N, N, N, N, N, N,
4628 N, N, N, N, N, N, N, N,
4630 N, N, N, N, N, N, N, N,
4633 static const struct escape escape_dd = { {
4634 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4637 N, N, N, N, N, N, N, N,
4639 N, N, N, N, N, N, N, N,
4641 N, N, N, N, N, N, N, N,
4643 N, N, N, N, N, N, N, N,
4645 N, N, N, N, N, N, N, N,
4647 N, N, N, N, N, N, N, N,
4649 N, N, N, N, N, N, N, N,
4651 N, N, N, N, N, N, N, N,
4654 static const struct instr_dual instr_dual_0f_c3 = {
4655 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
4658 static const struct mode_dual mode_dual_63 = {
4659 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
4662 static const struct opcode opcode_table[256] = {
4664 F6ALU(Lock, em_add),
4665 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4666 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4668 F6ALU(Lock | PageTable, em_or),
4669 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4672 F6ALU(Lock, em_adc),
4673 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4674 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4676 F6ALU(Lock, em_sbb),
4677 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4678 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4680 F6ALU(Lock | PageTable, em_and), N, N,
4682 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4684 F6ALU(Lock, em_xor), N, N,
4686 F6ALU(NoWrite, em_cmp), N, N,
4688 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4690 X8(I(SrcReg | Stack, em_push)),
4692 X8(I(DstReg | Stack, em_pop)),
4694 I(ImplicitOps | Stack | No64, em_pusha),
4695 I(ImplicitOps | Stack | No64, em_popa),
4696 N, MD(ModRM, &mode_dual_63),
4699 I(SrcImm | Mov | Stack, em_push),
4700 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4701 I(SrcImmByte | Mov | Stack, em_push),
4702 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4703 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4704 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4706 X16(D(SrcImmByte | NearBranch)),
4708 G(ByteOp | DstMem | SrcImm, group1),
4709 G(DstMem | SrcImm, group1),
4710 G(ByteOp | DstMem | SrcImm | No64, group1),
4711 G(DstMem | SrcImmByte, group1),
4712 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4713 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4715 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4716 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4717 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4718 D(ModRM | SrcMem | NoAccess | DstReg),
4719 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4722 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4724 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4725 I(SrcImmFAddr | No64, em_call_far), N,
4726 II(ImplicitOps | Stack, em_pushf, pushf),
4727 II(ImplicitOps | Stack, em_popf, popf),
4728 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4730 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4731 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4732 I2bv(SrcSI | DstDI | Mov | String | TwoMemOp, em_mov),
4733 F2bv(SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r),
4735 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4736 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4737 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4738 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4740 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4742 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4744 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4745 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4746 I(ImplicitOps | NearBranch, em_ret),
4747 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4748 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4749 G(ByteOp, group11), G(0, group11),
4751 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4752 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4753 I(ImplicitOps, em_ret_far),
4754 D(ImplicitOps), DI(SrcImmByte, intn),
4755 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4757 G(Src2One | ByteOp, group2), G(Src2One, group2),
4758 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4759 I(DstAcc | SrcImmUByte | No64, em_aam),
4760 I(DstAcc | SrcImmUByte | No64, em_aad),
4761 F(DstAcc | ByteOp | No64, em_salc),
4762 I(DstAcc | SrcXLat | ByteOp, em_mov),
4764 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4766 X3(I(SrcImmByte | NearBranch, em_loop)),
4767 I(SrcImmByte | NearBranch, em_jcxz),
4768 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4769 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4771 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4772 I(SrcImmFAddr | No64, em_jmp_far),
4773 D(SrcImmByte | ImplicitOps | NearBranch),
4774 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4775 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4777 N, DI(ImplicitOps, icebp), N, N,
4778 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4779 G(ByteOp, group3), G(0, group3),
4781 D(ImplicitOps), D(ImplicitOps),
4782 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4783 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4786 static const struct opcode twobyte_table[256] = {
4788 G(0, group6), GD(0, &group7), N, N,
4789 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4790 II(ImplicitOps | Priv, em_clts, clts), N,
4791 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4792 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4794 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_10_0f_11),
4795 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_10_0f_11),
4797 D(ImplicitOps | ModRM | SrcMem | NoAccess),
4798 N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4800 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4801 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4802 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4804 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4807 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4808 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4809 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4812 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4813 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4814 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4815 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4816 I(ImplicitOps | EmulateOnUD, em_sysenter),
4817 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4819 N, N, N, N, N, N, N, N,
4821 X16(D(DstReg | SrcMem | ModRM)),
4823 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4828 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4833 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4835 X16(D(SrcImm | NearBranch)),
4837 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4839 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4840 II(ImplicitOps, em_cpuid, cpuid),
4841 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4842 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4843 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4845 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4846 II(EmulateOnUD | ImplicitOps, em_rsm, rsm),
4847 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4848 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4849 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4850 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4852 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4853 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4854 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4855 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4856 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4857 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4861 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4862 I(DstReg | SrcMem | ModRM, em_bsf_c),
4863 I(DstReg | SrcMem | ModRM, em_bsr_c),
4864 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4866 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4867 N, ID(0, &instr_dual_0f_c3),
4868 N, N, N, GD(0, &group9),
4870 X8(I(DstReg, em_bswap)),
4872 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4874 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4875 N, N, N, N, N, N, N, N,
4877 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4880 static const struct instr_dual instr_dual_0f_38_f0 = {
4881 I(DstReg | SrcMem | Mov, em_movbe), N
4884 static const struct instr_dual instr_dual_0f_38_f1 = {
4885 I(DstMem | SrcReg | Mov, em_movbe), N
4888 static const struct gprefix three_byte_0f_38_f0 = {
4889 ID(0, &instr_dual_0f_38_f0), N, N, N
4892 static const struct gprefix three_byte_0f_38_f1 = {
4893 ID(0, &instr_dual_0f_38_f1), N, N, N
4897 * Insns below are selected by the prefix which indexed by the third opcode
4900 static const struct opcode opcode_map_0f_38[256] = {
4902 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4904 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4906 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4907 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4928 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4932 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4938 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4939 unsigned size, bool sign_extension)
4941 int rc = X86EMUL_CONTINUE;
4945 op->addr.mem.ea = ctxt->_eip;
4946 /* NB. Immediates are sign-extended as necessary. */
4947 switch (op->bytes) {
4949 op->val = insn_fetch(s8, ctxt);
4952 op->val = insn_fetch(s16, ctxt);
4955 op->val = insn_fetch(s32, ctxt);
4958 op->val = insn_fetch(s64, ctxt);
4961 if (!sign_extension) {
4962 switch (op->bytes) {
4970 op->val &= 0xffffffff;
4978 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4981 int rc = X86EMUL_CONTINUE;
4985 decode_register_operand(ctxt, op);
4988 rc = decode_imm(ctxt, op, 1, false);
4991 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4995 if (ctxt->d & BitOp)
4996 fetch_bit_operand(ctxt);
4997 op->orig_val = op->val;
5000 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
5004 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5005 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
5006 fetch_register_operand(op);
5007 op->orig_val = op->val;
5011 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
5012 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
5013 fetch_register_operand(op);
5014 op->orig_val = op->val;
5017 if (ctxt->d & ByteOp) {
5022 op->bytes = ctxt->op_bytes;
5023 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
5024 fetch_register_operand(op);
5025 op->orig_val = op->val;
5029 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5031 register_address(ctxt, VCPU_REGS_RDI);
5032 op->addr.mem.seg = VCPU_SREG_ES;
5039 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
5040 fetch_register_operand(op);
5045 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
5048 rc = decode_imm(ctxt, op, 1, true);
5056 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
5059 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
5062 ctxt->memop.bytes = 1;
5063 if (ctxt->memop.type == OP_REG) {
5064 ctxt->memop.addr.reg = decode_register(ctxt,
5065 ctxt->modrm_rm, true);
5066 fetch_register_operand(&ctxt->memop);
5070 ctxt->memop.bytes = 2;
5073 ctxt->memop.bytes = 4;
5076 rc = decode_imm(ctxt, op, 2, false);
5079 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
5083 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5085 register_address(ctxt, VCPU_REGS_RSI);
5086 op->addr.mem.seg = ctxt->seg_override;
5092 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5095 reg_read(ctxt, VCPU_REGS_RBX) +
5096 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
5097 op->addr.mem.seg = ctxt->seg_override;
5102 op->addr.mem.ea = ctxt->_eip;
5103 op->bytes = ctxt->op_bytes + 2;
5104 insn_fetch_arr(op->valptr, op->bytes, ctxt);
5107 ctxt->memop.bytes = ctxt->op_bytes + 2;
5111 op->val = VCPU_SREG_ES;
5115 op->val = VCPU_SREG_CS;
5119 op->val = VCPU_SREG_SS;
5123 op->val = VCPU_SREG_DS;
5127 op->val = VCPU_SREG_FS;
5131 op->val = VCPU_SREG_GS;
5134 /* Special instructions do their own operand decoding. */
5136 op->type = OP_NONE; /* Disable writeback. */
5144 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
5146 int rc = X86EMUL_CONTINUE;
5147 int mode = ctxt->mode;
5148 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
5149 bool op_prefix = false;
5150 bool has_seg_override = false;
5151 struct opcode opcode;
5153 struct desc_struct desc;
5155 ctxt->memop.type = OP_NONE;
5156 ctxt->memopp = NULL;
5157 ctxt->_eip = ctxt->eip;
5158 ctxt->fetch.ptr = ctxt->fetch.data;
5159 ctxt->fetch.end = ctxt->fetch.data + insn_len;
5160 ctxt->opcode_len = 1;
5162 memcpy(ctxt->fetch.data, insn, insn_len);
5164 rc = __do_insn_fetch_bytes(ctxt, 1);
5165 if (rc != X86EMUL_CONTINUE)
5170 case X86EMUL_MODE_REAL:
5171 case X86EMUL_MODE_VM86:
5172 def_op_bytes = def_ad_bytes = 2;
5173 ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS);
5175 def_op_bytes = def_ad_bytes = 4;
5177 case X86EMUL_MODE_PROT16:
5178 def_op_bytes = def_ad_bytes = 2;
5180 case X86EMUL_MODE_PROT32:
5181 def_op_bytes = def_ad_bytes = 4;
5183 #ifdef CONFIG_X86_64
5184 case X86EMUL_MODE_PROT64:
5190 return EMULATION_FAILED;
5193 ctxt->op_bytes = def_op_bytes;
5194 ctxt->ad_bytes = def_ad_bytes;
5196 /* Legacy prefixes. */
5198 switch (ctxt->b = insn_fetch(u8, ctxt)) {
5199 case 0x66: /* operand-size override */
5201 /* switch between 2/4 bytes */
5202 ctxt->op_bytes = def_op_bytes ^ 6;
5204 case 0x67: /* address-size override */
5205 if (mode == X86EMUL_MODE_PROT64)
5206 /* switch between 4/8 bytes */
5207 ctxt->ad_bytes = def_ad_bytes ^ 12;
5209 /* switch between 2/4 bytes */
5210 ctxt->ad_bytes = def_ad_bytes ^ 6;
5212 case 0x26: /* ES override */
5213 case 0x2e: /* CS override */
5214 case 0x36: /* SS override */
5215 case 0x3e: /* DS override */
5216 has_seg_override = true;
5217 ctxt->seg_override = (ctxt->b >> 3) & 3;
5219 case 0x64: /* FS override */
5220 case 0x65: /* GS override */
5221 has_seg_override = true;
5222 ctxt->seg_override = ctxt->b & 7;
5224 case 0x40 ... 0x4f: /* REX */
5225 if (mode != X86EMUL_MODE_PROT64)
5227 ctxt->rex_prefix = ctxt->b;
5229 case 0xf0: /* LOCK */
5230 ctxt->lock_prefix = 1;
5232 case 0xf2: /* REPNE/REPNZ */
5233 case 0xf3: /* REP/REPE/REPZ */
5234 ctxt->rep_prefix = ctxt->b;
5240 /* Any legacy prefix after a REX prefix nullifies its effect. */
5242 ctxt->rex_prefix = 0;
5248 if (ctxt->rex_prefix & 8)
5249 ctxt->op_bytes = 8; /* REX.W */
5251 /* Opcode byte(s). */
5252 opcode = opcode_table[ctxt->b];
5253 /* Two-byte opcode? */
5254 if (ctxt->b == 0x0f) {
5255 ctxt->opcode_len = 2;
5256 ctxt->b = insn_fetch(u8, ctxt);
5257 opcode = twobyte_table[ctxt->b];
5259 /* 0F_38 opcode map */
5260 if (ctxt->b == 0x38) {
5261 ctxt->opcode_len = 3;
5262 ctxt->b = insn_fetch(u8, ctxt);
5263 opcode = opcode_map_0f_38[ctxt->b];
5266 ctxt->d = opcode.flags;
5268 if (ctxt->d & ModRM)
5269 ctxt->modrm = insn_fetch(u8, ctxt);
5271 /* vex-prefix instructions are not implemented */
5272 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
5273 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
5277 while (ctxt->d & GroupMask) {
5278 switch (ctxt->d & GroupMask) {
5280 goffset = (ctxt->modrm >> 3) & 7;
5281 opcode = opcode.u.group[goffset];
5284 goffset = (ctxt->modrm >> 3) & 7;
5285 if ((ctxt->modrm >> 6) == 3)
5286 opcode = opcode.u.gdual->mod3[goffset];
5288 opcode = opcode.u.gdual->mod012[goffset];
5291 goffset = ctxt->modrm & 7;
5292 opcode = opcode.u.group[goffset];
5295 if (ctxt->rep_prefix && op_prefix)
5296 return EMULATION_FAILED;
5297 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
5298 switch (simd_prefix) {
5299 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
5300 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
5301 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
5302 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
5306 if (ctxt->modrm > 0xbf)
5307 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
5309 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
5312 if ((ctxt->modrm >> 6) == 3)
5313 opcode = opcode.u.idual->mod3;
5315 opcode = opcode.u.idual->mod012;
5318 if (ctxt->mode == X86EMUL_MODE_PROT64)
5319 opcode = opcode.u.mdual->mode64;
5321 opcode = opcode.u.mdual->mode32;
5324 return EMULATION_FAILED;
5327 ctxt->d &= ~(u64)GroupMask;
5328 ctxt->d |= opcode.flags;
5333 return EMULATION_FAILED;
5335 ctxt->execute = opcode.u.execute;
5337 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
5338 return EMULATION_FAILED;
5340 if (unlikely(ctxt->d &
5341 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
5344 * These are copied unconditionally here, and checked unconditionally
5345 * in x86_emulate_insn.
5347 ctxt->check_perm = opcode.check_perm;
5348 ctxt->intercept = opcode.intercept;
5350 if (ctxt->d & NotImpl)
5351 return EMULATION_FAILED;
5353 if (mode == X86EMUL_MODE_PROT64) {
5354 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
5356 else if (ctxt->d & NearBranch)
5360 if (ctxt->d & Op3264) {
5361 if (mode == X86EMUL_MODE_PROT64)
5367 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
5371 ctxt->op_bytes = 16;
5372 else if (ctxt->d & Mmx)
5376 /* ModRM and SIB bytes. */
5377 if (ctxt->d & ModRM) {
5378 rc = decode_modrm(ctxt, &ctxt->memop);
5379 if (!has_seg_override) {
5380 has_seg_override = true;
5381 ctxt->seg_override = ctxt->modrm_seg;
5383 } else if (ctxt->d & MemAbs)
5384 rc = decode_abs(ctxt, &ctxt->memop);
5385 if (rc != X86EMUL_CONTINUE)
5388 if (!has_seg_override)
5389 ctxt->seg_override = VCPU_SREG_DS;
5391 ctxt->memop.addr.mem.seg = ctxt->seg_override;
5394 * Decode and fetch the source operand: register, memory
5397 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
5398 if (rc != X86EMUL_CONTINUE)
5402 * Decode and fetch the second source operand: register, memory
5405 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
5406 if (rc != X86EMUL_CONTINUE)
5409 /* Decode and fetch the destination operand: register or memory. */
5410 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
5412 if (ctxt->rip_relative && likely(ctxt->memopp))
5413 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
5414 ctxt->memopp->addr.mem.ea + ctxt->_eip);
5417 if (rc == X86EMUL_PROPAGATE_FAULT)
5418 ctxt->have_exception = true;
5419 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5422 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
5424 return ctxt->d & PageTable;
5427 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
5429 /* The second termination condition only applies for REPE
5430 * and REPNE. Test if the repeat string operation prefix is
5431 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
5432 * corresponding termination condition according to:
5433 * - if REPE/REPZ and ZF = 0 then done
5434 * - if REPNE/REPNZ and ZF = 1 then done
5436 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
5437 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
5438 && (((ctxt->rep_prefix == REPE_PREFIX) &&
5439 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
5440 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
5441 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
5447 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
5451 rc = asm_safe("fwait");
5453 if (unlikely(rc != X86EMUL_CONTINUE))
5454 return emulate_exception(ctxt, MF_VECTOR, 0, false);
5456 return X86EMUL_CONTINUE;
5459 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
5462 if (op->type == OP_MM)
5463 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
5466 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
5468 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5470 if (!(ctxt->d & ByteOp))
5471 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5473 asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n"
5474 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5475 [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT
5476 : "c"(ctxt->src2.val));
5478 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5479 if (!fop) /* exception is returned in fop variable */
5480 return emulate_de(ctxt);
5481 return X86EMUL_CONTINUE;
5484 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5486 memset(&ctxt->rip_relative, 0,
5487 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
5489 ctxt->io_read.pos = 0;
5490 ctxt->io_read.end = 0;
5491 ctxt->mem_read.end = 0;
5494 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
5496 const struct x86_emulate_ops *ops = ctxt->ops;
5497 int rc = X86EMUL_CONTINUE;
5498 int saved_dst_type = ctxt->dst.type;
5499 unsigned emul_flags;
5501 ctxt->mem_read.pos = 0;
5503 /* LOCK prefix is allowed only with some instructions */
5504 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
5505 rc = emulate_ud(ctxt);
5509 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
5510 rc = emulate_ud(ctxt);
5514 emul_flags = ctxt->ops->get_hflags(ctxt);
5515 if (unlikely(ctxt->d &
5516 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
5517 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
5518 (ctxt->d & Undefined)) {
5519 rc = emulate_ud(ctxt);
5523 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
5524 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
5525 rc = emulate_ud(ctxt);
5529 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
5530 rc = emulate_nm(ctxt);
5534 if (ctxt->d & Mmx) {
5535 rc = flush_pending_x87_faults(ctxt);
5536 if (rc != X86EMUL_CONTINUE)
5539 * Now that we know the fpu is exception safe, we can fetch
5542 fetch_possible_mmx_operand(ctxt, &ctxt->src);
5543 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
5544 if (!(ctxt->d & Mov))
5545 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
5548 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
5549 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5550 X86_ICPT_PRE_EXCEPT);
5551 if (rc != X86EMUL_CONTINUE)
5555 /* Instruction can only be executed in protected mode */
5556 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
5557 rc = emulate_ud(ctxt);
5561 /* Privileged instruction can be executed only in CPL=0 */
5562 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
5563 if (ctxt->d & PrivUD)
5564 rc = emulate_ud(ctxt);
5566 rc = emulate_gp(ctxt, 0);
5570 /* Do instruction specific permission checks */
5571 if (ctxt->d & CheckPerm) {
5572 rc = ctxt->check_perm(ctxt);
5573 if (rc != X86EMUL_CONTINUE)
5577 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5578 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5579 X86_ICPT_POST_EXCEPT);
5580 if (rc != X86EMUL_CONTINUE)
5584 if (ctxt->rep_prefix && (ctxt->d & String)) {
5585 /* All REP prefixes have the same first termination condition */
5586 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
5587 string_registers_quirk(ctxt);
5588 ctxt->eip = ctxt->_eip;
5589 ctxt->eflags &= ~X86_EFLAGS_RF;
5595 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
5596 rc = segmented_read(ctxt, ctxt->src.addr.mem,
5597 ctxt->src.valptr, ctxt->src.bytes);
5598 if (rc != X86EMUL_CONTINUE)
5600 ctxt->src.orig_val64 = ctxt->src.val64;
5603 if (ctxt->src2.type == OP_MEM) {
5604 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
5605 &ctxt->src2.val, ctxt->src2.bytes);
5606 if (rc != X86EMUL_CONTINUE)
5610 if ((ctxt->d & DstMask) == ImplicitOps)
5614 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
5615 /* optimisation - avoid slow emulated read if Mov */
5616 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
5617 &ctxt->dst.val, ctxt->dst.bytes);
5618 if (rc != X86EMUL_CONTINUE) {
5619 if (!(ctxt->d & NoWrite) &&
5620 rc == X86EMUL_PROPAGATE_FAULT &&
5621 ctxt->exception.vector == PF_VECTOR)
5622 ctxt->exception.error_code |= PFERR_WRITE_MASK;
5626 /* Copy full 64-bit value for CMPXCHG8B. */
5627 ctxt->dst.orig_val64 = ctxt->dst.val64;
5631 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5632 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5633 X86_ICPT_POST_MEMACCESS);
5634 if (rc != X86EMUL_CONTINUE)
5638 if (ctxt->rep_prefix && (ctxt->d & String))
5639 ctxt->eflags |= X86_EFLAGS_RF;
5641 ctxt->eflags &= ~X86_EFLAGS_RF;
5643 if (ctxt->execute) {
5644 if (ctxt->d & Fastop) {
5645 void (*fop)(struct fastop *) = (void *)ctxt->execute;
5646 rc = fastop(ctxt, fop);
5647 if (rc != X86EMUL_CONTINUE)
5651 rc = ctxt->execute(ctxt);
5652 if (rc != X86EMUL_CONTINUE)
5657 if (ctxt->opcode_len == 2)
5659 else if (ctxt->opcode_len == 3)
5660 goto threebyte_insn;
5663 case 0x70 ... 0x7f: /* jcc (short) */
5664 if (test_cc(ctxt->b, ctxt->eflags))
5665 rc = jmp_rel(ctxt, ctxt->src.val);
5667 case 0x8d: /* lea r16/r32, m */
5668 ctxt->dst.val = ctxt->src.addr.mem.ea;
5670 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5671 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5672 ctxt->dst.type = OP_NONE;
5676 case 0x98: /* cbw/cwde/cdqe */
5677 switch (ctxt->op_bytes) {
5678 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5679 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5680 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5683 case 0xcc: /* int3 */
5684 rc = emulate_int(ctxt, 3);
5686 case 0xcd: /* int n */
5687 rc = emulate_int(ctxt, ctxt->src.val);
5689 case 0xce: /* into */
5690 if (ctxt->eflags & X86_EFLAGS_OF)
5691 rc = emulate_int(ctxt, 4);
5693 case 0xe9: /* jmp rel */
5694 case 0xeb: /* jmp rel short */
5695 rc = jmp_rel(ctxt, ctxt->src.val);
5696 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5698 case 0xf4: /* hlt */
5699 ctxt->ops->halt(ctxt);
5701 case 0xf5: /* cmc */
5702 /* complement carry flag from eflags reg */
5703 ctxt->eflags ^= X86_EFLAGS_CF;
5705 case 0xf8: /* clc */
5706 ctxt->eflags &= ~X86_EFLAGS_CF;
5708 case 0xf9: /* stc */
5709 ctxt->eflags |= X86_EFLAGS_CF;
5711 case 0xfc: /* cld */
5712 ctxt->eflags &= ~X86_EFLAGS_DF;
5714 case 0xfd: /* std */
5715 ctxt->eflags |= X86_EFLAGS_DF;
5718 goto cannot_emulate;
5721 if (rc != X86EMUL_CONTINUE)
5725 if (ctxt->d & SrcWrite) {
5726 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5727 rc = writeback(ctxt, &ctxt->src);
5728 if (rc != X86EMUL_CONTINUE)
5731 if (!(ctxt->d & NoWrite)) {
5732 rc = writeback(ctxt, &ctxt->dst);
5733 if (rc != X86EMUL_CONTINUE)
5738 * restore dst type in case the decoding will be reused
5739 * (happens for string instruction )
5741 ctxt->dst.type = saved_dst_type;
5743 if ((ctxt->d & SrcMask) == SrcSI)
5744 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5746 if ((ctxt->d & DstMask) == DstDI)
5747 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5749 if (ctxt->rep_prefix && (ctxt->d & String)) {
5751 struct read_cache *r = &ctxt->io_read;
5752 if ((ctxt->d & SrcMask) == SrcSI)
5753 count = ctxt->src.count;
5755 count = ctxt->dst.count;
5756 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5758 if (!string_insn_completed(ctxt)) {
5760 * Re-enter guest when pio read ahead buffer is empty
5761 * or, if it is not used, after each 1024 iteration.
5763 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5764 (r->end == 0 || r->end != r->pos)) {
5766 * Reset read cache. Usually happens before
5767 * decode, but since instruction is restarted
5768 * we have to do it here.
5770 ctxt->mem_read.end = 0;
5771 writeback_registers(ctxt);
5772 return EMULATION_RESTART;
5774 goto done; /* skip rip writeback */
5776 ctxt->eflags &= ~X86_EFLAGS_RF;
5779 ctxt->eip = ctxt->_eip;
5782 if (rc == X86EMUL_PROPAGATE_FAULT) {
5783 WARN_ON(ctxt->exception.vector > 0x1f);
5784 ctxt->have_exception = true;
5786 if (rc == X86EMUL_INTERCEPTED)
5787 return EMULATION_INTERCEPTED;
5789 if (rc == X86EMUL_CONTINUE)
5790 writeback_registers(ctxt);
5792 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5796 case 0x09: /* wbinvd */
5797 (ctxt->ops->wbinvd)(ctxt);
5799 case 0x08: /* invd */
5800 case 0x0d: /* GrpP (prefetch) */
5801 case 0x18: /* Grp16 (prefetch/nop) */
5802 case 0x1f: /* nop */
5804 case 0x20: /* mov cr, reg */
5805 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5807 case 0x21: /* mov from dr to reg */
5808 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5810 case 0x40 ... 0x4f: /* cmov */
5811 if (test_cc(ctxt->b, ctxt->eflags))
5812 ctxt->dst.val = ctxt->src.val;
5813 else if (ctxt->op_bytes != 4)
5814 ctxt->dst.type = OP_NONE; /* no writeback */
5816 case 0x80 ... 0x8f: /* jnz rel, etc*/
5817 if (test_cc(ctxt->b, ctxt->eflags))
5818 rc = jmp_rel(ctxt, ctxt->src.val);
5820 case 0x90 ... 0x9f: /* setcc r/m8 */
5821 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5823 case 0xb6 ... 0xb7: /* movzx */
5824 ctxt->dst.bytes = ctxt->op_bytes;
5825 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5826 : (u16) ctxt->src.val;
5828 case 0xbe ... 0xbf: /* movsx */
5829 ctxt->dst.bytes = ctxt->op_bytes;
5830 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5831 (s16) ctxt->src.val;
5834 goto cannot_emulate;
5839 if (rc != X86EMUL_CONTINUE)
5845 return EMULATION_FAILED;
5848 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5850 invalidate_registers(ctxt);
5853 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5855 writeback_registers(ctxt);
5858 bool emulator_can_use_gpa(struct x86_emulate_ctxt *ctxt)
5860 if (ctxt->rep_prefix && (ctxt->d & String))
5863 if (ctxt->d & TwoMemOp)