1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstMem16 (OpMem16 << DstShift)
90 #define DstImmUByte (OpImmUByte << DstShift)
91 #define DstDX (OpDX << DstShift)
92 #define DstAccLo (OpAccLo << DstShift)
93 #define DstMask (OpMask << DstShift)
94 /* Source operand type. */
96 #define SrcNone (OpNone << SrcShift)
97 #define SrcReg (OpReg << SrcShift)
98 #define SrcMem (OpMem << SrcShift)
99 #define SrcMem16 (OpMem16 << SrcShift)
100 #define SrcMem32 (OpMem32 << SrcShift)
101 #define SrcImm (OpImm << SrcShift)
102 #define SrcImmByte (OpImmByte << SrcShift)
103 #define SrcOne (OpOne << SrcShift)
104 #define SrcImmUByte (OpImmUByte << SrcShift)
105 #define SrcImmU (OpImmU << SrcShift)
106 #define SrcSI (OpSI << SrcShift)
107 #define SrcXLat (OpXLat << SrcShift)
108 #define SrcImmFAddr (OpImmFAddr << SrcShift)
109 #define SrcMemFAddr (OpMemFAddr << SrcShift)
110 #define SrcAcc (OpAcc << SrcShift)
111 #define SrcImmU16 (OpImmU16 << SrcShift)
112 #define SrcImm64 (OpImm64 << SrcShift)
113 #define SrcDX (OpDX << SrcShift)
114 #define SrcMem8 (OpMem8 << SrcShift)
115 #define SrcAccHi (OpAccHi << SrcShift)
116 #define SrcMask (OpMask << SrcShift)
117 #define BitOp (1<<11)
118 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
119 #define String (1<<13) /* String instruction (rep capable) */
120 #define Stack (1<<14) /* Stack instruction (push/pop) */
121 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
122 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
123 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
124 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
125 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
126 #define Escape (5<<15) /* Escape to coprocessor instruction */
127 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
128 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
129 #define Sse (1<<18) /* SSE Vector instruction */
130 /* Generic ModRM decode. */
131 #define ModRM (1<<19)
132 /* Destination is only written; never read. */
135 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
136 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
137 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
138 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
139 #define Undefined (1<<25) /* No Such Instruction */
140 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
141 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
143 #define PageTable (1 << 29) /* instruction used to write page table */
144 #define NotImpl (1 << 30) /* instruction is not implemented */
145 /* Source 2 operand type */
146 #define Src2Shift (31)
147 #define Src2None (OpNone << Src2Shift)
148 #define Src2Mem (OpMem << Src2Shift)
149 #define Src2CL (OpCL << Src2Shift)
150 #define Src2ImmByte (OpImmByte << Src2Shift)
151 #define Src2One (OpOne << Src2Shift)
152 #define Src2Imm (OpImm << Src2Shift)
153 #define Src2ES (OpES << Src2Shift)
154 #define Src2CS (OpCS << Src2Shift)
155 #define Src2SS (OpSS << Src2Shift)
156 #define Src2DS (OpDS << Src2Shift)
157 #define Src2FS (OpFS << Src2Shift)
158 #define Src2GS (OpGS << Src2Shift)
159 #define Src2Mask (OpMask << Src2Shift)
160 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
161 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
162 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
163 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
164 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
165 #define NoWrite ((u64)1 << 45) /* No writeback */
166 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
167 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
168 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
169 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
170 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
171 #define NearBranch ((u64)1 << 52) /* Near branches */
172 #define No16 ((u64)1 << 53) /* No 16 bit operand */
173 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
175 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
177 #define X2(x...) x, x
178 #define X3(x...) X2(x), x
179 #define X4(x...) X2(x), X2(x)
180 #define X5(x...) X4(x), x
181 #define X6(x...) X4(x), X2(x)
182 #define X7(x...) X4(x), X3(x)
183 #define X8(x...) X4(x), X4(x)
184 #define X16(x...) X8(x), X8(x)
186 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
187 #define FASTOP_SIZE 8
190 * fastop functions have a special calling convention:
195 * flags: rflags (in/out)
196 * ex: rsi (in:fastop pointer, out:zero if exception)
198 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
199 * different operand sizes can be reached by calculation, rather than a jump
200 * table (which would be bigger than the code).
202 * fastop functions are declared as taking a never-defined fastop parameter,
203 * so they can't be called from C directly.
212 int (*execute)(struct x86_emulate_ctxt *ctxt);
213 const struct opcode *group;
214 const struct group_dual *gdual;
215 const struct gprefix *gprefix;
216 const struct escape *esc;
217 const struct instr_dual *idual;
218 const struct mode_dual *mdual;
219 void (*fastop)(struct fastop *fake);
221 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
225 struct opcode mod012[8];
226 struct opcode mod3[8];
230 struct opcode pfx_no;
231 struct opcode pfx_66;
232 struct opcode pfx_f2;
233 struct opcode pfx_f3;
238 struct opcode high[64];
242 struct opcode mod012;
247 struct opcode mode32;
248 struct opcode mode64;
251 /* EFLAGS bit definitions. */
252 #define EFLG_ID (1<<21)
253 #define EFLG_VIP (1<<20)
254 #define EFLG_VIF (1<<19)
255 #define EFLG_AC (1<<18)
256 #define EFLG_VM (1<<17)
257 #define EFLG_RF (1<<16)
258 #define EFLG_IOPL (3<<12)
259 #define EFLG_NT (1<<14)
260 #define EFLG_OF (1<<11)
261 #define EFLG_DF (1<<10)
262 #define EFLG_IF (1<<9)
263 #define EFLG_TF (1<<8)
264 #define EFLG_SF (1<<7)
265 #define EFLG_ZF (1<<6)
266 #define EFLG_AF (1<<4)
267 #define EFLG_PF (1<<2)
268 #define EFLG_CF (1<<0)
270 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
271 #define EFLG_RESERVED_ONE_MASK 2
273 enum x86_transfer_type {
275 X86_TRANSFER_CALL_JMP,
277 X86_TRANSFER_TASK_SWITCH,
280 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
282 if (!(ctxt->regs_valid & (1 << nr))) {
283 ctxt->regs_valid |= 1 << nr;
284 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
286 return ctxt->_regs[nr];
289 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
291 ctxt->regs_valid |= 1 << nr;
292 ctxt->regs_dirty |= 1 << nr;
293 return &ctxt->_regs[nr];
296 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
299 return reg_write(ctxt, nr);
302 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
306 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
307 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
310 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
312 ctxt->regs_dirty = 0;
313 ctxt->regs_valid = 0;
317 * These EFLAGS bits are restored from saved value during emulation, and
318 * any changes are written back to the saved value after emulation.
320 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
328 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
330 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
331 #define FOP_RET "ret \n\t"
333 #define FOP_START(op) \
334 extern void em_##op(struct fastop *fake); \
335 asm(".pushsection .text, \"ax\" \n\t" \
336 ".global em_" #op " \n\t" \
343 #define FOPNOP() FOP_ALIGN FOP_RET
345 #define FOP1E(op, dst) \
346 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
348 #define FOP1EEX(op, dst) \
349 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
351 #define FASTOP1(op) \
356 ON64(FOP1E(op##q, rax)) \
359 /* 1-operand, using src2 (for MUL/DIV r/m) */
360 #define FASTOP1SRC2(op, name) \
365 ON64(FOP1E(op, rcx)) \
368 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
369 #define FASTOP1SRC2EX(op, name) \
374 ON64(FOP1EEX(op, rcx)) \
377 #define FOP2E(op, dst, src) \
378 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
380 #define FASTOP2(op) \
382 FOP2E(op##b, al, dl) \
383 FOP2E(op##w, ax, dx) \
384 FOP2E(op##l, eax, edx) \
385 ON64(FOP2E(op##q, rax, rdx)) \
388 /* 2 operand, word only */
389 #define FASTOP2W(op) \
392 FOP2E(op##w, ax, dx) \
393 FOP2E(op##l, eax, edx) \
394 ON64(FOP2E(op##q, rax, rdx)) \
397 /* 2 operand, src is CL */
398 #define FASTOP2CL(op) \
400 FOP2E(op##b, al, cl) \
401 FOP2E(op##w, ax, cl) \
402 FOP2E(op##l, eax, cl) \
403 ON64(FOP2E(op##q, rax, cl)) \
406 /* 2 operand, src and dest are reversed */
407 #define FASTOP2R(op, name) \
409 FOP2E(op##b, dl, al) \
410 FOP2E(op##w, dx, ax) \
411 FOP2E(op##l, edx, eax) \
412 ON64(FOP2E(op##q, rdx, rax)) \
415 #define FOP3E(op, dst, src, src2) \
416 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
418 /* 3-operand, word-only, src2=cl */
419 #define FASTOP3WCL(op) \
422 FOP3E(op##w, ax, dx, cl) \
423 FOP3E(op##l, eax, edx, cl) \
424 ON64(FOP3E(op##q, rax, rdx, cl)) \
427 /* Special case for SETcc - 1 instruction per cc */
428 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
430 asm(".global kvm_fastop_exception \n"
431 "kvm_fastop_exception: xor %esi, %esi; ret");
452 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
455 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
456 enum x86_intercept intercept,
457 enum x86_intercept_stage stage)
459 struct x86_instruction_info info = {
460 .intercept = intercept,
461 .rep_prefix = ctxt->rep_prefix,
462 .modrm_mod = ctxt->modrm_mod,
463 .modrm_reg = ctxt->modrm_reg,
464 .modrm_rm = ctxt->modrm_rm,
465 .src_val = ctxt->src.val64,
466 .dst_val = ctxt->dst.val64,
467 .src_bytes = ctxt->src.bytes,
468 .dst_bytes = ctxt->dst.bytes,
469 .ad_bytes = ctxt->ad_bytes,
470 .next_rip = ctxt->eip,
473 return ctxt->ops->intercept(ctxt, &info, stage);
476 static void assign_masked(ulong *dest, ulong src, ulong mask)
478 *dest = (*dest & ~mask) | (src & mask);
481 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
483 return (1UL << (ctxt->ad_bytes << 3)) - 1;
486 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
489 struct desc_struct ss;
491 if (ctxt->mode == X86EMUL_MODE_PROT64)
493 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
494 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
497 static int stack_size(struct x86_emulate_ctxt *ctxt)
499 return (__fls(stack_mask(ctxt)) + 1) >> 3;
502 /* Access/update address held in a register, based on addressing mode. */
503 static inline unsigned long
504 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
506 if (ctxt->ad_bytes == sizeof(unsigned long))
509 return reg & ad_mask(ctxt);
512 static inline unsigned long
513 register_address(struct x86_emulate_ctxt *ctxt, int reg)
515 return address_mask(ctxt, reg_read(ctxt, reg));
518 static void masked_increment(ulong *reg, ulong mask, int inc)
520 assign_masked(reg, *reg + inc, mask);
524 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
528 if (ctxt->ad_bytes == sizeof(unsigned long))
531 mask = ad_mask(ctxt);
532 masked_increment(reg_rmw(ctxt, reg), mask, inc);
535 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
537 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
540 static u32 desc_limit_scaled(struct desc_struct *desc)
542 u32 limit = get_desc_limit(desc);
544 return desc->g ? (limit << 12) | 0xfff : limit;
547 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
549 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
552 return ctxt->ops->get_cached_segment_base(ctxt, seg);
555 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
556 u32 error, bool valid)
559 ctxt->exception.vector = vec;
560 ctxt->exception.error_code = error;
561 ctxt->exception.error_code_valid = valid;
562 return X86EMUL_PROPAGATE_FAULT;
565 static int emulate_db(struct x86_emulate_ctxt *ctxt)
567 return emulate_exception(ctxt, DB_VECTOR, 0, false);
570 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
572 return emulate_exception(ctxt, GP_VECTOR, err, true);
575 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
577 return emulate_exception(ctxt, SS_VECTOR, err, true);
580 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
582 return emulate_exception(ctxt, UD_VECTOR, 0, false);
585 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
587 return emulate_exception(ctxt, TS_VECTOR, err, true);
590 static int emulate_de(struct x86_emulate_ctxt *ctxt)
592 return emulate_exception(ctxt, DE_VECTOR, 0, false);
595 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
597 return emulate_exception(ctxt, NM_VECTOR, 0, false);
600 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
603 struct desc_struct desc;
605 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
609 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
614 struct desc_struct desc;
616 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
617 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
621 * x86 defines three classes of vector instructions: explicitly
622 * aligned, explicitly unaligned, and the rest, which change behaviour
623 * depending on whether they're AVX encoded or not.
625 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
626 * subject to the same check.
628 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
630 if (likely(size < 16))
633 if (ctxt->d & Aligned)
635 else if (ctxt->d & Unaligned)
637 else if (ctxt->d & Avx)
643 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
644 struct segmented_address addr,
645 unsigned *max_size, unsigned size,
646 bool write, bool fetch,
647 enum x86emul_mode mode, ulong *linear)
649 struct desc_struct desc;
655 la = seg_base(ctxt, addr.seg) + addr.ea;
658 case X86EMUL_MODE_PROT64:
659 if (is_noncanonical_address(la))
662 *max_size = min_t(u64, ~0u, (1ull << 48) - la);
663 if (size > *max_size)
667 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
671 /* code segment in protected mode or read-only data segment */
672 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
673 || !(desc.type & 2)) && write)
675 /* unreadable code segment */
676 if (!fetch && (desc.type & 8) && !(desc.type & 2))
678 lim = desc_limit_scaled(&desc);
679 if (!(desc.type & 8) && (desc.type & 4)) {
680 /* expand-down segment */
683 lim = desc.d ? 0xffffffff : 0xffff;
687 if (lim == 0xffffffff)
690 *max_size = (u64)lim + 1 - addr.ea;
691 if (size > *max_size)
697 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
698 return emulate_gp(ctxt, 0);
700 return X86EMUL_CONTINUE;
702 if (addr.seg == VCPU_SREG_SS)
703 return emulate_ss(ctxt, 0);
705 return emulate_gp(ctxt, 0);
708 static int linearize(struct x86_emulate_ctxt *ctxt,
709 struct segmented_address addr,
710 unsigned size, bool write,
714 return __linearize(ctxt, addr, &max_size, size, write, false,
718 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
719 enum x86emul_mode mode)
724 struct segmented_address addr = { .seg = VCPU_SREG_CS,
727 if (ctxt->op_bytes != sizeof(unsigned long))
728 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
729 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
730 if (rc == X86EMUL_CONTINUE)
731 ctxt->_eip = addr.ea;
735 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
737 return assign_eip(ctxt, dst, ctxt->mode);
740 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
741 const struct desc_struct *cs_desc)
743 enum x86emul_mode mode = ctxt->mode;
746 if (ctxt->mode >= X86EMUL_MODE_PROT32 && cs_desc->l) {
749 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
751 mode = X86EMUL_MODE_PROT64;
754 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
755 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
756 return assign_eip(ctxt, dst, mode);
759 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
761 return assign_eip_near(ctxt, ctxt->_eip + rel);
764 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
765 struct segmented_address addr,
772 rc = linearize(ctxt, addr, size, false, &linear);
773 if (rc != X86EMUL_CONTINUE)
775 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
779 * Prefetch the remaining bytes of the instruction without crossing page
780 * boundary if they are not in fetch_cache yet.
782 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
785 unsigned size, max_size;
786 unsigned long linear;
787 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
788 struct segmented_address addr = { .seg = VCPU_SREG_CS,
789 .ea = ctxt->eip + cur_size };
792 * We do not know exactly how many bytes will be needed, and
793 * __linearize is expensive, so fetch as much as possible. We
794 * just have to avoid going beyond the 15 byte limit, the end
795 * of the segment, or the end of the page.
797 * __linearize is called with size 0 so that it does not do any
798 * boundary check itself. Instead, we use max_size to check
801 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
803 if (unlikely(rc != X86EMUL_CONTINUE))
806 size = min_t(unsigned, 15UL ^ cur_size, max_size);
807 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
810 * One instruction can only straddle two pages,
811 * and one has been loaded at the beginning of
812 * x86_decode_insn. So, if not enough bytes
813 * still, we must have hit the 15-byte boundary.
815 if (unlikely(size < op_size))
816 return emulate_gp(ctxt, 0);
818 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
819 size, &ctxt->exception);
820 if (unlikely(rc != X86EMUL_CONTINUE))
822 ctxt->fetch.end += size;
823 return X86EMUL_CONTINUE;
826 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
829 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
831 if (unlikely(done_size < size))
832 return __do_insn_fetch_bytes(ctxt, size - done_size);
834 return X86EMUL_CONTINUE;
837 /* Fetch next part of the instruction being emulated. */
838 #define insn_fetch(_type, _ctxt) \
841 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
842 if (rc != X86EMUL_CONTINUE) \
844 ctxt->_eip += sizeof(_type); \
845 _x = *(_type __aligned(1) *) ctxt->fetch.ptr; \
846 ctxt->fetch.ptr += sizeof(_type); \
850 #define insn_fetch_arr(_arr, _size, _ctxt) \
852 rc = do_insn_fetch_bytes(_ctxt, _size); \
853 if (rc != X86EMUL_CONTINUE) \
855 ctxt->_eip += (_size); \
856 memcpy(_arr, ctxt->fetch.ptr, _size); \
857 ctxt->fetch.ptr += (_size); \
861 * Given the 'reg' portion of a ModRM byte, and a register block, return a
862 * pointer into the block that addresses the relevant register.
863 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
865 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
869 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
871 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
872 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
874 p = reg_rmw(ctxt, modrm_reg);
878 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
879 struct segmented_address addr,
880 u16 *size, unsigned long *address, int op_bytes)
887 rc = segmented_read_std(ctxt, addr, size, 2);
888 if (rc != X86EMUL_CONTINUE)
891 rc = segmented_read_std(ctxt, addr, address, op_bytes);
905 FASTOP1SRC2(mul, mul_ex);
906 FASTOP1SRC2(imul, imul_ex);
907 FASTOP1SRC2EX(div, div_ex);
908 FASTOP1SRC2EX(idiv, idiv_ex);
937 FASTOP2R(cmp, cmp_r);
939 static u8 test_cc(unsigned int condition, unsigned long flags)
942 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
944 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
945 asm("push %[flags]; popf; call *%[fastop]"
946 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
950 static void fetch_register_operand(struct operand *op)
954 op->val = *(u8 *)op->addr.reg;
957 op->val = *(u16 *)op->addr.reg;
960 op->val = *(u32 *)op->addr.reg;
963 op->val = *(u64 *)op->addr.reg;
968 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
970 ctxt->ops->get_fpu(ctxt);
972 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
973 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
974 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
975 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
976 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
977 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
978 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
979 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
981 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
982 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
983 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
984 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
985 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
986 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
987 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
988 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
992 ctxt->ops->put_fpu(ctxt);
995 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
998 ctxt->ops->get_fpu(ctxt);
1000 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1001 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1002 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1003 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1004 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1005 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1006 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1007 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1008 #ifdef CONFIG_X86_64
1009 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1010 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1011 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1012 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1013 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1014 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1015 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1016 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1020 ctxt->ops->put_fpu(ctxt);
1023 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1025 ctxt->ops->get_fpu(ctxt);
1027 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1028 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1029 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1030 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1031 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1032 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1033 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1034 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1037 ctxt->ops->put_fpu(ctxt);
1040 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1042 ctxt->ops->get_fpu(ctxt);
1044 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1045 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1046 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1047 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1048 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1049 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1050 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1051 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1054 ctxt->ops->put_fpu(ctxt);
1057 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1059 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1060 return emulate_nm(ctxt);
1062 ctxt->ops->get_fpu(ctxt);
1063 asm volatile("fninit");
1064 ctxt->ops->put_fpu(ctxt);
1065 return X86EMUL_CONTINUE;
1068 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1072 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1073 return emulate_nm(ctxt);
1075 ctxt->ops->get_fpu(ctxt);
1076 asm volatile("fnstcw %0": "+m"(fcw));
1077 ctxt->ops->put_fpu(ctxt);
1079 ctxt->dst.val = fcw;
1081 return X86EMUL_CONTINUE;
1084 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1088 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1089 return emulate_nm(ctxt);
1091 ctxt->ops->get_fpu(ctxt);
1092 asm volatile("fnstsw %0": "+m"(fsw));
1093 ctxt->ops->put_fpu(ctxt);
1095 ctxt->dst.val = fsw;
1097 return X86EMUL_CONTINUE;
1100 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1103 unsigned reg = ctxt->modrm_reg;
1105 if (!(ctxt->d & ModRM))
1106 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1108 if (ctxt->d & Sse) {
1112 read_sse_reg(ctxt, &op->vec_val, reg);
1115 if (ctxt->d & Mmx) {
1124 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1125 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1127 fetch_register_operand(op);
1128 op->orig_val = op->val;
1131 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1133 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1134 ctxt->modrm_seg = VCPU_SREG_SS;
1137 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1141 int index_reg, base_reg, scale;
1142 int rc = X86EMUL_CONTINUE;
1145 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1146 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1147 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1149 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1150 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1151 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1152 ctxt->modrm_seg = VCPU_SREG_DS;
1154 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1156 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1157 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1159 if (ctxt->d & Sse) {
1162 op->addr.xmm = ctxt->modrm_rm;
1163 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1166 if (ctxt->d & Mmx) {
1169 op->addr.mm = ctxt->modrm_rm & 7;
1172 fetch_register_operand(op);
1178 if (ctxt->ad_bytes == 2) {
1179 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1180 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1181 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1182 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1184 /* 16-bit ModR/M decode. */
1185 switch (ctxt->modrm_mod) {
1187 if (ctxt->modrm_rm == 6)
1188 modrm_ea += insn_fetch(u16, ctxt);
1191 modrm_ea += insn_fetch(s8, ctxt);
1194 modrm_ea += insn_fetch(u16, ctxt);
1197 switch (ctxt->modrm_rm) {
1199 modrm_ea += bx + si;
1202 modrm_ea += bx + di;
1205 modrm_ea += bp + si;
1208 modrm_ea += bp + di;
1217 if (ctxt->modrm_mod != 0)
1224 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1225 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1226 ctxt->modrm_seg = VCPU_SREG_SS;
1227 modrm_ea = (u16)modrm_ea;
1229 /* 32/64-bit ModR/M decode. */
1230 if ((ctxt->modrm_rm & 7) == 4) {
1231 sib = insn_fetch(u8, ctxt);
1232 index_reg |= (sib >> 3) & 7;
1233 base_reg |= sib & 7;
1236 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1237 modrm_ea += insn_fetch(s32, ctxt);
1239 modrm_ea += reg_read(ctxt, base_reg);
1240 adjust_modrm_seg(ctxt, base_reg);
1241 /* Increment ESP on POP [ESP] */
1242 if ((ctxt->d & IncSP) &&
1243 base_reg == VCPU_REGS_RSP)
1244 modrm_ea += ctxt->op_bytes;
1247 modrm_ea += reg_read(ctxt, index_reg) << scale;
1248 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1249 modrm_ea += insn_fetch(s32, ctxt);
1250 if (ctxt->mode == X86EMUL_MODE_PROT64)
1251 ctxt->rip_relative = 1;
1253 base_reg = ctxt->modrm_rm;
1254 modrm_ea += reg_read(ctxt, base_reg);
1255 adjust_modrm_seg(ctxt, base_reg);
1257 switch (ctxt->modrm_mod) {
1259 modrm_ea += insn_fetch(s8, ctxt);
1262 modrm_ea += insn_fetch(s32, ctxt);
1266 op->addr.mem.ea = modrm_ea;
1267 if (ctxt->ad_bytes != 8)
1268 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1274 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1277 int rc = X86EMUL_CONTINUE;
1280 switch (ctxt->ad_bytes) {
1282 op->addr.mem.ea = insn_fetch(u16, ctxt);
1285 op->addr.mem.ea = insn_fetch(u32, ctxt);
1288 op->addr.mem.ea = insn_fetch(u64, ctxt);
1295 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1299 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1300 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1302 if (ctxt->src.bytes == 2)
1303 sv = (s16)ctxt->src.val & (s16)mask;
1304 else if (ctxt->src.bytes == 4)
1305 sv = (s32)ctxt->src.val & (s32)mask;
1307 sv = (s64)ctxt->src.val & (s64)mask;
1309 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1310 ctxt->dst.addr.mem.ea + (sv >> 3));
1313 /* only subword offset */
1314 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1317 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1318 unsigned long addr, void *dest, unsigned size)
1321 struct read_cache *mc = &ctxt->mem_read;
1323 if (mc->pos < mc->end)
1326 WARN_ON((mc->end + size) >= sizeof(mc->data));
1328 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1330 if (rc != X86EMUL_CONTINUE)
1336 memcpy(dest, mc->data + mc->pos, size);
1338 return X86EMUL_CONTINUE;
1341 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1342 struct segmented_address addr,
1349 rc = linearize(ctxt, addr, size, false, &linear);
1350 if (rc != X86EMUL_CONTINUE)
1352 return read_emulated(ctxt, linear, data, size);
1355 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1356 struct segmented_address addr,
1363 rc = linearize(ctxt, addr, size, true, &linear);
1364 if (rc != X86EMUL_CONTINUE)
1366 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1370 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1371 struct segmented_address addr,
1372 const void *orig_data, const void *data,
1378 rc = linearize(ctxt, addr, size, true, &linear);
1379 if (rc != X86EMUL_CONTINUE)
1381 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1382 size, &ctxt->exception);
1385 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1386 unsigned int size, unsigned short port,
1389 struct read_cache *rc = &ctxt->io_read;
1391 if (rc->pos == rc->end) { /* refill pio read ahead */
1392 unsigned int in_page, n;
1393 unsigned int count = ctxt->rep_prefix ?
1394 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1395 in_page = (ctxt->eflags & EFLG_DF) ?
1396 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1397 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1398 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1401 rc->pos = rc->end = 0;
1402 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1407 if (ctxt->rep_prefix && (ctxt->d & String) &&
1408 !(ctxt->eflags & EFLG_DF)) {
1409 ctxt->dst.data = rc->data + rc->pos;
1410 ctxt->dst.type = OP_MEM_STR;
1411 ctxt->dst.count = (rc->end - rc->pos) / size;
1414 memcpy(dest, rc->data + rc->pos, size);
1420 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1421 u16 index, struct desc_struct *desc)
1426 ctxt->ops->get_idt(ctxt, &dt);
1428 if (dt.size < index * 8 + 7)
1429 return emulate_gp(ctxt, index << 3 | 0x2);
1431 addr = dt.address + index * 8;
1432 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1436 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1437 u16 selector, struct desc_ptr *dt)
1439 const struct x86_emulate_ops *ops = ctxt->ops;
1442 if (selector & 1 << 2) {
1443 struct desc_struct desc;
1446 memset (dt, 0, sizeof *dt);
1447 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1451 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1452 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1454 ops->get_gdt(ctxt, dt);
1457 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1458 u16 selector, ulong *desc_addr_p)
1461 u16 index = selector >> 3;
1464 get_descriptor_table_ptr(ctxt, selector, &dt);
1466 if (dt.size < index * 8 + 7)
1467 return emulate_gp(ctxt, selector & 0xfffc);
1469 addr = dt.address + index * 8;
1471 #ifdef CONFIG_X86_64
1472 if (addr >> 32 != 0) {
1475 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1476 if (!(efer & EFER_LMA))
1481 *desc_addr_p = addr;
1482 return X86EMUL_CONTINUE;
1485 /* allowed just for 8 bytes segments */
1486 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1487 u16 selector, struct desc_struct *desc,
1492 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1493 if (rc != X86EMUL_CONTINUE)
1496 return ctxt->ops->read_std(ctxt, *desc_addr_p, desc, sizeof(*desc),
1500 /* allowed just for 8 bytes segments */
1501 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1502 u16 selector, struct desc_struct *desc)
1507 rc = get_descriptor_ptr(ctxt, selector, &addr);
1508 if (rc != X86EMUL_CONTINUE)
1511 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1515 /* Does not support long mode */
1516 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1517 u16 selector, int seg, u8 cpl,
1518 enum x86_transfer_type transfer,
1519 struct desc_struct *desc)
1521 struct desc_struct seg_desc, old_desc;
1523 unsigned err_vec = GP_VECTOR;
1525 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1531 memset(&seg_desc, 0, sizeof seg_desc);
1533 if (ctxt->mode == X86EMUL_MODE_REAL) {
1534 /* set real mode segment descriptor (keep limit etc. for
1536 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1537 set_desc_base(&seg_desc, selector << 4);
1539 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1540 /* VM86 needs a clean new segment descriptor */
1541 set_desc_base(&seg_desc, selector << 4);
1542 set_desc_limit(&seg_desc, 0xffff);
1552 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1553 if ((seg == VCPU_SREG_CS
1554 || (seg == VCPU_SREG_SS
1555 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1556 || seg == VCPU_SREG_TR)
1560 /* TR should be in GDT only */
1561 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1564 if (null_selector) /* for NULL selector skip all following checks */
1567 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1568 if (ret != X86EMUL_CONTINUE)
1571 err_code = selector & 0xfffc;
1572 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1575 /* can't load system descriptor into segment selector */
1576 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1577 if (transfer == X86_TRANSFER_CALL_JMP)
1578 return X86EMUL_UNHANDLEABLE;
1583 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1592 * segment is not a writable data segment or segment
1593 * selector's RPL != CPL or segment selector's RPL != CPL
1595 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1599 if (!(seg_desc.type & 8))
1602 if (seg_desc.type & 4) {
1608 if (rpl > cpl || dpl != cpl)
1611 /* in long-mode d/b must be clear if l is set */
1612 if (seg_desc.d && seg_desc.l) {
1615 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1616 if (efer & EFER_LMA)
1620 /* CS(RPL) <- CPL */
1621 selector = (selector & 0xfffc) | cpl;
1624 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1626 old_desc = seg_desc;
1627 seg_desc.type |= 2; /* busy */
1628 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1629 sizeof(seg_desc), &ctxt->exception);
1630 if (ret != X86EMUL_CONTINUE)
1633 case VCPU_SREG_LDTR:
1634 if (seg_desc.s || seg_desc.type != 2)
1637 default: /* DS, ES, FS, or GS */
1639 * segment is not a data or readable code segment or
1640 * ((segment is a data or nonconforming code segment)
1641 * and (both RPL and CPL > DPL))
1643 if ((seg_desc.type & 0xa) == 0x8 ||
1644 (((seg_desc.type & 0xc) != 0xc) &&
1645 (rpl > dpl && cpl > dpl)))
1651 /* mark segment as accessed */
1652 if (!(seg_desc.type & 1)) {
1654 ret = write_segment_descriptor(ctxt, selector,
1656 if (ret != X86EMUL_CONTINUE)
1659 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1660 ret = ctxt->ops->read_std(ctxt, desc_addr+8, &base3,
1661 sizeof(base3), &ctxt->exception);
1662 if (ret != X86EMUL_CONTINUE)
1664 if (is_noncanonical_address(get_desc_base(&seg_desc) |
1665 ((u64)base3 << 32)))
1666 return emulate_gp(ctxt, 0);
1669 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1672 return X86EMUL_CONTINUE;
1674 return emulate_exception(ctxt, err_vec, err_code, true);
1677 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1678 u16 selector, int seg)
1680 u8 cpl = ctxt->ops->cpl(ctxt);
1681 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1682 X86_TRANSFER_NONE, NULL);
1685 static void write_register_operand(struct operand *op)
1687 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1688 switch (op->bytes) {
1690 *(u8 *)op->addr.reg = (u8)op->val;
1693 *(u16 *)op->addr.reg = (u16)op->val;
1696 *op->addr.reg = (u32)op->val;
1697 break; /* 64b: zero-extend */
1699 *op->addr.reg = op->val;
1704 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1708 write_register_operand(op);
1711 if (ctxt->lock_prefix)
1712 return segmented_cmpxchg(ctxt,
1718 return segmented_write(ctxt,
1724 return segmented_write(ctxt,
1727 op->bytes * op->count);
1730 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1733 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1741 return X86EMUL_CONTINUE;
1744 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1746 struct segmented_address addr;
1748 rsp_increment(ctxt, -bytes);
1749 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1750 addr.seg = VCPU_SREG_SS;
1752 return segmented_write(ctxt, addr, data, bytes);
1755 static int em_push(struct x86_emulate_ctxt *ctxt)
1757 /* Disable writeback. */
1758 ctxt->dst.type = OP_NONE;
1759 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1762 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1763 void *dest, int len)
1766 struct segmented_address addr;
1768 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1769 addr.seg = VCPU_SREG_SS;
1770 rc = segmented_read(ctxt, addr, dest, len);
1771 if (rc != X86EMUL_CONTINUE)
1774 rsp_increment(ctxt, len);
1778 static int em_pop(struct x86_emulate_ctxt *ctxt)
1780 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1783 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1784 void *dest, int len)
1787 unsigned long val, change_mask;
1788 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1789 int cpl = ctxt->ops->cpl(ctxt);
1791 rc = emulate_pop(ctxt, &val, len);
1792 if (rc != X86EMUL_CONTINUE)
1795 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1796 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_AC | EFLG_ID;
1798 switch(ctxt->mode) {
1799 case X86EMUL_MODE_PROT64:
1800 case X86EMUL_MODE_PROT32:
1801 case X86EMUL_MODE_PROT16:
1803 change_mask |= EFLG_IOPL;
1805 change_mask |= EFLG_IF;
1807 case X86EMUL_MODE_VM86:
1809 return emulate_gp(ctxt, 0);
1810 change_mask |= EFLG_IF;
1812 default: /* real mode */
1813 change_mask |= (EFLG_IOPL | EFLG_IF);
1817 *(unsigned long *)dest =
1818 (ctxt->eflags & ~change_mask) | (val & change_mask);
1823 static int em_popf(struct x86_emulate_ctxt *ctxt)
1825 ctxt->dst.type = OP_REG;
1826 ctxt->dst.addr.reg = &ctxt->eflags;
1827 ctxt->dst.bytes = ctxt->op_bytes;
1828 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1831 static int em_enter(struct x86_emulate_ctxt *ctxt)
1834 unsigned frame_size = ctxt->src.val;
1835 unsigned nesting_level = ctxt->src2.val & 31;
1839 return X86EMUL_UNHANDLEABLE;
1841 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1842 rc = push(ctxt, &rbp, stack_size(ctxt));
1843 if (rc != X86EMUL_CONTINUE)
1845 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1847 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1848 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1850 return X86EMUL_CONTINUE;
1853 static int em_leave(struct x86_emulate_ctxt *ctxt)
1855 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1857 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1860 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1862 int seg = ctxt->src2.val;
1864 ctxt->src.val = get_segment_selector(ctxt, seg);
1865 if (ctxt->op_bytes == 4) {
1866 rsp_increment(ctxt, -2);
1870 return em_push(ctxt);
1873 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1875 int seg = ctxt->src2.val;
1876 unsigned long selector;
1879 rc = emulate_pop(ctxt, &selector, 2);
1880 if (rc != X86EMUL_CONTINUE)
1883 if (ctxt->modrm_reg == VCPU_SREG_SS)
1884 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1885 if (ctxt->op_bytes > 2)
1886 rsp_increment(ctxt, ctxt->op_bytes - 2);
1888 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1892 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1894 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1895 int rc = X86EMUL_CONTINUE;
1896 int reg = VCPU_REGS_RAX;
1898 while (reg <= VCPU_REGS_RDI) {
1899 (reg == VCPU_REGS_RSP) ?
1900 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1903 if (rc != X86EMUL_CONTINUE)
1912 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1914 ctxt->src.val = (unsigned long)ctxt->eflags & ~EFLG_VM;
1915 return em_push(ctxt);
1918 static int em_popa(struct x86_emulate_ctxt *ctxt)
1920 int rc = X86EMUL_CONTINUE;
1921 int reg = VCPU_REGS_RDI;
1923 while (reg >= VCPU_REGS_RAX) {
1924 if (reg == VCPU_REGS_RSP) {
1925 rsp_increment(ctxt, ctxt->op_bytes);
1929 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1930 if (rc != X86EMUL_CONTINUE)
1937 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1939 const struct x86_emulate_ops *ops = ctxt->ops;
1946 /* TODO: Add limit checks */
1947 ctxt->src.val = ctxt->eflags;
1949 if (rc != X86EMUL_CONTINUE)
1952 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1954 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1956 if (rc != X86EMUL_CONTINUE)
1959 ctxt->src.val = ctxt->_eip;
1961 if (rc != X86EMUL_CONTINUE)
1964 ops->get_idt(ctxt, &dt);
1966 eip_addr = dt.address + (irq << 2);
1967 cs_addr = dt.address + (irq << 2) + 2;
1969 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1970 if (rc != X86EMUL_CONTINUE)
1973 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1974 if (rc != X86EMUL_CONTINUE)
1977 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1978 if (rc != X86EMUL_CONTINUE)
1986 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1990 invalidate_registers(ctxt);
1991 rc = __emulate_int_real(ctxt, irq);
1992 if (rc == X86EMUL_CONTINUE)
1993 writeback_registers(ctxt);
1997 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1999 switch(ctxt->mode) {
2000 case X86EMUL_MODE_REAL:
2001 return __emulate_int_real(ctxt, irq);
2002 case X86EMUL_MODE_VM86:
2003 case X86EMUL_MODE_PROT16:
2004 case X86EMUL_MODE_PROT32:
2005 case X86EMUL_MODE_PROT64:
2007 /* Protected mode interrupts unimplemented yet */
2008 return X86EMUL_UNHANDLEABLE;
2012 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2014 int rc = X86EMUL_CONTINUE;
2015 unsigned long temp_eip = 0;
2016 unsigned long temp_eflags = 0;
2017 unsigned long cs = 0;
2018 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
2019 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
2020 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
2021 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
2023 /* TODO: Add stack limit check */
2025 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2027 if (rc != X86EMUL_CONTINUE)
2030 if (temp_eip & ~0xffff)
2031 return emulate_gp(ctxt, 0);
2033 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2035 if (rc != X86EMUL_CONTINUE)
2038 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2040 if (rc != X86EMUL_CONTINUE)
2043 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2045 if (rc != X86EMUL_CONTINUE)
2048 ctxt->_eip = temp_eip;
2051 if (ctxt->op_bytes == 4)
2052 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2053 else if (ctxt->op_bytes == 2) {
2054 ctxt->eflags &= ~0xffff;
2055 ctxt->eflags |= temp_eflags;
2058 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2059 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
2060 ctxt->ops->set_nmi_mask(ctxt, false);
2065 static int em_iret(struct x86_emulate_ctxt *ctxt)
2067 switch(ctxt->mode) {
2068 case X86EMUL_MODE_REAL:
2069 return emulate_iret_real(ctxt);
2070 case X86EMUL_MODE_VM86:
2071 case X86EMUL_MODE_PROT16:
2072 case X86EMUL_MODE_PROT32:
2073 case X86EMUL_MODE_PROT64:
2075 /* iret from protected mode unimplemented yet */
2076 return X86EMUL_UNHANDLEABLE;
2080 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2083 unsigned short sel, old_sel;
2084 struct desc_struct old_desc, new_desc;
2085 const struct x86_emulate_ops *ops = ctxt->ops;
2086 u8 cpl = ctxt->ops->cpl(ctxt);
2088 /* Assignment of RIP may only fail in 64-bit mode */
2089 if (ctxt->mode == X86EMUL_MODE_PROT64)
2090 ops->get_segment(ctxt, &old_sel, &old_desc, NULL,
2093 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2095 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2096 X86_TRANSFER_CALL_JMP,
2098 if (rc != X86EMUL_CONTINUE)
2101 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2102 if (rc != X86EMUL_CONTINUE) {
2103 WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64);
2104 /* assigning eip failed; restore the old cs */
2105 ops->set_segment(ctxt, old_sel, &old_desc, 0, VCPU_SREG_CS);
2111 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2113 return assign_eip_near(ctxt, ctxt->src.val);
2116 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2121 old_eip = ctxt->_eip;
2122 rc = assign_eip_near(ctxt, ctxt->src.val);
2123 if (rc != X86EMUL_CONTINUE)
2125 ctxt->src.val = old_eip;
2130 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2132 u64 old = ctxt->dst.orig_val64;
2134 if (ctxt->dst.bytes == 16)
2135 return X86EMUL_UNHANDLEABLE;
2137 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2138 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2139 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2140 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2141 ctxt->eflags &= ~EFLG_ZF;
2143 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2144 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2146 ctxt->eflags |= EFLG_ZF;
2148 return X86EMUL_CONTINUE;
2151 static int em_ret(struct x86_emulate_ctxt *ctxt)
2156 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2157 if (rc != X86EMUL_CONTINUE)
2160 return assign_eip_near(ctxt, eip);
2163 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2166 unsigned long eip, cs;
2168 int cpl = ctxt->ops->cpl(ctxt);
2169 struct desc_struct old_desc, new_desc;
2170 const struct x86_emulate_ops *ops = ctxt->ops;
2172 if (ctxt->mode == X86EMUL_MODE_PROT64)
2173 ops->get_segment(ctxt, &old_cs, &old_desc, NULL,
2176 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2177 if (rc != X86EMUL_CONTINUE)
2179 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2180 if (rc != X86EMUL_CONTINUE)
2182 /* Outer-privilege level return is not implemented */
2183 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2184 return X86EMUL_UNHANDLEABLE;
2185 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2188 if (rc != X86EMUL_CONTINUE)
2190 rc = assign_eip_far(ctxt, eip, &new_desc);
2191 if (rc != X86EMUL_CONTINUE) {
2192 WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64);
2193 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
2198 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2202 rc = em_ret_far(ctxt);
2203 if (rc != X86EMUL_CONTINUE)
2205 rsp_increment(ctxt, ctxt->src.val);
2206 return X86EMUL_CONTINUE;
2209 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2211 /* Save real source value, then compare EAX against destination. */
2212 ctxt->dst.orig_val = ctxt->dst.val;
2213 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2214 ctxt->src.orig_val = ctxt->src.val;
2215 ctxt->src.val = ctxt->dst.orig_val;
2216 fastop(ctxt, em_cmp);
2218 if (ctxt->eflags & EFLG_ZF) {
2219 /* Success: write back to memory; no update of EAX */
2220 ctxt->src.type = OP_NONE;
2221 ctxt->dst.val = ctxt->src.orig_val;
2223 /* Failure: write the value we saw to EAX. */
2224 ctxt->src.type = OP_REG;
2225 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2226 ctxt->src.val = ctxt->dst.orig_val;
2227 /* Create write-cycle to dest by writing the same value */
2228 ctxt->dst.val = ctxt->dst.orig_val;
2230 return X86EMUL_CONTINUE;
2233 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2235 int seg = ctxt->src2.val;
2239 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2241 rc = load_segment_descriptor(ctxt, sel, seg);
2242 if (rc != X86EMUL_CONTINUE)
2245 ctxt->dst.val = ctxt->src.val;
2250 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2251 struct desc_struct *cs, struct desc_struct *ss)
2253 cs->l = 0; /* will be adjusted later */
2254 set_desc_base(cs, 0); /* flat segment */
2255 cs->g = 1; /* 4kb granularity */
2256 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2257 cs->type = 0x0b; /* Read, Execute, Accessed */
2259 cs->dpl = 0; /* will be adjusted later */
2264 set_desc_base(ss, 0); /* flat segment */
2265 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2266 ss->g = 1; /* 4kb granularity */
2268 ss->type = 0x03; /* Read/Write, Accessed */
2269 ss->d = 1; /* 32bit stack segment */
2276 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2278 u32 eax, ebx, ecx, edx;
2281 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2282 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2283 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2284 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2287 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2289 const struct x86_emulate_ops *ops = ctxt->ops;
2290 u32 eax, ebx, ecx, edx;
2293 * syscall should always be enabled in longmode - so only become
2294 * vendor specific (cpuid) if other modes are active...
2296 if (ctxt->mode == X86EMUL_MODE_PROT64)
2301 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2303 * Intel ("GenuineIntel")
2304 * remark: Intel CPUs only support "syscall" in 64bit
2305 * longmode. Also an 64bit guest with a
2306 * 32bit compat-app running will #UD !! While this
2307 * behaviour can be fixed (by emulating) into AMD
2308 * response - CPUs of AMD can't behave like Intel.
2310 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2311 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2312 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2315 /* AMD ("AuthenticAMD") */
2316 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2317 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2318 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2321 /* AMD ("AMDisbetter!") */
2322 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2323 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2324 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2327 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2331 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2333 const struct x86_emulate_ops *ops = ctxt->ops;
2334 struct desc_struct cs, ss;
2339 /* syscall is not available in real mode */
2340 if (ctxt->mode == X86EMUL_MODE_REAL ||
2341 ctxt->mode == X86EMUL_MODE_VM86)
2342 return emulate_ud(ctxt);
2344 if (!(em_syscall_is_enabled(ctxt)))
2345 return emulate_ud(ctxt);
2347 ops->get_msr(ctxt, MSR_EFER, &efer);
2348 setup_syscalls_segments(ctxt, &cs, &ss);
2350 if (!(efer & EFER_SCE))
2351 return emulate_ud(ctxt);
2353 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2355 cs_sel = (u16)(msr_data & 0xfffc);
2356 ss_sel = (u16)(msr_data + 8);
2358 if (efer & EFER_LMA) {
2362 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2363 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2365 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2366 if (efer & EFER_LMA) {
2367 #ifdef CONFIG_X86_64
2368 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2371 ctxt->mode == X86EMUL_MODE_PROT64 ?
2372 MSR_LSTAR : MSR_CSTAR, &msr_data);
2373 ctxt->_eip = msr_data;
2375 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2376 ctxt->eflags &= ~msr_data;
2377 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
2381 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2382 ctxt->_eip = (u32)msr_data;
2384 ctxt->eflags &= ~(EFLG_VM | EFLG_IF);
2387 return X86EMUL_CONTINUE;
2390 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2392 const struct x86_emulate_ops *ops = ctxt->ops;
2393 struct desc_struct cs, ss;
2398 ops->get_msr(ctxt, MSR_EFER, &efer);
2399 /* inject #GP if in real mode */
2400 if (ctxt->mode == X86EMUL_MODE_REAL)
2401 return emulate_gp(ctxt, 0);
2404 * Not recognized on AMD in compat mode (but is recognized in legacy
2407 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2408 && !vendor_intel(ctxt))
2409 return emulate_ud(ctxt);
2411 /* sysenter/sysexit have not been tested in 64bit mode. */
2412 if (ctxt->mode == X86EMUL_MODE_PROT64)
2413 return X86EMUL_UNHANDLEABLE;
2415 setup_syscalls_segments(ctxt, &cs, &ss);
2417 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2418 switch (ctxt->mode) {
2419 case X86EMUL_MODE_PROT32:
2420 if ((msr_data & 0xfffc) == 0x0)
2421 return emulate_gp(ctxt, 0);
2423 case X86EMUL_MODE_PROT64:
2424 if (msr_data == 0x0)
2425 return emulate_gp(ctxt, 0);
2431 ctxt->eflags &= ~(EFLG_VM | EFLG_IF);
2432 cs_sel = (u16)msr_data;
2433 cs_sel &= ~SELECTOR_RPL_MASK;
2434 ss_sel = cs_sel + 8;
2435 ss_sel &= ~SELECTOR_RPL_MASK;
2436 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2441 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2442 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2444 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2445 ctxt->_eip = msr_data;
2447 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2448 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2450 return X86EMUL_CONTINUE;
2453 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2455 const struct x86_emulate_ops *ops = ctxt->ops;
2456 struct desc_struct cs, ss;
2457 u64 msr_data, rcx, rdx;
2459 u16 cs_sel = 0, ss_sel = 0;
2461 /* inject #GP if in real mode or Virtual 8086 mode */
2462 if (ctxt->mode == X86EMUL_MODE_REAL ||
2463 ctxt->mode == X86EMUL_MODE_VM86)
2464 return emulate_gp(ctxt, 0);
2466 setup_syscalls_segments(ctxt, &cs, &ss);
2468 if ((ctxt->rex_prefix & 0x8) != 0x0)
2469 usermode = X86EMUL_MODE_PROT64;
2471 usermode = X86EMUL_MODE_PROT32;
2473 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2474 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2478 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2480 case X86EMUL_MODE_PROT32:
2481 cs_sel = (u16)(msr_data + 16);
2482 if ((msr_data & 0xfffc) == 0x0)
2483 return emulate_gp(ctxt, 0);
2484 ss_sel = (u16)(msr_data + 24);
2488 case X86EMUL_MODE_PROT64:
2489 cs_sel = (u16)(msr_data + 32);
2490 if (msr_data == 0x0)
2491 return emulate_gp(ctxt, 0);
2492 ss_sel = cs_sel + 8;
2495 if (is_noncanonical_address(rcx) ||
2496 is_noncanonical_address(rdx))
2497 return emulate_gp(ctxt, 0);
2500 cs_sel |= SELECTOR_RPL_MASK;
2501 ss_sel |= SELECTOR_RPL_MASK;
2503 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2504 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2507 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2509 return X86EMUL_CONTINUE;
2512 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2515 if (ctxt->mode == X86EMUL_MODE_REAL)
2517 if (ctxt->mode == X86EMUL_MODE_VM86)
2519 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2520 return ctxt->ops->cpl(ctxt) > iopl;
2523 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2526 const struct x86_emulate_ops *ops = ctxt->ops;
2527 struct desc_struct tr_seg;
2530 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2531 unsigned mask = (1 << len) - 1;
2534 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2537 if (desc_limit_scaled(&tr_seg) < 103)
2539 base = get_desc_base(&tr_seg);
2540 #ifdef CONFIG_X86_64
2541 base |= ((u64)base3) << 32;
2543 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2544 if (r != X86EMUL_CONTINUE)
2546 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2548 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2549 if (r != X86EMUL_CONTINUE)
2551 if ((perm >> bit_idx) & mask)
2556 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2562 if (emulator_bad_iopl(ctxt))
2563 if (!emulator_io_port_access_allowed(ctxt, port, len))
2566 ctxt->perm_ok = true;
2571 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2572 struct tss_segment_16 *tss)
2574 tss->ip = ctxt->_eip;
2575 tss->flag = ctxt->eflags;
2576 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2577 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2578 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2579 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2580 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2581 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2582 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2583 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2585 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2586 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2587 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2588 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2589 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2592 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2593 struct tss_segment_16 *tss)
2598 ctxt->_eip = tss->ip;
2599 ctxt->eflags = tss->flag | 2;
2600 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2601 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2602 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2603 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2604 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2605 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2606 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2607 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2610 * SDM says that segment selectors are loaded before segment
2613 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2614 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2615 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2616 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2617 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2622 * Now load segment descriptors. If fault happens at this stage
2623 * it is handled in a context of new task
2625 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
2626 X86_TRANSFER_TASK_SWITCH, NULL);
2627 if (ret != X86EMUL_CONTINUE)
2629 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
2630 X86_TRANSFER_TASK_SWITCH, NULL);
2631 if (ret != X86EMUL_CONTINUE)
2633 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
2634 X86_TRANSFER_TASK_SWITCH, NULL);
2635 if (ret != X86EMUL_CONTINUE)
2637 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
2638 X86_TRANSFER_TASK_SWITCH, NULL);
2639 if (ret != X86EMUL_CONTINUE)
2641 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
2642 X86_TRANSFER_TASK_SWITCH, NULL);
2643 if (ret != X86EMUL_CONTINUE)
2646 return X86EMUL_CONTINUE;
2649 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2650 u16 tss_selector, u16 old_tss_sel,
2651 ulong old_tss_base, struct desc_struct *new_desc)
2653 const struct x86_emulate_ops *ops = ctxt->ops;
2654 struct tss_segment_16 tss_seg;
2656 u32 new_tss_base = get_desc_base(new_desc);
2658 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2660 if (ret != X86EMUL_CONTINUE)
2663 save_state_to_tss16(ctxt, &tss_seg);
2665 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2667 if (ret != X86EMUL_CONTINUE)
2670 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2672 if (ret != X86EMUL_CONTINUE)
2675 if (old_tss_sel != 0xffff) {
2676 tss_seg.prev_task_link = old_tss_sel;
2678 ret = ops->write_std(ctxt, new_tss_base,
2679 &tss_seg.prev_task_link,
2680 sizeof tss_seg.prev_task_link,
2682 if (ret != X86EMUL_CONTINUE)
2686 return load_state_from_tss16(ctxt, &tss_seg);
2689 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2690 struct tss_segment_32 *tss)
2692 /* CR3 and ldt selector are not saved intentionally */
2693 tss->eip = ctxt->_eip;
2694 tss->eflags = ctxt->eflags;
2695 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2696 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2697 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2698 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2699 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2700 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2701 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2702 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2704 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2705 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2706 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2707 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2708 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2709 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2712 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2713 struct tss_segment_32 *tss)
2718 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2719 return emulate_gp(ctxt, 0);
2720 ctxt->_eip = tss->eip;
2721 ctxt->eflags = tss->eflags | 2;
2723 /* General purpose registers */
2724 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2725 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2726 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2727 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2728 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2729 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2730 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2731 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2734 * SDM says that segment selectors are loaded before segment
2735 * descriptors. This is important because CPL checks will
2738 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2739 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2740 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2741 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2742 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2743 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2744 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2747 * If we're switching between Protected Mode and VM86, we need to make
2748 * sure to update the mode before loading the segment descriptors so
2749 * that the selectors are interpreted correctly.
2751 if (ctxt->eflags & X86_EFLAGS_VM) {
2752 ctxt->mode = X86EMUL_MODE_VM86;
2755 ctxt->mode = X86EMUL_MODE_PROT32;
2760 * Now load segment descriptors. If fault happenes at this stage
2761 * it is handled in a context of new task
2763 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
2764 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
2765 if (ret != X86EMUL_CONTINUE)
2767 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
2768 X86_TRANSFER_TASK_SWITCH, NULL);
2769 if (ret != X86EMUL_CONTINUE)
2771 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
2772 X86_TRANSFER_TASK_SWITCH, NULL);
2773 if (ret != X86EMUL_CONTINUE)
2775 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
2776 X86_TRANSFER_TASK_SWITCH, NULL);
2777 if (ret != X86EMUL_CONTINUE)
2779 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
2780 X86_TRANSFER_TASK_SWITCH, NULL);
2781 if (ret != X86EMUL_CONTINUE)
2783 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
2784 X86_TRANSFER_TASK_SWITCH, NULL);
2785 if (ret != X86EMUL_CONTINUE)
2787 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
2788 X86_TRANSFER_TASK_SWITCH, NULL);
2789 if (ret != X86EMUL_CONTINUE)
2792 return X86EMUL_CONTINUE;
2795 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2796 u16 tss_selector, u16 old_tss_sel,
2797 ulong old_tss_base, struct desc_struct *new_desc)
2799 const struct x86_emulate_ops *ops = ctxt->ops;
2800 struct tss_segment_32 tss_seg;
2802 u32 new_tss_base = get_desc_base(new_desc);
2803 u32 eip_offset = offsetof(struct tss_segment_32, eip);
2804 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
2806 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2808 if (ret != X86EMUL_CONTINUE)
2811 save_state_to_tss32(ctxt, &tss_seg);
2813 /* Only GP registers and segment selectors are saved */
2814 ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
2815 ldt_sel_offset - eip_offset, &ctxt->exception);
2816 if (ret != X86EMUL_CONTINUE)
2819 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2821 if (ret != X86EMUL_CONTINUE)
2824 if (old_tss_sel != 0xffff) {
2825 tss_seg.prev_task_link = old_tss_sel;
2827 ret = ops->write_std(ctxt, new_tss_base,
2828 &tss_seg.prev_task_link,
2829 sizeof tss_seg.prev_task_link,
2831 if (ret != X86EMUL_CONTINUE)
2835 return load_state_from_tss32(ctxt, &tss_seg);
2838 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2839 u16 tss_selector, int idt_index, int reason,
2840 bool has_error_code, u32 error_code)
2842 const struct x86_emulate_ops *ops = ctxt->ops;
2843 struct desc_struct curr_tss_desc, next_tss_desc;
2845 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2846 ulong old_tss_base =
2847 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2851 /* FIXME: old_tss_base == ~0 ? */
2853 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2854 if (ret != X86EMUL_CONTINUE)
2856 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2857 if (ret != X86EMUL_CONTINUE)
2860 /* FIXME: check that next_tss_desc is tss */
2863 * Check privileges. The three cases are task switch caused by...
2865 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2866 * 2. Exception/IRQ/iret: No check is performed
2867 * 3. jmp/call to TSS/task-gate: No check is performed since the
2868 * hardware checks it before exiting.
2870 if (reason == TASK_SWITCH_GATE) {
2871 if (idt_index != -1) {
2872 /* Software interrupts */
2873 struct desc_struct task_gate_desc;
2876 ret = read_interrupt_descriptor(ctxt, idt_index,
2878 if (ret != X86EMUL_CONTINUE)
2881 dpl = task_gate_desc.dpl;
2882 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2883 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2887 desc_limit = desc_limit_scaled(&next_tss_desc);
2888 if (!next_tss_desc.p ||
2889 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2890 desc_limit < 0x2b)) {
2891 return emulate_ts(ctxt, tss_selector & 0xfffc);
2894 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2895 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2896 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2899 if (reason == TASK_SWITCH_IRET)
2900 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2902 /* set back link to prev task only if NT bit is set in eflags
2903 note that old_tss_sel is not used after this point */
2904 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2905 old_tss_sel = 0xffff;
2907 if (next_tss_desc.type & 8)
2908 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2909 old_tss_base, &next_tss_desc);
2911 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2912 old_tss_base, &next_tss_desc);
2913 if (ret != X86EMUL_CONTINUE)
2916 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2917 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2919 if (reason != TASK_SWITCH_IRET) {
2920 next_tss_desc.type |= (1 << 1); /* set busy flag */
2921 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2924 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2925 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2927 if (has_error_code) {
2928 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2929 ctxt->lock_prefix = 0;
2930 ctxt->src.val = (unsigned long) error_code;
2931 ret = em_push(ctxt);
2937 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2938 u16 tss_selector, int idt_index, int reason,
2939 bool has_error_code, u32 error_code)
2943 invalidate_registers(ctxt);
2944 ctxt->_eip = ctxt->eip;
2945 ctxt->dst.type = OP_NONE;
2947 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2948 has_error_code, error_code);
2950 if (rc == X86EMUL_CONTINUE) {
2951 ctxt->eip = ctxt->_eip;
2952 writeback_registers(ctxt);
2955 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2958 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2961 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2963 register_address_increment(ctxt, reg, df * op->bytes);
2964 op->addr.mem.ea = register_address(ctxt, reg);
2967 static int em_das(struct x86_emulate_ctxt *ctxt)
2970 bool af, cf, old_cf;
2972 cf = ctxt->eflags & X86_EFLAGS_CF;
2978 af = ctxt->eflags & X86_EFLAGS_AF;
2979 if ((al & 0x0f) > 9 || af) {
2981 cf = old_cf | (al >= 250);
2986 if (old_al > 0x99 || old_cf) {
2992 /* Set PF, ZF, SF */
2993 ctxt->src.type = OP_IMM;
2995 ctxt->src.bytes = 1;
2996 fastop(ctxt, em_or);
2997 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2999 ctxt->eflags |= X86_EFLAGS_CF;
3001 ctxt->eflags |= X86_EFLAGS_AF;
3002 return X86EMUL_CONTINUE;
3005 static int em_aam(struct x86_emulate_ctxt *ctxt)
3009 if (ctxt->src.val == 0)
3010 return emulate_de(ctxt);
3012 al = ctxt->dst.val & 0xff;
3013 ah = al / ctxt->src.val;
3014 al %= ctxt->src.val;
3016 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3018 /* Set PF, ZF, SF */
3019 ctxt->src.type = OP_IMM;
3021 ctxt->src.bytes = 1;
3022 fastop(ctxt, em_or);
3024 return X86EMUL_CONTINUE;
3027 static int em_aad(struct x86_emulate_ctxt *ctxt)
3029 u8 al = ctxt->dst.val & 0xff;
3030 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3032 al = (al + (ah * ctxt->src.val)) & 0xff;
3034 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3036 /* Set PF, ZF, SF */
3037 ctxt->src.type = OP_IMM;
3039 ctxt->src.bytes = 1;
3040 fastop(ctxt, em_or);
3042 return X86EMUL_CONTINUE;
3045 static int em_call(struct x86_emulate_ctxt *ctxt)
3048 long rel = ctxt->src.val;
3050 ctxt->src.val = (unsigned long)ctxt->_eip;
3051 rc = jmp_rel(ctxt, rel);
3052 if (rc != X86EMUL_CONTINUE)
3054 return em_push(ctxt);
3057 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3062 struct desc_struct old_desc, new_desc;
3063 const struct x86_emulate_ops *ops = ctxt->ops;
3064 int cpl = ctxt->ops->cpl(ctxt);
3066 old_eip = ctxt->_eip;
3067 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3069 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3070 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3071 X86_TRANSFER_CALL_JMP, &new_desc);
3072 if (rc != X86EMUL_CONTINUE)
3075 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3076 if (rc != X86EMUL_CONTINUE)
3079 ctxt->src.val = old_cs;
3081 if (rc != X86EMUL_CONTINUE)
3084 ctxt->src.val = old_eip;
3086 /* If we failed, we tainted the memory, but the very least we should
3088 if (rc != X86EMUL_CONTINUE)
3092 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3097 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3102 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3103 if (rc != X86EMUL_CONTINUE)
3105 rc = assign_eip_near(ctxt, eip);
3106 if (rc != X86EMUL_CONTINUE)
3108 rsp_increment(ctxt, ctxt->src.val);
3109 return X86EMUL_CONTINUE;
3112 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3114 /* Write back the register source. */
3115 ctxt->src.val = ctxt->dst.val;
3116 write_register_operand(&ctxt->src);
3118 /* Write back the memory destination with implicit LOCK prefix. */
3119 ctxt->dst.val = ctxt->src.orig_val;
3120 ctxt->lock_prefix = 1;
3121 return X86EMUL_CONTINUE;
3124 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3126 ctxt->dst.val = ctxt->src2.val;
3127 return fastop(ctxt, em_imul);
3130 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3132 ctxt->dst.type = OP_REG;
3133 ctxt->dst.bytes = ctxt->src.bytes;
3134 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3135 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3137 return X86EMUL_CONTINUE;
3140 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3144 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3145 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3146 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3147 return X86EMUL_CONTINUE;
3150 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3154 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3155 return emulate_gp(ctxt, 0);
3156 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3157 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3158 return X86EMUL_CONTINUE;
3161 static int em_mov(struct x86_emulate_ctxt *ctxt)
3163 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3164 return X86EMUL_CONTINUE;
3167 #define FFL(x) bit(X86_FEATURE_##x)
3169 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3171 u32 ebx, ecx, edx, eax = 1;
3175 * Check MOVBE is set in the guest-visible CPUID leaf.
3177 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3178 if (!(ecx & FFL(MOVBE)))
3179 return emulate_ud(ctxt);
3181 switch (ctxt->op_bytes) {
3184 * From MOVBE definition: "...When the operand size is 16 bits,
3185 * the upper word of the destination register remains unchanged
3188 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3189 * rules so we have to do the operation almost per hand.
3191 tmp = (u16)ctxt->src.val;
3192 ctxt->dst.val &= ~0xffffUL;
3193 ctxt->dst.val |= (unsigned long)swab16(tmp);
3196 ctxt->dst.val = swab32((u32)ctxt->src.val);
3199 ctxt->dst.val = swab64(ctxt->src.val);
3204 return X86EMUL_CONTINUE;
3207 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3209 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3210 return emulate_gp(ctxt, 0);
3212 /* Disable writeback. */
3213 ctxt->dst.type = OP_NONE;
3214 return X86EMUL_CONTINUE;
3217 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3221 if (ctxt->mode == X86EMUL_MODE_PROT64)
3222 val = ctxt->src.val & ~0ULL;
3224 val = ctxt->src.val & ~0U;
3226 /* #UD condition is already handled. */
3227 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3228 return emulate_gp(ctxt, 0);
3230 /* Disable writeback. */
3231 ctxt->dst.type = OP_NONE;
3232 return X86EMUL_CONTINUE;
3235 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3239 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3240 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3241 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3242 return emulate_gp(ctxt, 0);
3244 return X86EMUL_CONTINUE;
3247 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3251 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3252 return emulate_gp(ctxt, 0);
3254 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3255 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3256 return X86EMUL_CONTINUE;
3259 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3261 if (ctxt->modrm_reg > VCPU_SREG_GS)
3262 return emulate_ud(ctxt);
3264 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3265 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3266 ctxt->dst.bytes = 2;
3267 return X86EMUL_CONTINUE;
3270 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3272 u16 sel = ctxt->src.val;
3274 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3275 return emulate_ud(ctxt);
3277 if (ctxt->modrm_reg == VCPU_SREG_SS)
3278 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3280 /* Disable writeback. */
3281 ctxt->dst.type = OP_NONE;
3282 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3285 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3287 u16 sel = ctxt->src.val;
3289 /* Disable writeback. */
3290 ctxt->dst.type = OP_NONE;
3291 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3294 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3296 u16 sel = ctxt->src.val;
3298 /* Disable writeback. */
3299 ctxt->dst.type = OP_NONE;
3300 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3303 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3308 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3309 if (rc == X86EMUL_CONTINUE)
3310 ctxt->ops->invlpg(ctxt, linear);
3311 /* Disable writeback. */
3312 ctxt->dst.type = OP_NONE;
3313 return X86EMUL_CONTINUE;
3316 static int em_clts(struct x86_emulate_ctxt *ctxt)
3320 cr0 = ctxt->ops->get_cr(ctxt, 0);
3322 ctxt->ops->set_cr(ctxt, 0, cr0);
3323 return X86EMUL_CONTINUE;
3326 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3328 int rc = ctxt->ops->fix_hypercall(ctxt);
3330 if (rc != X86EMUL_CONTINUE)
3333 /* Let the processor re-execute the fixed hypercall */
3334 ctxt->_eip = ctxt->eip;
3335 /* Disable writeback. */
3336 ctxt->dst.type = OP_NONE;
3337 return X86EMUL_CONTINUE;
3340 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3341 void (*get)(struct x86_emulate_ctxt *ctxt,
3342 struct desc_ptr *ptr))
3344 struct desc_ptr desc_ptr;
3346 if (ctxt->mode == X86EMUL_MODE_PROT64)
3348 get(ctxt, &desc_ptr);
3349 if (ctxt->op_bytes == 2) {
3351 desc_ptr.address &= 0x00ffffff;
3353 /* Disable writeback. */
3354 ctxt->dst.type = OP_NONE;
3355 return segmented_write(ctxt, ctxt->dst.addr.mem,
3356 &desc_ptr, 2 + ctxt->op_bytes);
3359 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3361 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3364 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3366 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3369 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3371 struct desc_ptr desc_ptr;
3374 if (ctxt->mode == X86EMUL_MODE_PROT64)
3376 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3377 &desc_ptr.size, &desc_ptr.address,
3379 if (rc != X86EMUL_CONTINUE)
3381 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3382 is_noncanonical_address(desc_ptr.address))
3383 return emulate_gp(ctxt, 0);
3385 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3387 ctxt->ops->set_idt(ctxt, &desc_ptr);
3388 /* Disable writeback. */
3389 ctxt->dst.type = OP_NONE;
3390 return X86EMUL_CONTINUE;
3393 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3395 return em_lgdt_lidt(ctxt, true);
3398 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3402 rc = ctxt->ops->fix_hypercall(ctxt);
3404 /* Disable writeback. */
3405 ctxt->dst.type = OP_NONE;
3409 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3411 return em_lgdt_lidt(ctxt, false);
3414 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3416 if (ctxt->dst.type == OP_MEM)
3417 ctxt->dst.bytes = 2;
3418 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3419 return X86EMUL_CONTINUE;
3422 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3424 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3425 | (ctxt->src.val & 0x0f));
3426 ctxt->dst.type = OP_NONE;
3427 return X86EMUL_CONTINUE;
3430 static int em_loop(struct x86_emulate_ctxt *ctxt)
3432 int rc = X86EMUL_CONTINUE;
3434 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3435 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3436 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3437 rc = jmp_rel(ctxt, ctxt->src.val);
3442 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3444 int rc = X86EMUL_CONTINUE;
3446 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3447 rc = jmp_rel(ctxt, ctxt->src.val);
3452 static int em_in(struct x86_emulate_ctxt *ctxt)
3454 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3456 return X86EMUL_IO_NEEDED;
3458 return X86EMUL_CONTINUE;
3461 static int em_out(struct x86_emulate_ctxt *ctxt)
3463 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3465 /* Disable writeback. */
3466 ctxt->dst.type = OP_NONE;
3467 return X86EMUL_CONTINUE;
3470 static int em_cli(struct x86_emulate_ctxt *ctxt)
3472 if (emulator_bad_iopl(ctxt))
3473 return emulate_gp(ctxt, 0);
3475 ctxt->eflags &= ~X86_EFLAGS_IF;
3476 return X86EMUL_CONTINUE;
3479 static int em_sti(struct x86_emulate_ctxt *ctxt)
3481 if (emulator_bad_iopl(ctxt))
3482 return emulate_gp(ctxt, 0);
3484 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3485 ctxt->eflags |= X86_EFLAGS_IF;
3486 return X86EMUL_CONTINUE;
3489 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3491 u32 eax, ebx, ecx, edx;
3493 eax = reg_read(ctxt, VCPU_REGS_RAX);
3494 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3495 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3496 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3497 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3498 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3499 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3500 return X86EMUL_CONTINUE;
3503 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3507 flags = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF;
3508 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3510 ctxt->eflags &= ~0xffUL;
3511 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3512 return X86EMUL_CONTINUE;
3515 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3517 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3518 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3519 return X86EMUL_CONTINUE;
3522 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3524 switch (ctxt->op_bytes) {
3525 #ifdef CONFIG_X86_64
3527 asm("bswap %0" : "+r"(ctxt->dst.val));
3531 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3534 return X86EMUL_CONTINUE;
3537 static int em_clflush(struct x86_emulate_ctxt *ctxt)
3539 /* emulating clflush regardless of cpuid */
3540 return X86EMUL_CONTINUE;
3543 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
3545 ctxt->dst.val = (s32) ctxt->src.val;
3546 return X86EMUL_CONTINUE;
3549 static bool valid_cr(int nr)
3561 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3563 if (!valid_cr(ctxt->modrm_reg))
3564 return emulate_ud(ctxt);
3566 return X86EMUL_CONTINUE;
3569 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3571 u64 new_val = ctxt->src.val64;
3572 int cr = ctxt->modrm_reg;
3575 static u64 cr_reserved_bits[] = {
3576 0xffffffff00000000ULL,
3577 0, 0, 0, /* CR3 checked later */
3584 return emulate_ud(ctxt);
3586 if (new_val & cr_reserved_bits[cr])
3587 return emulate_gp(ctxt, 0);
3592 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3593 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3594 return emulate_gp(ctxt, 0);
3596 cr4 = ctxt->ops->get_cr(ctxt, 4);
3597 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3599 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3600 !(cr4 & X86_CR4_PAE))
3601 return emulate_gp(ctxt, 0);
3608 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3609 if (efer & EFER_LMA)
3610 rsvd = CR3_L_MODE_RESERVED_BITS & ~CR3_PCID_INVD;
3613 return emulate_gp(ctxt, 0);
3618 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3620 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3621 return emulate_gp(ctxt, 0);
3627 return X86EMUL_CONTINUE;
3630 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3634 ctxt->ops->get_dr(ctxt, 7, &dr7);
3636 /* Check if DR7.Global_Enable is set */
3637 return dr7 & (1 << 13);
3640 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3642 int dr = ctxt->modrm_reg;
3646 return emulate_ud(ctxt);
3648 cr4 = ctxt->ops->get_cr(ctxt, 4);
3649 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3650 return emulate_ud(ctxt);
3652 if (check_dr7_gd(ctxt)) {
3655 ctxt->ops->get_dr(ctxt, 6, &dr6);
3657 dr6 |= DR6_BD | DR6_RTM;
3658 ctxt->ops->set_dr(ctxt, 6, dr6);
3659 return emulate_db(ctxt);
3662 return X86EMUL_CONTINUE;
3665 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3667 u64 new_val = ctxt->src.val64;
3668 int dr = ctxt->modrm_reg;
3670 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3671 return emulate_gp(ctxt, 0);
3673 return check_dr_read(ctxt);
3676 static int check_svme(struct x86_emulate_ctxt *ctxt)
3680 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3682 if (!(efer & EFER_SVME))
3683 return emulate_ud(ctxt);
3685 return X86EMUL_CONTINUE;
3688 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3690 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3692 /* Valid physical address? */
3693 if (rax & 0xffff000000000000ULL)
3694 return emulate_gp(ctxt, 0);
3696 return check_svme(ctxt);
3699 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3701 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3703 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3704 return emulate_ud(ctxt);
3706 return X86EMUL_CONTINUE;
3709 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3711 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3712 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3714 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3715 ctxt->ops->check_pmc(ctxt, rcx))
3716 return emulate_gp(ctxt, 0);
3718 return X86EMUL_CONTINUE;
3721 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3723 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3724 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3725 return emulate_gp(ctxt, 0);
3727 return X86EMUL_CONTINUE;
3730 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3732 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3733 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3734 return emulate_gp(ctxt, 0);
3736 return X86EMUL_CONTINUE;
3739 #define D(_y) { .flags = (_y) }
3740 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
3741 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
3742 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3743 #define N D(NotImpl)
3744 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3745 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3746 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3747 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
3748 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
3749 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3750 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3751 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3752 #define II(_f, _e, _i) \
3753 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
3754 #define IIP(_f, _e, _i, _p) \
3755 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
3756 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3757 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3759 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3760 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3761 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3762 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3763 #define I2bvIP(_f, _e, _i, _p) \
3764 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3766 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3767 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3768 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3770 static const struct opcode group7_rm0[] = {
3772 I(SrcNone | Priv | EmulateOnUD, em_vmcall),
3776 static const struct opcode group7_rm1[] = {
3777 DI(SrcNone | Priv, monitor),
3778 DI(SrcNone | Priv, mwait),
3782 static const struct opcode group7_rm3[] = {
3783 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3784 II(SrcNone | Prot | EmulateOnUD, em_vmmcall, vmmcall),
3785 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3786 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3787 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3788 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3789 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3790 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3793 static const struct opcode group7_rm7[] = {
3795 DIP(SrcNone, rdtscp, check_rdtsc),
3799 static const struct opcode group1[] = {
3801 F(Lock | PageTable, em_or),
3804 F(Lock | PageTable, em_and),
3810 static const struct opcode group1A[] = {
3811 I(DstMem | SrcNone | Mov | Stack | IncSP, em_pop), N, N, N, N, N, N, N,
3814 static const struct opcode group2[] = {
3815 F(DstMem | ModRM, em_rol),
3816 F(DstMem | ModRM, em_ror),
3817 F(DstMem | ModRM, em_rcl),
3818 F(DstMem | ModRM, em_rcr),
3819 F(DstMem | ModRM, em_shl),
3820 F(DstMem | ModRM, em_shr),
3821 F(DstMem | ModRM, em_shl),
3822 F(DstMem | ModRM, em_sar),
3825 static const struct opcode group3[] = {
3826 F(DstMem | SrcImm | NoWrite, em_test),
3827 F(DstMem | SrcImm | NoWrite, em_test),
3828 F(DstMem | SrcNone | Lock, em_not),
3829 F(DstMem | SrcNone | Lock, em_neg),
3830 F(DstXacc | Src2Mem, em_mul_ex),
3831 F(DstXacc | Src2Mem, em_imul_ex),
3832 F(DstXacc | Src2Mem, em_div_ex),
3833 F(DstXacc | Src2Mem, em_idiv_ex),
3836 static const struct opcode group4[] = {
3837 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3838 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3842 static const struct opcode group5[] = {
3843 F(DstMem | SrcNone | Lock, em_inc),
3844 F(DstMem | SrcNone | Lock, em_dec),
3845 I(SrcMem | NearBranch, em_call_near_abs),
3846 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3847 I(SrcMem | NearBranch, em_jmp_abs),
3848 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
3849 I(SrcMem | Stack, em_push), D(Undefined),
3852 static const struct opcode group6[] = {
3855 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3856 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3860 static const struct group_dual group7 = { {
3861 II(Mov | DstMem, em_sgdt, sgdt),
3862 II(Mov | DstMem, em_sidt, sidt),
3863 II(SrcMem | Priv, em_lgdt, lgdt),
3864 II(SrcMem | Priv, em_lidt, lidt),
3865 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3866 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3867 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3871 N, EXT(0, group7_rm3),
3872 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3873 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3877 static const struct opcode group8[] = {
3879 F(DstMem | SrcImmByte | NoWrite, em_bt),
3880 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3881 F(DstMem | SrcImmByte | Lock, em_btr),
3882 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3885 static const struct group_dual group9 = { {
3886 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3888 N, N, N, N, N, N, N, N,
3891 static const struct opcode group11[] = {
3892 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3896 static const struct gprefix pfx_0f_ae_7 = {
3897 I(SrcMem | ByteOp, em_clflush), N, N, N,
3900 static const struct group_dual group15 = { {
3901 N, N, N, N, N, N, N, GP(0, &pfx_0f_ae_7),
3903 N, N, N, N, N, N, N, N,
3906 static const struct gprefix pfx_0f_6f_0f_7f = {
3907 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3910 static const struct instr_dual instr_dual_0f_2b = {
3914 static const struct gprefix pfx_0f_2b = {
3915 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
3918 static const struct gprefix pfx_0f_28_0f_29 = {
3919 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
3922 static const struct gprefix pfx_0f_e7 = {
3923 N, I(Sse, em_mov), N, N,
3926 static const struct escape escape_d9 = { {
3927 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
3930 N, N, N, N, N, N, N, N,
3932 N, N, N, N, N, N, N, N,
3934 N, N, N, N, N, N, N, N,
3936 N, N, N, N, N, N, N, N,
3938 N, N, N, N, N, N, N, N,
3940 N, N, N, N, N, N, N, N,
3942 N, N, N, N, N, N, N, N,
3944 N, N, N, N, N, N, N, N,
3947 static const struct escape escape_db = { {
3948 N, N, N, N, N, N, N, N,
3951 N, N, N, N, N, N, N, N,
3953 N, N, N, N, N, N, N, N,
3955 N, N, N, N, N, N, N, N,
3957 N, N, N, N, N, N, N, N,
3959 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3961 N, N, N, N, N, N, N, N,
3963 N, N, N, N, N, N, N, N,
3965 N, N, N, N, N, N, N, N,
3968 static const struct escape escape_dd = { {
3969 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
3972 N, N, N, N, N, N, N, N,
3974 N, N, N, N, N, N, N, N,
3976 N, N, N, N, N, N, N, N,
3978 N, N, N, N, N, N, N, N,
3980 N, N, N, N, N, N, N, N,
3982 N, N, N, N, N, N, N, N,
3984 N, N, N, N, N, N, N, N,
3986 N, N, N, N, N, N, N, N,
3989 static const struct instr_dual instr_dual_0f_c3 = {
3990 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
3993 static const struct mode_dual mode_dual_63 = {
3994 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
3997 static const struct opcode opcode_table[256] = {
3999 F6ALU(Lock, em_add),
4000 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4001 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4003 F6ALU(Lock | PageTable, em_or),
4004 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4007 F6ALU(Lock, em_adc),
4008 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4009 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4011 F6ALU(Lock, em_sbb),
4012 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4013 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4015 F6ALU(Lock | PageTable, em_and), N, N,
4017 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4019 F6ALU(Lock, em_xor), N, N,
4021 F6ALU(NoWrite, em_cmp), N, N,
4023 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4025 X8(I(SrcReg | Stack, em_push)),
4027 X8(I(DstReg | Stack, em_pop)),
4029 I(ImplicitOps | Stack | No64, em_pusha),
4030 I(ImplicitOps | Stack | No64, em_popa),
4031 N, MD(ModRM, &mode_dual_63),
4034 I(SrcImm | Mov | Stack, em_push),
4035 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4036 I(SrcImmByte | Mov | Stack, em_push),
4037 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4038 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4039 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4041 X16(D(SrcImmByte | NearBranch)),
4043 G(ByteOp | DstMem | SrcImm, group1),
4044 G(DstMem | SrcImm, group1),
4045 G(ByteOp | DstMem | SrcImm | No64, group1),
4046 G(DstMem | SrcImmByte, group1),
4047 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4048 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4050 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4051 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4052 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4053 D(ModRM | SrcMem | NoAccess | DstReg),
4054 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4057 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4059 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4060 I(SrcImmFAddr | No64, em_call_far), N,
4061 II(ImplicitOps | Stack, em_pushf, pushf),
4062 II(ImplicitOps | Stack, em_popf, popf),
4063 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4065 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4066 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4067 I2bv(SrcSI | DstDI | Mov | String, em_mov),
4068 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp_r),
4070 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4071 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4072 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4073 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4075 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4077 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4079 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4080 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4081 I(ImplicitOps | NearBranch, em_ret),
4082 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4083 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4084 G(ByteOp, group11), G(0, group11),
4086 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4087 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4088 I(ImplicitOps, em_ret_far),
4089 D(ImplicitOps), DI(SrcImmByte, intn),
4090 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4092 G(Src2One | ByteOp, group2), G(Src2One, group2),
4093 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4094 I(DstAcc | SrcImmUByte | No64, em_aam),
4095 I(DstAcc | SrcImmUByte | No64, em_aad),
4096 F(DstAcc | ByteOp | No64, em_salc),
4097 I(DstAcc | SrcXLat | ByteOp, em_mov),
4099 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4101 X3(I(SrcImmByte | NearBranch, em_loop)),
4102 I(SrcImmByte | NearBranch, em_jcxz),
4103 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4104 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4106 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4107 I(SrcImmFAddr | No64, em_jmp_far),
4108 D(SrcImmByte | ImplicitOps | NearBranch),
4109 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4110 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4112 N, DI(ImplicitOps, icebp), N, N,
4113 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4114 G(ByteOp, group3), G(0, group3),
4116 D(ImplicitOps), D(ImplicitOps),
4117 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4118 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4121 static const struct opcode twobyte_table[256] = {
4123 G(0, group6), GD(0, &group7), N, N,
4124 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4125 II(ImplicitOps | Priv, em_clts, clts), N,
4126 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4127 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4129 N, N, N, N, N, N, N, N,
4130 D(ImplicitOps | ModRM | SrcMem | NoAccess),
4131 N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4133 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4134 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4135 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4137 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4140 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4141 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4142 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4145 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4146 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4147 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4148 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4149 I(ImplicitOps | EmulateOnUD, em_sysenter),
4150 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4152 N, N, N, N, N, N, N, N,
4154 X16(D(DstReg | SrcMem | ModRM)),
4156 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4161 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4166 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4168 X16(D(SrcImm | NearBranch)),
4170 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4172 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4173 II(ImplicitOps, em_cpuid, cpuid),
4174 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4175 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4176 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4178 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4179 DI(ImplicitOps, rsm),
4180 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4181 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4182 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4183 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4185 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4186 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4187 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4188 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4189 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4190 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4194 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4195 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
4196 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4198 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4199 N, ID(0, &instr_dual_0f_c3),
4200 N, N, N, GD(0, &group9),
4202 X8(I(DstReg, em_bswap)),
4204 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4206 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4207 N, N, N, N, N, N, N, N,
4209 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4212 static const struct instr_dual instr_dual_0f_38_f0 = {
4213 I(DstReg | SrcMem | Mov, em_movbe), N
4216 static const struct instr_dual instr_dual_0f_38_f1 = {
4217 I(DstMem | SrcReg | Mov, em_movbe), N
4220 static const struct gprefix three_byte_0f_38_f0 = {
4221 ID(0, &instr_dual_0f_38_f0), N, N, N
4224 static const struct gprefix three_byte_0f_38_f1 = {
4225 ID(0, &instr_dual_0f_38_f1), N, N, N
4229 * Insns below are selected by the prefix which indexed by the third opcode
4232 static const struct opcode opcode_map_0f_38[256] = {
4234 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4236 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4238 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4239 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4260 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4264 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4270 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4271 unsigned size, bool sign_extension)
4273 int rc = X86EMUL_CONTINUE;
4277 op->addr.mem.ea = ctxt->_eip;
4278 /* NB. Immediates are sign-extended as necessary. */
4279 switch (op->bytes) {
4281 op->val = insn_fetch(s8, ctxt);
4284 op->val = insn_fetch(s16, ctxt);
4287 op->val = insn_fetch(s32, ctxt);
4290 op->val = insn_fetch(s64, ctxt);
4293 if (!sign_extension) {
4294 switch (op->bytes) {
4302 op->val &= 0xffffffff;
4310 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4313 int rc = X86EMUL_CONTINUE;
4317 decode_register_operand(ctxt, op);
4320 rc = decode_imm(ctxt, op, 1, false);
4323 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4327 if (ctxt->d & BitOp)
4328 fetch_bit_operand(ctxt);
4329 op->orig_val = op->val;
4332 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4336 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4337 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4338 fetch_register_operand(op);
4339 op->orig_val = op->val;
4343 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4344 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4345 fetch_register_operand(op);
4346 op->orig_val = op->val;
4349 if (ctxt->d & ByteOp) {
4354 op->bytes = ctxt->op_bytes;
4355 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4356 fetch_register_operand(op);
4357 op->orig_val = op->val;
4361 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4363 register_address(ctxt, VCPU_REGS_RDI);
4364 op->addr.mem.seg = VCPU_SREG_ES;
4371 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4372 fetch_register_operand(op);
4377 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4380 rc = decode_imm(ctxt, op, 1, true);
4388 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4391 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4394 ctxt->memop.bytes = 1;
4395 if (ctxt->memop.type == OP_REG) {
4396 ctxt->memop.addr.reg = decode_register(ctxt,
4397 ctxt->modrm_rm, true);
4398 fetch_register_operand(&ctxt->memop);
4402 ctxt->memop.bytes = 2;
4405 ctxt->memop.bytes = 4;
4408 rc = decode_imm(ctxt, op, 2, false);
4411 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4415 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4417 register_address(ctxt, VCPU_REGS_RSI);
4418 op->addr.mem.seg = ctxt->seg_override;
4424 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4427 reg_read(ctxt, VCPU_REGS_RBX) +
4428 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4429 op->addr.mem.seg = ctxt->seg_override;
4434 op->addr.mem.ea = ctxt->_eip;
4435 op->bytes = ctxt->op_bytes + 2;
4436 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4439 ctxt->memop.bytes = ctxt->op_bytes + 2;
4443 op->val = VCPU_SREG_ES;
4447 op->val = VCPU_SREG_CS;
4451 op->val = VCPU_SREG_SS;
4455 op->val = VCPU_SREG_DS;
4459 op->val = VCPU_SREG_FS;
4463 op->val = VCPU_SREG_GS;
4466 /* Special instructions do their own operand decoding. */
4468 op->type = OP_NONE; /* Disable writeback. */
4476 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4478 int rc = X86EMUL_CONTINUE;
4479 int mode = ctxt->mode;
4480 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4481 bool op_prefix = false;
4482 bool has_seg_override = false;
4483 struct opcode opcode;
4485 ctxt->memop.type = OP_NONE;
4486 ctxt->memopp = NULL;
4487 ctxt->_eip = ctxt->eip;
4488 ctxt->fetch.ptr = ctxt->fetch.data;
4489 ctxt->fetch.end = ctxt->fetch.data + insn_len;
4490 ctxt->opcode_len = 1;
4492 memcpy(ctxt->fetch.data, insn, insn_len);
4494 rc = __do_insn_fetch_bytes(ctxt, 1);
4495 if (rc != X86EMUL_CONTINUE)
4500 case X86EMUL_MODE_REAL:
4501 case X86EMUL_MODE_VM86:
4502 case X86EMUL_MODE_PROT16:
4503 def_op_bytes = def_ad_bytes = 2;
4505 case X86EMUL_MODE_PROT32:
4506 def_op_bytes = def_ad_bytes = 4;
4508 #ifdef CONFIG_X86_64
4509 case X86EMUL_MODE_PROT64:
4515 return EMULATION_FAILED;
4518 ctxt->op_bytes = def_op_bytes;
4519 ctxt->ad_bytes = def_ad_bytes;
4521 /* Legacy prefixes. */
4523 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4524 case 0x66: /* operand-size override */
4526 /* switch between 2/4 bytes */
4527 ctxt->op_bytes = def_op_bytes ^ 6;
4529 case 0x67: /* address-size override */
4530 if (mode == X86EMUL_MODE_PROT64)
4531 /* switch between 4/8 bytes */
4532 ctxt->ad_bytes = def_ad_bytes ^ 12;
4534 /* switch between 2/4 bytes */
4535 ctxt->ad_bytes = def_ad_bytes ^ 6;
4537 case 0x26: /* ES override */
4538 case 0x2e: /* CS override */
4539 case 0x36: /* SS override */
4540 case 0x3e: /* DS override */
4541 has_seg_override = true;
4542 ctxt->seg_override = (ctxt->b >> 3) & 3;
4544 case 0x64: /* FS override */
4545 case 0x65: /* GS override */
4546 has_seg_override = true;
4547 ctxt->seg_override = ctxt->b & 7;
4549 case 0x40 ... 0x4f: /* REX */
4550 if (mode != X86EMUL_MODE_PROT64)
4552 ctxt->rex_prefix = ctxt->b;
4554 case 0xf0: /* LOCK */
4555 ctxt->lock_prefix = 1;
4557 case 0xf2: /* REPNE/REPNZ */
4558 case 0xf3: /* REP/REPE/REPZ */
4559 ctxt->rep_prefix = ctxt->b;
4565 /* Any legacy prefix after a REX prefix nullifies its effect. */
4567 ctxt->rex_prefix = 0;
4573 if (ctxt->rex_prefix & 8)
4574 ctxt->op_bytes = 8; /* REX.W */
4576 /* Opcode byte(s). */
4577 opcode = opcode_table[ctxt->b];
4578 /* Two-byte opcode? */
4579 if (ctxt->b == 0x0f) {
4580 ctxt->opcode_len = 2;
4581 ctxt->b = insn_fetch(u8, ctxt);
4582 opcode = twobyte_table[ctxt->b];
4584 /* 0F_38 opcode map */
4585 if (ctxt->b == 0x38) {
4586 ctxt->opcode_len = 3;
4587 ctxt->b = insn_fetch(u8, ctxt);
4588 opcode = opcode_map_0f_38[ctxt->b];
4591 ctxt->d = opcode.flags;
4593 if (ctxt->d & ModRM)
4594 ctxt->modrm = insn_fetch(u8, ctxt);
4596 /* vex-prefix instructions are not implemented */
4597 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
4598 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
4602 while (ctxt->d & GroupMask) {
4603 switch (ctxt->d & GroupMask) {
4605 goffset = (ctxt->modrm >> 3) & 7;
4606 opcode = opcode.u.group[goffset];
4609 goffset = (ctxt->modrm >> 3) & 7;
4610 if ((ctxt->modrm >> 6) == 3)
4611 opcode = opcode.u.gdual->mod3[goffset];
4613 opcode = opcode.u.gdual->mod012[goffset];
4616 goffset = ctxt->modrm & 7;
4617 opcode = opcode.u.group[goffset];
4620 if (ctxt->rep_prefix && op_prefix)
4621 return EMULATION_FAILED;
4622 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4623 switch (simd_prefix) {
4624 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4625 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4626 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4627 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4631 if (ctxt->modrm > 0xbf)
4632 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4634 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4637 if ((ctxt->modrm >> 6) == 3)
4638 opcode = opcode.u.idual->mod3;
4640 opcode = opcode.u.idual->mod012;
4643 if (ctxt->mode == X86EMUL_MODE_PROT64)
4644 opcode = opcode.u.mdual->mode64;
4646 opcode = opcode.u.mdual->mode32;
4649 return EMULATION_FAILED;
4652 ctxt->d &= ~(u64)GroupMask;
4653 ctxt->d |= opcode.flags;
4658 return EMULATION_FAILED;
4660 ctxt->execute = opcode.u.execute;
4662 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
4663 return EMULATION_FAILED;
4665 if (unlikely(ctxt->d &
4666 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
4669 * These are copied unconditionally here, and checked unconditionally
4670 * in x86_emulate_insn.
4672 ctxt->check_perm = opcode.check_perm;
4673 ctxt->intercept = opcode.intercept;
4675 if (ctxt->d & NotImpl)
4676 return EMULATION_FAILED;
4678 if (mode == X86EMUL_MODE_PROT64) {
4679 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
4681 else if (ctxt->d & NearBranch)
4685 if (ctxt->d & Op3264) {
4686 if (mode == X86EMUL_MODE_PROT64)
4692 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
4696 ctxt->op_bytes = 16;
4697 else if (ctxt->d & Mmx)
4701 /* ModRM and SIB bytes. */
4702 if (ctxt->d & ModRM) {
4703 rc = decode_modrm(ctxt, &ctxt->memop);
4704 if (!has_seg_override) {
4705 has_seg_override = true;
4706 ctxt->seg_override = ctxt->modrm_seg;
4708 } else if (ctxt->d & MemAbs)
4709 rc = decode_abs(ctxt, &ctxt->memop);
4710 if (rc != X86EMUL_CONTINUE)
4713 if (!has_seg_override)
4714 ctxt->seg_override = VCPU_SREG_DS;
4716 ctxt->memop.addr.mem.seg = ctxt->seg_override;
4719 * Decode and fetch the source operand: register, memory
4722 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4723 if (rc != X86EMUL_CONTINUE)
4727 * Decode and fetch the second source operand: register, memory
4730 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4731 if (rc != X86EMUL_CONTINUE)
4734 /* Decode and fetch the destination operand: register or memory. */
4735 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4737 if (ctxt->rip_relative)
4738 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
4739 ctxt->memopp->addr.mem.ea + ctxt->_eip);
4742 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4745 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4747 return ctxt->d & PageTable;
4750 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4752 /* The second termination condition only applies for REPE
4753 * and REPNE. Test if the repeat string operation prefix is
4754 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4755 * corresponding termination condition according to:
4756 * - if REPE/REPZ and ZF = 0 then done
4757 * - if REPNE/REPNZ and ZF = 1 then done
4759 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4760 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4761 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4762 ((ctxt->eflags & EFLG_ZF) == 0))
4763 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4764 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4770 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4774 ctxt->ops->get_fpu(ctxt);
4775 asm volatile("1: fwait \n\t"
4777 ".pushsection .fixup,\"ax\" \n\t"
4779 "movb $1, %[fault] \n\t"
4782 _ASM_EXTABLE(1b, 3b)
4783 : [fault]"+qm"(fault));
4784 ctxt->ops->put_fpu(ctxt);
4786 if (unlikely(fault))
4787 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4789 return X86EMUL_CONTINUE;
4792 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4795 if (op->type == OP_MM)
4796 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4799 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4801 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4802 if (!(ctxt->d & ByteOp))
4803 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4804 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4805 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
4807 : "c"(ctxt->src2.val));
4808 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4809 if (!fop) /* exception is returned in fop variable */
4810 return emulate_de(ctxt);
4811 return X86EMUL_CONTINUE;
4814 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
4816 memset(&ctxt->rip_relative, 0,
4817 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
4819 ctxt->io_read.pos = 0;
4820 ctxt->io_read.end = 0;
4821 ctxt->mem_read.end = 0;
4824 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4826 const struct x86_emulate_ops *ops = ctxt->ops;
4827 int rc = X86EMUL_CONTINUE;
4828 int saved_dst_type = ctxt->dst.type;
4830 ctxt->mem_read.pos = 0;
4832 /* LOCK prefix is allowed only with some instructions */
4833 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4834 rc = emulate_ud(ctxt);
4838 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4839 rc = emulate_ud(ctxt);
4843 if (unlikely(ctxt->d &
4844 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
4845 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4846 (ctxt->d & Undefined)) {
4847 rc = emulate_ud(ctxt);
4851 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4852 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4853 rc = emulate_ud(ctxt);
4857 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4858 rc = emulate_nm(ctxt);
4862 if (ctxt->d & Mmx) {
4863 rc = flush_pending_x87_faults(ctxt);
4864 if (rc != X86EMUL_CONTINUE)
4867 * Now that we know the fpu is exception safe, we can fetch
4870 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4871 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4872 if (!(ctxt->d & Mov))
4873 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4876 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4877 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4878 X86_ICPT_PRE_EXCEPT);
4879 if (rc != X86EMUL_CONTINUE)
4883 /* Instruction can only be executed in protected mode */
4884 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4885 rc = emulate_ud(ctxt);
4889 /* Privileged instruction can be executed only in CPL=0 */
4890 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4891 if (ctxt->d & PrivUD)
4892 rc = emulate_ud(ctxt);
4894 rc = emulate_gp(ctxt, 0);
4898 /* Do instruction specific permission checks */
4899 if (ctxt->d & CheckPerm) {
4900 rc = ctxt->check_perm(ctxt);
4901 if (rc != X86EMUL_CONTINUE)
4905 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4906 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4907 X86_ICPT_POST_EXCEPT);
4908 if (rc != X86EMUL_CONTINUE)
4912 if (ctxt->rep_prefix && (ctxt->d & String)) {
4913 /* All REP prefixes have the same first termination condition */
4914 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4915 ctxt->eip = ctxt->_eip;
4916 ctxt->eflags &= ~EFLG_RF;
4922 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4923 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4924 ctxt->src.valptr, ctxt->src.bytes);
4925 if (rc != X86EMUL_CONTINUE)
4927 ctxt->src.orig_val64 = ctxt->src.val64;
4930 if (ctxt->src2.type == OP_MEM) {
4931 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4932 &ctxt->src2.val, ctxt->src2.bytes);
4933 if (rc != X86EMUL_CONTINUE)
4937 if ((ctxt->d & DstMask) == ImplicitOps)
4941 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4942 /* optimisation - avoid slow emulated read if Mov */
4943 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4944 &ctxt->dst.val, ctxt->dst.bytes);
4945 if (rc != X86EMUL_CONTINUE) {
4946 if (rc == X86EMUL_PROPAGATE_FAULT &&
4947 ctxt->exception.vector == PF_VECTOR)
4948 ctxt->exception.error_code |= PFERR_WRITE_MASK;
4952 ctxt->dst.orig_val = ctxt->dst.val;
4956 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4957 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4958 X86_ICPT_POST_MEMACCESS);
4959 if (rc != X86EMUL_CONTINUE)
4963 if (ctxt->rep_prefix && (ctxt->d & String))
4964 ctxt->eflags |= EFLG_RF;
4966 ctxt->eflags &= ~EFLG_RF;
4968 if (ctxt->execute) {
4969 if (ctxt->d & Fastop) {
4970 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4971 rc = fastop(ctxt, fop);
4972 if (rc != X86EMUL_CONTINUE)
4976 rc = ctxt->execute(ctxt);
4977 if (rc != X86EMUL_CONTINUE)
4982 if (ctxt->opcode_len == 2)
4984 else if (ctxt->opcode_len == 3)
4985 goto threebyte_insn;
4988 case 0x70 ... 0x7f: /* jcc (short) */
4989 if (test_cc(ctxt->b, ctxt->eflags))
4990 rc = jmp_rel(ctxt, ctxt->src.val);
4992 case 0x8d: /* lea r16/r32, m */
4993 ctxt->dst.val = ctxt->src.addr.mem.ea;
4995 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4996 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4997 ctxt->dst.type = OP_NONE;
5001 case 0x98: /* cbw/cwde/cdqe */
5002 switch (ctxt->op_bytes) {
5003 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5004 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5005 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5008 case 0xcc: /* int3 */
5009 rc = emulate_int(ctxt, 3);
5011 case 0xcd: /* int n */
5012 rc = emulate_int(ctxt, ctxt->src.val);
5014 case 0xce: /* into */
5015 if (ctxt->eflags & EFLG_OF)
5016 rc = emulate_int(ctxt, 4);
5018 case 0xe9: /* jmp rel */
5019 case 0xeb: /* jmp rel short */
5020 rc = jmp_rel(ctxt, ctxt->src.val);
5021 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5023 case 0xf4: /* hlt */
5024 ctxt->ops->halt(ctxt);
5026 case 0xf5: /* cmc */
5027 /* complement carry flag from eflags reg */
5028 ctxt->eflags ^= EFLG_CF;
5030 case 0xf8: /* clc */
5031 ctxt->eflags &= ~EFLG_CF;
5033 case 0xf9: /* stc */
5034 ctxt->eflags |= EFLG_CF;
5036 case 0xfc: /* cld */
5037 ctxt->eflags &= ~EFLG_DF;
5039 case 0xfd: /* std */
5040 ctxt->eflags |= EFLG_DF;
5043 goto cannot_emulate;
5046 if (rc != X86EMUL_CONTINUE)
5050 if (ctxt->d & SrcWrite) {
5051 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5052 rc = writeback(ctxt, &ctxt->src);
5053 if (rc != X86EMUL_CONTINUE)
5056 if (!(ctxt->d & NoWrite)) {
5057 rc = writeback(ctxt, &ctxt->dst);
5058 if (rc != X86EMUL_CONTINUE)
5063 * restore dst type in case the decoding will be reused
5064 * (happens for string instruction )
5066 ctxt->dst.type = saved_dst_type;
5068 if ((ctxt->d & SrcMask) == SrcSI)
5069 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5071 if ((ctxt->d & DstMask) == DstDI)
5072 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5074 if (ctxt->rep_prefix && (ctxt->d & String)) {
5076 struct read_cache *r = &ctxt->io_read;
5077 if ((ctxt->d & SrcMask) == SrcSI)
5078 count = ctxt->src.count;
5080 count = ctxt->dst.count;
5081 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5083 if (!string_insn_completed(ctxt)) {
5085 * Re-enter guest when pio read ahead buffer is empty
5086 * or, if it is not used, after each 1024 iteration.
5088 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5089 (r->end == 0 || r->end != r->pos)) {
5091 * Reset read cache. Usually happens before
5092 * decode, but since instruction is restarted
5093 * we have to do it here.
5095 ctxt->mem_read.end = 0;
5096 writeback_registers(ctxt);
5097 return EMULATION_RESTART;
5099 goto done; /* skip rip writeback */
5101 ctxt->eflags &= ~EFLG_RF;
5104 ctxt->eip = ctxt->_eip;
5107 if (rc == X86EMUL_PROPAGATE_FAULT) {
5108 WARN_ON(ctxt->exception.vector > 0x1f);
5109 ctxt->have_exception = true;
5111 if (rc == X86EMUL_INTERCEPTED)
5112 return EMULATION_INTERCEPTED;
5114 if (rc == X86EMUL_CONTINUE)
5115 writeback_registers(ctxt);
5117 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5121 case 0x09: /* wbinvd */
5122 (ctxt->ops->wbinvd)(ctxt);
5124 case 0x08: /* invd */
5125 case 0x0d: /* GrpP (prefetch) */
5126 case 0x18: /* Grp16 (prefetch/nop) */
5127 case 0x1f: /* nop */
5129 case 0x20: /* mov cr, reg */
5130 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5132 case 0x21: /* mov from dr to reg */
5133 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5135 case 0x40 ... 0x4f: /* cmov */
5136 if (test_cc(ctxt->b, ctxt->eflags))
5137 ctxt->dst.val = ctxt->src.val;
5138 else if (ctxt->mode != X86EMUL_MODE_PROT64 ||
5139 ctxt->op_bytes != 4)
5140 ctxt->dst.type = OP_NONE; /* no writeback */
5142 case 0x80 ... 0x8f: /* jnz rel, etc*/
5143 if (test_cc(ctxt->b, ctxt->eflags))
5144 rc = jmp_rel(ctxt, ctxt->src.val);
5146 case 0x90 ... 0x9f: /* setcc r/m8 */
5147 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5149 case 0xb6 ... 0xb7: /* movzx */
5150 ctxt->dst.bytes = ctxt->op_bytes;
5151 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5152 : (u16) ctxt->src.val;
5154 case 0xbe ... 0xbf: /* movsx */
5155 ctxt->dst.bytes = ctxt->op_bytes;
5156 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5157 (s16) ctxt->src.val;
5160 goto cannot_emulate;
5165 if (rc != X86EMUL_CONTINUE)
5171 return EMULATION_FAILED;
5174 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5176 invalidate_registers(ctxt);
5179 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5181 writeback_registers(ctxt);