1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstMem16 (OpMem16 << DstShift)
90 #define DstImmUByte (OpImmUByte << DstShift)
91 #define DstDX (OpDX << DstShift)
92 #define DstAccLo (OpAccLo << DstShift)
93 #define DstMask (OpMask << DstShift)
94 /* Source operand type. */
96 #define SrcNone (OpNone << SrcShift)
97 #define SrcReg (OpReg << SrcShift)
98 #define SrcMem (OpMem << SrcShift)
99 #define SrcMem16 (OpMem16 << SrcShift)
100 #define SrcMem32 (OpMem32 << SrcShift)
101 #define SrcImm (OpImm << SrcShift)
102 #define SrcImmByte (OpImmByte << SrcShift)
103 #define SrcOne (OpOne << SrcShift)
104 #define SrcImmUByte (OpImmUByte << SrcShift)
105 #define SrcImmU (OpImmU << SrcShift)
106 #define SrcSI (OpSI << SrcShift)
107 #define SrcXLat (OpXLat << SrcShift)
108 #define SrcImmFAddr (OpImmFAddr << SrcShift)
109 #define SrcMemFAddr (OpMemFAddr << SrcShift)
110 #define SrcAcc (OpAcc << SrcShift)
111 #define SrcImmU16 (OpImmU16 << SrcShift)
112 #define SrcImm64 (OpImm64 << SrcShift)
113 #define SrcDX (OpDX << SrcShift)
114 #define SrcMem8 (OpMem8 << SrcShift)
115 #define SrcAccHi (OpAccHi << SrcShift)
116 #define SrcMask (OpMask << SrcShift)
117 #define BitOp (1<<11)
118 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
119 #define String (1<<13) /* String instruction (rep capable) */
120 #define Stack (1<<14) /* Stack instruction (push/pop) */
121 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
122 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
123 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
124 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
125 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
126 #define Escape (5<<15) /* Escape to coprocessor instruction */
127 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
128 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
129 #define Sse (1<<18) /* SSE Vector instruction */
130 /* Generic ModRM decode. */
131 #define ModRM (1<<19)
132 /* Destination is only written; never read. */
135 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
136 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
137 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
138 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
139 #define Undefined (1<<25) /* No Such Instruction */
140 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
141 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
143 #define PageTable (1 << 29) /* instruction used to write page table */
144 #define NotImpl (1 << 30) /* instruction is not implemented */
145 /* Source 2 operand type */
146 #define Src2Shift (31)
147 #define Src2None (OpNone << Src2Shift)
148 #define Src2Mem (OpMem << Src2Shift)
149 #define Src2CL (OpCL << Src2Shift)
150 #define Src2ImmByte (OpImmByte << Src2Shift)
151 #define Src2One (OpOne << Src2Shift)
152 #define Src2Imm (OpImm << Src2Shift)
153 #define Src2ES (OpES << Src2Shift)
154 #define Src2CS (OpCS << Src2Shift)
155 #define Src2SS (OpSS << Src2Shift)
156 #define Src2DS (OpDS << Src2Shift)
157 #define Src2FS (OpFS << Src2Shift)
158 #define Src2GS (OpGS << Src2Shift)
159 #define Src2Mask (OpMask << Src2Shift)
160 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
161 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
162 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
163 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
164 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
165 #define NoWrite ((u64)1 << 45) /* No writeback */
166 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
167 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
168 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
169 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
170 #define NoBigReal ((u64)1 << 50) /* No big real mode */
171 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
172 #define NearBranch ((u64)1 << 52) /* Near branches */
173 #define No16 ((u64)1 << 53) /* No 16 bit operand */
174 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
176 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
178 #define X2(x...) x, x
179 #define X3(x...) X2(x), x
180 #define X4(x...) X2(x), X2(x)
181 #define X5(x...) X4(x), x
182 #define X6(x...) X4(x), X2(x)
183 #define X7(x...) X4(x), X3(x)
184 #define X8(x...) X4(x), X4(x)
185 #define X16(x...) X8(x), X8(x)
187 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
188 #define FASTOP_SIZE 8
191 * fastop functions have a special calling convention:
196 * flags: rflags (in/out)
197 * ex: rsi (in:fastop pointer, out:zero if exception)
199 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
200 * different operand sizes can be reached by calculation, rather than a jump
201 * table (which would be bigger than the code).
203 * fastop functions are declared as taking a never-defined fastop parameter,
204 * so they can't be called from C directly.
213 int (*execute)(struct x86_emulate_ctxt *ctxt);
214 const struct opcode *group;
215 const struct group_dual *gdual;
216 const struct gprefix *gprefix;
217 const struct escape *esc;
218 const struct instr_dual *idual;
219 const struct mode_dual *mdual;
220 void (*fastop)(struct fastop *fake);
222 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
226 struct opcode mod012[8];
227 struct opcode mod3[8];
231 struct opcode pfx_no;
232 struct opcode pfx_66;
233 struct opcode pfx_f2;
234 struct opcode pfx_f3;
239 struct opcode high[64];
243 struct opcode mod012;
248 struct opcode mode32;
249 struct opcode mode64;
252 /* EFLAGS bit definitions. */
253 #define EFLG_ID (1<<21)
254 #define EFLG_VIP (1<<20)
255 #define EFLG_VIF (1<<19)
256 #define EFLG_AC (1<<18)
257 #define EFLG_VM (1<<17)
258 #define EFLG_RF (1<<16)
259 #define EFLG_IOPL (3<<12)
260 #define EFLG_NT (1<<14)
261 #define EFLG_OF (1<<11)
262 #define EFLG_DF (1<<10)
263 #define EFLG_IF (1<<9)
264 #define EFLG_TF (1<<8)
265 #define EFLG_SF (1<<7)
266 #define EFLG_ZF (1<<6)
267 #define EFLG_AF (1<<4)
268 #define EFLG_PF (1<<2)
269 #define EFLG_CF (1<<0)
271 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
272 #define EFLG_RESERVED_ONE_MASK 2
274 enum x86_transfer_type {
276 X86_TRANSFER_CALL_JMP,
278 X86_TRANSFER_TASK_SWITCH,
281 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
283 if (!(ctxt->regs_valid & (1 << nr))) {
284 ctxt->regs_valid |= 1 << nr;
285 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
287 return ctxt->_regs[nr];
290 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
292 ctxt->regs_valid |= 1 << nr;
293 ctxt->regs_dirty |= 1 << nr;
294 return &ctxt->_regs[nr];
297 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
300 return reg_write(ctxt, nr);
303 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
307 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
308 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
311 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
313 ctxt->regs_dirty = 0;
314 ctxt->regs_valid = 0;
318 * These EFLAGS bits are restored from saved value during emulation, and
319 * any changes are written back to the saved value after emulation.
321 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
329 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
331 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
332 #define FOP_RET "ret \n\t"
334 #define FOP_START(op) \
335 extern void em_##op(struct fastop *fake); \
336 asm(".pushsection .text, \"ax\" \n\t" \
337 ".global em_" #op " \n\t" \
344 #define FOPNOP() FOP_ALIGN FOP_RET
346 #define FOP1E(op, dst) \
347 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
349 #define FOP1EEX(op, dst) \
350 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
352 #define FASTOP1(op) \
357 ON64(FOP1E(op##q, rax)) \
360 /* 1-operand, using src2 (for MUL/DIV r/m) */
361 #define FASTOP1SRC2(op, name) \
366 ON64(FOP1E(op, rcx)) \
369 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
370 #define FASTOP1SRC2EX(op, name) \
375 ON64(FOP1EEX(op, rcx)) \
378 #define FOP2E(op, dst, src) \
379 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
381 #define FASTOP2(op) \
383 FOP2E(op##b, al, dl) \
384 FOP2E(op##w, ax, dx) \
385 FOP2E(op##l, eax, edx) \
386 ON64(FOP2E(op##q, rax, rdx)) \
389 /* 2 operand, word only */
390 #define FASTOP2W(op) \
393 FOP2E(op##w, ax, dx) \
394 FOP2E(op##l, eax, edx) \
395 ON64(FOP2E(op##q, rax, rdx)) \
398 /* 2 operand, src is CL */
399 #define FASTOP2CL(op) \
401 FOP2E(op##b, al, cl) \
402 FOP2E(op##w, ax, cl) \
403 FOP2E(op##l, eax, cl) \
404 ON64(FOP2E(op##q, rax, cl)) \
407 /* 2 operand, src and dest are reversed */
408 #define FASTOP2R(op, name) \
410 FOP2E(op##b, dl, al) \
411 FOP2E(op##w, dx, ax) \
412 FOP2E(op##l, edx, eax) \
413 ON64(FOP2E(op##q, rdx, rax)) \
416 #define FOP3E(op, dst, src, src2) \
417 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
419 /* 3-operand, word-only, src2=cl */
420 #define FASTOP3WCL(op) \
423 FOP3E(op##w, ax, dx, cl) \
424 FOP3E(op##l, eax, edx, cl) \
425 ON64(FOP3E(op##q, rax, rdx, cl)) \
428 /* Special case for SETcc - 1 instruction per cc */
429 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
431 asm(".global kvm_fastop_exception \n"
432 "kvm_fastop_exception: xor %esi, %esi; ret");
453 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
456 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
457 enum x86_intercept intercept,
458 enum x86_intercept_stage stage)
460 struct x86_instruction_info info = {
461 .intercept = intercept,
462 .rep_prefix = ctxt->rep_prefix,
463 .modrm_mod = ctxt->modrm_mod,
464 .modrm_reg = ctxt->modrm_reg,
465 .modrm_rm = ctxt->modrm_rm,
466 .src_val = ctxt->src.val64,
467 .dst_val = ctxt->dst.val64,
468 .src_bytes = ctxt->src.bytes,
469 .dst_bytes = ctxt->dst.bytes,
470 .ad_bytes = ctxt->ad_bytes,
471 .next_rip = ctxt->eip,
474 return ctxt->ops->intercept(ctxt, &info, stage);
477 static void assign_masked(ulong *dest, ulong src, ulong mask)
479 *dest = (*dest & ~mask) | (src & mask);
482 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
484 return (1UL << (ctxt->ad_bytes << 3)) - 1;
487 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
490 struct desc_struct ss;
492 if (ctxt->mode == X86EMUL_MODE_PROT64)
494 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
495 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
498 static int stack_size(struct x86_emulate_ctxt *ctxt)
500 return (__fls(stack_mask(ctxt)) + 1) >> 3;
503 /* Access/update address held in a register, based on addressing mode. */
504 static inline unsigned long
505 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
507 if (ctxt->ad_bytes == sizeof(unsigned long))
510 return reg & ad_mask(ctxt);
513 static inline unsigned long
514 register_address(struct x86_emulate_ctxt *ctxt, int reg)
516 return address_mask(ctxt, reg_read(ctxt, reg));
519 static void masked_increment(ulong *reg, ulong mask, int inc)
521 assign_masked(reg, *reg + inc, mask);
525 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
529 if (ctxt->ad_bytes == sizeof(unsigned long))
532 mask = ad_mask(ctxt);
533 masked_increment(reg_rmw(ctxt, reg), mask, inc);
536 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
538 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
541 static u32 desc_limit_scaled(struct desc_struct *desc)
543 u32 limit = get_desc_limit(desc);
545 return desc->g ? (limit << 12) | 0xfff : limit;
548 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
550 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
553 return ctxt->ops->get_cached_segment_base(ctxt, seg);
556 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
557 u32 error, bool valid)
560 ctxt->exception.vector = vec;
561 ctxt->exception.error_code = error;
562 ctxt->exception.error_code_valid = valid;
563 return X86EMUL_PROPAGATE_FAULT;
566 static int emulate_db(struct x86_emulate_ctxt *ctxt)
568 return emulate_exception(ctxt, DB_VECTOR, 0, false);
571 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
573 return emulate_exception(ctxt, GP_VECTOR, err, true);
576 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
578 return emulate_exception(ctxt, SS_VECTOR, err, true);
581 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
583 return emulate_exception(ctxt, UD_VECTOR, 0, false);
586 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
588 return emulate_exception(ctxt, TS_VECTOR, err, true);
591 static int emulate_de(struct x86_emulate_ctxt *ctxt)
593 return emulate_exception(ctxt, DE_VECTOR, 0, false);
596 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
598 return emulate_exception(ctxt, NM_VECTOR, 0, false);
601 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
604 struct desc_struct desc;
606 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
610 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
615 struct desc_struct desc;
617 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
618 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
622 * x86 defines three classes of vector instructions: explicitly
623 * aligned, explicitly unaligned, and the rest, which change behaviour
624 * depending on whether they're AVX encoded or not.
626 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
627 * subject to the same check.
629 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
631 if (likely(size < 16))
634 if (ctxt->d & Aligned)
636 else if (ctxt->d & Unaligned)
638 else if (ctxt->d & Avx)
644 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
645 struct segmented_address addr,
646 unsigned *max_size, unsigned size,
647 bool write, bool fetch,
648 enum x86emul_mode mode, ulong *linear)
650 struct desc_struct desc;
656 la = seg_base(ctxt, addr.seg) + addr.ea;
659 case X86EMUL_MODE_PROT64:
660 if (is_noncanonical_address(la))
663 *max_size = min_t(u64, ~0u, (1ull << 48) - la);
664 if (size > *max_size)
668 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
672 /* code segment in protected mode or read-only data segment */
673 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
674 || !(desc.type & 2)) && write)
676 /* unreadable code segment */
677 if (!fetch && (desc.type & 8) && !(desc.type & 2))
679 lim = desc_limit_scaled(&desc);
680 if (!(desc.type & 8) && (desc.type & 4)) {
681 /* expand-down segment */
684 lim = desc.d ? 0xffffffff : 0xffff;
688 *max_size = min_t(u64, ~0u, (u64)lim + 1 - addr.ea);
689 if (size > *max_size)
694 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
695 return emulate_gp(ctxt, 0);
697 return X86EMUL_CONTINUE;
699 if (addr.seg == VCPU_SREG_SS)
700 return emulate_ss(ctxt, 0);
702 return emulate_gp(ctxt, 0);
705 static int linearize(struct x86_emulate_ctxt *ctxt,
706 struct segmented_address addr,
707 unsigned size, bool write,
711 return __linearize(ctxt, addr, &max_size, size, write, false,
715 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
716 enum x86emul_mode mode)
721 struct segmented_address addr = { .seg = VCPU_SREG_CS,
724 if (ctxt->op_bytes != sizeof(unsigned long))
725 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
726 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
727 if (rc == X86EMUL_CONTINUE)
728 ctxt->_eip = addr.ea;
732 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
734 return assign_eip(ctxt, dst, ctxt->mode);
737 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
738 const struct desc_struct *cs_desc)
740 enum x86emul_mode mode = ctxt->mode;
743 if (ctxt->mode >= X86EMUL_MODE_PROT32 && cs_desc->l) {
746 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
748 mode = X86EMUL_MODE_PROT64;
751 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
752 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
753 return assign_eip(ctxt, dst, mode);
756 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
758 return assign_eip_near(ctxt, ctxt->_eip + rel);
761 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
762 struct segmented_address addr,
769 rc = linearize(ctxt, addr, size, false, &linear);
770 if (rc != X86EMUL_CONTINUE)
772 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
776 * Prefetch the remaining bytes of the instruction without crossing page
777 * boundary if they are not in fetch_cache yet.
779 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
782 unsigned size, max_size;
783 unsigned long linear;
784 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
785 struct segmented_address addr = { .seg = VCPU_SREG_CS,
786 .ea = ctxt->eip + cur_size };
789 * We do not know exactly how many bytes will be needed, and
790 * __linearize is expensive, so fetch as much as possible. We
791 * just have to avoid going beyond the 15 byte limit, the end
792 * of the segment, or the end of the page.
794 * __linearize is called with size 0 so that it does not do any
795 * boundary check itself. Instead, we use max_size to check
798 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
800 if (unlikely(rc != X86EMUL_CONTINUE))
803 size = min_t(unsigned, 15UL ^ cur_size, max_size);
804 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
807 * One instruction can only straddle two pages,
808 * and one has been loaded at the beginning of
809 * x86_decode_insn. So, if not enough bytes
810 * still, we must have hit the 15-byte boundary.
812 if (unlikely(size < op_size))
813 return emulate_gp(ctxt, 0);
815 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
816 size, &ctxt->exception);
817 if (unlikely(rc != X86EMUL_CONTINUE))
819 ctxt->fetch.end += size;
820 return X86EMUL_CONTINUE;
823 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
826 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
828 if (unlikely(done_size < size))
829 return __do_insn_fetch_bytes(ctxt, size - done_size);
831 return X86EMUL_CONTINUE;
834 /* Fetch next part of the instruction being emulated. */
835 #define insn_fetch(_type, _ctxt) \
838 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
839 if (rc != X86EMUL_CONTINUE) \
841 ctxt->_eip += sizeof(_type); \
842 _x = *(_type __aligned(1) *) ctxt->fetch.ptr; \
843 ctxt->fetch.ptr += sizeof(_type); \
847 #define insn_fetch_arr(_arr, _size, _ctxt) \
849 rc = do_insn_fetch_bytes(_ctxt, _size); \
850 if (rc != X86EMUL_CONTINUE) \
852 ctxt->_eip += (_size); \
853 memcpy(_arr, ctxt->fetch.ptr, _size); \
854 ctxt->fetch.ptr += (_size); \
858 * Given the 'reg' portion of a ModRM byte, and a register block, return a
859 * pointer into the block that addresses the relevant register.
860 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
862 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
866 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
868 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
869 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
871 p = reg_rmw(ctxt, modrm_reg);
875 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
876 struct segmented_address addr,
877 u16 *size, unsigned long *address, int op_bytes)
884 rc = segmented_read_std(ctxt, addr, size, 2);
885 if (rc != X86EMUL_CONTINUE)
888 rc = segmented_read_std(ctxt, addr, address, op_bytes);
902 FASTOP1SRC2(mul, mul_ex);
903 FASTOP1SRC2(imul, imul_ex);
904 FASTOP1SRC2EX(div, div_ex);
905 FASTOP1SRC2EX(idiv, idiv_ex);
934 FASTOP2R(cmp, cmp_r);
936 static u8 test_cc(unsigned int condition, unsigned long flags)
939 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
941 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
942 asm("push %[flags]; popf; call *%[fastop]"
943 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
947 static void fetch_register_operand(struct operand *op)
951 op->val = *(u8 *)op->addr.reg;
954 op->val = *(u16 *)op->addr.reg;
957 op->val = *(u32 *)op->addr.reg;
960 op->val = *(u64 *)op->addr.reg;
965 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
967 ctxt->ops->get_fpu(ctxt);
969 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
970 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
971 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
972 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
973 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
974 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
975 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
976 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
978 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
979 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
980 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
981 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
982 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
983 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
984 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
985 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
989 ctxt->ops->put_fpu(ctxt);
992 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
995 ctxt->ops->get_fpu(ctxt);
997 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
998 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
999 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1000 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1001 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1002 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1003 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1004 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1005 #ifdef CONFIG_X86_64
1006 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1007 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1008 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1009 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1010 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1011 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1012 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1013 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1017 ctxt->ops->put_fpu(ctxt);
1020 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1022 ctxt->ops->get_fpu(ctxt);
1024 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1025 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1026 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1027 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1028 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1029 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1030 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1031 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1034 ctxt->ops->put_fpu(ctxt);
1037 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1039 ctxt->ops->get_fpu(ctxt);
1041 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1042 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1043 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1044 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1045 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1046 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1047 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1048 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1051 ctxt->ops->put_fpu(ctxt);
1054 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1056 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1057 return emulate_nm(ctxt);
1059 ctxt->ops->get_fpu(ctxt);
1060 asm volatile("fninit");
1061 ctxt->ops->put_fpu(ctxt);
1062 return X86EMUL_CONTINUE;
1065 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1069 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1070 return emulate_nm(ctxt);
1072 ctxt->ops->get_fpu(ctxt);
1073 asm volatile("fnstcw %0": "+m"(fcw));
1074 ctxt->ops->put_fpu(ctxt);
1076 ctxt->dst.val = fcw;
1078 return X86EMUL_CONTINUE;
1081 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1085 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1086 return emulate_nm(ctxt);
1088 ctxt->ops->get_fpu(ctxt);
1089 asm volatile("fnstsw %0": "+m"(fsw));
1090 ctxt->ops->put_fpu(ctxt);
1092 ctxt->dst.val = fsw;
1094 return X86EMUL_CONTINUE;
1097 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1100 unsigned reg = ctxt->modrm_reg;
1102 if (!(ctxt->d & ModRM))
1103 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1105 if (ctxt->d & Sse) {
1109 read_sse_reg(ctxt, &op->vec_val, reg);
1112 if (ctxt->d & Mmx) {
1121 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1122 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1124 fetch_register_operand(op);
1125 op->orig_val = op->val;
1128 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1130 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1131 ctxt->modrm_seg = VCPU_SREG_SS;
1134 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1138 int index_reg, base_reg, scale;
1139 int rc = X86EMUL_CONTINUE;
1142 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1143 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1144 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1146 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1147 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1148 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1149 ctxt->modrm_seg = VCPU_SREG_DS;
1151 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1153 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1154 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1156 if (ctxt->d & Sse) {
1159 op->addr.xmm = ctxt->modrm_rm;
1160 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1163 if (ctxt->d & Mmx) {
1166 op->addr.mm = ctxt->modrm_rm & 7;
1169 fetch_register_operand(op);
1175 if (ctxt->ad_bytes == 2) {
1176 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1177 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1178 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1179 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1181 /* 16-bit ModR/M decode. */
1182 switch (ctxt->modrm_mod) {
1184 if (ctxt->modrm_rm == 6)
1185 modrm_ea += insn_fetch(u16, ctxt);
1188 modrm_ea += insn_fetch(s8, ctxt);
1191 modrm_ea += insn_fetch(u16, ctxt);
1194 switch (ctxt->modrm_rm) {
1196 modrm_ea += bx + si;
1199 modrm_ea += bx + di;
1202 modrm_ea += bp + si;
1205 modrm_ea += bp + di;
1214 if (ctxt->modrm_mod != 0)
1221 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1222 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1223 ctxt->modrm_seg = VCPU_SREG_SS;
1224 modrm_ea = (u16)modrm_ea;
1226 /* 32/64-bit ModR/M decode. */
1227 if ((ctxt->modrm_rm & 7) == 4) {
1228 sib = insn_fetch(u8, ctxt);
1229 index_reg |= (sib >> 3) & 7;
1230 base_reg |= sib & 7;
1233 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1234 modrm_ea += insn_fetch(s32, ctxt);
1236 modrm_ea += reg_read(ctxt, base_reg);
1237 adjust_modrm_seg(ctxt, base_reg);
1238 /* Increment ESP on POP [ESP] */
1239 if ((ctxt->d & IncSP) &&
1240 base_reg == VCPU_REGS_RSP)
1241 modrm_ea += ctxt->op_bytes;
1244 modrm_ea += reg_read(ctxt, index_reg) << scale;
1245 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1246 modrm_ea += insn_fetch(s32, ctxt);
1247 if (ctxt->mode == X86EMUL_MODE_PROT64)
1248 ctxt->rip_relative = 1;
1250 base_reg = ctxt->modrm_rm;
1251 modrm_ea += reg_read(ctxt, base_reg);
1252 adjust_modrm_seg(ctxt, base_reg);
1254 switch (ctxt->modrm_mod) {
1256 modrm_ea += insn_fetch(s8, ctxt);
1259 modrm_ea += insn_fetch(s32, ctxt);
1263 op->addr.mem.ea = modrm_ea;
1264 if (ctxt->ad_bytes != 8)
1265 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1271 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1274 int rc = X86EMUL_CONTINUE;
1277 switch (ctxt->ad_bytes) {
1279 op->addr.mem.ea = insn_fetch(u16, ctxt);
1282 op->addr.mem.ea = insn_fetch(u32, ctxt);
1285 op->addr.mem.ea = insn_fetch(u64, ctxt);
1292 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1296 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1297 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1299 if (ctxt->src.bytes == 2)
1300 sv = (s16)ctxt->src.val & (s16)mask;
1301 else if (ctxt->src.bytes == 4)
1302 sv = (s32)ctxt->src.val & (s32)mask;
1304 sv = (s64)ctxt->src.val & (s64)mask;
1306 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1307 ctxt->dst.addr.mem.ea + (sv >> 3));
1310 /* only subword offset */
1311 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1314 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1315 unsigned long addr, void *dest, unsigned size)
1318 struct read_cache *mc = &ctxt->mem_read;
1320 if (mc->pos < mc->end)
1323 WARN_ON((mc->end + size) >= sizeof(mc->data));
1325 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1327 if (rc != X86EMUL_CONTINUE)
1333 memcpy(dest, mc->data + mc->pos, size);
1335 return X86EMUL_CONTINUE;
1338 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1339 struct segmented_address addr,
1346 rc = linearize(ctxt, addr, size, false, &linear);
1347 if (rc != X86EMUL_CONTINUE)
1349 return read_emulated(ctxt, linear, data, size);
1352 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1353 struct segmented_address addr,
1360 rc = linearize(ctxt, addr, size, true, &linear);
1361 if (rc != X86EMUL_CONTINUE)
1363 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1367 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1368 struct segmented_address addr,
1369 const void *orig_data, const void *data,
1375 rc = linearize(ctxt, addr, size, true, &linear);
1376 if (rc != X86EMUL_CONTINUE)
1378 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1379 size, &ctxt->exception);
1382 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1383 unsigned int size, unsigned short port,
1386 struct read_cache *rc = &ctxt->io_read;
1388 if (rc->pos == rc->end) { /* refill pio read ahead */
1389 unsigned int in_page, n;
1390 unsigned int count = ctxt->rep_prefix ?
1391 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1392 in_page = (ctxt->eflags & EFLG_DF) ?
1393 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1394 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1395 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1398 rc->pos = rc->end = 0;
1399 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1404 if (ctxt->rep_prefix && (ctxt->d & String) &&
1405 !(ctxt->eflags & EFLG_DF)) {
1406 ctxt->dst.data = rc->data + rc->pos;
1407 ctxt->dst.type = OP_MEM_STR;
1408 ctxt->dst.count = (rc->end - rc->pos) / size;
1411 memcpy(dest, rc->data + rc->pos, size);
1417 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1418 u16 index, struct desc_struct *desc)
1423 ctxt->ops->get_idt(ctxt, &dt);
1425 if (dt.size < index * 8 + 7)
1426 return emulate_gp(ctxt, index << 3 | 0x2);
1428 addr = dt.address + index * 8;
1429 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1433 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1434 u16 selector, struct desc_ptr *dt)
1436 const struct x86_emulate_ops *ops = ctxt->ops;
1439 if (selector & 1 << 2) {
1440 struct desc_struct desc;
1443 memset (dt, 0, sizeof *dt);
1444 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1448 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1449 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1451 ops->get_gdt(ctxt, dt);
1454 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1455 u16 selector, ulong *desc_addr_p)
1458 u16 index = selector >> 3;
1461 get_descriptor_table_ptr(ctxt, selector, &dt);
1463 if (dt.size < index * 8 + 7)
1464 return emulate_gp(ctxt, selector & 0xfffc);
1466 addr = dt.address + index * 8;
1468 #ifdef CONFIG_X86_64
1469 if (addr >> 32 != 0) {
1472 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1473 if (!(efer & EFER_LMA))
1478 *desc_addr_p = addr;
1479 return X86EMUL_CONTINUE;
1482 /* allowed just for 8 bytes segments */
1483 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1484 u16 selector, struct desc_struct *desc,
1489 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1490 if (rc != X86EMUL_CONTINUE)
1493 return ctxt->ops->read_std(ctxt, *desc_addr_p, desc, sizeof(*desc),
1497 /* allowed just for 8 bytes segments */
1498 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1499 u16 selector, struct desc_struct *desc)
1504 rc = get_descriptor_ptr(ctxt, selector, &addr);
1505 if (rc != X86EMUL_CONTINUE)
1508 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1512 /* Does not support long mode */
1513 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1514 u16 selector, int seg, u8 cpl,
1515 enum x86_transfer_type transfer,
1516 struct desc_struct *desc)
1518 struct desc_struct seg_desc, old_desc;
1520 unsigned err_vec = GP_VECTOR;
1522 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1528 memset(&seg_desc, 0, sizeof seg_desc);
1530 if (ctxt->mode == X86EMUL_MODE_REAL) {
1531 /* set real mode segment descriptor (keep limit etc. for
1533 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1534 set_desc_base(&seg_desc, selector << 4);
1536 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1537 /* VM86 needs a clean new segment descriptor */
1538 set_desc_base(&seg_desc, selector << 4);
1539 set_desc_limit(&seg_desc, 0xffff);
1549 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1550 if ((seg == VCPU_SREG_CS
1551 || (seg == VCPU_SREG_SS
1552 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1553 || seg == VCPU_SREG_TR)
1557 /* TR should be in GDT only */
1558 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1561 if (null_selector) /* for NULL selector skip all following checks */
1564 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1565 if (ret != X86EMUL_CONTINUE)
1568 err_code = selector & 0xfffc;
1569 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1572 /* can't load system descriptor into segment selector */
1573 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1574 if (transfer == X86_TRANSFER_CALL_JMP)
1575 return X86EMUL_UNHANDLEABLE;
1580 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1589 * segment is not a writable data segment or segment
1590 * selector's RPL != CPL or segment selector's RPL != CPL
1592 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1596 if (!(seg_desc.type & 8))
1599 if (seg_desc.type & 4) {
1605 if (rpl > cpl || dpl != cpl)
1608 /* in long-mode d/b must be clear if l is set */
1609 if (seg_desc.d && seg_desc.l) {
1612 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1613 if (efer & EFER_LMA)
1617 /* CS(RPL) <- CPL */
1618 selector = (selector & 0xfffc) | cpl;
1621 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1623 old_desc = seg_desc;
1624 seg_desc.type |= 2; /* busy */
1625 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1626 sizeof(seg_desc), &ctxt->exception);
1627 if (ret != X86EMUL_CONTINUE)
1630 case VCPU_SREG_LDTR:
1631 if (seg_desc.s || seg_desc.type != 2)
1634 default: /* DS, ES, FS, or GS */
1636 * segment is not a data or readable code segment or
1637 * ((segment is a data or nonconforming code segment)
1638 * and (both RPL and CPL > DPL))
1640 if ((seg_desc.type & 0xa) == 0x8 ||
1641 (((seg_desc.type & 0xc) != 0xc) &&
1642 (rpl > dpl && cpl > dpl)))
1648 /* mark segment as accessed */
1649 if (!(seg_desc.type & 1)) {
1651 ret = write_segment_descriptor(ctxt, selector,
1653 if (ret != X86EMUL_CONTINUE)
1656 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1657 ret = ctxt->ops->read_std(ctxt, desc_addr+8, &base3,
1658 sizeof(base3), &ctxt->exception);
1659 if (ret != X86EMUL_CONTINUE)
1661 if (is_noncanonical_address(get_desc_base(&seg_desc) |
1662 ((u64)base3 << 32)))
1663 return emulate_gp(ctxt, 0);
1666 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1669 return X86EMUL_CONTINUE;
1671 return emulate_exception(ctxt, err_vec, err_code, true);
1674 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1675 u16 selector, int seg)
1677 u8 cpl = ctxt->ops->cpl(ctxt);
1678 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1679 X86_TRANSFER_NONE, NULL);
1682 static void write_register_operand(struct operand *op)
1684 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1685 switch (op->bytes) {
1687 *(u8 *)op->addr.reg = (u8)op->val;
1690 *(u16 *)op->addr.reg = (u16)op->val;
1693 *op->addr.reg = (u32)op->val;
1694 break; /* 64b: zero-extend */
1696 *op->addr.reg = op->val;
1701 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1705 write_register_operand(op);
1708 if (ctxt->lock_prefix)
1709 return segmented_cmpxchg(ctxt,
1715 return segmented_write(ctxt,
1721 return segmented_write(ctxt,
1724 op->bytes * op->count);
1727 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1730 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1738 return X86EMUL_CONTINUE;
1741 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1743 struct segmented_address addr;
1745 rsp_increment(ctxt, -bytes);
1746 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1747 addr.seg = VCPU_SREG_SS;
1749 return segmented_write(ctxt, addr, data, bytes);
1752 static int em_push(struct x86_emulate_ctxt *ctxt)
1754 /* Disable writeback. */
1755 ctxt->dst.type = OP_NONE;
1756 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1759 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1760 void *dest, int len)
1763 struct segmented_address addr;
1765 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1766 addr.seg = VCPU_SREG_SS;
1767 rc = segmented_read(ctxt, addr, dest, len);
1768 if (rc != X86EMUL_CONTINUE)
1771 rsp_increment(ctxt, len);
1775 static int em_pop(struct x86_emulate_ctxt *ctxt)
1777 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1780 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1781 void *dest, int len)
1784 unsigned long val, change_mask;
1785 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1786 int cpl = ctxt->ops->cpl(ctxt);
1788 rc = emulate_pop(ctxt, &val, len);
1789 if (rc != X86EMUL_CONTINUE)
1792 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1793 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_AC | EFLG_ID;
1795 switch(ctxt->mode) {
1796 case X86EMUL_MODE_PROT64:
1797 case X86EMUL_MODE_PROT32:
1798 case X86EMUL_MODE_PROT16:
1800 change_mask |= EFLG_IOPL;
1802 change_mask |= EFLG_IF;
1804 case X86EMUL_MODE_VM86:
1806 return emulate_gp(ctxt, 0);
1807 change_mask |= EFLG_IF;
1809 default: /* real mode */
1810 change_mask |= (EFLG_IOPL | EFLG_IF);
1814 *(unsigned long *)dest =
1815 (ctxt->eflags & ~change_mask) | (val & change_mask);
1820 static int em_popf(struct x86_emulate_ctxt *ctxt)
1822 ctxt->dst.type = OP_REG;
1823 ctxt->dst.addr.reg = &ctxt->eflags;
1824 ctxt->dst.bytes = ctxt->op_bytes;
1825 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1828 static int em_enter(struct x86_emulate_ctxt *ctxt)
1831 unsigned frame_size = ctxt->src.val;
1832 unsigned nesting_level = ctxt->src2.val & 31;
1836 return X86EMUL_UNHANDLEABLE;
1838 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1839 rc = push(ctxt, &rbp, stack_size(ctxt));
1840 if (rc != X86EMUL_CONTINUE)
1842 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1844 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1845 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1847 return X86EMUL_CONTINUE;
1850 static int em_leave(struct x86_emulate_ctxt *ctxt)
1852 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1854 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1857 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1859 int seg = ctxt->src2.val;
1861 ctxt->src.val = get_segment_selector(ctxt, seg);
1862 if (ctxt->op_bytes == 4) {
1863 rsp_increment(ctxt, -2);
1867 return em_push(ctxt);
1870 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1872 int seg = ctxt->src2.val;
1873 unsigned long selector;
1876 rc = emulate_pop(ctxt, &selector, 2);
1877 if (rc != X86EMUL_CONTINUE)
1880 if (ctxt->modrm_reg == VCPU_SREG_SS)
1881 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1882 if (ctxt->op_bytes > 2)
1883 rsp_increment(ctxt, ctxt->op_bytes - 2);
1885 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1889 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1891 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1892 int rc = X86EMUL_CONTINUE;
1893 int reg = VCPU_REGS_RAX;
1895 while (reg <= VCPU_REGS_RDI) {
1896 (reg == VCPU_REGS_RSP) ?
1897 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1900 if (rc != X86EMUL_CONTINUE)
1909 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1911 ctxt->src.val = (unsigned long)ctxt->eflags & ~EFLG_VM;
1912 return em_push(ctxt);
1915 static int em_popa(struct x86_emulate_ctxt *ctxt)
1917 int rc = X86EMUL_CONTINUE;
1918 int reg = VCPU_REGS_RDI;
1920 while (reg >= VCPU_REGS_RAX) {
1921 if (reg == VCPU_REGS_RSP) {
1922 rsp_increment(ctxt, ctxt->op_bytes);
1926 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1927 if (rc != X86EMUL_CONTINUE)
1934 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1936 const struct x86_emulate_ops *ops = ctxt->ops;
1943 /* TODO: Add limit checks */
1944 ctxt->src.val = ctxt->eflags;
1946 if (rc != X86EMUL_CONTINUE)
1949 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1951 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1953 if (rc != X86EMUL_CONTINUE)
1956 ctxt->src.val = ctxt->_eip;
1958 if (rc != X86EMUL_CONTINUE)
1961 ops->get_idt(ctxt, &dt);
1963 eip_addr = dt.address + (irq << 2);
1964 cs_addr = dt.address + (irq << 2) + 2;
1966 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1967 if (rc != X86EMUL_CONTINUE)
1970 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1971 if (rc != X86EMUL_CONTINUE)
1974 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1975 if (rc != X86EMUL_CONTINUE)
1983 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1987 invalidate_registers(ctxt);
1988 rc = __emulate_int_real(ctxt, irq);
1989 if (rc == X86EMUL_CONTINUE)
1990 writeback_registers(ctxt);
1994 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1996 switch(ctxt->mode) {
1997 case X86EMUL_MODE_REAL:
1998 return __emulate_int_real(ctxt, irq);
1999 case X86EMUL_MODE_VM86:
2000 case X86EMUL_MODE_PROT16:
2001 case X86EMUL_MODE_PROT32:
2002 case X86EMUL_MODE_PROT64:
2004 /* Protected mode interrupts unimplemented yet */
2005 return X86EMUL_UNHANDLEABLE;
2009 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2011 int rc = X86EMUL_CONTINUE;
2012 unsigned long temp_eip = 0;
2013 unsigned long temp_eflags = 0;
2014 unsigned long cs = 0;
2015 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
2016 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
2017 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
2018 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
2020 /* TODO: Add stack limit check */
2022 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2024 if (rc != X86EMUL_CONTINUE)
2027 if (temp_eip & ~0xffff)
2028 return emulate_gp(ctxt, 0);
2030 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2032 if (rc != X86EMUL_CONTINUE)
2035 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2037 if (rc != X86EMUL_CONTINUE)
2040 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2042 if (rc != X86EMUL_CONTINUE)
2045 ctxt->_eip = temp_eip;
2048 if (ctxt->op_bytes == 4)
2049 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2050 else if (ctxt->op_bytes == 2) {
2051 ctxt->eflags &= ~0xffff;
2052 ctxt->eflags |= temp_eflags;
2055 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2056 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
2057 ctxt->ops->set_nmi_mask(ctxt, false);
2062 static int em_iret(struct x86_emulate_ctxt *ctxt)
2064 switch(ctxt->mode) {
2065 case X86EMUL_MODE_REAL:
2066 return emulate_iret_real(ctxt);
2067 case X86EMUL_MODE_VM86:
2068 case X86EMUL_MODE_PROT16:
2069 case X86EMUL_MODE_PROT32:
2070 case X86EMUL_MODE_PROT64:
2072 /* iret from protected mode unimplemented yet */
2073 return X86EMUL_UNHANDLEABLE;
2077 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2080 unsigned short sel, old_sel;
2081 struct desc_struct old_desc, new_desc;
2082 const struct x86_emulate_ops *ops = ctxt->ops;
2083 u8 cpl = ctxt->ops->cpl(ctxt);
2085 /* Assignment of RIP may only fail in 64-bit mode */
2086 if (ctxt->mode == X86EMUL_MODE_PROT64)
2087 ops->get_segment(ctxt, &old_sel, &old_desc, NULL,
2090 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2092 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2093 X86_TRANSFER_CALL_JMP,
2095 if (rc != X86EMUL_CONTINUE)
2098 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2099 if (rc != X86EMUL_CONTINUE) {
2100 WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64);
2101 /* assigning eip failed; restore the old cs */
2102 ops->set_segment(ctxt, old_sel, &old_desc, 0, VCPU_SREG_CS);
2108 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2110 return assign_eip_near(ctxt, ctxt->src.val);
2113 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2118 old_eip = ctxt->_eip;
2119 rc = assign_eip_near(ctxt, ctxt->src.val);
2120 if (rc != X86EMUL_CONTINUE)
2122 ctxt->src.val = old_eip;
2127 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2129 u64 old = ctxt->dst.orig_val64;
2131 if (ctxt->dst.bytes == 16)
2132 return X86EMUL_UNHANDLEABLE;
2134 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2135 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2136 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2137 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2138 ctxt->eflags &= ~EFLG_ZF;
2140 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2141 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2143 ctxt->eflags |= EFLG_ZF;
2145 return X86EMUL_CONTINUE;
2148 static int em_ret(struct x86_emulate_ctxt *ctxt)
2153 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2154 if (rc != X86EMUL_CONTINUE)
2157 return assign_eip_near(ctxt, eip);
2160 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2163 unsigned long eip, cs;
2165 int cpl = ctxt->ops->cpl(ctxt);
2166 struct desc_struct old_desc, new_desc;
2167 const struct x86_emulate_ops *ops = ctxt->ops;
2169 if (ctxt->mode == X86EMUL_MODE_PROT64)
2170 ops->get_segment(ctxt, &old_cs, &old_desc, NULL,
2173 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2174 if (rc != X86EMUL_CONTINUE)
2176 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2177 if (rc != X86EMUL_CONTINUE)
2179 /* Outer-privilege level return is not implemented */
2180 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2181 return X86EMUL_UNHANDLEABLE;
2182 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2185 if (rc != X86EMUL_CONTINUE)
2187 rc = assign_eip_far(ctxt, eip, &new_desc);
2188 if (rc != X86EMUL_CONTINUE) {
2189 WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64);
2190 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
2195 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2199 rc = em_ret_far(ctxt);
2200 if (rc != X86EMUL_CONTINUE)
2202 rsp_increment(ctxt, ctxt->src.val);
2203 return X86EMUL_CONTINUE;
2206 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2208 /* Save real source value, then compare EAX against destination. */
2209 ctxt->dst.orig_val = ctxt->dst.val;
2210 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2211 ctxt->src.orig_val = ctxt->src.val;
2212 ctxt->src.val = ctxt->dst.orig_val;
2213 fastop(ctxt, em_cmp);
2215 if (ctxt->eflags & EFLG_ZF) {
2216 /* Success: write back to memory; no update of EAX */
2217 ctxt->src.type = OP_NONE;
2218 ctxt->dst.val = ctxt->src.orig_val;
2220 /* Failure: write the value we saw to EAX. */
2221 ctxt->src.type = OP_REG;
2222 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2223 ctxt->src.val = ctxt->dst.orig_val;
2224 /* Create write-cycle to dest by writing the same value */
2225 ctxt->dst.val = ctxt->dst.orig_val;
2227 return X86EMUL_CONTINUE;
2230 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2232 int seg = ctxt->src2.val;
2236 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2238 rc = load_segment_descriptor(ctxt, sel, seg);
2239 if (rc != X86EMUL_CONTINUE)
2242 ctxt->dst.val = ctxt->src.val;
2247 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2248 struct desc_struct *cs, struct desc_struct *ss)
2250 cs->l = 0; /* will be adjusted later */
2251 set_desc_base(cs, 0); /* flat segment */
2252 cs->g = 1; /* 4kb granularity */
2253 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2254 cs->type = 0x0b; /* Read, Execute, Accessed */
2256 cs->dpl = 0; /* will be adjusted later */
2261 set_desc_base(ss, 0); /* flat segment */
2262 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2263 ss->g = 1; /* 4kb granularity */
2265 ss->type = 0x03; /* Read/Write, Accessed */
2266 ss->d = 1; /* 32bit stack segment */
2273 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2275 u32 eax, ebx, ecx, edx;
2278 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2279 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2280 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2281 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2284 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2286 const struct x86_emulate_ops *ops = ctxt->ops;
2287 u32 eax, ebx, ecx, edx;
2290 * syscall should always be enabled in longmode - so only become
2291 * vendor specific (cpuid) if other modes are active...
2293 if (ctxt->mode == X86EMUL_MODE_PROT64)
2298 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2300 * Intel ("GenuineIntel")
2301 * remark: Intel CPUs only support "syscall" in 64bit
2302 * longmode. Also an 64bit guest with a
2303 * 32bit compat-app running will #UD !! While this
2304 * behaviour can be fixed (by emulating) into AMD
2305 * response - CPUs of AMD can't behave like Intel.
2307 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2308 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2309 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2312 /* AMD ("AuthenticAMD") */
2313 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2314 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2315 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2318 /* AMD ("AMDisbetter!") */
2319 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2320 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2321 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2324 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2328 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2330 const struct x86_emulate_ops *ops = ctxt->ops;
2331 struct desc_struct cs, ss;
2336 /* syscall is not available in real mode */
2337 if (ctxt->mode == X86EMUL_MODE_REAL ||
2338 ctxt->mode == X86EMUL_MODE_VM86)
2339 return emulate_ud(ctxt);
2341 if (!(em_syscall_is_enabled(ctxt)))
2342 return emulate_ud(ctxt);
2344 ops->get_msr(ctxt, MSR_EFER, &efer);
2345 setup_syscalls_segments(ctxt, &cs, &ss);
2347 if (!(efer & EFER_SCE))
2348 return emulate_ud(ctxt);
2350 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2352 cs_sel = (u16)(msr_data & 0xfffc);
2353 ss_sel = (u16)(msr_data + 8);
2355 if (efer & EFER_LMA) {
2359 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2360 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2362 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2363 if (efer & EFER_LMA) {
2364 #ifdef CONFIG_X86_64
2365 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2368 ctxt->mode == X86EMUL_MODE_PROT64 ?
2369 MSR_LSTAR : MSR_CSTAR, &msr_data);
2370 ctxt->_eip = msr_data;
2372 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2373 ctxt->eflags &= ~msr_data;
2374 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
2378 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2379 ctxt->_eip = (u32)msr_data;
2381 ctxt->eflags &= ~(EFLG_VM | EFLG_IF);
2384 return X86EMUL_CONTINUE;
2387 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2389 const struct x86_emulate_ops *ops = ctxt->ops;
2390 struct desc_struct cs, ss;
2395 ops->get_msr(ctxt, MSR_EFER, &efer);
2396 /* inject #GP if in real mode */
2397 if (ctxt->mode == X86EMUL_MODE_REAL)
2398 return emulate_gp(ctxt, 0);
2401 * Not recognized on AMD in compat mode (but is recognized in legacy
2404 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2405 && !vendor_intel(ctxt))
2406 return emulate_ud(ctxt);
2408 /* sysenter/sysexit have not been tested in 64bit mode. */
2409 if (ctxt->mode == X86EMUL_MODE_PROT64)
2410 return X86EMUL_UNHANDLEABLE;
2412 setup_syscalls_segments(ctxt, &cs, &ss);
2414 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2415 switch (ctxt->mode) {
2416 case X86EMUL_MODE_PROT32:
2417 if ((msr_data & 0xfffc) == 0x0)
2418 return emulate_gp(ctxt, 0);
2420 case X86EMUL_MODE_PROT64:
2421 if (msr_data == 0x0)
2422 return emulate_gp(ctxt, 0);
2428 ctxt->eflags &= ~(EFLG_VM | EFLG_IF);
2429 cs_sel = (u16)msr_data;
2430 cs_sel &= ~SELECTOR_RPL_MASK;
2431 ss_sel = cs_sel + 8;
2432 ss_sel &= ~SELECTOR_RPL_MASK;
2433 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2438 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2439 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2441 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2442 ctxt->_eip = msr_data;
2444 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2445 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2447 return X86EMUL_CONTINUE;
2450 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2452 const struct x86_emulate_ops *ops = ctxt->ops;
2453 struct desc_struct cs, ss;
2454 u64 msr_data, rcx, rdx;
2456 u16 cs_sel = 0, ss_sel = 0;
2458 /* inject #GP if in real mode or Virtual 8086 mode */
2459 if (ctxt->mode == X86EMUL_MODE_REAL ||
2460 ctxt->mode == X86EMUL_MODE_VM86)
2461 return emulate_gp(ctxt, 0);
2463 setup_syscalls_segments(ctxt, &cs, &ss);
2465 if ((ctxt->rex_prefix & 0x8) != 0x0)
2466 usermode = X86EMUL_MODE_PROT64;
2468 usermode = X86EMUL_MODE_PROT32;
2470 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2471 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2475 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2477 case X86EMUL_MODE_PROT32:
2478 cs_sel = (u16)(msr_data + 16);
2479 if ((msr_data & 0xfffc) == 0x0)
2480 return emulate_gp(ctxt, 0);
2481 ss_sel = (u16)(msr_data + 24);
2485 case X86EMUL_MODE_PROT64:
2486 cs_sel = (u16)(msr_data + 32);
2487 if (msr_data == 0x0)
2488 return emulate_gp(ctxt, 0);
2489 ss_sel = cs_sel + 8;
2492 if (is_noncanonical_address(rcx) ||
2493 is_noncanonical_address(rdx))
2494 return emulate_gp(ctxt, 0);
2497 cs_sel |= SELECTOR_RPL_MASK;
2498 ss_sel |= SELECTOR_RPL_MASK;
2500 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2501 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2504 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2506 return X86EMUL_CONTINUE;
2509 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2512 if (ctxt->mode == X86EMUL_MODE_REAL)
2514 if (ctxt->mode == X86EMUL_MODE_VM86)
2516 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2517 return ctxt->ops->cpl(ctxt) > iopl;
2520 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2523 const struct x86_emulate_ops *ops = ctxt->ops;
2524 struct desc_struct tr_seg;
2527 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2528 unsigned mask = (1 << len) - 1;
2531 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2534 if (desc_limit_scaled(&tr_seg) < 103)
2536 base = get_desc_base(&tr_seg);
2537 #ifdef CONFIG_X86_64
2538 base |= ((u64)base3) << 32;
2540 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2541 if (r != X86EMUL_CONTINUE)
2543 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2545 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2546 if (r != X86EMUL_CONTINUE)
2548 if ((perm >> bit_idx) & mask)
2553 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2559 if (emulator_bad_iopl(ctxt))
2560 if (!emulator_io_port_access_allowed(ctxt, port, len))
2563 ctxt->perm_ok = true;
2568 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2569 struct tss_segment_16 *tss)
2571 tss->ip = ctxt->_eip;
2572 tss->flag = ctxt->eflags;
2573 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2574 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2575 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2576 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2577 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2578 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2579 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2580 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2582 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2583 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2584 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2585 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2586 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2589 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2590 struct tss_segment_16 *tss)
2595 ctxt->_eip = tss->ip;
2596 ctxt->eflags = tss->flag | 2;
2597 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2598 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2599 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2600 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2601 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2602 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2603 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2604 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2607 * SDM says that segment selectors are loaded before segment
2610 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2611 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2612 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2613 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2614 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2619 * Now load segment descriptors. If fault happens at this stage
2620 * it is handled in a context of new task
2622 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
2623 X86_TRANSFER_TASK_SWITCH, NULL);
2624 if (ret != X86EMUL_CONTINUE)
2626 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
2627 X86_TRANSFER_TASK_SWITCH, NULL);
2628 if (ret != X86EMUL_CONTINUE)
2630 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
2631 X86_TRANSFER_TASK_SWITCH, NULL);
2632 if (ret != X86EMUL_CONTINUE)
2634 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
2635 X86_TRANSFER_TASK_SWITCH, NULL);
2636 if (ret != X86EMUL_CONTINUE)
2638 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
2639 X86_TRANSFER_TASK_SWITCH, NULL);
2640 if (ret != X86EMUL_CONTINUE)
2643 return X86EMUL_CONTINUE;
2646 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2647 u16 tss_selector, u16 old_tss_sel,
2648 ulong old_tss_base, struct desc_struct *new_desc)
2650 const struct x86_emulate_ops *ops = ctxt->ops;
2651 struct tss_segment_16 tss_seg;
2653 u32 new_tss_base = get_desc_base(new_desc);
2655 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2657 if (ret != X86EMUL_CONTINUE)
2660 save_state_to_tss16(ctxt, &tss_seg);
2662 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2664 if (ret != X86EMUL_CONTINUE)
2667 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2669 if (ret != X86EMUL_CONTINUE)
2672 if (old_tss_sel != 0xffff) {
2673 tss_seg.prev_task_link = old_tss_sel;
2675 ret = ops->write_std(ctxt, new_tss_base,
2676 &tss_seg.prev_task_link,
2677 sizeof tss_seg.prev_task_link,
2679 if (ret != X86EMUL_CONTINUE)
2683 return load_state_from_tss16(ctxt, &tss_seg);
2686 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2687 struct tss_segment_32 *tss)
2689 /* CR3 and ldt selector are not saved intentionally */
2690 tss->eip = ctxt->_eip;
2691 tss->eflags = ctxt->eflags;
2692 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2693 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2694 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2695 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2696 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2697 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2698 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2699 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2701 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2702 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2703 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2704 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2705 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2706 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2709 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2710 struct tss_segment_32 *tss)
2715 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2716 return emulate_gp(ctxt, 0);
2717 ctxt->_eip = tss->eip;
2718 ctxt->eflags = tss->eflags | 2;
2720 /* General purpose registers */
2721 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2722 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2723 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2724 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2725 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2726 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2727 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2728 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2731 * SDM says that segment selectors are loaded before segment
2732 * descriptors. This is important because CPL checks will
2735 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2736 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2737 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2738 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2739 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2740 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2741 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2744 * If we're switching between Protected Mode and VM86, we need to make
2745 * sure to update the mode before loading the segment descriptors so
2746 * that the selectors are interpreted correctly.
2748 if (ctxt->eflags & X86_EFLAGS_VM) {
2749 ctxt->mode = X86EMUL_MODE_VM86;
2752 ctxt->mode = X86EMUL_MODE_PROT32;
2757 * Now load segment descriptors. If fault happenes at this stage
2758 * it is handled in a context of new task
2760 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
2761 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
2762 if (ret != X86EMUL_CONTINUE)
2764 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
2765 X86_TRANSFER_TASK_SWITCH, NULL);
2766 if (ret != X86EMUL_CONTINUE)
2768 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
2769 X86_TRANSFER_TASK_SWITCH, NULL);
2770 if (ret != X86EMUL_CONTINUE)
2772 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
2773 X86_TRANSFER_TASK_SWITCH, NULL);
2774 if (ret != X86EMUL_CONTINUE)
2776 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
2777 X86_TRANSFER_TASK_SWITCH, NULL);
2778 if (ret != X86EMUL_CONTINUE)
2780 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
2781 X86_TRANSFER_TASK_SWITCH, NULL);
2782 if (ret != X86EMUL_CONTINUE)
2784 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
2785 X86_TRANSFER_TASK_SWITCH, NULL);
2786 if (ret != X86EMUL_CONTINUE)
2789 return X86EMUL_CONTINUE;
2792 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2793 u16 tss_selector, u16 old_tss_sel,
2794 ulong old_tss_base, struct desc_struct *new_desc)
2796 const struct x86_emulate_ops *ops = ctxt->ops;
2797 struct tss_segment_32 tss_seg;
2799 u32 new_tss_base = get_desc_base(new_desc);
2800 u32 eip_offset = offsetof(struct tss_segment_32, eip);
2801 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
2803 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2805 if (ret != X86EMUL_CONTINUE)
2808 save_state_to_tss32(ctxt, &tss_seg);
2810 /* Only GP registers and segment selectors are saved */
2811 ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
2812 ldt_sel_offset - eip_offset, &ctxt->exception);
2813 if (ret != X86EMUL_CONTINUE)
2816 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2818 if (ret != X86EMUL_CONTINUE)
2821 if (old_tss_sel != 0xffff) {
2822 tss_seg.prev_task_link = old_tss_sel;
2824 ret = ops->write_std(ctxt, new_tss_base,
2825 &tss_seg.prev_task_link,
2826 sizeof tss_seg.prev_task_link,
2828 if (ret != X86EMUL_CONTINUE)
2832 return load_state_from_tss32(ctxt, &tss_seg);
2835 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2836 u16 tss_selector, int idt_index, int reason,
2837 bool has_error_code, u32 error_code)
2839 const struct x86_emulate_ops *ops = ctxt->ops;
2840 struct desc_struct curr_tss_desc, next_tss_desc;
2842 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2843 ulong old_tss_base =
2844 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2848 /* FIXME: old_tss_base == ~0 ? */
2850 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2851 if (ret != X86EMUL_CONTINUE)
2853 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2854 if (ret != X86EMUL_CONTINUE)
2857 /* FIXME: check that next_tss_desc is tss */
2860 * Check privileges. The three cases are task switch caused by...
2862 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2863 * 2. Exception/IRQ/iret: No check is performed
2864 * 3. jmp/call to TSS/task-gate: No check is performed since the
2865 * hardware checks it before exiting.
2867 if (reason == TASK_SWITCH_GATE) {
2868 if (idt_index != -1) {
2869 /* Software interrupts */
2870 struct desc_struct task_gate_desc;
2873 ret = read_interrupt_descriptor(ctxt, idt_index,
2875 if (ret != X86EMUL_CONTINUE)
2878 dpl = task_gate_desc.dpl;
2879 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2880 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2884 desc_limit = desc_limit_scaled(&next_tss_desc);
2885 if (!next_tss_desc.p ||
2886 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2887 desc_limit < 0x2b)) {
2888 return emulate_ts(ctxt, tss_selector & 0xfffc);
2891 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2892 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2893 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2896 if (reason == TASK_SWITCH_IRET)
2897 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2899 /* set back link to prev task only if NT bit is set in eflags
2900 note that old_tss_sel is not used after this point */
2901 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2902 old_tss_sel = 0xffff;
2904 if (next_tss_desc.type & 8)
2905 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2906 old_tss_base, &next_tss_desc);
2908 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2909 old_tss_base, &next_tss_desc);
2910 if (ret != X86EMUL_CONTINUE)
2913 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2914 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2916 if (reason != TASK_SWITCH_IRET) {
2917 next_tss_desc.type |= (1 << 1); /* set busy flag */
2918 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2921 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2922 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2924 if (has_error_code) {
2925 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2926 ctxt->lock_prefix = 0;
2927 ctxt->src.val = (unsigned long) error_code;
2928 ret = em_push(ctxt);
2934 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2935 u16 tss_selector, int idt_index, int reason,
2936 bool has_error_code, u32 error_code)
2940 invalidate_registers(ctxt);
2941 ctxt->_eip = ctxt->eip;
2942 ctxt->dst.type = OP_NONE;
2944 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2945 has_error_code, error_code);
2947 if (rc == X86EMUL_CONTINUE) {
2948 ctxt->eip = ctxt->_eip;
2949 writeback_registers(ctxt);
2952 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2955 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2958 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2960 register_address_increment(ctxt, reg, df * op->bytes);
2961 op->addr.mem.ea = register_address(ctxt, reg);
2964 static int em_das(struct x86_emulate_ctxt *ctxt)
2967 bool af, cf, old_cf;
2969 cf = ctxt->eflags & X86_EFLAGS_CF;
2975 af = ctxt->eflags & X86_EFLAGS_AF;
2976 if ((al & 0x0f) > 9 || af) {
2978 cf = old_cf | (al >= 250);
2983 if (old_al > 0x99 || old_cf) {
2989 /* Set PF, ZF, SF */
2990 ctxt->src.type = OP_IMM;
2992 ctxt->src.bytes = 1;
2993 fastop(ctxt, em_or);
2994 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2996 ctxt->eflags |= X86_EFLAGS_CF;
2998 ctxt->eflags |= X86_EFLAGS_AF;
2999 return X86EMUL_CONTINUE;
3002 static int em_aam(struct x86_emulate_ctxt *ctxt)
3006 if (ctxt->src.val == 0)
3007 return emulate_de(ctxt);
3009 al = ctxt->dst.val & 0xff;
3010 ah = al / ctxt->src.val;
3011 al %= ctxt->src.val;
3013 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3015 /* Set PF, ZF, SF */
3016 ctxt->src.type = OP_IMM;
3018 ctxt->src.bytes = 1;
3019 fastop(ctxt, em_or);
3021 return X86EMUL_CONTINUE;
3024 static int em_aad(struct x86_emulate_ctxt *ctxt)
3026 u8 al = ctxt->dst.val & 0xff;
3027 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3029 al = (al + (ah * ctxt->src.val)) & 0xff;
3031 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3033 /* Set PF, ZF, SF */
3034 ctxt->src.type = OP_IMM;
3036 ctxt->src.bytes = 1;
3037 fastop(ctxt, em_or);
3039 return X86EMUL_CONTINUE;
3042 static int em_call(struct x86_emulate_ctxt *ctxt)
3045 long rel = ctxt->src.val;
3047 ctxt->src.val = (unsigned long)ctxt->_eip;
3048 rc = jmp_rel(ctxt, rel);
3049 if (rc != X86EMUL_CONTINUE)
3051 return em_push(ctxt);
3054 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3059 struct desc_struct old_desc, new_desc;
3060 const struct x86_emulate_ops *ops = ctxt->ops;
3061 int cpl = ctxt->ops->cpl(ctxt);
3063 old_eip = ctxt->_eip;
3064 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3066 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3067 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3068 X86_TRANSFER_CALL_JMP, &new_desc);
3069 if (rc != X86EMUL_CONTINUE)
3072 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3073 if (rc != X86EMUL_CONTINUE)
3076 ctxt->src.val = old_cs;
3078 if (rc != X86EMUL_CONTINUE)
3081 ctxt->src.val = old_eip;
3083 /* If we failed, we tainted the memory, but the very least we should
3085 if (rc != X86EMUL_CONTINUE)
3089 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3094 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3099 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3100 if (rc != X86EMUL_CONTINUE)
3102 rc = assign_eip_near(ctxt, eip);
3103 if (rc != X86EMUL_CONTINUE)
3105 rsp_increment(ctxt, ctxt->src.val);
3106 return X86EMUL_CONTINUE;
3109 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3111 /* Write back the register source. */
3112 ctxt->src.val = ctxt->dst.val;
3113 write_register_operand(&ctxt->src);
3115 /* Write back the memory destination with implicit LOCK prefix. */
3116 ctxt->dst.val = ctxt->src.orig_val;
3117 ctxt->lock_prefix = 1;
3118 return X86EMUL_CONTINUE;
3121 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3123 ctxt->dst.val = ctxt->src2.val;
3124 return fastop(ctxt, em_imul);
3127 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3129 ctxt->dst.type = OP_REG;
3130 ctxt->dst.bytes = ctxt->src.bytes;
3131 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3132 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3134 return X86EMUL_CONTINUE;
3137 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3141 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3142 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3143 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3144 return X86EMUL_CONTINUE;
3147 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3151 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3152 return emulate_gp(ctxt, 0);
3153 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3154 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3155 return X86EMUL_CONTINUE;
3158 static int em_mov(struct x86_emulate_ctxt *ctxt)
3160 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3161 return X86EMUL_CONTINUE;
3164 #define FFL(x) bit(X86_FEATURE_##x)
3166 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3168 u32 ebx, ecx, edx, eax = 1;
3172 * Check MOVBE is set in the guest-visible CPUID leaf.
3174 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3175 if (!(ecx & FFL(MOVBE)))
3176 return emulate_ud(ctxt);
3178 switch (ctxt->op_bytes) {
3181 * From MOVBE definition: "...When the operand size is 16 bits,
3182 * the upper word of the destination register remains unchanged
3185 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3186 * rules so we have to do the operation almost per hand.
3188 tmp = (u16)ctxt->src.val;
3189 ctxt->dst.val &= ~0xffffUL;
3190 ctxt->dst.val |= (unsigned long)swab16(tmp);
3193 ctxt->dst.val = swab32((u32)ctxt->src.val);
3196 ctxt->dst.val = swab64(ctxt->src.val);
3201 return X86EMUL_CONTINUE;
3204 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3206 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3207 return emulate_gp(ctxt, 0);
3209 /* Disable writeback. */
3210 ctxt->dst.type = OP_NONE;
3211 return X86EMUL_CONTINUE;
3214 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3218 if (ctxt->mode == X86EMUL_MODE_PROT64)
3219 val = ctxt->src.val & ~0ULL;
3221 val = ctxt->src.val & ~0U;
3223 /* #UD condition is already handled. */
3224 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3225 return emulate_gp(ctxt, 0);
3227 /* Disable writeback. */
3228 ctxt->dst.type = OP_NONE;
3229 return X86EMUL_CONTINUE;
3232 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3236 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3237 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3238 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3239 return emulate_gp(ctxt, 0);
3241 return X86EMUL_CONTINUE;
3244 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3248 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3249 return emulate_gp(ctxt, 0);
3251 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3252 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3253 return X86EMUL_CONTINUE;
3256 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3258 if (ctxt->modrm_reg > VCPU_SREG_GS)
3259 return emulate_ud(ctxt);
3261 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3262 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3263 ctxt->dst.bytes = 2;
3264 return X86EMUL_CONTINUE;
3267 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3269 u16 sel = ctxt->src.val;
3271 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3272 return emulate_ud(ctxt);
3274 if (ctxt->modrm_reg == VCPU_SREG_SS)
3275 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3277 /* Disable writeback. */
3278 ctxt->dst.type = OP_NONE;
3279 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3282 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3284 u16 sel = ctxt->src.val;
3286 /* Disable writeback. */
3287 ctxt->dst.type = OP_NONE;
3288 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3291 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3293 u16 sel = ctxt->src.val;
3295 /* Disable writeback. */
3296 ctxt->dst.type = OP_NONE;
3297 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3300 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3305 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3306 if (rc == X86EMUL_CONTINUE)
3307 ctxt->ops->invlpg(ctxt, linear);
3308 /* Disable writeback. */
3309 ctxt->dst.type = OP_NONE;
3310 return X86EMUL_CONTINUE;
3313 static int em_clts(struct x86_emulate_ctxt *ctxt)
3317 cr0 = ctxt->ops->get_cr(ctxt, 0);
3319 ctxt->ops->set_cr(ctxt, 0, cr0);
3320 return X86EMUL_CONTINUE;
3323 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3325 int rc = ctxt->ops->fix_hypercall(ctxt);
3327 if (rc != X86EMUL_CONTINUE)
3330 /* Let the processor re-execute the fixed hypercall */
3331 ctxt->_eip = ctxt->eip;
3332 /* Disable writeback. */
3333 ctxt->dst.type = OP_NONE;
3334 return X86EMUL_CONTINUE;
3337 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3338 void (*get)(struct x86_emulate_ctxt *ctxt,
3339 struct desc_ptr *ptr))
3341 struct desc_ptr desc_ptr;
3343 if (ctxt->mode == X86EMUL_MODE_PROT64)
3345 get(ctxt, &desc_ptr);
3346 if (ctxt->op_bytes == 2) {
3348 desc_ptr.address &= 0x00ffffff;
3350 /* Disable writeback. */
3351 ctxt->dst.type = OP_NONE;
3352 return segmented_write(ctxt, ctxt->dst.addr.mem,
3353 &desc_ptr, 2 + ctxt->op_bytes);
3356 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3358 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3361 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3363 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3366 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3368 struct desc_ptr desc_ptr;
3371 if (ctxt->mode == X86EMUL_MODE_PROT64)
3373 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3374 &desc_ptr.size, &desc_ptr.address,
3376 if (rc != X86EMUL_CONTINUE)
3378 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3379 is_noncanonical_address(desc_ptr.address))
3380 return emulate_gp(ctxt, 0);
3382 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3384 ctxt->ops->set_idt(ctxt, &desc_ptr);
3385 /* Disable writeback. */
3386 ctxt->dst.type = OP_NONE;
3387 return X86EMUL_CONTINUE;
3390 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3392 return em_lgdt_lidt(ctxt, true);
3395 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3399 rc = ctxt->ops->fix_hypercall(ctxt);
3401 /* Disable writeback. */
3402 ctxt->dst.type = OP_NONE;
3406 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3408 return em_lgdt_lidt(ctxt, false);
3411 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3413 if (ctxt->dst.type == OP_MEM)
3414 ctxt->dst.bytes = 2;
3415 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3416 return X86EMUL_CONTINUE;
3419 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3421 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3422 | (ctxt->src.val & 0x0f));
3423 ctxt->dst.type = OP_NONE;
3424 return X86EMUL_CONTINUE;
3427 static int em_loop(struct x86_emulate_ctxt *ctxt)
3429 int rc = X86EMUL_CONTINUE;
3431 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3432 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3433 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3434 rc = jmp_rel(ctxt, ctxt->src.val);
3439 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3441 int rc = X86EMUL_CONTINUE;
3443 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3444 rc = jmp_rel(ctxt, ctxt->src.val);
3449 static int em_in(struct x86_emulate_ctxt *ctxt)
3451 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3453 return X86EMUL_IO_NEEDED;
3455 return X86EMUL_CONTINUE;
3458 static int em_out(struct x86_emulate_ctxt *ctxt)
3460 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3462 /* Disable writeback. */
3463 ctxt->dst.type = OP_NONE;
3464 return X86EMUL_CONTINUE;
3467 static int em_cli(struct x86_emulate_ctxt *ctxt)
3469 if (emulator_bad_iopl(ctxt))
3470 return emulate_gp(ctxt, 0);
3472 ctxt->eflags &= ~X86_EFLAGS_IF;
3473 return X86EMUL_CONTINUE;
3476 static int em_sti(struct x86_emulate_ctxt *ctxt)
3478 if (emulator_bad_iopl(ctxt))
3479 return emulate_gp(ctxt, 0);
3481 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3482 ctxt->eflags |= X86_EFLAGS_IF;
3483 return X86EMUL_CONTINUE;
3486 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3488 u32 eax, ebx, ecx, edx;
3490 eax = reg_read(ctxt, VCPU_REGS_RAX);
3491 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3492 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3493 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3494 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3495 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3496 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3497 return X86EMUL_CONTINUE;
3500 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3504 flags = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF;
3505 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3507 ctxt->eflags &= ~0xffUL;
3508 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3509 return X86EMUL_CONTINUE;
3512 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3514 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3515 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3516 return X86EMUL_CONTINUE;
3519 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3521 switch (ctxt->op_bytes) {
3522 #ifdef CONFIG_X86_64
3524 asm("bswap %0" : "+r"(ctxt->dst.val));
3528 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3531 return X86EMUL_CONTINUE;
3534 static int em_clflush(struct x86_emulate_ctxt *ctxt)
3536 /* emulating clflush regardless of cpuid */
3537 return X86EMUL_CONTINUE;
3540 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
3542 ctxt->dst.val = (s32) ctxt->src.val;
3543 return X86EMUL_CONTINUE;
3546 static bool valid_cr(int nr)
3558 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3560 if (!valid_cr(ctxt->modrm_reg))
3561 return emulate_ud(ctxt);
3563 return X86EMUL_CONTINUE;
3566 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3568 u64 new_val = ctxt->src.val64;
3569 int cr = ctxt->modrm_reg;
3572 static u64 cr_reserved_bits[] = {
3573 0xffffffff00000000ULL,
3574 0, 0, 0, /* CR3 checked later */
3581 return emulate_ud(ctxt);
3583 if (new_val & cr_reserved_bits[cr])
3584 return emulate_gp(ctxt, 0);
3589 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3590 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3591 return emulate_gp(ctxt, 0);
3593 cr4 = ctxt->ops->get_cr(ctxt, 4);
3594 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3596 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3597 !(cr4 & X86_CR4_PAE))
3598 return emulate_gp(ctxt, 0);
3605 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3606 if (efer & EFER_LMA)
3607 rsvd = CR3_L_MODE_RESERVED_BITS & ~CR3_PCID_INVD;
3610 return emulate_gp(ctxt, 0);
3615 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3617 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3618 return emulate_gp(ctxt, 0);
3624 return X86EMUL_CONTINUE;
3627 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3631 ctxt->ops->get_dr(ctxt, 7, &dr7);
3633 /* Check if DR7.Global_Enable is set */
3634 return dr7 & (1 << 13);
3637 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3639 int dr = ctxt->modrm_reg;
3643 return emulate_ud(ctxt);
3645 cr4 = ctxt->ops->get_cr(ctxt, 4);
3646 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3647 return emulate_ud(ctxt);
3649 if (check_dr7_gd(ctxt)) {
3652 ctxt->ops->get_dr(ctxt, 6, &dr6);
3654 dr6 |= DR6_BD | DR6_RTM;
3655 ctxt->ops->set_dr(ctxt, 6, dr6);
3656 return emulate_db(ctxt);
3659 return X86EMUL_CONTINUE;
3662 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3664 u64 new_val = ctxt->src.val64;
3665 int dr = ctxt->modrm_reg;
3667 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3668 return emulate_gp(ctxt, 0);
3670 return check_dr_read(ctxt);
3673 static int check_svme(struct x86_emulate_ctxt *ctxt)
3677 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3679 if (!(efer & EFER_SVME))
3680 return emulate_ud(ctxt);
3682 return X86EMUL_CONTINUE;
3685 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3687 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3689 /* Valid physical address? */
3690 if (rax & 0xffff000000000000ULL)
3691 return emulate_gp(ctxt, 0);
3693 return check_svme(ctxt);
3696 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3698 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3700 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3701 return emulate_ud(ctxt);
3703 return X86EMUL_CONTINUE;
3706 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3708 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3709 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3711 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3712 ctxt->ops->check_pmc(ctxt, rcx))
3713 return emulate_gp(ctxt, 0);
3715 return X86EMUL_CONTINUE;
3718 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3720 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3721 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3722 return emulate_gp(ctxt, 0);
3724 return X86EMUL_CONTINUE;
3727 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3729 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3730 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3731 return emulate_gp(ctxt, 0);
3733 return X86EMUL_CONTINUE;
3736 #define D(_y) { .flags = (_y) }
3737 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
3738 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
3739 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3740 #define N D(NotImpl)
3741 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3742 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3743 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3744 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
3745 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
3746 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3747 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3748 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3749 #define II(_f, _e, _i) \
3750 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
3751 #define IIP(_f, _e, _i, _p) \
3752 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
3753 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3754 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3756 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3757 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3758 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3759 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3760 #define I2bvIP(_f, _e, _i, _p) \
3761 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3763 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3764 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3765 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3767 static const struct opcode group7_rm0[] = {
3769 I(SrcNone | Priv | EmulateOnUD, em_vmcall),
3773 static const struct opcode group7_rm1[] = {
3774 DI(SrcNone | Priv, monitor),
3775 DI(SrcNone | Priv, mwait),
3779 static const struct opcode group7_rm3[] = {
3780 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3781 II(SrcNone | Prot | EmulateOnUD, em_vmmcall, vmmcall),
3782 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3783 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3784 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3785 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3786 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3787 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3790 static const struct opcode group7_rm7[] = {
3792 DIP(SrcNone, rdtscp, check_rdtsc),
3796 static const struct opcode group1[] = {
3798 F(Lock | PageTable, em_or),
3801 F(Lock | PageTable, em_and),
3807 static const struct opcode group1A[] = {
3808 I(DstMem | SrcNone | Mov | Stack | IncSP, em_pop), N, N, N, N, N, N, N,
3811 static const struct opcode group2[] = {
3812 F(DstMem | ModRM, em_rol),
3813 F(DstMem | ModRM, em_ror),
3814 F(DstMem | ModRM, em_rcl),
3815 F(DstMem | ModRM, em_rcr),
3816 F(DstMem | ModRM, em_shl),
3817 F(DstMem | ModRM, em_shr),
3818 F(DstMem | ModRM, em_shl),
3819 F(DstMem | ModRM, em_sar),
3822 static const struct opcode group3[] = {
3823 F(DstMem | SrcImm | NoWrite, em_test),
3824 F(DstMem | SrcImm | NoWrite, em_test),
3825 F(DstMem | SrcNone | Lock, em_not),
3826 F(DstMem | SrcNone | Lock, em_neg),
3827 F(DstXacc | Src2Mem, em_mul_ex),
3828 F(DstXacc | Src2Mem, em_imul_ex),
3829 F(DstXacc | Src2Mem, em_div_ex),
3830 F(DstXacc | Src2Mem, em_idiv_ex),
3833 static const struct opcode group4[] = {
3834 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3835 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3839 static const struct opcode group5[] = {
3840 F(DstMem | SrcNone | Lock, em_inc),
3841 F(DstMem | SrcNone | Lock, em_dec),
3842 I(SrcMem | NearBranch, em_call_near_abs),
3843 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3844 I(SrcMem | NearBranch, em_jmp_abs),
3845 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
3846 I(SrcMem | Stack, em_push), D(Undefined),
3849 static const struct opcode group6[] = {
3852 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3853 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3857 static const struct group_dual group7 = { {
3858 II(Mov | DstMem, em_sgdt, sgdt),
3859 II(Mov | DstMem, em_sidt, sidt),
3860 II(SrcMem | Priv, em_lgdt, lgdt),
3861 II(SrcMem | Priv, em_lidt, lidt),
3862 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3863 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3864 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3868 N, EXT(0, group7_rm3),
3869 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3870 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3874 static const struct opcode group8[] = {
3876 F(DstMem | SrcImmByte | NoWrite, em_bt),
3877 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3878 F(DstMem | SrcImmByte | Lock, em_btr),
3879 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3882 static const struct group_dual group9 = { {
3883 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3885 N, N, N, N, N, N, N, N,
3888 static const struct opcode group11[] = {
3889 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3893 static const struct gprefix pfx_0f_ae_7 = {
3894 I(SrcMem | ByteOp, em_clflush), N, N, N,
3897 static const struct group_dual group15 = { {
3898 N, N, N, N, N, N, N, GP(0, &pfx_0f_ae_7),
3900 N, N, N, N, N, N, N, N,
3903 static const struct gprefix pfx_0f_6f_0f_7f = {
3904 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3907 static const struct instr_dual instr_dual_0f_2b = {
3911 static const struct gprefix pfx_0f_2b = {
3912 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
3915 static const struct gprefix pfx_0f_28_0f_29 = {
3916 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
3919 static const struct gprefix pfx_0f_e7 = {
3920 N, I(Sse, em_mov), N, N,
3923 static const struct escape escape_d9 = { {
3924 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
3927 N, N, N, N, N, N, N, N,
3929 N, N, N, N, N, N, N, N,
3931 N, N, N, N, N, N, N, N,
3933 N, N, N, N, N, N, N, N,
3935 N, N, N, N, N, N, N, N,
3937 N, N, N, N, N, N, N, N,
3939 N, N, N, N, N, N, N, N,
3941 N, N, N, N, N, N, N, N,
3944 static const struct escape escape_db = { {
3945 N, N, N, N, N, N, N, N,
3948 N, N, N, N, N, N, N, N,
3950 N, N, N, N, N, N, N, N,
3952 N, N, N, N, N, N, N, N,
3954 N, N, N, N, N, N, N, N,
3956 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3958 N, N, N, N, N, N, N, N,
3960 N, N, N, N, N, N, N, N,
3962 N, N, N, N, N, N, N, N,
3965 static const struct escape escape_dd = { {
3966 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
3969 N, N, N, N, N, N, N, N,
3971 N, N, N, N, N, N, N, N,
3973 N, N, N, N, N, N, N, N,
3975 N, N, N, N, N, N, N, N,
3977 N, N, N, N, N, N, N, N,
3979 N, N, N, N, N, N, N, N,
3981 N, N, N, N, N, N, N, N,
3983 N, N, N, N, N, N, N, N,
3986 static const struct instr_dual instr_dual_0f_c3 = {
3987 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
3990 static const struct mode_dual mode_dual_63 = {
3991 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
3994 static const struct opcode opcode_table[256] = {
3996 F6ALU(Lock, em_add),
3997 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3998 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4000 F6ALU(Lock | PageTable, em_or),
4001 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4004 F6ALU(Lock, em_adc),
4005 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4006 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4008 F6ALU(Lock, em_sbb),
4009 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4010 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4012 F6ALU(Lock | PageTable, em_and), N, N,
4014 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4016 F6ALU(Lock, em_xor), N, N,
4018 F6ALU(NoWrite, em_cmp), N, N,
4020 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4022 X8(I(SrcReg | Stack, em_push)),
4024 X8(I(DstReg | Stack, em_pop)),
4026 I(ImplicitOps | Stack | No64, em_pusha),
4027 I(ImplicitOps | Stack | No64, em_popa),
4028 N, MD(ModRM, &mode_dual_63),
4031 I(SrcImm | Mov | Stack, em_push),
4032 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4033 I(SrcImmByte | Mov | Stack, em_push),
4034 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4035 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4036 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4038 X16(D(SrcImmByte | NearBranch)),
4040 G(ByteOp | DstMem | SrcImm, group1),
4041 G(DstMem | SrcImm, group1),
4042 G(ByteOp | DstMem | SrcImm | No64, group1),
4043 G(DstMem | SrcImmByte, group1),
4044 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4045 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4047 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4048 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4049 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4050 D(ModRM | SrcMem | NoAccess | DstReg),
4051 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4054 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4056 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4057 I(SrcImmFAddr | No64, em_call_far), N,
4058 II(ImplicitOps | Stack, em_pushf, pushf),
4059 II(ImplicitOps | Stack, em_popf, popf),
4060 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4062 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4063 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4064 I2bv(SrcSI | DstDI | Mov | String, em_mov),
4065 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp_r),
4067 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4068 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4069 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4070 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4072 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4074 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4076 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4077 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4078 I(ImplicitOps | NearBranch, em_ret),
4079 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4080 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4081 G(ByteOp, group11), G(0, group11),
4083 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4084 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4085 I(ImplicitOps, em_ret_far),
4086 D(ImplicitOps), DI(SrcImmByte, intn),
4087 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4089 G(Src2One | ByteOp, group2), G(Src2One, group2),
4090 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4091 I(DstAcc | SrcImmUByte | No64, em_aam),
4092 I(DstAcc | SrcImmUByte | No64, em_aad),
4093 F(DstAcc | ByteOp | No64, em_salc),
4094 I(DstAcc | SrcXLat | ByteOp, em_mov),
4096 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4098 X3(I(SrcImmByte | NearBranch, em_loop)),
4099 I(SrcImmByte | NearBranch, em_jcxz),
4100 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4101 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4103 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4104 I(SrcImmFAddr | No64, em_jmp_far),
4105 D(SrcImmByte | ImplicitOps | NearBranch),
4106 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4107 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4109 N, DI(ImplicitOps, icebp), N, N,
4110 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4111 G(ByteOp, group3), G(0, group3),
4113 D(ImplicitOps), D(ImplicitOps),
4114 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4115 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4118 static const struct opcode twobyte_table[256] = {
4120 G(0, group6), GD(0, &group7), N, N,
4121 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4122 II(ImplicitOps | Priv, em_clts, clts), N,
4123 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4124 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4126 N, N, N, N, N, N, N, N,
4127 D(ImplicitOps | ModRM | SrcMem | NoAccess),
4128 N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4130 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4131 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4132 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4134 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4137 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4138 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4139 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4142 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4143 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4144 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4145 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4146 I(ImplicitOps | EmulateOnUD, em_sysenter),
4147 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4149 N, N, N, N, N, N, N, N,
4151 X16(D(DstReg | SrcMem | ModRM)),
4153 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4158 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4163 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4165 X16(D(SrcImm | NearBranch)),
4167 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4169 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4170 II(ImplicitOps, em_cpuid, cpuid),
4171 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4172 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4173 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4175 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4176 DI(ImplicitOps, rsm),
4177 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4178 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4179 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4180 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4182 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4183 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4184 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4185 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4186 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4187 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4191 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4192 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
4193 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4195 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4196 N, ID(0, &instr_dual_0f_c3),
4197 N, N, N, GD(0, &group9),
4199 X8(I(DstReg, em_bswap)),
4201 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4203 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4204 N, N, N, N, N, N, N, N,
4206 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4209 static const struct instr_dual instr_dual_0f_38_f0 = {
4210 I(DstReg | SrcMem | Mov, em_movbe), N
4213 static const struct instr_dual instr_dual_0f_38_f1 = {
4214 I(DstMem | SrcReg | Mov, em_movbe), N
4217 static const struct gprefix three_byte_0f_38_f0 = {
4218 ID(0, &instr_dual_0f_38_f0), N, N, N
4221 static const struct gprefix three_byte_0f_38_f1 = {
4222 ID(0, &instr_dual_0f_38_f1), N, N, N
4226 * Insns below are selected by the prefix which indexed by the third opcode
4229 static const struct opcode opcode_map_0f_38[256] = {
4231 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4233 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4235 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4236 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4256 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4260 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4266 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4267 unsigned size, bool sign_extension)
4269 int rc = X86EMUL_CONTINUE;
4273 op->addr.mem.ea = ctxt->_eip;
4274 /* NB. Immediates are sign-extended as necessary. */
4275 switch (op->bytes) {
4277 op->val = insn_fetch(s8, ctxt);
4280 op->val = insn_fetch(s16, ctxt);
4283 op->val = insn_fetch(s32, ctxt);
4286 op->val = insn_fetch(s64, ctxt);
4289 if (!sign_extension) {
4290 switch (op->bytes) {
4298 op->val &= 0xffffffff;
4306 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4309 int rc = X86EMUL_CONTINUE;
4313 decode_register_operand(ctxt, op);
4316 rc = decode_imm(ctxt, op, 1, false);
4319 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4323 if (ctxt->d & BitOp)
4324 fetch_bit_operand(ctxt);
4325 op->orig_val = op->val;
4328 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4332 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4333 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4334 fetch_register_operand(op);
4335 op->orig_val = op->val;
4339 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4340 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4341 fetch_register_operand(op);
4342 op->orig_val = op->val;
4345 if (ctxt->d & ByteOp) {
4350 op->bytes = ctxt->op_bytes;
4351 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4352 fetch_register_operand(op);
4353 op->orig_val = op->val;
4357 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4359 register_address(ctxt, VCPU_REGS_RDI);
4360 op->addr.mem.seg = VCPU_SREG_ES;
4367 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4368 fetch_register_operand(op);
4373 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4376 rc = decode_imm(ctxt, op, 1, true);
4384 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4387 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4390 ctxt->memop.bytes = 1;
4391 if (ctxt->memop.type == OP_REG) {
4392 ctxt->memop.addr.reg = decode_register(ctxt,
4393 ctxt->modrm_rm, true);
4394 fetch_register_operand(&ctxt->memop);
4398 ctxt->memop.bytes = 2;
4401 ctxt->memop.bytes = 4;
4404 rc = decode_imm(ctxt, op, 2, false);
4407 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4411 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4413 register_address(ctxt, VCPU_REGS_RSI);
4414 op->addr.mem.seg = ctxt->seg_override;
4420 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4423 reg_read(ctxt, VCPU_REGS_RBX) +
4424 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4425 op->addr.mem.seg = ctxt->seg_override;
4430 op->addr.mem.ea = ctxt->_eip;
4431 op->bytes = ctxt->op_bytes + 2;
4432 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4435 ctxt->memop.bytes = ctxt->op_bytes + 2;
4439 op->val = VCPU_SREG_ES;
4443 op->val = VCPU_SREG_CS;
4447 op->val = VCPU_SREG_SS;
4451 op->val = VCPU_SREG_DS;
4455 op->val = VCPU_SREG_FS;
4459 op->val = VCPU_SREG_GS;
4462 /* Special instructions do their own operand decoding. */
4464 op->type = OP_NONE; /* Disable writeback. */
4472 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4474 int rc = X86EMUL_CONTINUE;
4475 int mode = ctxt->mode;
4476 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4477 bool op_prefix = false;
4478 bool has_seg_override = false;
4479 struct opcode opcode;
4481 ctxt->memop.type = OP_NONE;
4482 ctxt->memopp = NULL;
4483 ctxt->_eip = ctxt->eip;
4484 ctxt->fetch.ptr = ctxt->fetch.data;
4485 ctxt->fetch.end = ctxt->fetch.data + insn_len;
4486 ctxt->opcode_len = 1;
4488 memcpy(ctxt->fetch.data, insn, insn_len);
4490 rc = __do_insn_fetch_bytes(ctxt, 1);
4491 if (rc != X86EMUL_CONTINUE)
4496 case X86EMUL_MODE_REAL:
4497 case X86EMUL_MODE_VM86:
4498 case X86EMUL_MODE_PROT16:
4499 def_op_bytes = def_ad_bytes = 2;
4501 case X86EMUL_MODE_PROT32:
4502 def_op_bytes = def_ad_bytes = 4;
4504 #ifdef CONFIG_X86_64
4505 case X86EMUL_MODE_PROT64:
4511 return EMULATION_FAILED;
4514 ctxt->op_bytes = def_op_bytes;
4515 ctxt->ad_bytes = def_ad_bytes;
4517 /* Legacy prefixes. */
4519 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4520 case 0x66: /* operand-size override */
4522 /* switch between 2/4 bytes */
4523 ctxt->op_bytes = def_op_bytes ^ 6;
4525 case 0x67: /* address-size override */
4526 if (mode == X86EMUL_MODE_PROT64)
4527 /* switch between 4/8 bytes */
4528 ctxt->ad_bytes = def_ad_bytes ^ 12;
4530 /* switch between 2/4 bytes */
4531 ctxt->ad_bytes = def_ad_bytes ^ 6;
4533 case 0x26: /* ES override */
4534 case 0x2e: /* CS override */
4535 case 0x36: /* SS override */
4536 case 0x3e: /* DS override */
4537 has_seg_override = true;
4538 ctxt->seg_override = (ctxt->b >> 3) & 3;
4540 case 0x64: /* FS override */
4541 case 0x65: /* GS override */
4542 has_seg_override = true;
4543 ctxt->seg_override = ctxt->b & 7;
4545 case 0x40 ... 0x4f: /* REX */
4546 if (mode != X86EMUL_MODE_PROT64)
4548 ctxt->rex_prefix = ctxt->b;
4550 case 0xf0: /* LOCK */
4551 ctxt->lock_prefix = 1;
4553 case 0xf2: /* REPNE/REPNZ */
4554 case 0xf3: /* REP/REPE/REPZ */
4555 ctxt->rep_prefix = ctxt->b;
4561 /* Any legacy prefix after a REX prefix nullifies its effect. */
4563 ctxt->rex_prefix = 0;
4569 if (ctxt->rex_prefix & 8)
4570 ctxt->op_bytes = 8; /* REX.W */
4572 /* Opcode byte(s). */
4573 opcode = opcode_table[ctxt->b];
4574 /* Two-byte opcode? */
4575 if (ctxt->b == 0x0f) {
4576 ctxt->opcode_len = 2;
4577 ctxt->b = insn_fetch(u8, ctxt);
4578 opcode = twobyte_table[ctxt->b];
4580 /* 0F_38 opcode map */
4581 if (ctxt->b == 0x38) {
4582 ctxt->opcode_len = 3;
4583 ctxt->b = insn_fetch(u8, ctxt);
4584 opcode = opcode_map_0f_38[ctxt->b];
4587 ctxt->d = opcode.flags;
4589 if (ctxt->d & ModRM)
4590 ctxt->modrm = insn_fetch(u8, ctxt);
4592 /* vex-prefix instructions are not implemented */
4593 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
4594 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
4598 while (ctxt->d & GroupMask) {
4599 switch (ctxt->d & GroupMask) {
4601 goffset = (ctxt->modrm >> 3) & 7;
4602 opcode = opcode.u.group[goffset];
4605 goffset = (ctxt->modrm >> 3) & 7;
4606 if ((ctxt->modrm >> 6) == 3)
4607 opcode = opcode.u.gdual->mod3[goffset];
4609 opcode = opcode.u.gdual->mod012[goffset];
4612 goffset = ctxt->modrm & 7;
4613 opcode = opcode.u.group[goffset];
4616 if (ctxt->rep_prefix && op_prefix)
4617 return EMULATION_FAILED;
4618 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4619 switch (simd_prefix) {
4620 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4621 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4622 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4623 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4627 if (ctxt->modrm > 0xbf)
4628 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4630 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4633 if ((ctxt->modrm >> 6) == 3)
4634 opcode = opcode.u.idual->mod3;
4636 opcode = opcode.u.idual->mod012;
4639 if (ctxt->mode == X86EMUL_MODE_PROT64)
4640 opcode = opcode.u.mdual->mode64;
4642 opcode = opcode.u.mdual->mode32;
4645 return EMULATION_FAILED;
4648 ctxt->d &= ~(u64)GroupMask;
4649 ctxt->d |= opcode.flags;
4654 return EMULATION_FAILED;
4656 ctxt->execute = opcode.u.execute;
4658 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
4659 return EMULATION_FAILED;
4661 if (unlikely(ctxt->d &
4662 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
4665 * These are copied unconditionally here, and checked unconditionally
4666 * in x86_emulate_insn.
4668 ctxt->check_perm = opcode.check_perm;
4669 ctxt->intercept = opcode.intercept;
4671 if (ctxt->d & NotImpl)
4672 return EMULATION_FAILED;
4674 if (mode == X86EMUL_MODE_PROT64) {
4675 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
4677 else if (ctxt->d & NearBranch)
4681 if (ctxt->d & Op3264) {
4682 if (mode == X86EMUL_MODE_PROT64)
4688 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
4692 ctxt->op_bytes = 16;
4693 else if (ctxt->d & Mmx)
4697 /* ModRM and SIB bytes. */
4698 if (ctxt->d & ModRM) {
4699 rc = decode_modrm(ctxt, &ctxt->memop);
4700 if (!has_seg_override) {
4701 has_seg_override = true;
4702 ctxt->seg_override = ctxt->modrm_seg;
4704 } else if (ctxt->d & MemAbs)
4705 rc = decode_abs(ctxt, &ctxt->memop);
4706 if (rc != X86EMUL_CONTINUE)
4709 if (!has_seg_override)
4710 ctxt->seg_override = VCPU_SREG_DS;
4712 ctxt->memop.addr.mem.seg = ctxt->seg_override;
4715 * Decode and fetch the source operand: register, memory
4718 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4719 if (rc != X86EMUL_CONTINUE)
4723 * Decode and fetch the second source operand: register, memory
4726 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4727 if (rc != X86EMUL_CONTINUE)
4730 /* Decode and fetch the destination operand: register or memory. */
4731 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4733 if (ctxt->rip_relative)
4734 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
4735 ctxt->memopp->addr.mem.ea + ctxt->_eip);
4738 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4741 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4743 return ctxt->d & PageTable;
4746 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4748 /* The second termination condition only applies for REPE
4749 * and REPNE. Test if the repeat string operation prefix is
4750 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4751 * corresponding termination condition according to:
4752 * - if REPE/REPZ and ZF = 0 then done
4753 * - if REPNE/REPNZ and ZF = 1 then done
4755 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4756 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4757 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4758 ((ctxt->eflags & EFLG_ZF) == 0))
4759 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4760 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4766 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4770 ctxt->ops->get_fpu(ctxt);
4771 asm volatile("1: fwait \n\t"
4773 ".pushsection .fixup,\"ax\" \n\t"
4775 "movb $1, %[fault] \n\t"
4778 _ASM_EXTABLE(1b, 3b)
4779 : [fault]"+qm"(fault));
4780 ctxt->ops->put_fpu(ctxt);
4782 if (unlikely(fault))
4783 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4785 return X86EMUL_CONTINUE;
4788 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4791 if (op->type == OP_MM)
4792 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4795 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4797 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4798 if (!(ctxt->d & ByteOp))
4799 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4800 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4801 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
4803 : "c"(ctxt->src2.val));
4804 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4805 if (!fop) /* exception is returned in fop variable */
4806 return emulate_de(ctxt);
4807 return X86EMUL_CONTINUE;
4810 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
4812 memset(&ctxt->rip_relative, 0,
4813 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
4815 ctxt->io_read.pos = 0;
4816 ctxt->io_read.end = 0;
4817 ctxt->mem_read.end = 0;
4820 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4822 const struct x86_emulate_ops *ops = ctxt->ops;
4823 int rc = X86EMUL_CONTINUE;
4824 int saved_dst_type = ctxt->dst.type;
4826 ctxt->mem_read.pos = 0;
4828 /* LOCK prefix is allowed only with some instructions */
4829 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4830 rc = emulate_ud(ctxt);
4834 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4835 rc = emulate_ud(ctxt);
4839 if (unlikely(ctxt->d &
4840 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
4841 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4842 (ctxt->d & Undefined)) {
4843 rc = emulate_ud(ctxt);
4847 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4848 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4849 rc = emulate_ud(ctxt);
4853 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4854 rc = emulate_nm(ctxt);
4858 if (ctxt->d & Mmx) {
4859 rc = flush_pending_x87_faults(ctxt);
4860 if (rc != X86EMUL_CONTINUE)
4863 * Now that we know the fpu is exception safe, we can fetch
4866 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4867 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4868 if (!(ctxt->d & Mov))
4869 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4872 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4873 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4874 X86_ICPT_PRE_EXCEPT);
4875 if (rc != X86EMUL_CONTINUE)
4879 /* Instruction can only be executed in protected mode */
4880 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4881 rc = emulate_ud(ctxt);
4885 /* Privileged instruction can be executed only in CPL=0 */
4886 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4887 if (ctxt->d & PrivUD)
4888 rc = emulate_ud(ctxt);
4890 rc = emulate_gp(ctxt, 0);
4894 /* Do instruction specific permission checks */
4895 if (ctxt->d & CheckPerm) {
4896 rc = ctxt->check_perm(ctxt);
4897 if (rc != X86EMUL_CONTINUE)
4901 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4902 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4903 X86_ICPT_POST_EXCEPT);
4904 if (rc != X86EMUL_CONTINUE)
4908 if (ctxt->rep_prefix && (ctxt->d & String)) {
4909 /* All REP prefixes have the same first termination condition */
4910 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4911 ctxt->eip = ctxt->_eip;
4912 ctxt->eflags &= ~EFLG_RF;
4918 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4919 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4920 ctxt->src.valptr, ctxt->src.bytes);
4921 if (rc != X86EMUL_CONTINUE)
4923 ctxt->src.orig_val64 = ctxt->src.val64;
4926 if (ctxt->src2.type == OP_MEM) {
4927 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4928 &ctxt->src2.val, ctxt->src2.bytes);
4929 if (rc != X86EMUL_CONTINUE)
4933 if ((ctxt->d & DstMask) == ImplicitOps)
4937 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4938 /* optimisation - avoid slow emulated read if Mov */
4939 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4940 &ctxt->dst.val, ctxt->dst.bytes);
4941 if (rc != X86EMUL_CONTINUE) {
4942 if (rc == X86EMUL_PROPAGATE_FAULT &&
4943 ctxt->exception.vector == PF_VECTOR)
4944 ctxt->exception.error_code |= PFERR_WRITE_MASK;
4948 ctxt->dst.orig_val = ctxt->dst.val;
4952 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4953 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4954 X86_ICPT_POST_MEMACCESS);
4955 if (rc != X86EMUL_CONTINUE)
4959 if (ctxt->rep_prefix && (ctxt->d & String))
4960 ctxt->eflags |= EFLG_RF;
4962 ctxt->eflags &= ~EFLG_RF;
4964 if (ctxt->execute) {
4965 if (ctxt->d & Fastop) {
4966 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4967 rc = fastop(ctxt, fop);
4968 if (rc != X86EMUL_CONTINUE)
4972 rc = ctxt->execute(ctxt);
4973 if (rc != X86EMUL_CONTINUE)
4978 if (ctxt->opcode_len == 2)
4980 else if (ctxt->opcode_len == 3)
4981 goto threebyte_insn;
4984 case 0x70 ... 0x7f: /* jcc (short) */
4985 if (test_cc(ctxt->b, ctxt->eflags))
4986 rc = jmp_rel(ctxt, ctxt->src.val);
4988 case 0x8d: /* lea r16/r32, m */
4989 ctxt->dst.val = ctxt->src.addr.mem.ea;
4991 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4992 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4993 ctxt->dst.type = OP_NONE;
4997 case 0x98: /* cbw/cwde/cdqe */
4998 switch (ctxt->op_bytes) {
4999 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5000 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5001 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5004 case 0xcc: /* int3 */
5005 rc = emulate_int(ctxt, 3);
5007 case 0xcd: /* int n */
5008 rc = emulate_int(ctxt, ctxt->src.val);
5010 case 0xce: /* into */
5011 if (ctxt->eflags & EFLG_OF)
5012 rc = emulate_int(ctxt, 4);
5014 case 0xe9: /* jmp rel */
5015 case 0xeb: /* jmp rel short */
5016 rc = jmp_rel(ctxt, ctxt->src.val);
5017 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5019 case 0xf4: /* hlt */
5020 ctxt->ops->halt(ctxt);
5022 case 0xf5: /* cmc */
5023 /* complement carry flag from eflags reg */
5024 ctxt->eflags ^= EFLG_CF;
5026 case 0xf8: /* clc */
5027 ctxt->eflags &= ~EFLG_CF;
5029 case 0xf9: /* stc */
5030 ctxt->eflags |= EFLG_CF;
5032 case 0xfc: /* cld */
5033 ctxt->eflags &= ~EFLG_DF;
5035 case 0xfd: /* std */
5036 ctxt->eflags |= EFLG_DF;
5039 goto cannot_emulate;
5042 if (rc != X86EMUL_CONTINUE)
5046 if (ctxt->d & SrcWrite) {
5047 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5048 rc = writeback(ctxt, &ctxt->src);
5049 if (rc != X86EMUL_CONTINUE)
5052 if (!(ctxt->d & NoWrite)) {
5053 rc = writeback(ctxt, &ctxt->dst);
5054 if (rc != X86EMUL_CONTINUE)
5059 * restore dst type in case the decoding will be reused
5060 * (happens for string instruction )
5062 ctxt->dst.type = saved_dst_type;
5064 if ((ctxt->d & SrcMask) == SrcSI)
5065 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5067 if ((ctxt->d & DstMask) == DstDI)
5068 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5070 if (ctxt->rep_prefix && (ctxt->d & String)) {
5072 struct read_cache *r = &ctxt->io_read;
5073 if ((ctxt->d & SrcMask) == SrcSI)
5074 count = ctxt->src.count;
5076 count = ctxt->dst.count;
5077 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5079 if (!string_insn_completed(ctxt)) {
5081 * Re-enter guest when pio read ahead buffer is empty
5082 * or, if it is not used, after each 1024 iteration.
5084 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5085 (r->end == 0 || r->end != r->pos)) {
5087 * Reset read cache. Usually happens before
5088 * decode, but since instruction is restarted
5089 * we have to do it here.
5091 ctxt->mem_read.end = 0;
5092 writeback_registers(ctxt);
5093 return EMULATION_RESTART;
5095 goto done; /* skip rip writeback */
5097 ctxt->eflags &= ~EFLG_RF;
5100 ctxt->eip = ctxt->_eip;
5103 if (rc == X86EMUL_PROPAGATE_FAULT) {
5104 WARN_ON(ctxt->exception.vector > 0x1f);
5105 ctxt->have_exception = true;
5107 if (rc == X86EMUL_INTERCEPTED)
5108 return EMULATION_INTERCEPTED;
5110 if (rc == X86EMUL_CONTINUE)
5111 writeback_registers(ctxt);
5113 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5117 case 0x09: /* wbinvd */
5118 (ctxt->ops->wbinvd)(ctxt);
5120 case 0x08: /* invd */
5121 case 0x0d: /* GrpP (prefetch) */
5122 case 0x18: /* Grp16 (prefetch/nop) */
5123 case 0x1f: /* nop */
5125 case 0x20: /* mov cr, reg */
5126 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5128 case 0x21: /* mov from dr to reg */
5129 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5131 case 0x40 ... 0x4f: /* cmov */
5132 if (test_cc(ctxt->b, ctxt->eflags))
5133 ctxt->dst.val = ctxt->src.val;
5134 else if (ctxt->mode != X86EMUL_MODE_PROT64 ||
5135 ctxt->op_bytes != 4)
5136 ctxt->dst.type = OP_NONE; /* no writeback */
5138 case 0x80 ... 0x8f: /* jnz rel, etc*/
5139 if (test_cc(ctxt->b, ctxt->eflags))
5140 rc = jmp_rel(ctxt, ctxt->src.val);
5142 case 0x90 ... 0x9f: /* setcc r/m8 */
5143 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5145 case 0xb6 ... 0xb7: /* movzx */
5146 ctxt->dst.bytes = ctxt->op_bytes;
5147 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5148 : (u16) ctxt->src.val;
5150 case 0xbe ... 0xbf: /* movsx */
5151 ctxt->dst.bytes = ctxt->op_bytes;
5152 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5153 (s16) ctxt->src.val;
5156 goto cannot_emulate;
5161 if (rc != X86EMUL_CONTINUE)
5167 return EMULATION_FAILED;
5170 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5172 invalidate_registers(ctxt);
5175 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5177 writeback_registers(ctxt);