1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstMem16 (OpMem16 << DstShift)
90 #define DstImmUByte (OpImmUByte << DstShift)
91 #define DstDX (OpDX << DstShift)
92 #define DstAccLo (OpAccLo << DstShift)
93 #define DstMask (OpMask << DstShift)
94 /* Source operand type. */
96 #define SrcNone (OpNone << SrcShift)
97 #define SrcReg (OpReg << SrcShift)
98 #define SrcMem (OpMem << SrcShift)
99 #define SrcMem16 (OpMem16 << SrcShift)
100 #define SrcMem32 (OpMem32 << SrcShift)
101 #define SrcImm (OpImm << SrcShift)
102 #define SrcImmByte (OpImmByte << SrcShift)
103 #define SrcOne (OpOne << SrcShift)
104 #define SrcImmUByte (OpImmUByte << SrcShift)
105 #define SrcImmU (OpImmU << SrcShift)
106 #define SrcSI (OpSI << SrcShift)
107 #define SrcXLat (OpXLat << SrcShift)
108 #define SrcImmFAddr (OpImmFAddr << SrcShift)
109 #define SrcMemFAddr (OpMemFAddr << SrcShift)
110 #define SrcAcc (OpAcc << SrcShift)
111 #define SrcImmU16 (OpImmU16 << SrcShift)
112 #define SrcImm64 (OpImm64 << SrcShift)
113 #define SrcDX (OpDX << SrcShift)
114 #define SrcMem8 (OpMem8 << SrcShift)
115 #define SrcAccHi (OpAccHi << SrcShift)
116 #define SrcMask (OpMask << SrcShift)
117 #define BitOp (1<<11)
118 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
119 #define String (1<<13) /* String instruction (rep capable) */
120 #define Stack (1<<14) /* Stack instruction (push/pop) */
121 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
122 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
123 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
124 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
125 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
126 #define Escape (5<<15) /* Escape to coprocessor instruction */
127 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
128 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
129 #define Sse (1<<18) /* SSE Vector instruction */
130 /* Generic ModRM decode. */
131 #define ModRM (1<<19)
132 /* Destination is only written; never read. */
135 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
136 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
137 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
138 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
139 #define Undefined (1<<25) /* No Such Instruction */
140 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
141 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
143 #define PageTable (1 << 29) /* instruction used to write page table */
144 #define NotImpl (1 << 30) /* instruction is not implemented */
145 /* Source 2 operand type */
146 #define Src2Shift (31)
147 #define Src2None (OpNone << Src2Shift)
148 #define Src2Mem (OpMem << Src2Shift)
149 #define Src2CL (OpCL << Src2Shift)
150 #define Src2ImmByte (OpImmByte << Src2Shift)
151 #define Src2One (OpOne << Src2Shift)
152 #define Src2Imm (OpImm << Src2Shift)
153 #define Src2ES (OpES << Src2Shift)
154 #define Src2CS (OpCS << Src2Shift)
155 #define Src2SS (OpSS << Src2Shift)
156 #define Src2DS (OpDS << Src2Shift)
157 #define Src2FS (OpFS << Src2Shift)
158 #define Src2GS (OpGS << Src2Shift)
159 #define Src2Mask (OpMask << Src2Shift)
160 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
161 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
162 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
163 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
164 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
165 #define NoWrite ((u64)1 << 45) /* No writeback */
166 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
167 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
168 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
169 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
170 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
171 #define NearBranch ((u64)1 << 52) /* Near branches */
172 #define No16 ((u64)1 << 53) /* No 16 bit operand */
173 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
175 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
177 #define X2(x...) x, x
178 #define X3(x...) X2(x), x
179 #define X4(x...) X2(x), X2(x)
180 #define X5(x...) X4(x), x
181 #define X6(x...) X4(x), X2(x)
182 #define X7(x...) X4(x), X3(x)
183 #define X8(x...) X4(x), X4(x)
184 #define X16(x...) X8(x), X8(x)
186 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
187 #define FASTOP_SIZE 8
190 * fastop functions have a special calling convention:
195 * flags: rflags (in/out)
196 * ex: rsi (in:fastop pointer, out:zero if exception)
198 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
199 * different operand sizes can be reached by calculation, rather than a jump
200 * table (which would be bigger than the code).
202 * fastop functions are declared as taking a never-defined fastop parameter,
203 * so they can't be called from C directly.
212 int (*execute)(struct x86_emulate_ctxt *ctxt);
213 const struct opcode *group;
214 const struct group_dual *gdual;
215 const struct gprefix *gprefix;
216 const struct escape *esc;
217 const struct instr_dual *idual;
218 const struct mode_dual *mdual;
219 void (*fastop)(struct fastop *fake);
221 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
225 struct opcode mod012[8];
226 struct opcode mod3[8];
230 struct opcode pfx_no;
231 struct opcode pfx_66;
232 struct opcode pfx_f2;
233 struct opcode pfx_f3;
238 struct opcode high[64];
242 struct opcode mod012;
247 struct opcode mode32;
248 struct opcode mode64;
251 /* EFLAGS bit definitions. */
252 #define EFLG_ID (1<<21)
253 #define EFLG_VIP (1<<20)
254 #define EFLG_VIF (1<<19)
255 #define EFLG_AC (1<<18)
256 #define EFLG_VM (1<<17)
257 #define EFLG_RF (1<<16)
258 #define EFLG_IOPL (3<<12)
259 #define EFLG_NT (1<<14)
260 #define EFLG_OF (1<<11)
261 #define EFLG_DF (1<<10)
262 #define EFLG_IF (1<<9)
263 #define EFLG_TF (1<<8)
264 #define EFLG_SF (1<<7)
265 #define EFLG_ZF (1<<6)
266 #define EFLG_AF (1<<4)
267 #define EFLG_PF (1<<2)
268 #define EFLG_CF (1<<0)
270 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
271 #define EFLG_RESERVED_ONE_MASK 2
273 enum x86_transfer_type {
275 X86_TRANSFER_CALL_JMP,
277 X86_TRANSFER_TASK_SWITCH,
280 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
282 if (!(ctxt->regs_valid & (1 << nr))) {
283 ctxt->regs_valid |= 1 << nr;
284 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
286 return ctxt->_regs[nr];
289 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
291 ctxt->regs_valid |= 1 << nr;
292 ctxt->regs_dirty |= 1 << nr;
293 return &ctxt->_regs[nr];
296 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
299 return reg_write(ctxt, nr);
302 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
306 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
307 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
310 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
312 ctxt->regs_dirty = 0;
313 ctxt->regs_valid = 0;
317 * These EFLAGS bits are restored from saved value during emulation, and
318 * any changes are written back to the saved value after emulation.
320 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
328 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
330 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
331 #define FOP_RET "ret \n\t"
333 #define FOP_START(op) \
334 extern void em_##op(struct fastop *fake); \
335 asm(".pushsection .text, \"ax\" \n\t" \
336 ".global em_" #op " \n\t" \
343 #define FOPNOP() FOP_ALIGN FOP_RET
345 #define FOP1E(op, dst) \
346 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
348 #define FOP1EEX(op, dst) \
349 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
351 #define FASTOP1(op) \
356 ON64(FOP1E(op##q, rax)) \
359 /* 1-operand, using src2 (for MUL/DIV r/m) */
360 #define FASTOP1SRC2(op, name) \
365 ON64(FOP1E(op, rcx)) \
368 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
369 #define FASTOP1SRC2EX(op, name) \
374 ON64(FOP1EEX(op, rcx)) \
377 #define FOP2E(op, dst, src) \
378 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
380 #define FASTOP2(op) \
382 FOP2E(op##b, al, dl) \
383 FOP2E(op##w, ax, dx) \
384 FOP2E(op##l, eax, edx) \
385 ON64(FOP2E(op##q, rax, rdx)) \
388 /* 2 operand, word only */
389 #define FASTOP2W(op) \
392 FOP2E(op##w, ax, dx) \
393 FOP2E(op##l, eax, edx) \
394 ON64(FOP2E(op##q, rax, rdx)) \
397 /* 2 operand, src is CL */
398 #define FASTOP2CL(op) \
400 FOP2E(op##b, al, cl) \
401 FOP2E(op##w, ax, cl) \
402 FOP2E(op##l, eax, cl) \
403 ON64(FOP2E(op##q, rax, cl)) \
406 /* 2 operand, src and dest are reversed */
407 #define FASTOP2R(op, name) \
409 FOP2E(op##b, dl, al) \
410 FOP2E(op##w, dx, ax) \
411 FOP2E(op##l, edx, eax) \
412 ON64(FOP2E(op##q, rdx, rax)) \
415 #define FOP3E(op, dst, src, src2) \
416 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
418 /* 3-operand, word-only, src2=cl */
419 #define FASTOP3WCL(op) \
422 FOP3E(op##w, ax, dx, cl) \
423 FOP3E(op##l, eax, edx, cl) \
424 ON64(FOP3E(op##q, rax, rdx, cl)) \
427 /* Special case for SETcc - 1 instruction per cc */
428 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
430 asm(".global kvm_fastop_exception \n"
431 "kvm_fastop_exception: xor %esi, %esi; ret");
452 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
455 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
456 enum x86_intercept intercept,
457 enum x86_intercept_stage stage)
459 struct x86_instruction_info info = {
460 .intercept = intercept,
461 .rep_prefix = ctxt->rep_prefix,
462 .modrm_mod = ctxt->modrm_mod,
463 .modrm_reg = ctxt->modrm_reg,
464 .modrm_rm = ctxt->modrm_rm,
465 .src_val = ctxt->src.val64,
466 .dst_val = ctxt->dst.val64,
467 .src_bytes = ctxt->src.bytes,
468 .dst_bytes = ctxt->dst.bytes,
469 .ad_bytes = ctxt->ad_bytes,
470 .next_rip = ctxt->eip,
473 return ctxt->ops->intercept(ctxt, &info, stage);
476 static void assign_masked(ulong *dest, ulong src, ulong mask)
478 *dest = (*dest & ~mask) | (src & mask);
481 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
483 return (1UL << (ctxt->ad_bytes << 3)) - 1;
486 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
489 struct desc_struct ss;
491 if (ctxt->mode == X86EMUL_MODE_PROT64)
493 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
494 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
497 static int stack_size(struct x86_emulate_ctxt *ctxt)
499 return (__fls(stack_mask(ctxt)) + 1) >> 3;
502 /* Access/update address held in a register, based on addressing mode. */
503 static inline unsigned long
504 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
506 if (ctxt->ad_bytes == sizeof(unsigned long))
509 return reg & ad_mask(ctxt);
512 static inline unsigned long
513 register_address(struct x86_emulate_ctxt *ctxt, int reg)
515 return address_mask(ctxt, reg_read(ctxt, reg));
518 static void masked_increment(ulong *reg, ulong mask, int inc)
520 assign_masked(reg, *reg + inc, mask);
524 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
528 if (ctxt->ad_bytes == sizeof(unsigned long))
531 mask = ad_mask(ctxt);
532 masked_increment(reg_rmw(ctxt, reg), mask, inc);
535 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
537 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
540 static u32 desc_limit_scaled(struct desc_struct *desc)
542 u32 limit = get_desc_limit(desc);
544 return desc->g ? (limit << 12) | 0xfff : limit;
547 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
549 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
552 return ctxt->ops->get_cached_segment_base(ctxt, seg);
555 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
556 u32 error, bool valid)
559 ctxt->exception.vector = vec;
560 ctxt->exception.error_code = error;
561 ctxt->exception.error_code_valid = valid;
562 return X86EMUL_PROPAGATE_FAULT;
565 static int emulate_db(struct x86_emulate_ctxt *ctxt)
567 return emulate_exception(ctxt, DB_VECTOR, 0, false);
570 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
572 return emulate_exception(ctxt, GP_VECTOR, err, true);
575 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
577 return emulate_exception(ctxt, SS_VECTOR, err, true);
580 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
582 return emulate_exception(ctxt, UD_VECTOR, 0, false);
585 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
587 return emulate_exception(ctxt, TS_VECTOR, err, true);
590 static int emulate_de(struct x86_emulate_ctxt *ctxt)
592 return emulate_exception(ctxt, DE_VECTOR, 0, false);
595 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
597 return emulate_exception(ctxt, NM_VECTOR, 0, false);
600 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
603 struct desc_struct desc;
605 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
609 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
614 struct desc_struct desc;
616 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
617 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
621 * x86 defines three classes of vector instructions: explicitly
622 * aligned, explicitly unaligned, and the rest, which change behaviour
623 * depending on whether they're AVX encoded or not.
625 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
626 * subject to the same check.
628 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
630 if (likely(size < 16))
633 if (ctxt->d & Aligned)
635 else if (ctxt->d & Unaligned)
637 else if (ctxt->d & Avx)
643 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
644 struct segmented_address addr,
645 unsigned *max_size, unsigned size,
646 bool write, bool fetch,
647 enum x86emul_mode mode, ulong *linear)
649 struct desc_struct desc;
655 la = seg_base(ctxt, addr.seg) + addr.ea;
658 case X86EMUL_MODE_PROT64:
659 if (is_noncanonical_address(la))
662 *max_size = min_t(u64, ~0u, (1ull << 48) - la);
663 if (size > *max_size)
667 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
671 /* code segment in protected mode or read-only data segment */
672 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
673 || !(desc.type & 2)) && write)
675 /* unreadable code segment */
676 if (!fetch && (desc.type & 8) && !(desc.type & 2))
678 lim = desc_limit_scaled(&desc);
679 if (!(desc.type & 8) && (desc.type & 4)) {
680 /* expand-down segment */
683 lim = desc.d ? 0xffffffff : 0xffff;
687 if (lim == 0xffffffff)
690 *max_size = (u64)lim + 1 - addr.ea;
691 if (size > *max_size)
697 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
698 return emulate_gp(ctxt, 0);
700 return X86EMUL_CONTINUE;
702 if (addr.seg == VCPU_SREG_SS)
703 return emulate_ss(ctxt, 0);
705 return emulate_gp(ctxt, 0);
708 static int linearize(struct x86_emulate_ctxt *ctxt,
709 struct segmented_address addr,
710 unsigned size, bool write,
714 return __linearize(ctxt, addr, &max_size, size, write, false,
718 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
719 enum x86emul_mode mode)
724 struct segmented_address addr = { .seg = VCPU_SREG_CS,
727 if (ctxt->op_bytes != sizeof(unsigned long))
728 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
729 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
730 if (rc == X86EMUL_CONTINUE)
731 ctxt->_eip = addr.ea;
735 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
737 return assign_eip(ctxt, dst, ctxt->mode);
740 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
741 const struct desc_struct *cs_desc)
743 enum x86emul_mode mode = ctxt->mode;
747 if (ctxt->mode >= X86EMUL_MODE_PROT16) {
751 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
753 mode = X86EMUL_MODE_PROT64;
755 mode = X86EMUL_MODE_PROT32; /* temporary value */
758 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
759 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
760 rc = assign_eip(ctxt, dst, mode);
761 if (rc == X86EMUL_CONTINUE)
766 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
768 return assign_eip_near(ctxt, ctxt->_eip + rel);
771 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
772 struct segmented_address addr,
779 rc = linearize(ctxt, addr, size, false, &linear);
780 if (rc != X86EMUL_CONTINUE)
782 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
786 * Prefetch the remaining bytes of the instruction without crossing page
787 * boundary if they are not in fetch_cache yet.
789 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
792 unsigned size, max_size;
793 unsigned long linear;
794 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
795 struct segmented_address addr = { .seg = VCPU_SREG_CS,
796 .ea = ctxt->eip + cur_size };
799 * We do not know exactly how many bytes will be needed, and
800 * __linearize is expensive, so fetch as much as possible. We
801 * just have to avoid going beyond the 15 byte limit, the end
802 * of the segment, or the end of the page.
804 * __linearize is called with size 0 so that it does not do any
805 * boundary check itself. Instead, we use max_size to check
808 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
810 if (unlikely(rc != X86EMUL_CONTINUE))
813 size = min_t(unsigned, 15UL ^ cur_size, max_size);
814 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
817 * One instruction can only straddle two pages,
818 * and one has been loaded at the beginning of
819 * x86_decode_insn. So, if not enough bytes
820 * still, we must have hit the 15-byte boundary.
822 if (unlikely(size < op_size))
823 return emulate_gp(ctxt, 0);
825 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
826 size, &ctxt->exception);
827 if (unlikely(rc != X86EMUL_CONTINUE))
829 ctxt->fetch.end += size;
830 return X86EMUL_CONTINUE;
833 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
836 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
838 if (unlikely(done_size < size))
839 return __do_insn_fetch_bytes(ctxt, size - done_size);
841 return X86EMUL_CONTINUE;
844 /* Fetch next part of the instruction being emulated. */
845 #define insn_fetch(_type, _ctxt) \
848 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
849 if (rc != X86EMUL_CONTINUE) \
851 ctxt->_eip += sizeof(_type); \
852 _x = *(_type __aligned(1) *) ctxt->fetch.ptr; \
853 ctxt->fetch.ptr += sizeof(_type); \
857 #define insn_fetch_arr(_arr, _size, _ctxt) \
859 rc = do_insn_fetch_bytes(_ctxt, _size); \
860 if (rc != X86EMUL_CONTINUE) \
862 ctxt->_eip += (_size); \
863 memcpy(_arr, ctxt->fetch.ptr, _size); \
864 ctxt->fetch.ptr += (_size); \
868 * Given the 'reg' portion of a ModRM byte, and a register block, return a
869 * pointer into the block that addresses the relevant register.
870 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
872 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
876 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
878 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
879 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
881 p = reg_rmw(ctxt, modrm_reg);
885 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
886 struct segmented_address addr,
887 u16 *size, unsigned long *address, int op_bytes)
894 rc = segmented_read_std(ctxt, addr, size, 2);
895 if (rc != X86EMUL_CONTINUE)
898 rc = segmented_read_std(ctxt, addr, address, op_bytes);
912 FASTOP1SRC2(mul, mul_ex);
913 FASTOP1SRC2(imul, imul_ex);
914 FASTOP1SRC2EX(div, div_ex);
915 FASTOP1SRC2EX(idiv, idiv_ex);
944 FASTOP2R(cmp, cmp_r);
946 static u8 test_cc(unsigned int condition, unsigned long flags)
949 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
951 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
952 asm("push %[flags]; popf; call *%[fastop]"
953 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
957 static void fetch_register_operand(struct operand *op)
961 op->val = *(u8 *)op->addr.reg;
964 op->val = *(u16 *)op->addr.reg;
967 op->val = *(u32 *)op->addr.reg;
970 op->val = *(u64 *)op->addr.reg;
975 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
977 ctxt->ops->get_fpu(ctxt);
979 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
980 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
981 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
982 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
983 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
984 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
985 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
986 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
988 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
989 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
990 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
991 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
992 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
993 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
994 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
995 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
999 ctxt->ops->put_fpu(ctxt);
1002 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1005 ctxt->ops->get_fpu(ctxt);
1007 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1008 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1009 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1010 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1011 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1012 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1013 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1014 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1015 #ifdef CONFIG_X86_64
1016 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1017 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1018 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1019 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1020 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1021 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1022 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1023 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1027 ctxt->ops->put_fpu(ctxt);
1030 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1032 ctxt->ops->get_fpu(ctxt);
1034 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1035 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1036 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1037 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1038 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1039 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1040 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1041 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1044 ctxt->ops->put_fpu(ctxt);
1047 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1049 ctxt->ops->get_fpu(ctxt);
1051 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1052 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1053 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1054 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1055 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1056 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1057 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1058 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1061 ctxt->ops->put_fpu(ctxt);
1064 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1066 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1067 return emulate_nm(ctxt);
1069 ctxt->ops->get_fpu(ctxt);
1070 asm volatile("fninit");
1071 ctxt->ops->put_fpu(ctxt);
1072 return X86EMUL_CONTINUE;
1075 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1079 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1080 return emulate_nm(ctxt);
1082 ctxt->ops->get_fpu(ctxt);
1083 asm volatile("fnstcw %0": "+m"(fcw));
1084 ctxt->ops->put_fpu(ctxt);
1086 ctxt->dst.val = fcw;
1088 return X86EMUL_CONTINUE;
1091 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1095 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1096 return emulate_nm(ctxt);
1098 ctxt->ops->get_fpu(ctxt);
1099 asm volatile("fnstsw %0": "+m"(fsw));
1100 ctxt->ops->put_fpu(ctxt);
1102 ctxt->dst.val = fsw;
1104 return X86EMUL_CONTINUE;
1107 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1110 unsigned reg = ctxt->modrm_reg;
1112 if (!(ctxt->d & ModRM))
1113 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1115 if (ctxt->d & Sse) {
1119 read_sse_reg(ctxt, &op->vec_val, reg);
1122 if (ctxt->d & Mmx) {
1131 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1132 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1134 fetch_register_operand(op);
1135 op->orig_val = op->val;
1138 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1140 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1141 ctxt->modrm_seg = VCPU_SREG_SS;
1144 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1148 int index_reg, base_reg, scale;
1149 int rc = X86EMUL_CONTINUE;
1152 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1153 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1154 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1156 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1157 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1158 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1159 ctxt->modrm_seg = VCPU_SREG_DS;
1161 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1163 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1164 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1166 if (ctxt->d & Sse) {
1169 op->addr.xmm = ctxt->modrm_rm;
1170 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1173 if (ctxt->d & Mmx) {
1176 op->addr.mm = ctxt->modrm_rm & 7;
1179 fetch_register_operand(op);
1185 if (ctxt->ad_bytes == 2) {
1186 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1187 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1188 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1189 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1191 /* 16-bit ModR/M decode. */
1192 switch (ctxt->modrm_mod) {
1194 if (ctxt->modrm_rm == 6)
1195 modrm_ea += insn_fetch(u16, ctxt);
1198 modrm_ea += insn_fetch(s8, ctxt);
1201 modrm_ea += insn_fetch(u16, ctxt);
1204 switch (ctxt->modrm_rm) {
1206 modrm_ea += bx + si;
1209 modrm_ea += bx + di;
1212 modrm_ea += bp + si;
1215 modrm_ea += bp + di;
1224 if (ctxt->modrm_mod != 0)
1231 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1232 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1233 ctxt->modrm_seg = VCPU_SREG_SS;
1234 modrm_ea = (u16)modrm_ea;
1236 /* 32/64-bit ModR/M decode. */
1237 if ((ctxt->modrm_rm & 7) == 4) {
1238 sib = insn_fetch(u8, ctxt);
1239 index_reg |= (sib >> 3) & 7;
1240 base_reg |= sib & 7;
1243 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1244 modrm_ea += insn_fetch(s32, ctxt);
1246 modrm_ea += reg_read(ctxt, base_reg);
1247 adjust_modrm_seg(ctxt, base_reg);
1248 /* Increment ESP on POP [ESP] */
1249 if ((ctxt->d & IncSP) &&
1250 base_reg == VCPU_REGS_RSP)
1251 modrm_ea += ctxt->op_bytes;
1254 modrm_ea += reg_read(ctxt, index_reg) << scale;
1255 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1256 modrm_ea += insn_fetch(s32, ctxt);
1257 if (ctxt->mode == X86EMUL_MODE_PROT64)
1258 ctxt->rip_relative = 1;
1260 base_reg = ctxt->modrm_rm;
1261 modrm_ea += reg_read(ctxt, base_reg);
1262 adjust_modrm_seg(ctxt, base_reg);
1264 switch (ctxt->modrm_mod) {
1266 modrm_ea += insn_fetch(s8, ctxt);
1269 modrm_ea += insn_fetch(s32, ctxt);
1273 op->addr.mem.ea = modrm_ea;
1274 if (ctxt->ad_bytes != 8)
1275 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1281 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1284 int rc = X86EMUL_CONTINUE;
1287 switch (ctxt->ad_bytes) {
1289 op->addr.mem.ea = insn_fetch(u16, ctxt);
1292 op->addr.mem.ea = insn_fetch(u32, ctxt);
1295 op->addr.mem.ea = insn_fetch(u64, ctxt);
1302 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1306 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1307 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1309 if (ctxt->src.bytes == 2)
1310 sv = (s16)ctxt->src.val & (s16)mask;
1311 else if (ctxt->src.bytes == 4)
1312 sv = (s32)ctxt->src.val & (s32)mask;
1314 sv = (s64)ctxt->src.val & (s64)mask;
1316 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1317 ctxt->dst.addr.mem.ea + (sv >> 3));
1320 /* only subword offset */
1321 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1324 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1325 unsigned long addr, void *dest, unsigned size)
1328 struct read_cache *mc = &ctxt->mem_read;
1330 if (mc->pos < mc->end)
1333 WARN_ON((mc->end + size) >= sizeof(mc->data));
1335 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1337 if (rc != X86EMUL_CONTINUE)
1343 memcpy(dest, mc->data + mc->pos, size);
1345 return X86EMUL_CONTINUE;
1348 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1349 struct segmented_address addr,
1356 rc = linearize(ctxt, addr, size, false, &linear);
1357 if (rc != X86EMUL_CONTINUE)
1359 return read_emulated(ctxt, linear, data, size);
1362 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1363 struct segmented_address addr,
1370 rc = linearize(ctxt, addr, size, true, &linear);
1371 if (rc != X86EMUL_CONTINUE)
1373 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1377 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1378 struct segmented_address addr,
1379 const void *orig_data, const void *data,
1385 rc = linearize(ctxt, addr, size, true, &linear);
1386 if (rc != X86EMUL_CONTINUE)
1388 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1389 size, &ctxt->exception);
1392 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1393 unsigned int size, unsigned short port,
1396 struct read_cache *rc = &ctxt->io_read;
1398 if (rc->pos == rc->end) { /* refill pio read ahead */
1399 unsigned int in_page, n;
1400 unsigned int count = ctxt->rep_prefix ?
1401 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1402 in_page = (ctxt->eflags & EFLG_DF) ?
1403 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1404 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1405 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1408 rc->pos = rc->end = 0;
1409 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1414 if (ctxt->rep_prefix && (ctxt->d & String) &&
1415 !(ctxt->eflags & EFLG_DF)) {
1416 ctxt->dst.data = rc->data + rc->pos;
1417 ctxt->dst.type = OP_MEM_STR;
1418 ctxt->dst.count = (rc->end - rc->pos) / size;
1421 memcpy(dest, rc->data + rc->pos, size);
1427 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1428 u16 index, struct desc_struct *desc)
1433 ctxt->ops->get_idt(ctxt, &dt);
1435 if (dt.size < index * 8 + 7)
1436 return emulate_gp(ctxt, index << 3 | 0x2);
1438 addr = dt.address + index * 8;
1439 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1443 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1444 u16 selector, struct desc_ptr *dt)
1446 const struct x86_emulate_ops *ops = ctxt->ops;
1449 if (selector & 1 << 2) {
1450 struct desc_struct desc;
1453 memset (dt, 0, sizeof *dt);
1454 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1458 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1459 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1461 ops->get_gdt(ctxt, dt);
1464 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1465 u16 selector, ulong *desc_addr_p)
1468 u16 index = selector >> 3;
1471 get_descriptor_table_ptr(ctxt, selector, &dt);
1473 if (dt.size < index * 8 + 7)
1474 return emulate_gp(ctxt, selector & 0xfffc);
1476 addr = dt.address + index * 8;
1478 #ifdef CONFIG_X86_64
1479 if (addr >> 32 != 0) {
1482 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1483 if (!(efer & EFER_LMA))
1488 *desc_addr_p = addr;
1489 return X86EMUL_CONTINUE;
1492 /* allowed just for 8 bytes segments */
1493 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1494 u16 selector, struct desc_struct *desc,
1499 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1500 if (rc != X86EMUL_CONTINUE)
1503 return ctxt->ops->read_std(ctxt, *desc_addr_p, desc, sizeof(*desc),
1507 /* allowed just for 8 bytes segments */
1508 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1509 u16 selector, struct desc_struct *desc)
1514 rc = get_descriptor_ptr(ctxt, selector, &addr);
1515 if (rc != X86EMUL_CONTINUE)
1518 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1522 /* Does not support long mode */
1523 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1524 u16 selector, int seg, u8 cpl,
1525 enum x86_transfer_type transfer,
1526 struct desc_struct *desc)
1528 struct desc_struct seg_desc, old_desc;
1530 unsigned err_vec = GP_VECTOR;
1532 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1538 memset(&seg_desc, 0, sizeof seg_desc);
1540 if (ctxt->mode == X86EMUL_MODE_REAL) {
1541 /* set real mode segment descriptor (keep limit etc. for
1543 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1544 set_desc_base(&seg_desc, selector << 4);
1546 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1547 /* VM86 needs a clean new segment descriptor */
1548 set_desc_base(&seg_desc, selector << 4);
1549 set_desc_limit(&seg_desc, 0xffff);
1559 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1560 if ((seg == VCPU_SREG_CS
1561 || (seg == VCPU_SREG_SS
1562 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1563 || seg == VCPU_SREG_TR)
1567 /* TR should be in GDT only */
1568 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1571 if (null_selector) /* for NULL selector skip all following checks */
1574 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1575 if (ret != X86EMUL_CONTINUE)
1578 err_code = selector & 0xfffc;
1579 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1582 /* can't load system descriptor into segment selector */
1583 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1584 if (transfer == X86_TRANSFER_CALL_JMP)
1585 return X86EMUL_UNHANDLEABLE;
1590 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1599 * segment is not a writable data segment or segment
1600 * selector's RPL != CPL or segment selector's RPL != CPL
1602 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1606 if (!(seg_desc.type & 8))
1609 if (seg_desc.type & 4) {
1615 if (rpl > cpl || dpl != cpl)
1618 /* in long-mode d/b must be clear if l is set */
1619 if (seg_desc.d && seg_desc.l) {
1622 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1623 if (efer & EFER_LMA)
1627 /* CS(RPL) <- CPL */
1628 selector = (selector & 0xfffc) | cpl;
1631 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1633 old_desc = seg_desc;
1634 seg_desc.type |= 2; /* busy */
1635 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1636 sizeof(seg_desc), &ctxt->exception);
1637 if (ret != X86EMUL_CONTINUE)
1640 case VCPU_SREG_LDTR:
1641 if (seg_desc.s || seg_desc.type != 2)
1644 default: /* DS, ES, FS, or GS */
1646 * segment is not a data or readable code segment or
1647 * ((segment is a data or nonconforming code segment)
1648 * and (both RPL and CPL > DPL))
1650 if ((seg_desc.type & 0xa) == 0x8 ||
1651 (((seg_desc.type & 0xc) != 0xc) &&
1652 (rpl > dpl && cpl > dpl)))
1658 /* mark segment as accessed */
1659 if (!(seg_desc.type & 1)) {
1661 ret = write_segment_descriptor(ctxt, selector,
1663 if (ret != X86EMUL_CONTINUE)
1666 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1667 ret = ctxt->ops->read_std(ctxt, desc_addr+8, &base3,
1668 sizeof(base3), &ctxt->exception);
1669 if (ret != X86EMUL_CONTINUE)
1671 if (is_noncanonical_address(get_desc_base(&seg_desc) |
1672 ((u64)base3 << 32)))
1673 return emulate_gp(ctxt, 0);
1676 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1679 return X86EMUL_CONTINUE;
1681 return emulate_exception(ctxt, err_vec, err_code, true);
1684 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1685 u16 selector, int seg)
1687 u8 cpl = ctxt->ops->cpl(ctxt);
1688 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1689 X86_TRANSFER_NONE, NULL);
1692 static void write_register_operand(struct operand *op)
1694 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1695 switch (op->bytes) {
1697 *(u8 *)op->addr.reg = (u8)op->val;
1700 *(u16 *)op->addr.reg = (u16)op->val;
1703 *op->addr.reg = (u32)op->val;
1704 break; /* 64b: zero-extend */
1706 *op->addr.reg = op->val;
1711 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1715 write_register_operand(op);
1718 if (ctxt->lock_prefix)
1719 return segmented_cmpxchg(ctxt,
1725 return segmented_write(ctxt,
1731 return segmented_write(ctxt,
1734 op->bytes * op->count);
1737 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1740 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1748 return X86EMUL_CONTINUE;
1751 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1753 struct segmented_address addr;
1755 rsp_increment(ctxt, -bytes);
1756 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1757 addr.seg = VCPU_SREG_SS;
1759 return segmented_write(ctxt, addr, data, bytes);
1762 static int em_push(struct x86_emulate_ctxt *ctxt)
1764 /* Disable writeback. */
1765 ctxt->dst.type = OP_NONE;
1766 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1769 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1770 void *dest, int len)
1773 struct segmented_address addr;
1775 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1776 addr.seg = VCPU_SREG_SS;
1777 rc = segmented_read(ctxt, addr, dest, len);
1778 if (rc != X86EMUL_CONTINUE)
1781 rsp_increment(ctxt, len);
1785 static int em_pop(struct x86_emulate_ctxt *ctxt)
1787 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1790 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1791 void *dest, int len)
1794 unsigned long val, change_mask;
1795 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1796 int cpl = ctxt->ops->cpl(ctxt);
1798 rc = emulate_pop(ctxt, &val, len);
1799 if (rc != X86EMUL_CONTINUE)
1802 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1803 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_AC | EFLG_ID;
1805 switch(ctxt->mode) {
1806 case X86EMUL_MODE_PROT64:
1807 case X86EMUL_MODE_PROT32:
1808 case X86EMUL_MODE_PROT16:
1810 change_mask |= EFLG_IOPL;
1812 change_mask |= EFLG_IF;
1814 case X86EMUL_MODE_VM86:
1816 return emulate_gp(ctxt, 0);
1817 change_mask |= EFLG_IF;
1819 default: /* real mode */
1820 change_mask |= (EFLG_IOPL | EFLG_IF);
1824 *(unsigned long *)dest =
1825 (ctxt->eflags & ~change_mask) | (val & change_mask);
1830 static int em_popf(struct x86_emulate_ctxt *ctxt)
1832 ctxt->dst.type = OP_REG;
1833 ctxt->dst.addr.reg = &ctxt->eflags;
1834 ctxt->dst.bytes = ctxt->op_bytes;
1835 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1838 static int em_enter(struct x86_emulate_ctxt *ctxt)
1841 unsigned frame_size = ctxt->src.val;
1842 unsigned nesting_level = ctxt->src2.val & 31;
1846 return X86EMUL_UNHANDLEABLE;
1848 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1849 rc = push(ctxt, &rbp, stack_size(ctxt));
1850 if (rc != X86EMUL_CONTINUE)
1852 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1854 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1855 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1857 return X86EMUL_CONTINUE;
1860 static int em_leave(struct x86_emulate_ctxt *ctxt)
1862 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1864 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1867 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1869 int seg = ctxt->src2.val;
1871 ctxt->src.val = get_segment_selector(ctxt, seg);
1872 if (ctxt->op_bytes == 4) {
1873 rsp_increment(ctxt, -2);
1877 return em_push(ctxt);
1880 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1882 int seg = ctxt->src2.val;
1883 unsigned long selector;
1886 rc = emulate_pop(ctxt, &selector, 2);
1887 if (rc != X86EMUL_CONTINUE)
1890 if (ctxt->modrm_reg == VCPU_SREG_SS)
1891 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1892 if (ctxt->op_bytes > 2)
1893 rsp_increment(ctxt, ctxt->op_bytes - 2);
1895 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1899 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1901 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1902 int rc = X86EMUL_CONTINUE;
1903 int reg = VCPU_REGS_RAX;
1905 while (reg <= VCPU_REGS_RDI) {
1906 (reg == VCPU_REGS_RSP) ?
1907 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1910 if (rc != X86EMUL_CONTINUE)
1919 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1921 ctxt->src.val = (unsigned long)ctxt->eflags & ~EFLG_VM;
1922 return em_push(ctxt);
1925 static int em_popa(struct x86_emulate_ctxt *ctxt)
1927 int rc = X86EMUL_CONTINUE;
1928 int reg = VCPU_REGS_RDI;
1930 while (reg >= VCPU_REGS_RAX) {
1931 if (reg == VCPU_REGS_RSP) {
1932 rsp_increment(ctxt, ctxt->op_bytes);
1936 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1937 if (rc != X86EMUL_CONTINUE)
1944 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1946 const struct x86_emulate_ops *ops = ctxt->ops;
1953 /* TODO: Add limit checks */
1954 ctxt->src.val = ctxt->eflags;
1956 if (rc != X86EMUL_CONTINUE)
1959 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1961 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1963 if (rc != X86EMUL_CONTINUE)
1966 ctxt->src.val = ctxt->_eip;
1968 if (rc != X86EMUL_CONTINUE)
1971 ops->get_idt(ctxt, &dt);
1973 eip_addr = dt.address + (irq << 2);
1974 cs_addr = dt.address + (irq << 2) + 2;
1976 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1977 if (rc != X86EMUL_CONTINUE)
1980 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1981 if (rc != X86EMUL_CONTINUE)
1984 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1985 if (rc != X86EMUL_CONTINUE)
1993 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1997 invalidate_registers(ctxt);
1998 rc = __emulate_int_real(ctxt, irq);
1999 if (rc == X86EMUL_CONTINUE)
2000 writeback_registers(ctxt);
2004 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2006 switch(ctxt->mode) {
2007 case X86EMUL_MODE_REAL:
2008 return __emulate_int_real(ctxt, irq);
2009 case X86EMUL_MODE_VM86:
2010 case X86EMUL_MODE_PROT16:
2011 case X86EMUL_MODE_PROT32:
2012 case X86EMUL_MODE_PROT64:
2014 /* Protected mode interrupts unimplemented yet */
2015 return X86EMUL_UNHANDLEABLE;
2019 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2021 int rc = X86EMUL_CONTINUE;
2022 unsigned long temp_eip = 0;
2023 unsigned long temp_eflags = 0;
2024 unsigned long cs = 0;
2025 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
2026 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
2027 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
2028 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
2030 /* TODO: Add stack limit check */
2032 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2034 if (rc != X86EMUL_CONTINUE)
2037 if (temp_eip & ~0xffff)
2038 return emulate_gp(ctxt, 0);
2040 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2042 if (rc != X86EMUL_CONTINUE)
2045 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2047 if (rc != X86EMUL_CONTINUE)
2050 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2052 if (rc != X86EMUL_CONTINUE)
2055 ctxt->_eip = temp_eip;
2058 if (ctxt->op_bytes == 4)
2059 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2060 else if (ctxt->op_bytes == 2) {
2061 ctxt->eflags &= ~0xffff;
2062 ctxt->eflags |= temp_eflags;
2065 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2066 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
2067 ctxt->ops->set_nmi_mask(ctxt, false);
2072 static int em_iret(struct x86_emulate_ctxt *ctxt)
2074 switch(ctxt->mode) {
2075 case X86EMUL_MODE_REAL:
2076 return emulate_iret_real(ctxt);
2077 case X86EMUL_MODE_VM86:
2078 case X86EMUL_MODE_PROT16:
2079 case X86EMUL_MODE_PROT32:
2080 case X86EMUL_MODE_PROT64:
2082 /* iret from protected mode unimplemented yet */
2083 return X86EMUL_UNHANDLEABLE;
2087 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2090 unsigned short sel, old_sel;
2091 struct desc_struct old_desc, new_desc;
2092 const struct x86_emulate_ops *ops = ctxt->ops;
2093 u8 cpl = ctxt->ops->cpl(ctxt);
2095 /* Assignment of RIP may only fail in 64-bit mode */
2096 if (ctxt->mode == X86EMUL_MODE_PROT64)
2097 ops->get_segment(ctxt, &old_sel, &old_desc, NULL,
2100 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2102 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2103 X86_TRANSFER_CALL_JMP,
2105 if (rc != X86EMUL_CONTINUE)
2108 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2109 if (rc != X86EMUL_CONTINUE) {
2110 WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64);
2111 /* assigning eip failed; restore the old cs */
2112 ops->set_segment(ctxt, old_sel, &old_desc, 0, VCPU_SREG_CS);
2118 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2120 return assign_eip_near(ctxt, ctxt->src.val);
2123 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2128 old_eip = ctxt->_eip;
2129 rc = assign_eip_near(ctxt, ctxt->src.val);
2130 if (rc != X86EMUL_CONTINUE)
2132 ctxt->src.val = old_eip;
2137 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2139 u64 old = ctxt->dst.orig_val64;
2141 if (ctxt->dst.bytes == 16)
2142 return X86EMUL_UNHANDLEABLE;
2144 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2145 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2146 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2147 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2148 ctxt->eflags &= ~EFLG_ZF;
2150 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2151 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2153 ctxt->eflags |= EFLG_ZF;
2155 return X86EMUL_CONTINUE;
2158 static int em_ret(struct x86_emulate_ctxt *ctxt)
2163 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2164 if (rc != X86EMUL_CONTINUE)
2167 return assign_eip_near(ctxt, eip);
2170 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2173 unsigned long eip, cs;
2175 int cpl = ctxt->ops->cpl(ctxt);
2176 struct desc_struct old_desc, new_desc;
2177 const struct x86_emulate_ops *ops = ctxt->ops;
2179 if (ctxt->mode == X86EMUL_MODE_PROT64)
2180 ops->get_segment(ctxt, &old_cs, &old_desc, NULL,
2183 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2184 if (rc != X86EMUL_CONTINUE)
2186 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2187 if (rc != X86EMUL_CONTINUE)
2189 /* Outer-privilege level return is not implemented */
2190 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2191 return X86EMUL_UNHANDLEABLE;
2192 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2195 if (rc != X86EMUL_CONTINUE)
2197 rc = assign_eip_far(ctxt, eip, &new_desc);
2198 if (rc != X86EMUL_CONTINUE) {
2199 WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64);
2200 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
2205 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2209 rc = em_ret_far(ctxt);
2210 if (rc != X86EMUL_CONTINUE)
2212 rsp_increment(ctxt, ctxt->src.val);
2213 return X86EMUL_CONTINUE;
2216 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2218 /* Save real source value, then compare EAX against destination. */
2219 ctxt->dst.orig_val = ctxt->dst.val;
2220 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2221 ctxt->src.orig_val = ctxt->src.val;
2222 ctxt->src.val = ctxt->dst.orig_val;
2223 fastop(ctxt, em_cmp);
2225 if (ctxt->eflags & EFLG_ZF) {
2226 /* Success: write back to memory; no update of EAX */
2227 ctxt->src.type = OP_NONE;
2228 ctxt->dst.val = ctxt->src.orig_val;
2230 /* Failure: write the value we saw to EAX. */
2231 ctxt->src.type = OP_REG;
2232 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2233 ctxt->src.val = ctxt->dst.orig_val;
2234 /* Create write-cycle to dest by writing the same value */
2235 ctxt->dst.val = ctxt->dst.orig_val;
2237 return X86EMUL_CONTINUE;
2240 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2242 int seg = ctxt->src2.val;
2246 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2248 rc = load_segment_descriptor(ctxt, sel, seg);
2249 if (rc != X86EMUL_CONTINUE)
2252 ctxt->dst.val = ctxt->src.val;
2257 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2258 struct desc_struct *cs, struct desc_struct *ss)
2260 cs->l = 0; /* will be adjusted later */
2261 set_desc_base(cs, 0); /* flat segment */
2262 cs->g = 1; /* 4kb granularity */
2263 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2264 cs->type = 0x0b; /* Read, Execute, Accessed */
2266 cs->dpl = 0; /* will be adjusted later */
2271 set_desc_base(ss, 0); /* flat segment */
2272 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2273 ss->g = 1; /* 4kb granularity */
2275 ss->type = 0x03; /* Read/Write, Accessed */
2276 ss->d = 1; /* 32bit stack segment */
2283 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2285 u32 eax, ebx, ecx, edx;
2288 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2289 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2290 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2291 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2294 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2296 const struct x86_emulate_ops *ops = ctxt->ops;
2297 u32 eax, ebx, ecx, edx;
2300 * syscall should always be enabled in longmode - so only become
2301 * vendor specific (cpuid) if other modes are active...
2303 if (ctxt->mode == X86EMUL_MODE_PROT64)
2308 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2310 * Intel ("GenuineIntel")
2311 * remark: Intel CPUs only support "syscall" in 64bit
2312 * longmode. Also an 64bit guest with a
2313 * 32bit compat-app running will #UD !! While this
2314 * behaviour can be fixed (by emulating) into AMD
2315 * response - CPUs of AMD can't behave like Intel.
2317 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2318 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2319 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2322 /* AMD ("AuthenticAMD") */
2323 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2324 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2325 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2328 /* AMD ("AMDisbetter!") */
2329 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2330 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2331 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2334 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2338 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2340 const struct x86_emulate_ops *ops = ctxt->ops;
2341 struct desc_struct cs, ss;
2346 /* syscall is not available in real mode */
2347 if (ctxt->mode == X86EMUL_MODE_REAL ||
2348 ctxt->mode == X86EMUL_MODE_VM86)
2349 return emulate_ud(ctxt);
2351 if (!(em_syscall_is_enabled(ctxt)))
2352 return emulate_ud(ctxt);
2354 ops->get_msr(ctxt, MSR_EFER, &efer);
2355 setup_syscalls_segments(ctxt, &cs, &ss);
2357 if (!(efer & EFER_SCE))
2358 return emulate_ud(ctxt);
2360 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2362 cs_sel = (u16)(msr_data & 0xfffc);
2363 ss_sel = (u16)(msr_data + 8);
2365 if (efer & EFER_LMA) {
2369 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2370 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2372 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2373 if (efer & EFER_LMA) {
2374 #ifdef CONFIG_X86_64
2375 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2378 ctxt->mode == X86EMUL_MODE_PROT64 ?
2379 MSR_LSTAR : MSR_CSTAR, &msr_data);
2380 ctxt->_eip = msr_data;
2382 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2383 ctxt->eflags &= ~msr_data;
2384 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
2388 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2389 ctxt->_eip = (u32)msr_data;
2391 ctxt->eflags &= ~(EFLG_VM | EFLG_IF);
2394 return X86EMUL_CONTINUE;
2397 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2399 const struct x86_emulate_ops *ops = ctxt->ops;
2400 struct desc_struct cs, ss;
2405 ops->get_msr(ctxt, MSR_EFER, &efer);
2406 /* inject #GP if in real mode */
2407 if (ctxt->mode == X86EMUL_MODE_REAL)
2408 return emulate_gp(ctxt, 0);
2411 * Not recognized on AMD in compat mode (but is recognized in legacy
2414 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2415 && !vendor_intel(ctxt))
2416 return emulate_ud(ctxt);
2418 /* sysenter/sysexit have not been tested in 64bit mode. */
2419 if (ctxt->mode == X86EMUL_MODE_PROT64)
2420 return X86EMUL_UNHANDLEABLE;
2422 setup_syscalls_segments(ctxt, &cs, &ss);
2424 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2425 if ((msr_data & 0xfffc) == 0x0)
2426 return emulate_gp(ctxt, 0);
2428 ctxt->eflags &= ~(EFLG_VM | EFLG_IF);
2429 cs_sel = (u16)msr_data & ~SELECTOR_RPL_MASK;
2430 ss_sel = cs_sel + 8;
2431 if (efer & EFER_LMA) {
2436 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2437 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2439 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2440 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2442 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2443 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2446 return X86EMUL_CONTINUE;
2449 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2451 const struct x86_emulate_ops *ops = ctxt->ops;
2452 struct desc_struct cs, ss;
2453 u64 msr_data, rcx, rdx;
2455 u16 cs_sel = 0, ss_sel = 0;
2457 /* inject #GP if in real mode or Virtual 8086 mode */
2458 if (ctxt->mode == X86EMUL_MODE_REAL ||
2459 ctxt->mode == X86EMUL_MODE_VM86)
2460 return emulate_gp(ctxt, 0);
2462 setup_syscalls_segments(ctxt, &cs, &ss);
2464 if ((ctxt->rex_prefix & 0x8) != 0x0)
2465 usermode = X86EMUL_MODE_PROT64;
2467 usermode = X86EMUL_MODE_PROT32;
2469 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2470 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2474 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2476 case X86EMUL_MODE_PROT32:
2477 cs_sel = (u16)(msr_data + 16);
2478 if ((msr_data & 0xfffc) == 0x0)
2479 return emulate_gp(ctxt, 0);
2480 ss_sel = (u16)(msr_data + 24);
2484 case X86EMUL_MODE_PROT64:
2485 cs_sel = (u16)(msr_data + 32);
2486 if (msr_data == 0x0)
2487 return emulate_gp(ctxt, 0);
2488 ss_sel = cs_sel + 8;
2491 if (is_noncanonical_address(rcx) ||
2492 is_noncanonical_address(rdx))
2493 return emulate_gp(ctxt, 0);
2496 cs_sel |= SELECTOR_RPL_MASK;
2497 ss_sel |= SELECTOR_RPL_MASK;
2499 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2500 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2503 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2505 return X86EMUL_CONTINUE;
2508 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2511 if (ctxt->mode == X86EMUL_MODE_REAL)
2513 if (ctxt->mode == X86EMUL_MODE_VM86)
2515 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2516 return ctxt->ops->cpl(ctxt) > iopl;
2519 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2522 const struct x86_emulate_ops *ops = ctxt->ops;
2523 struct desc_struct tr_seg;
2526 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2527 unsigned mask = (1 << len) - 1;
2530 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2533 if (desc_limit_scaled(&tr_seg) < 103)
2535 base = get_desc_base(&tr_seg);
2536 #ifdef CONFIG_X86_64
2537 base |= ((u64)base3) << 32;
2539 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2540 if (r != X86EMUL_CONTINUE)
2542 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2544 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2545 if (r != X86EMUL_CONTINUE)
2547 if ((perm >> bit_idx) & mask)
2552 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2558 if (emulator_bad_iopl(ctxt))
2559 if (!emulator_io_port_access_allowed(ctxt, port, len))
2562 ctxt->perm_ok = true;
2567 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2568 struct tss_segment_16 *tss)
2570 tss->ip = ctxt->_eip;
2571 tss->flag = ctxt->eflags;
2572 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2573 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2574 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2575 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2576 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2577 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2578 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2579 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2581 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2582 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2583 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2584 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2585 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2588 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2589 struct tss_segment_16 *tss)
2594 ctxt->_eip = tss->ip;
2595 ctxt->eflags = tss->flag | 2;
2596 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2597 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2598 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2599 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2600 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2601 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2602 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2603 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2606 * SDM says that segment selectors are loaded before segment
2609 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2610 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2611 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2612 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2613 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2618 * Now load segment descriptors. If fault happens at this stage
2619 * it is handled in a context of new task
2621 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
2622 X86_TRANSFER_TASK_SWITCH, NULL);
2623 if (ret != X86EMUL_CONTINUE)
2625 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
2626 X86_TRANSFER_TASK_SWITCH, NULL);
2627 if (ret != X86EMUL_CONTINUE)
2629 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
2630 X86_TRANSFER_TASK_SWITCH, NULL);
2631 if (ret != X86EMUL_CONTINUE)
2633 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
2634 X86_TRANSFER_TASK_SWITCH, NULL);
2635 if (ret != X86EMUL_CONTINUE)
2637 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
2638 X86_TRANSFER_TASK_SWITCH, NULL);
2639 if (ret != X86EMUL_CONTINUE)
2642 return X86EMUL_CONTINUE;
2645 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2646 u16 tss_selector, u16 old_tss_sel,
2647 ulong old_tss_base, struct desc_struct *new_desc)
2649 const struct x86_emulate_ops *ops = ctxt->ops;
2650 struct tss_segment_16 tss_seg;
2652 u32 new_tss_base = get_desc_base(new_desc);
2654 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2656 if (ret != X86EMUL_CONTINUE)
2659 save_state_to_tss16(ctxt, &tss_seg);
2661 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2663 if (ret != X86EMUL_CONTINUE)
2666 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2668 if (ret != X86EMUL_CONTINUE)
2671 if (old_tss_sel != 0xffff) {
2672 tss_seg.prev_task_link = old_tss_sel;
2674 ret = ops->write_std(ctxt, new_tss_base,
2675 &tss_seg.prev_task_link,
2676 sizeof tss_seg.prev_task_link,
2678 if (ret != X86EMUL_CONTINUE)
2682 return load_state_from_tss16(ctxt, &tss_seg);
2685 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2686 struct tss_segment_32 *tss)
2688 /* CR3 and ldt selector are not saved intentionally */
2689 tss->eip = ctxt->_eip;
2690 tss->eflags = ctxt->eflags;
2691 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2692 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2693 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2694 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2695 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2696 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2697 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2698 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2700 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2701 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2702 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2703 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2704 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2705 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2708 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2709 struct tss_segment_32 *tss)
2714 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2715 return emulate_gp(ctxt, 0);
2716 ctxt->_eip = tss->eip;
2717 ctxt->eflags = tss->eflags | 2;
2719 /* General purpose registers */
2720 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2721 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2722 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2723 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2724 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2725 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2726 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2727 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2730 * SDM says that segment selectors are loaded before segment
2731 * descriptors. This is important because CPL checks will
2734 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2735 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2736 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2737 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2738 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2739 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2740 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2743 * If we're switching between Protected Mode and VM86, we need to make
2744 * sure to update the mode before loading the segment descriptors so
2745 * that the selectors are interpreted correctly.
2747 if (ctxt->eflags & X86_EFLAGS_VM) {
2748 ctxt->mode = X86EMUL_MODE_VM86;
2751 ctxt->mode = X86EMUL_MODE_PROT32;
2756 * Now load segment descriptors. If fault happenes at this stage
2757 * it is handled in a context of new task
2759 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
2760 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
2761 if (ret != X86EMUL_CONTINUE)
2763 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
2764 X86_TRANSFER_TASK_SWITCH, NULL);
2765 if (ret != X86EMUL_CONTINUE)
2767 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
2768 X86_TRANSFER_TASK_SWITCH, NULL);
2769 if (ret != X86EMUL_CONTINUE)
2771 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
2772 X86_TRANSFER_TASK_SWITCH, NULL);
2773 if (ret != X86EMUL_CONTINUE)
2775 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
2776 X86_TRANSFER_TASK_SWITCH, NULL);
2777 if (ret != X86EMUL_CONTINUE)
2779 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
2780 X86_TRANSFER_TASK_SWITCH, NULL);
2781 if (ret != X86EMUL_CONTINUE)
2783 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
2784 X86_TRANSFER_TASK_SWITCH, NULL);
2785 if (ret != X86EMUL_CONTINUE)
2788 return X86EMUL_CONTINUE;
2791 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2792 u16 tss_selector, u16 old_tss_sel,
2793 ulong old_tss_base, struct desc_struct *new_desc)
2795 const struct x86_emulate_ops *ops = ctxt->ops;
2796 struct tss_segment_32 tss_seg;
2798 u32 new_tss_base = get_desc_base(new_desc);
2799 u32 eip_offset = offsetof(struct tss_segment_32, eip);
2800 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
2802 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2804 if (ret != X86EMUL_CONTINUE)
2807 save_state_to_tss32(ctxt, &tss_seg);
2809 /* Only GP registers and segment selectors are saved */
2810 ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
2811 ldt_sel_offset - eip_offset, &ctxt->exception);
2812 if (ret != X86EMUL_CONTINUE)
2815 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2817 if (ret != X86EMUL_CONTINUE)
2820 if (old_tss_sel != 0xffff) {
2821 tss_seg.prev_task_link = old_tss_sel;
2823 ret = ops->write_std(ctxt, new_tss_base,
2824 &tss_seg.prev_task_link,
2825 sizeof tss_seg.prev_task_link,
2827 if (ret != X86EMUL_CONTINUE)
2831 return load_state_from_tss32(ctxt, &tss_seg);
2834 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2835 u16 tss_selector, int idt_index, int reason,
2836 bool has_error_code, u32 error_code)
2838 const struct x86_emulate_ops *ops = ctxt->ops;
2839 struct desc_struct curr_tss_desc, next_tss_desc;
2841 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2842 ulong old_tss_base =
2843 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2847 /* FIXME: old_tss_base == ~0 ? */
2849 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2850 if (ret != X86EMUL_CONTINUE)
2852 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2853 if (ret != X86EMUL_CONTINUE)
2856 /* FIXME: check that next_tss_desc is tss */
2859 * Check privileges. The three cases are task switch caused by...
2861 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2862 * 2. Exception/IRQ/iret: No check is performed
2863 * 3. jmp/call to TSS/task-gate: No check is performed since the
2864 * hardware checks it before exiting.
2866 if (reason == TASK_SWITCH_GATE) {
2867 if (idt_index != -1) {
2868 /* Software interrupts */
2869 struct desc_struct task_gate_desc;
2872 ret = read_interrupt_descriptor(ctxt, idt_index,
2874 if (ret != X86EMUL_CONTINUE)
2877 dpl = task_gate_desc.dpl;
2878 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2879 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2883 desc_limit = desc_limit_scaled(&next_tss_desc);
2884 if (!next_tss_desc.p ||
2885 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2886 desc_limit < 0x2b)) {
2887 return emulate_ts(ctxt, tss_selector & 0xfffc);
2890 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2891 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2892 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2895 if (reason == TASK_SWITCH_IRET)
2896 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2898 /* set back link to prev task only if NT bit is set in eflags
2899 note that old_tss_sel is not used after this point */
2900 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2901 old_tss_sel = 0xffff;
2903 if (next_tss_desc.type & 8)
2904 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2905 old_tss_base, &next_tss_desc);
2907 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2908 old_tss_base, &next_tss_desc);
2909 if (ret != X86EMUL_CONTINUE)
2912 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2913 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2915 if (reason != TASK_SWITCH_IRET) {
2916 next_tss_desc.type |= (1 << 1); /* set busy flag */
2917 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2920 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2921 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2923 if (has_error_code) {
2924 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2925 ctxt->lock_prefix = 0;
2926 ctxt->src.val = (unsigned long) error_code;
2927 ret = em_push(ctxt);
2933 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2934 u16 tss_selector, int idt_index, int reason,
2935 bool has_error_code, u32 error_code)
2939 invalidate_registers(ctxt);
2940 ctxt->_eip = ctxt->eip;
2941 ctxt->dst.type = OP_NONE;
2943 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2944 has_error_code, error_code);
2946 if (rc == X86EMUL_CONTINUE) {
2947 ctxt->eip = ctxt->_eip;
2948 writeback_registers(ctxt);
2951 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2954 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2957 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2959 register_address_increment(ctxt, reg, df * op->bytes);
2960 op->addr.mem.ea = register_address(ctxt, reg);
2963 static int em_das(struct x86_emulate_ctxt *ctxt)
2966 bool af, cf, old_cf;
2968 cf = ctxt->eflags & X86_EFLAGS_CF;
2974 af = ctxt->eflags & X86_EFLAGS_AF;
2975 if ((al & 0x0f) > 9 || af) {
2977 cf = old_cf | (al >= 250);
2982 if (old_al > 0x99 || old_cf) {
2988 /* Set PF, ZF, SF */
2989 ctxt->src.type = OP_IMM;
2991 ctxt->src.bytes = 1;
2992 fastop(ctxt, em_or);
2993 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2995 ctxt->eflags |= X86_EFLAGS_CF;
2997 ctxt->eflags |= X86_EFLAGS_AF;
2998 return X86EMUL_CONTINUE;
3001 static int em_aam(struct x86_emulate_ctxt *ctxt)
3005 if (ctxt->src.val == 0)
3006 return emulate_de(ctxt);
3008 al = ctxt->dst.val & 0xff;
3009 ah = al / ctxt->src.val;
3010 al %= ctxt->src.val;
3012 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3014 /* Set PF, ZF, SF */
3015 ctxt->src.type = OP_IMM;
3017 ctxt->src.bytes = 1;
3018 fastop(ctxt, em_or);
3020 return X86EMUL_CONTINUE;
3023 static int em_aad(struct x86_emulate_ctxt *ctxt)
3025 u8 al = ctxt->dst.val & 0xff;
3026 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3028 al = (al + (ah * ctxt->src.val)) & 0xff;
3030 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3032 /* Set PF, ZF, SF */
3033 ctxt->src.type = OP_IMM;
3035 ctxt->src.bytes = 1;
3036 fastop(ctxt, em_or);
3038 return X86EMUL_CONTINUE;
3041 static int em_call(struct x86_emulate_ctxt *ctxt)
3044 long rel = ctxt->src.val;
3046 ctxt->src.val = (unsigned long)ctxt->_eip;
3047 rc = jmp_rel(ctxt, rel);
3048 if (rc != X86EMUL_CONTINUE)
3050 return em_push(ctxt);
3053 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3058 struct desc_struct old_desc, new_desc;
3059 const struct x86_emulate_ops *ops = ctxt->ops;
3060 int cpl = ctxt->ops->cpl(ctxt);
3061 enum x86emul_mode prev_mode = ctxt->mode;
3063 old_eip = ctxt->_eip;
3064 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3066 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3067 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3068 X86_TRANSFER_CALL_JMP, &new_desc);
3069 if (rc != X86EMUL_CONTINUE)
3072 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3073 if (rc != X86EMUL_CONTINUE)
3076 ctxt->src.val = old_cs;
3078 if (rc != X86EMUL_CONTINUE)
3081 ctxt->src.val = old_eip;
3083 /* If we failed, we tainted the memory, but the very least we should
3085 if (rc != X86EMUL_CONTINUE) {
3086 pr_warn_once("faulting far call emulation tainted memory\n");
3091 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3092 ctxt->mode = prev_mode;
3097 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3102 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3103 if (rc != X86EMUL_CONTINUE)
3105 rc = assign_eip_near(ctxt, eip);
3106 if (rc != X86EMUL_CONTINUE)
3108 rsp_increment(ctxt, ctxt->src.val);
3109 return X86EMUL_CONTINUE;
3112 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3114 /* Write back the register source. */
3115 ctxt->src.val = ctxt->dst.val;
3116 write_register_operand(&ctxt->src);
3118 /* Write back the memory destination with implicit LOCK prefix. */
3119 ctxt->dst.val = ctxt->src.orig_val;
3120 ctxt->lock_prefix = 1;
3121 return X86EMUL_CONTINUE;
3124 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3126 ctxt->dst.val = ctxt->src2.val;
3127 return fastop(ctxt, em_imul);
3130 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3132 ctxt->dst.type = OP_REG;
3133 ctxt->dst.bytes = ctxt->src.bytes;
3134 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3135 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3137 return X86EMUL_CONTINUE;
3140 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3144 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3145 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3146 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3147 return X86EMUL_CONTINUE;
3150 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3154 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3155 return emulate_gp(ctxt, 0);
3156 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3157 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3158 return X86EMUL_CONTINUE;
3161 static int em_mov(struct x86_emulate_ctxt *ctxt)
3163 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3164 return X86EMUL_CONTINUE;
3167 #define FFL(x) bit(X86_FEATURE_##x)
3169 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3171 u32 ebx, ecx, edx, eax = 1;
3175 * Check MOVBE is set in the guest-visible CPUID leaf.
3177 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3178 if (!(ecx & FFL(MOVBE)))
3179 return emulate_ud(ctxt);
3181 switch (ctxt->op_bytes) {
3184 * From MOVBE definition: "...When the operand size is 16 bits,
3185 * the upper word of the destination register remains unchanged
3188 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3189 * rules so we have to do the operation almost per hand.
3191 tmp = (u16)ctxt->src.val;
3192 ctxt->dst.val &= ~0xffffUL;
3193 ctxt->dst.val |= (unsigned long)swab16(tmp);
3196 ctxt->dst.val = swab32((u32)ctxt->src.val);
3199 ctxt->dst.val = swab64(ctxt->src.val);
3204 return X86EMUL_CONTINUE;
3207 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3209 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3210 return emulate_gp(ctxt, 0);
3212 /* Disable writeback. */
3213 ctxt->dst.type = OP_NONE;
3214 return X86EMUL_CONTINUE;
3217 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3221 if (ctxt->mode == X86EMUL_MODE_PROT64)
3222 val = ctxt->src.val & ~0ULL;
3224 val = ctxt->src.val & ~0U;
3226 /* #UD condition is already handled. */
3227 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3228 return emulate_gp(ctxt, 0);
3230 /* Disable writeback. */
3231 ctxt->dst.type = OP_NONE;
3232 return X86EMUL_CONTINUE;
3235 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3239 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3240 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3241 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3242 return emulate_gp(ctxt, 0);
3244 return X86EMUL_CONTINUE;
3247 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3251 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3252 return emulate_gp(ctxt, 0);
3254 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3255 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3256 return X86EMUL_CONTINUE;
3259 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3261 if (ctxt->modrm_reg > VCPU_SREG_GS)
3262 return emulate_ud(ctxt);
3264 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3265 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3266 ctxt->dst.bytes = 2;
3267 return X86EMUL_CONTINUE;
3270 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3272 u16 sel = ctxt->src.val;
3274 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3275 return emulate_ud(ctxt);
3277 if (ctxt->modrm_reg == VCPU_SREG_SS)
3278 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3280 /* Disable writeback. */
3281 ctxt->dst.type = OP_NONE;
3282 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3285 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3287 u16 sel = ctxt->src.val;
3289 /* Disable writeback. */
3290 ctxt->dst.type = OP_NONE;
3291 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3294 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3296 u16 sel = ctxt->src.val;
3298 /* Disable writeback. */
3299 ctxt->dst.type = OP_NONE;
3300 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3303 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3308 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3309 if (rc == X86EMUL_CONTINUE)
3310 ctxt->ops->invlpg(ctxt, linear);
3311 /* Disable writeback. */
3312 ctxt->dst.type = OP_NONE;
3313 return X86EMUL_CONTINUE;
3316 static int em_clts(struct x86_emulate_ctxt *ctxt)
3320 cr0 = ctxt->ops->get_cr(ctxt, 0);
3322 ctxt->ops->set_cr(ctxt, 0, cr0);
3323 return X86EMUL_CONTINUE;
3326 static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3328 int rc = ctxt->ops->fix_hypercall(ctxt);
3330 if (rc != X86EMUL_CONTINUE)
3333 /* Let the processor re-execute the fixed hypercall */
3334 ctxt->_eip = ctxt->eip;
3335 /* Disable writeback. */
3336 ctxt->dst.type = OP_NONE;
3337 return X86EMUL_CONTINUE;
3340 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3341 void (*get)(struct x86_emulate_ctxt *ctxt,
3342 struct desc_ptr *ptr))
3344 struct desc_ptr desc_ptr;
3346 if (ctxt->mode == X86EMUL_MODE_PROT64)
3348 get(ctxt, &desc_ptr);
3349 if (ctxt->op_bytes == 2) {
3351 desc_ptr.address &= 0x00ffffff;
3353 /* Disable writeback. */
3354 ctxt->dst.type = OP_NONE;
3355 return segmented_write(ctxt, ctxt->dst.addr.mem,
3356 &desc_ptr, 2 + ctxt->op_bytes);
3359 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3361 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3364 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3366 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3369 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3371 struct desc_ptr desc_ptr;
3374 if (ctxt->mode == X86EMUL_MODE_PROT64)
3376 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3377 &desc_ptr.size, &desc_ptr.address,
3379 if (rc != X86EMUL_CONTINUE)
3381 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3382 is_noncanonical_address(desc_ptr.address))
3383 return emulate_gp(ctxt, 0);
3385 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3387 ctxt->ops->set_idt(ctxt, &desc_ptr);
3388 /* Disable writeback. */
3389 ctxt->dst.type = OP_NONE;
3390 return X86EMUL_CONTINUE;
3393 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3395 return em_lgdt_lidt(ctxt, true);
3398 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3400 return em_lgdt_lidt(ctxt, false);
3403 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3405 if (ctxt->dst.type == OP_MEM)
3406 ctxt->dst.bytes = 2;
3407 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3408 return X86EMUL_CONTINUE;
3411 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3413 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3414 | (ctxt->src.val & 0x0f));
3415 ctxt->dst.type = OP_NONE;
3416 return X86EMUL_CONTINUE;
3419 static int em_loop(struct x86_emulate_ctxt *ctxt)
3421 int rc = X86EMUL_CONTINUE;
3423 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3424 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3425 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3426 rc = jmp_rel(ctxt, ctxt->src.val);
3431 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3433 int rc = X86EMUL_CONTINUE;
3435 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3436 rc = jmp_rel(ctxt, ctxt->src.val);
3441 static int em_in(struct x86_emulate_ctxt *ctxt)
3443 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3445 return X86EMUL_IO_NEEDED;
3447 return X86EMUL_CONTINUE;
3450 static int em_out(struct x86_emulate_ctxt *ctxt)
3452 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3454 /* Disable writeback. */
3455 ctxt->dst.type = OP_NONE;
3456 return X86EMUL_CONTINUE;
3459 static int em_cli(struct x86_emulate_ctxt *ctxt)
3461 if (emulator_bad_iopl(ctxt))
3462 return emulate_gp(ctxt, 0);
3464 ctxt->eflags &= ~X86_EFLAGS_IF;
3465 return X86EMUL_CONTINUE;
3468 static int em_sti(struct x86_emulate_ctxt *ctxt)
3470 if (emulator_bad_iopl(ctxt))
3471 return emulate_gp(ctxt, 0);
3473 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3474 ctxt->eflags |= X86_EFLAGS_IF;
3475 return X86EMUL_CONTINUE;
3478 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3480 u32 eax, ebx, ecx, edx;
3482 eax = reg_read(ctxt, VCPU_REGS_RAX);
3483 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3484 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3485 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3486 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3487 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3488 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3489 return X86EMUL_CONTINUE;
3492 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3496 flags = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF;
3497 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3499 ctxt->eflags &= ~0xffUL;
3500 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3501 return X86EMUL_CONTINUE;
3504 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3506 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3507 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3508 return X86EMUL_CONTINUE;
3511 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3513 switch (ctxt->op_bytes) {
3514 #ifdef CONFIG_X86_64
3516 asm("bswap %0" : "+r"(ctxt->dst.val));
3520 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3523 return X86EMUL_CONTINUE;
3526 static int em_clflush(struct x86_emulate_ctxt *ctxt)
3528 /* emulating clflush regardless of cpuid */
3529 return X86EMUL_CONTINUE;
3532 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
3534 ctxt->dst.val = (s32) ctxt->src.val;
3535 return X86EMUL_CONTINUE;
3538 static bool valid_cr(int nr)
3550 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3552 if (!valid_cr(ctxt->modrm_reg))
3553 return emulate_ud(ctxt);
3555 return X86EMUL_CONTINUE;
3558 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3560 u64 new_val = ctxt->src.val64;
3561 int cr = ctxt->modrm_reg;
3564 static u64 cr_reserved_bits[] = {
3565 0xffffffff00000000ULL,
3566 0, 0, 0, /* CR3 checked later */
3573 return emulate_ud(ctxt);
3575 if (new_val & cr_reserved_bits[cr])
3576 return emulate_gp(ctxt, 0);
3581 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3582 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3583 return emulate_gp(ctxt, 0);
3585 cr4 = ctxt->ops->get_cr(ctxt, 4);
3586 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3588 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3589 !(cr4 & X86_CR4_PAE))
3590 return emulate_gp(ctxt, 0);
3597 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3598 if (efer & EFER_LMA)
3599 rsvd = CR3_L_MODE_RESERVED_BITS & ~CR3_PCID_INVD;
3602 return emulate_gp(ctxt, 0);
3607 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3609 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3610 return emulate_gp(ctxt, 0);
3616 return X86EMUL_CONTINUE;
3619 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3623 ctxt->ops->get_dr(ctxt, 7, &dr7);
3625 /* Check if DR7.Global_Enable is set */
3626 return dr7 & (1 << 13);
3629 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3631 int dr = ctxt->modrm_reg;
3635 return emulate_ud(ctxt);
3637 cr4 = ctxt->ops->get_cr(ctxt, 4);
3638 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3639 return emulate_ud(ctxt);
3641 if (check_dr7_gd(ctxt)) {
3644 ctxt->ops->get_dr(ctxt, 6, &dr6);
3646 dr6 |= DR6_BD | DR6_RTM;
3647 ctxt->ops->set_dr(ctxt, 6, dr6);
3648 return emulate_db(ctxt);
3651 return X86EMUL_CONTINUE;
3654 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3656 u64 new_val = ctxt->src.val64;
3657 int dr = ctxt->modrm_reg;
3659 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3660 return emulate_gp(ctxt, 0);
3662 return check_dr_read(ctxt);
3665 static int check_svme(struct x86_emulate_ctxt *ctxt)
3669 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3671 if (!(efer & EFER_SVME))
3672 return emulate_ud(ctxt);
3674 return X86EMUL_CONTINUE;
3677 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3679 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3681 /* Valid physical address? */
3682 if (rax & 0xffff000000000000ULL)
3683 return emulate_gp(ctxt, 0);
3685 return check_svme(ctxt);
3688 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3690 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3692 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3693 return emulate_ud(ctxt);
3695 return X86EMUL_CONTINUE;
3698 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3700 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3701 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3703 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3704 ctxt->ops->check_pmc(ctxt, rcx))
3705 return emulate_gp(ctxt, 0);
3707 return X86EMUL_CONTINUE;
3710 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3712 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3713 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3714 return emulate_gp(ctxt, 0);
3716 return X86EMUL_CONTINUE;
3719 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3721 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3722 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3723 return emulate_gp(ctxt, 0);
3725 return X86EMUL_CONTINUE;
3728 #define D(_y) { .flags = (_y) }
3729 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
3730 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
3731 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3732 #define N D(NotImpl)
3733 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3734 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3735 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3736 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
3737 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
3738 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3739 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3740 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3741 #define II(_f, _e, _i) \
3742 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
3743 #define IIP(_f, _e, _i, _p) \
3744 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
3745 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3746 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3748 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3749 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3750 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3751 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3752 #define I2bvIP(_f, _e, _i, _p) \
3753 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3755 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3756 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3757 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3759 static const struct opcode group7_rm0[] = {
3761 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
3765 static const struct opcode group7_rm1[] = {
3766 DI(SrcNone | Priv, monitor),
3767 DI(SrcNone | Priv, mwait),
3771 static const struct opcode group7_rm3[] = {
3772 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3773 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
3774 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3775 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3776 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3777 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3778 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3779 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3782 static const struct opcode group7_rm7[] = {
3784 DIP(SrcNone, rdtscp, check_rdtsc),
3788 static const struct opcode group1[] = {
3790 F(Lock | PageTable, em_or),
3793 F(Lock | PageTable, em_and),
3799 static const struct opcode group1A[] = {
3800 I(DstMem | SrcNone | Mov | Stack | IncSP, em_pop), N, N, N, N, N, N, N,
3803 static const struct opcode group2[] = {
3804 F(DstMem | ModRM, em_rol),
3805 F(DstMem | ModRM, em_ror),
3806 F(DstMem | ModRM, em_rcl),
3807 F(DstMem | ModRM, em_rcr),
3808 F(DstMem | ModRM, em_shl),
3809 F(DstMem | ModRM, em_shr),
3810 F(DstMem | ModRM, em_shl),
3811 F(DstMem | ModRM, em_sar),
3814 static const struct opcode group3[] = {
3815 F(DstMem | SrcImm | NoWrite, em_test),
3816 F(DstMem | SrcImm | NoWrite, em_test),
3817 F(DstMem | SrcNone | Lock, em_not),
3818 F(DstMem | SrcNone | Lock, em_neg),
3819 F(DstXacc | Src2Mem, em_mul_ex),
3820 F(DstXacc | Src2Mem, em_imul_ex),
3821 F(DstXacc | Src2Mem, em_div_ex),
3822 F(DstXacc | Src2Mem, em_idiv_ex),
3825 static const struct opcode group4[] = {
3826 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3827 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3831 static const struct opcode group5[] = {
3832 F(DstMem | SrcNone | Lock, em_inc),
3833 F(DstMem | SrcNone | Lock, em_dec),
3834 I(SrcMem | NearBranch, em_call_near_abs),
3835 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3836 I(SrcMem | NearBranch, em_jmp_abs),
3837 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
3838 I(SrcMem | Stack, em_push), D(Undefined),
3841 static const struct opcode group6[] = {
3842 DI(Prot | DstMem, sldt),
3843 DI(Prot | DstMem, str),
3844 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3845 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3849 static const struct group_dual group7 = { {
3850 II(Mov | DstMem, em_sgdt, sgdt),
3851 II(Mov | DstMem, em_sidt, sidt),
3852 II(SrcMem | Priv, em_lgdt, lgdt),
3853 II(SrcMem | Priv, em_lidt, lidt),
3854 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3855 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3856 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3860 N, EXT(0, group7_rm3),
3861 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3862 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3866 static const struct opcode group8[] = {
3868 F(DstMem | SrcImmByte | NoWrite, em_bt),
3869 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3870 F(DstMem | SrcImmByte | Lock, em_btr),
3871 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3874 static const struct group_dual group9 = { {
3875 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3877 N, N, N, N, N, N, N, N,
3880 static const struct opcode group11[] = {
3881 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3885 static const struct gprefix pfx_0f_ae_7 = {
3886 I(SrcMem | ByteOp, em_clflush), N, N, N,
3889 static const struct group_dual group15 = { {
3890 N, N, N, N, N, N, N, GP(0, &pfx_0f_ae_7),
3892 N, N, N, N, N, N, N, N,
3895 static const struct gprefix pfx_0f_6f_0f_7f = {
3896 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3899 static const struct instr_dual instr_dual_0f_2b = {
3903 static const struct gprefix pfx_0f_2b = {
3904 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
3907 static const struct gprefix pfx_0f_28_0f_29 = {
3908 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
3911 static const struct gprefix pfx_0f_e7 = {
3912 N, I(Sse, em_mov), N, N,
3915 static const struct escape escape_d9 = { {
3916 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
3919 N, N, N, N, N, N, N, N,
3921 N, N, N, N, N, N, N, N,
3923 N, N, N, N, N, N, N, N,
3925 N, N, N, N, N, N, N, N,
3927 N, N, N, N, N, N, N, N,
3929 N, N, N, N, N, N, N, N,
3931 N, N, N, N, N, N, N, N,
3933 N, N, N, N, N, N, N, N,
3936 static const struct escape escape_db = { {
3937 N, N, N, N, N, N, N, N,
3940 N, N, N, N, N, N, N, N,
3942 N, N, N, N, N, N, N, N,
3944 N, N, N, N, N, N, N, N,
3946 N, N, N, N, N, N, N, N,
3948 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3950 N, N, N, N, N, N, N, N,
3952 N, N, N, N, N, N, N, N,
3954 N, N, N, N, N, N, N, N,
3957 static const struct escape escape_dd = { {
3958 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
3961 N, N, N, N, N, N, N, N,
3963 N, N, N, N, N, N, N, N,
3965 N, N, N, N, N, N, N, N,
3967 N, N, N, N, N, N, N, N,
3969 N, N, N, N, N, N, N, N,
3971 N, N, N, N, N, N, N, N,
3973 N, N, N, N, N, N, N, N,
3975 N, N, N, N, N, N, N, N,
3978 static const struct instr_dual instr_dual_0f_c3 = {
3979 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
3982 static const struct mode_dual mode_dual_63 = {
3983 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
3986 static const struct opcode opcode_table[256] = {
3988 F6ALU(Lock, em_add),
3989 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3990 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3992 F6ALU(Lock | PageTable, em_or),
3993 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3996 F6ALU(Lock, em_adc),
3997 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3998 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4000 F6ALU(Lock, em_sbb),
4001 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4002 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4004 F6ALU(Lock | PageTable, em_and), N, N,
4006 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4008 F6ALU(Lock, em_xor), N, N,
4010 F6ALU(NoWrite, em_cmp), N, N,
4012 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4014 X8(I(SrcReg | Stack, em_push)),
4016 X8(I(DstReg | Stack, em_pop)),
4018 I(ImplicitOps | Stack | No64, em_pusha),
4019 I(ImplicitOps | Stack | No64, em_popa),
4020 N, MD(ModRM, &mode_dual_63),
4023 I(SrcImm | Mov | Stack, em_push),
4024 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4025 I(SrcImmByte | Mov | Stack, em_push),
4026 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4027 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4028 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4030 X16(D(SrcImmByte | NearBranch)),
4032 G(ByteOp | DstMem | SrcImm, group1),
4033 G(DstMem | SrcImm, group1),
4034 G(ByteOp | DstMem | SrcImm | No64, group1),
4035 G(DstMem | SrcImmByte, group1),
4036 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4037 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4039 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4040 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4041 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4042 D(ModRM | SrcMem | NoAccess | DstReg),
4043 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4046 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4048 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4049 I(SrcImmFAddr | No64, em_call_far), N,
4050 II(ImplicitOps | Stack, em_pushf, pushf),
4051 II(ImplicitOps | Stack, em_popf, popf),
4052 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4054 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4055 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4056 I2bv(SrcSI | DstDI | Mov | String, em_mov),
4057 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp_r),
4059 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4060 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4061 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4062 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4064 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4066 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4068 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4069 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4070 I(ImplicitOps | NearBranch, em_ret),
4071 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4072 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4073 G(ByteOp, group11), G(0, group11),
4075 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4076 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4077 I(ImplicitOps, em_ret_far),
4078 D(ImplicitOps), DI(SrcImmByte, intn),
4079 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4081 G(Src2One | ByteOp, group2), G(Src2One, group2),
4082 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4083 I(DstAcc | SrcImmUByte | No64, em_aam),
4084 I(DstAcc | SrcImmUByte | No64, em_aad),
4085 F(DstAcc | ByteOp | No64, em_salc),
4086 I(DstAcc | SrcXLat | ByteOp, em_mov),
4088 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4090 X3(I(SrcImmByte | NearBranch, em_loop)),
4091 I(SrcImmByte | NearBranch, em_jcxz),
4092 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4093 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4095 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4096 I(SrcImmFAddr | No64, em_jmp_far),
4097 D(SrcImmByte | ImplicitOps | NearBranch),
4098 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4099 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4101 N, DI(ImplicitOps, icebp), N, N,
4102 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4103 G(ByteOp, group3), G(0, group3),
4105 D(ImplicitOps), D(ImplicitOps),
4106 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4107 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4110 static const struct opcode twobyte_table[256] = {
4112 G(0, group6), GD(0, &group7), N, N,
4113 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4114 II(ImplicitOps | Priv, em_clts, clts), N,
4115 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4116 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4118 N, N, N, N, N, N, N, N,
4119 D(ImplicitOps | ModRM | SrcMem | NoAccess),
4120 N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4122 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4123 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4124 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4126 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4129 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4130 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4131 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4134 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4135 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4136 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4137 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4138 I(ImplicitOps | EmulateOnUD, em_sysenter),
4139 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4141 N, N, N, N, N, N, N, N,
4143 X16(D(DstReg | SrcMem | ModRM)),
4145 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4150 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4155 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4157 X16(D(SrcImm | NearBranch)),
4159 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4161 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4162 II(ImplicitOps, em_cpuid, cpuid),
4163 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4164 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4165 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4167 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4168 DI(ImplicitOps, rsm),
4169 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4170 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4171 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4172 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4174 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4175 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4176 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4177 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4178 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4179 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4183 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4184 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
4185 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4187 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4188 N, ID(0, &instr_dual_0f_c3),
4189 N, N, N, GD(0, &group9),
4191 X8(I(DstReg, em_bswap)),
4193 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4195 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4196 N, N, N, N, N, N, N, N,
4198 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4201 static const struct instr_dual instr_dual_0f_38_f0 = {
4202 I(DstReg | SrcMem | Mov, em_movbe), N
4205 static const struct instr_dual instr_dual_0f_38_f1 = {
4206 I(DstMem | SrcReg | Mov, em_movbe), N
4209 static const struct gprefix three_byte_0f_38_f0 = {
4210 ID(0, &instr_dual_0f_38_f0), N, N, N
4213 static const struct gprefix three_byte_0f_38_f1 = {
4214 ID(0, &instr_dual_0f_38_f1), N, N, N
4218 * Insns below are selected by the prefix which indexed by the third opcode
4221 static const struct opcode opcode_map_0f_38[256] = {
4223 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4225 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4227 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4228 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4249 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4253 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4259 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4260 unsigned size, bool sign_extension)
4262 int rc = X86EMUL_CONTINUE;
4266 op->addr.mem.ea = ctxt->_eip;
4267 /* NB. Immediates are sign-extended as necessary. */
4268 switch (op->bytes) {
4270 op->val = insn_fetch(s8, ctxt);
4273 op->val = insn_fetch(s16, ctxt);
4276 op->val = insn_fetch(s32, ctxt);
4279 op->val = insn_fetch(s64, ctxt);
4282 if (!sign_extension) {
4283 switch (op->bytes) {
4291 op->val &= 0xffffffff;
4299 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4302 int rc = X86EMUL_CONTINUE;
4306 decode_register_operand(ctxt, op);
4309 rc = decode_imm(ctxt, op, 1, false);
4312 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4316 if (ctxt->d & BitOp)
4317 fetch_bit_operand(ctxt);
4318 op->orig_val = op->val;
4321 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4325 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4326 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4327 fetch_register_operand(op);
4328 op->orig_val = op->val;
4332 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4333 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4334 fetch_register_operand(op);
4335 op->orig_val = op->val;
4338 if (ctxt->d & ByteOp) {
4343 op->bytes = ctxt->op_bytes;
4344 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4345 fetch_register_operand(op);
4346 op->orig_val = op->val;
4350 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4352 register_address(ctxt, VCPU_REGS_RDI);
4353 op->addr.mem.seg = VCPU_SREG_ES;
4360 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4361 fetch_register_operand(op);
4366 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4369 rc = decode_imm(ctxt, op, 1, true);
4377 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4380 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4383 ctxt->memop.bytes = 1;
4384 if (ctxt->memop.type == OP_REG) {
4385 ctxt->memop.addr.reg = decode_register(ctxt,
4386 ctxt->modrm_rm, true);
4387 fetch_register_operand(&ctxt->memop);
4391 ctxt->memop.bytes = 2;
4394 ctxt->memop.bytes = 4;
4397 rc = decode_imm(ctxt, op, 2, false);
4400 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4404 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4406 register_address(ctxt, VCPU_REGS_RSI);
4407 op->addr.mem.seg = ctxt->seg_override;
4413 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4416 reg_read(ctxt, VCPU_REGS_RBX) +
4417 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4418 op->addr.mem.seg = ctxt->seg_override;
4423 op->addr.mem.ea = ctxt->_eip;
4424 op->bytes = ctxt->op_bytes + 2;
4425 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4428 ctxt->memop.bytes = ctxt->op_bytes + 2;
4432 op->val = VCPU_SREG_ES;
4436 op->val = VCPU_SREG_CS;
4440 op->val = VCPU_SREG_SS;
4444 op->val = VCPU_SREG_DS;
4448 op->val = VCPU_SREG_FS;
4452 op->val = VCPU_SREG_GS;
4455 /* Special instructions do their own operand decoding. */
4457 op->type = OP_NONE; /* Disable writeback. */
4465 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4467 int rc = X86EMUL_CONTINUE;
4468 int mode = ctxt->mode;
4469 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4470 bool op_prefix = false;
4471 bool has_seg_override = false;
4472 struct opcode opcode;
4474 ctxt->memop.type = OP_NONE;
4475 ctxt->memopp = NULL;
4476 ctxt->_eip = ctxt->eip;
4477 ctxt->fetch.ptr = ctxt->fetch.data;
4478 ctxt->fetch.end = ctxt->fetch.data + insn_len;
4479 ctxt->opcode_len = 1;
4481 memcpy(ctxt->fetch.data, insn, insn_len);
4483 rc = __do_insn_fetch_bytes(ctxt, 1);
4484 if (rc != X86EMUL_CONTINUE)
4489 case X86EMUL_MODE_REAL:
4490 case X86EMUL_MODE_VM86:
4491 case X86EMUL_MODE_PROT16:
4492 def_op_bytes = def_ad_bytes = 2;
4494 case X86EMUL_MODE_PROT32:
4495 def_op_bytes = def_ad_bytes = 4;
4497 #ifdef CONFIG_X86_64
4498 case X86EMUL_MODE_PROT64:
4504 return EMULATION_FAILED;
4507 ctxt->op_bytes = def_op_bytes;
4508 ctxt->ad_bytes = def_ad_bytes;
4510 /* Legacy prefixes. */
4512 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4513 case 0x66: /* operand-size override */
4515 /* switch between 2/4 bytes */
4516 ctxt->op_bytes = def_op_bytes ^ 6;
4518 case 0x67: /* address-size override */
4519 if (mode == X86EMUL_MODE_PROT64)
4520 /* switch between 4/8 bytes */
4521 ctxt->ad_bytes = def_ad_bytes ^ 12;
4523 /* switch between 2/4 bytes */
4524 ctxt->ad_bytes = def_ad_bytes ^ 6;
4526 case 0x26: /* ES override */
4527 case 0x2e: /* CS override */
4528 case 0x36: /* SS override */
4529 case 0x3e: /* DS override */
4530 has_seg_override = true;
4531 ctxt->seg_override = (ctxt->b >> 3) & 3;
4533 case 0x64: /* FS override */
4534 case 0x65: /* GS override */
4535 has_seg_override = true;
4536 ctxt->seg_override = ctxt->b & 7;
4538 case 0x40 ... 0x4f: /* REX */
4539 if (mode != X86EMUL_MODE_PROT64)
4541 ctxt->rex_prefix = ctxt->b;
4543 case 0xf0: /* LOCK */
4544 ctxt->lock_prefix = 1;
4546 case 0xf2: /* REPNE/REPNZ */
4547 case 0xf3: /* REP/REPE/REPZ */
4548 ctxt->rep_prefix = ctxt->b;
4554 /* Any legacy prefix after a REX prefix nullifies its effect. */
4556 ctxt->rex_prefix = 0;
4562 if (ctxt->rex_prefix & 8)
4563 ctxt->op_bytes = 8; /* REX.W */
4565 /* Opcode byte(s). */
4566 opcode = opcode_table[ctxt->b];
4567 /* Two-byte opcode? */
4568 if (ctxt->b == 0x0f) {
4569 ctxt->opcode_len = 2;
4570 ctxt->b = insn_fetch(u8, ctxt);
4571 opcode = twobyte_table[ctxt->b];
4573 /* 0F_38 opcode map */
4574 if (ctxt->b == 0x38) {
4575 ctxt->opcode_len = 3;
4576 ctxt->b = insn_fetch(u8, ctxt);
4577 opcode = opcode_map_0f_38[ctxt->b];
4580 ctxt->d = opcode.flags;
4582 if (ctxt->d & ModRM)
4583 ctxt->modrm = insn_fetch(u8, ctxt);
4585 /* vex-prefix instructions are not implemented */
4586 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
4587 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
4591 while (ctxt->d & GroupMask) {
4592 switch (ctxt->d & GroupMask) {
4594 goffset = (ctxt->modrm >> 3) & 7;
4595 opcode = opcode.u.group[goffset];
4598 goffset = (ctxt->modrm >> 3) & 7;
4599 if ((ctxt->modrm >> 6) == 3)
4600 opcode = opcode.u.gdual->mod3[goffset];
4602 opcode = opcode.u.gdual->mod012[goffset];
4605 goffset = ctxt->modrm & 7;
4606 opcode = opcode.u.group[goffset];
4609 if (ctxt->rep_prefix && op_prefix)
4610 return EMULATION_FAILED;
4611 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4612 switch (simd_prefix) {
4613 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4614 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4615 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4616 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4620 if (ctxt->modrm > 0xbf)
4621 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4623 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4626 if ((ctxt->modrm >> 6) == 3)
4627 opcode = opcode.u.idual->mod3;
4629 opcode = opcode.u.idual->mod012;
4632 if (ctxt->mode == X86EMUL_MODE_PROT64)
4633 opcode = opcode.u.mdual->mode64;
4635 opcode = opcode.u.mdual->mode32;
4638 return EMULATION_FAILED;
4641 ctxt->d &= ~(u64)GroupMask;
4642 ctxt->d |= opcode.flags;
4647 return EMULATION_FAILED;
4649 ctxt->execute = opcode.u.execute;
4651 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
4652 return EMULATION_FAILED;
4654 if (unlikely(ctxt->d &
4655 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
4658 * These are copied unconditionally here, and checked unconditionally
4659 * in x86_emulate_insn.
4661 ctxt->check_perm = opcode.check_perm;
4662 ctxt->intercept = opcode.intercept;
4664 if (ctxt->d & NotImpl)
4665 return EMULATION_FAILED;
4667 if (mode == X86EMUL_MODE_PROT64) {
4668 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
4670 else if (ctxt->d & NearBranch)
4674 if (ctxt->d & Op3264) {
4675 if (mode == X86EMUL_MODE_PROT64)
4681 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
4685 ctxt->op_bytes = 16;
4686 else if (ctxt->d & Mmx)
4690 /* ModRM and SIB bytes. */
4691 if (ctxt->d & ModRM) {
4692 rc = decode_modrm(ctxt, &ctxt->memop);
4693 if (!has_seg_override) {
4694 has_seg_override = true;
4695 ctxt->seg_override = ctxt->modrm_seg;
4697 } else if (ctxt->d & MemAbs)
4698 rc = decode_abs(ctxt, &ctxt->memop);
4699 if (rc != X86EMUL_CONTINUE)
4702 if (!has_seg_override)
4703 ctxt->seg_override = VCPU_SREG_DS;
4705 ctxt->memop.addr.mem.seg = ctxt->seg_override;
4708 * Decode and fetch the source operand: register, memory
4711 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4712 if (rc != X86EMUL_CONTINUE)
4716 * Decode and fetch the second source operand: register, memory
4719 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4720 if (rc != X86EMUL_CONTINUE)
4723 /* Decode and fetch the destination operand: register or memory. */
4724 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4726 if (ctxt->rip_relative)
4727 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
4728 ctxt->memopp->addr.mem.ea + ctxt->_eip);
4731 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4734 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4736 return ctxt->d & PageTable;
4739 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4741 /* The second termination condition only applies for REPE
4742 * and REPNE. Test if the repeat string operation prefix is
4743 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4744 * corresponding termination condition according to:
4745 * - if REPE/REPZ and ZF = 0 then done
4746 * - if REPNE/REPNZ and ZF = 1 then done
4748 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4749 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4750 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4751 ((ctxt->eflags & EFLG_ZF) == 0))
4752 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4753 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4759 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4763 ctxt->ops->get_fpu(ctxt);
4764 asm volatile("1: fwait \n\t"
4766 ".pushsection .fixup,\"ax\" \n\t"
4768 "movb $1, %[fault] \n\t"
4771 _ASM_EXTABLE(1b, 3b)
4772 : [fault]"+qm"(fault));
4773 ctxt->ops->put_fpu(ctxt);
4775 if (unlikely(fault))
4776 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4778 return X86EMUL_CONTINUE;
4781 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4784 if (op->type == OP_MM)
4785 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4788 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4790 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4791 if (!(ctxt->d & ByteOp))
4792 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4793 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4794 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
4796 : "c"(ctxt->src2.val));
4797 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4798 if (!fop) /* exception is returned in fop variable */
4799 return emulate_de(ctxt);
4800 return X86EMUL_CONTINUE;
4803 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
4805 memset(&ctxt->rip_relative, 0,
4806 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
4808 ctxt->io_read.pos = 0;
4809 ctxt->io_read.end = 0;
4810 ctxt->mem_read.end = 0;
4813 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4815 const struct x86_emulate_ops *ops = ctxt->ops;
4816 int rc = X86EMUL_CONTINUE;
4817 int saved_dst_type = ctxt->dst.type;
4819 ctxt->mem_read.pos = 0;
4821 /* LOCK prefix is allowed only with some instructions */
4822 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4823 rc = emulate_ud(ctxt);
4827 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4828 rc = emulate_ud(ctxt);
4832 if (unlikely(ctxt->d &
4833 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
4834 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4835 (ctxt->d & Undefined)) {
4836 rc = emulate_ud(ctxt);
4840 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4841 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4842 rc = emulate_ud(ctxt);
4846 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4847 rc = emulate_nm(ctxt);
4851 if (ctxt->d & Mmx) {
4852 rc = flush_pending_x87_faults(ctxt);
4853 if (rc != X86EMUL_CONTINUE)
4856 * Now that we know the fpu is exception safe, we can fetch
4859 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4860 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4861 if (!(ctxt->d & Mov))
4862 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4865 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4866 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4867 X86_ICPT_PRE_EXCEPT);
4868 if (rc != X86EMUL_CONTINUE)
4872 /* Instruction can only be executed in protected mode */
4873 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4874 rc = emulate_ud(ctxt);
4878 /* Privileged instruction can be executed only in CPL=0 */
4879 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4880 if (ctxt->d & PrivUD)
4881 rc = emulate_ud(ctxt);
4883 rc = emulate_gp(ctxt, 0);
4887 /* Do instruction specific permission checks */
4888 if (ctxt->d & CheckPerm) {
4889 rc = ctxt->check_perm(ctxt);
4890 if (rc != X86EMUL_CONTINUE)
4894 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4895 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4896 X86_ICPT_POST_EXCEPT);
4897 if (rc != X86EMUL_CONTINUE)
4901 if (ctxt->rep_prefix && (ctxt->d & String)) {
4902 /* All REP prefixes have the same first termination condition */
4903 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4904 ctxt->eip = ctxt->_eip;
4905 ctxt->eflags &= ~EFLG_RF;
4911 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4912 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4913 ctxt->src.valptr, ctxt->src.bytes);
4914 if (rc != X86EMUL_CONTINUE)
4916 ctxt->src.orig_val64 = ctxt->src.val64;
4919 if (ctxt->src2.type == OP_MEM) {
4920 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4921 &ctxt->src2.val, ctxt->src2.bytes);
4922 if (rc != X86EMUL_CONTINUE)
4926 if ((ctxt->d & DstMask) == ImplicitOps)
4930 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4931 /* optimisation - avoid slow emulated read if Mov */
4932 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4933 &ctxt->dst.val, ctxt->dst.bytes);
4934 if (rc != X86EMUL_CONTINUE) {
4935 if (!(ctxt->d & NoWrite) &&
4936 rc == X86EMUL_PROPAGATE_FAULT &&
4937 ctxt->exception.vector == PF_VECTOR)
4938 ctxt->exception.error_code |= PFERR_WRITE_MASK;
4942 /* Copy full 64-bit value for CMPXCHG8B. */
4943 ctxt->dst.orig_val64 = ctxt->dst.val64;
4947 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4948 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4949 X86_ICPT_POST_MEMACCESS);
4950 if (rc != X86EMUL_CONTINUE)
4954 if (ctxt->rep_prefix && (ctxt->d & String))
4955 ctxt->eflags |= EFLG_RF;
4957 ctxt->eflags &= ~EFLG_RF;
4959 if (ctxt->execute) {
4960 if (ctxt->d & Fastop) {
4961 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4962 rc = fastop(ctxt, fop);
4963 if (rc != X86EMUL_CONTINUE)
4967 rc = ctxt->execute(ctxt);
4968 if (rc != X86EMUL_CONTINUE)
4973 if (ctxt->opcode_len == 2)
4975 else if (ctxt->opcode_len == 3)
4976 goto threebyte_insn;
4979 case 0x70 ... 0x7f: /* jcc (short) */
4980 if (test_cc(ctxt->b, ctxt->eflags))
4981 rc = jmp_rel(ctxt, ctxt->src.val);
4983 case 0x8d: /* lea r16/r32, m */
4984 ctxt->dst.val = ctxt->src.addr.mem.ea;
4986 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4987 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4988 ctxt->dst.type = OP_NONE;
4992 case 0x98: /* cbw/cwde/cdqe */
4993 switch (ctxt->op_bytes) {
4994 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4995 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4996 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4999 case 0xcc: /* int3 */
5000 rc = emulate_int(ctxt, 3);
5002 case 0xcd: /* int n */
5003 rc = emulate_int(ctxt, ctxt->src.val);
5005 case 0xce: /* into */
5006 if (ctxt->eflags & EFLG_OF)
5007 rc = emulate_int(ctxt, 4);
5009 case 0xe9: /* jmp rel */
5010 case 0xeb: /* jmp rel short */
5011 rc = jmp_rel(ctxt, ctxt->src.val);
5012 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5014 case 0xf4: /* hlt */
5015 ctxt->ops->halt(ctxt);
5017 case 0xf5: /* cmc */
5018 /* complement carry flag from eflags reg */
5019 ctxt->eflags ^= EFLG_CF;
5021 case 0xf8: /* clc */
5022 ctxt->eflags &= ~EFLG_CF;
5024 case 0xf9: /* stc */
5025 ctxt->eflags |= EFLG_CF;
5027 case 0xfc: /* cld */
5028 ctxt->eflags &= ~EFLG_DF;
5030 case 0xfd: /* std */
5031 ctxt->eflags |= EFLG_DF;
5034 goto cannot_emulate;
5037 if (rc != X86EMUL_CONTINUE)
5041 if (ctxt->d & SrcWrite) {
5042 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5043 rc = writeback(ctxt, &ctxt->src);
5044 if (rc != X86EMUL_CONTINUE)
5047 if (!(ctxt->d & NoWrite)) {
5048 rc = writeback(ctxt, &ctxt->dst);
5049 if (rc != X86EMUL_CONTINUE)
5054 * restore dst type in case the decoding will be reused
5055 * (happens for string instruction )
5057 ctxt->dst.type = saved_dst_type;
5059 if ((ctxt->d & SrcMask) == SrcSI)
5060 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5062 if ((ctxt->d & DstMask) == DstDI)
5063 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5065 if (ctxt->rep_prefix && (ctxt->d & String)) {
5067 struct read_cache *r = &ctxt->io_read;
5068 if ((ctxt->d & SrcMask) == SrcSI)
5069 count = ctxt->src.count;
5071 count = ctxt->dst.count;
5072 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5074 if (!string_insn_completed(ctxt)) {
5076 * Re-enter guest when pio read ahead buffer is empty
5077 * or, if it is not used, after each 1024 iteration.
5079 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5080 (r->end == 0 || r->end != r->pos)) {
5082 * Reset read cache. Usually happens before
5083 * decode, but since instruction is restarted
5084 * we have to do it here.
5086 ctxt->mem_read.end = 0;
5087 writeback_registers(ctxt);
5088 return EMULATION_RESTART;
5090 goto done; /* skip rip writeback */
5092 ctxt->eflags &= ~EFLG_RF;
5095 ctxt->eip = ctxt->_eip;
5098 if (rc == X86EMUL_PROPAGATE_FAULT) {
5099 WARN_ON(ctxt->exception.vector > 0x1f);
5100 ctxt->have_exception = true;
5102 if (rc == X86EMUL_INTERCEPTED)
5103 return EMULATION_INTERCEPTED;
5105 if (rc == X86EMUL_CONTINUE)
5106 writeback_registers(ctxt);
5108 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5112 case 0x09: /* wbinvd */
5113 (ctxt->ops->wbinvd)(ctxt);
5115 case 0x08: /* invd */
5116 case 0x0d: /* GrpP (prefetch) */
5117 case 0x18: /* Grp16 (prefetch/nop) */
5118 case 0x1f: /* nop */
5120 case 0x20: /* mov cr, reg */
5121 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5123 case 0x21: /* mov from dr to reg */
5124 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5126 case 0x40 ... 0x4f: /* cmov */
5127 if (test_cc(ctxt->b, ctxt->eflags))
5128 ctxt->dst.val = ctxt->src.val;
5129 else if (ctxt->op_bytes != 4)
5130 ctxt->dst.type = OP_NONE; /* no writeback */
5132 case 0x80 ... 0x8f: /* jnz rel, etc*/
5133 if (test_cc(ctxt->b, ctxt->eflags))
5134 rc = jmp_rel(ctxt, ctxt->src.val);
5136 case 0x90 ... 0x9f: /* setcc r/m8 */
5137 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5139 case 0xb6 ... 0xb7: /* movzx */
5140 ctxt->dst.bytes = ctxt->op_bytes;
5141 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5142 : (u16) ctxt->src.val;
5144 case 0xbe ... 0xbf: /* movsx */
5145 ctxt->dst.bytes = ctxt->op_bytes;
5146 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5147 (s16) ctxt->src.val;
5150 goto cannot_emulate;
5155 if (rc != X86EMUL_CONTINUE)
5161 return EMULATION_FAILED;
5164 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5166 invalidate_registers(ctxt);
5169 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5171 writeback_registers(ctxt);