1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstMem16 (OpMem16 << DstShift)
90 #define DstImmUByte (OpImmUByte << DstShift)
91 #define DstDX (OpDX << DstShift)
92 #define DstAccLo (OpAccLo << DstShift)
93 #define DstMask (OpMask << DstShift)
94 /* Source operand type. */
96 #define SrcNone (OpNone << SrcShift)
97 #define SrcReg (OpReg << SrcShift)
98 #define SrcMem (OpMem << SrcShift)
99 #define SrcMem16 (OpMem16 << SrcShift)
100 #define SrcMem32 (OpMem32 << SrcShift)
101 #define SrcImm (OpImm << SrcShift)
102 #define SrcImmByte (OpImmByte << SrcShift)
103 #define SrcOne (OpOne << SrcShift)
104 #define SrcImmUByte (OpImmUByte << SrcShift)
105 #define SrcImmU (OpImmU << SrcShift)
106 #define SrcSI (OpSI << SrcShift)
107 #define SrcXLat (OpXLat << SrcShift)
108 #define SrcImmFAddr (OpImmFAddr << SrcShift)
109 #define SrcMemFAddr (OpMemFAddr << SrcShift)
110 #define SrcAcc (OpAcc << SrcShift)
111 #define SrcImmU16 (OpImmU16 << SrcShift)
112 #define SrcImm64 (OpImm64 << SrcShift)
113 #define SrcDX (OpDX << SrcShift)
114 #define SrcMem8 (OpMem8 << SrcShift)
115 #define SrcAccHi (OpAccHi << SrcShift)
116 #define SrcMask (OpMask << SrcShift)
117 #define BitOp (1<<11)
118 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
119 #define String (1<<13) /* String instruction (rep capable) */
120 #define Stack (1<<14) /* Stack instruction (push/pop) */
121 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
122 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
123 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
124 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
125 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
126 #define Escape (5<<15) /* Escape to coprocessor instruction */
127 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
128 #define Sse (1<<18) /* SSE Vector instruction */
129 /* Generic ModRM decode. */
130 #define ModRM (1<<19)
131 /* Destination is only written; never read. */
134 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
135 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
136 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
137 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
138 #define Undefined (1<<25) /* No Such Instruction */
139 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
140 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
142 #define PageTable (1 << 29) /* instruction used to write page table */
143 #define NotImpl (1 << 30) /* instruction is not implemented */
144 /* Source 2 operand type */
145 #define Src2Shift (31)
146 #define Src2None (OpNone << Src2Shift)
147 #define Src2Mem (OpMem << Src2Shift)
148 #define Src2CL (OpCL << Src2Shift)
149 #define Src2ImmByte (OpImmByte << Src2Shift)
150 #define Src2One (OpOne << Src2Shift)
151 #define Src2Imm (OpImm << Src2Shift)
152 #define Src2ES (OpES << Src2Shift)
153 #define Src2CS (OpCS << Src2Shift)
154 #define Src2SS (OpSS << Src2Shift)
155 #define Src2DS (OpDS << Src2Shift)
156 #define Src2FS (OpFS << Src2Shift)
157 #define Src2GS (OpGS << Src2Shift)
158 #define Src2Mask (OpMask << Src2Shift)
159 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
160 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
161 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
162 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
163 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
164 #define NoWrite ((u64)1 << 45) /* No writeback */
165 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
166 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
167 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
168 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
169 #define NoBigReal ((u64)1 << 50) /* No big real mode */
170 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
171 #define NearBranch ((u64)1 << 52) /* Near branches */
172 #define No16 ((u64)1 << 53) /* No 16 bit operand */
173 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
175 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
177 #define X2(x...) x, x
178 #define X3(x...) X2(x), x
179 #define X4(x...) X2(x), X2(x)
180 #define X5(x...) X4(x), x
181 #define X6(x...) X4(x), X2(x)
182 #define X7(x...) X4(x), X3(x)
183 #define X8(x...) X4(x), X4(x)
184 #define X16(x...) X8(x), X8(x)
186 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
187 #define FASTOP_SIZE 8
190 * fastop functions have a special calling convention:
195 * flags: rflags (in/out)
196 * ex: rsi (in:fastop pointer, out:zero if exception)
198 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
199 * different operand sizes can be reached by calculation, rather than a jump
200 * table (which would be bigger than the code).
202 * fastop functions are declared as taking a never-defined fastop parameter,
203 * so they can't be called from C directly.
212 int (*execute)(struct x86_emulate_ctxt *ctxt);
213 const struct opcode *group;
214 const struct group_dual *gdual;
215 const struct gprefix *gprefix;
216 const struct escape *esc;
217 const struct instr_dual *idual;
218 void (*fastop)(struct fastop *fake);
220 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
224 struct opcode mod012[8];
225 struct opcode mod3[8];
229 struct opcode pfx_no;
230 struct opcode pfx_66;
231 struct opcode pfx_f2;
232 struct opcode pfx_f3;
237 struct opcode high[64];
241 struct opcode mod012;
245 /* EFLAGS bit definitions. */
246 #define EFLG_ID (1<<21)
247 #define EFLG_VIP (1<<20)
248 #define EFLG_VIF (1<<19)
249 #define EFLG_AC (1<<18)
250 #define EFLG_VM (1<<17)
251 #define EFLG_RF (1<<16)
252 #define EFLG_IOPL (3<<12)
253 #define EFLG_NT (1<<14)
254 #define EFLG_OF (1<<11)
255 #define EFLG_DF (1<<10)
256 #define EFLG_IF (1<<9)
257 #define EFLG_TF (1<<8)
258 #define EFLG_SF (1<<7)
259 #define EFLG_ZF (1<<6)
260 #define EFLG_AF (1<<4)
261 #define EFLG_PF (1<<2)
262 #define EFLG_CF (1<<0)
264 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
265 #define EFLG_RESERVED_ONE_MASK 2
267 enum x86_transfer_type {
269 X86_TRANSFER_CALL_JMP,
271 X86_TRANSFER_TASK_SWITCH,
274 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
276 if (!(ctxt->regs_valid & (1 << nr))) {
277 ctxt->regs_valid |= 1 << nr;
278 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
280 return ctxt->_regs[nr];
283 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
285 ctxt->regs_valid |= 1 << nr;
286 ctxt->regs_dirty |= 1 << nr;
287 return &ctxt->_regs[nr];
290 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
293 return reg_write(ctxt, nr);
296 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
300 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
301 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
304 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
306 ctxt->regs_dirty = 0;
307 ctxt->regs_valid = 0;
311 * These EFLAGS bits are restored from saved value during emulation, and
312 * any changes are written back to the saved value after emulation.
314 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
322 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
324 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
325 #define FOP_RET "ret \n\t"
327 #define FOP_START(op) \
328 extern void em_##op(struct fastop *fake); \
329 asm(".pushsection .text, \"ax\" \n\t" \
330 ".global em_" #op " \n\t" \
337 #define FOPNOP() FOP_ALIGN FOP_RET
339 #define FOP1E(op, dst) \
340 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
342 #define FOP1EEX(op, dst) \
343 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
345 #define FASTOP1(op) \
350 ON64(FOP1E(op##q, rax)) \
353 /* 1-operand, using src2 (for MUL/DIV r/m) */
354 #define FASTOP1SRC2(op, name) \
359 ON64(FOP1E(op, rcx)) \
362 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
363 #define FASTOP1SRC2EX(op, name) \
368 ON64(FOP1EEX(op, rcx)) \
371 #define FOP2E(op, dst, src) \
372 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
374 #define FASTOP2(op) \
376 FOP2E(op##b, al, dl) \
377 FOP2E(op##w, ax, dx) \
378 FOP2E(op##l, eax, edx) \
379 ON64(FOP2E(op##q, rax, rdx)) \
382 /* 2 operand, word only */
383 #define FASTOP2W(op) \
386 FOP2E(op##w, ax, dx) \
387 FOP2E(op##l, eax, edx) \
388 ON64(FOP2E(op##q, rax, rdx)) \
391 /* 2 operand, src is CL */
392 #define FASTOP2CL(op) \
394 FOP2E(op##b, al, cl) \
395 FOP2E(op##w, ax, cl) \
396 FOP2E(op##l, eax, cl) \
397 ON64(FOP2E(op##q, rax, cl)) \
400 /* 2 operand, src and dest are reversed */
401 #define FASTOP2R(op, name) \
403 FOP2E(op##b, dl, al) \
404 FOP2E(op##w, dx, ax) \
405 FOP2E(op##l, edx, eax) \
406 ON64(FOP2E(op##q, rdx, rax)) \
409 #define FOP3E(op, dst, src, src2) \
410 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
412 /* 3-operand, word-only, src2=cl */
413 #define FASTOP3WCL(op) \
416 FOP3E(op##w, ax, dx, cl) \
417 FOP3E(op##l, eax, edx, cl) \
418 ON64(FOP3E(op##q, rax, rdx, cl)) \
421 /* Special case for SETcc - 1 instruction per cc */
422 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
424 asm(".global kvm_fastop_exception \n"
425 "kvm_fastop_exception: xor %esi, %esi; ret");
446 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
449 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
450 enum x86_intercept intercept,
451 enum x86_intercept_stage stage)
453 struct x86_instruction_info info = {
454 .intercept = intercept,
455 .rep_prefix = ctxt->rep_prefix,
456 .modrm_mod = ctxt->modrm_mod,
457 .modrm_reg = ctxt->modrm_reg,
458 .modrm_rm = ctxt->modrm_rm,
459 .src_val = ctxt->src.val64,
460 .dst_val = ctxt->dst.val64,
461 .src_bytes = ctxt->src.bytes,
462 .dst_bytes = ctxt->dst.bytes,
463 .ad_bytes = ctxt->ad_bytes,
464 .next_rip = ctxt->eip,
467 return ctxt->ops->intercept(ctxt, &info, stage);
470 static void assign_masked(ulong *dest, ulong src, ulong mask)
472 *dest = (*dest & ~mask) | (src & mask);
475 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
477 return (1UL << (ctxt->ad_bytes << 3)) - 1;
480 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
483 struct desc_struct ss;
485 if (ctxt->mode == X86EMUL_MODE_PROT64)
487 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
488 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
491 static int stack_size(struct x86_emulate_ctxt *ctxt)
493 return (__fls(stack_mask(ctxt)) + 1) >> 3;
496 /* Access/update address held in a register, based on addressing mode. */
497 static inline unsigned long
498 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
500 if (ctxt->ad_bytes == sizeof(unsigned long))
503 return reg & ad_mask(ctxt);
506 static inline unsigned long
507 register_address(struct x86_emulate_ctxt *ctxt, int reg)
509 return address_mask(ctxt, reg_read(ctxt, reg));
512 static void masked_increment(ulong *reg, ulong mask, int inc)
514 assign_masked(reg, *reg + inc, mask);
518 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
522 if (ctxt->ad_bytes == sizeof(unsigned long))
525 mask = ad_mask(ctxt);
526 masked_increment(reg_rmw(ctxt, reg), mask, inc);
529 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
531 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
534 static u32 desc_limit_scaled(struct desc_struct *desc)
536 u32 limit = get_desc_limit(desc);
538 return desc->g ? (limit << 12) | 0xfff : limit;
541 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
543 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
546 return ctxt->ops->get_cached_segment_base(ctxt, seg);
549 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
550 u32 error, bool valid)
553 ctxt->exception.vector = vec;
554 ctxt->exception.error_code = error;
555 ctxt->exception.error_code_valid = valid;
556 return X86EMUL_PROPAGATE_FAULT;
559 static int emulate_db(struct x86_emulate_ctxt *ctxt)
561 return emulate_exception(ctxt, DB_VECTOR, 0, false);
564 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
566 return emulate_exception(ctxt, GP_VECTOR, err, true);
569 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
571 return emulate_exception(ctxt, SS_VECTOR, err, true);
574 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
576 return emulate_exception(ctxt, UD_VECTOR, 0, false);
579 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
581 return emulate_exception(ctxt, TS_VECTOR, err, true);
584 static int emulate_de(struct x86_emulate_ctxt *ctxt)
586 return emulate_exception(ctxt, DE_VECTOR, 0, false);
589 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
591 return emulate_exception(ctxt, NM_VECTOR, 0, false);
594 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
597 struct desc_struct desc;
599 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
603 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
608 struct desc_struct desc;
610 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
611 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
615 * x86 defines three classes of vector instructions: explicitly
616 * aligned, explicitly unaligned, and the rest, which change behaviour
617 * depending on whether they're AVX encoded or not.
619 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
620 * subject to the same check.
622 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
624 if (likely(size < 16))
627 if (ctxt->d & Aligned)
629 else if (ctxt->d & Unaligned)
631 else if (ctxt->d & Avx)
637 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
638 struct segmented_address addr,
639 unsigned *max_size, unsigned size,
640 bool write, bool fetch,
641 enum x86emul_mode mode, ulong *linear)
643 struct desc_struct desc;
649 la = seg_base(ctxt, addr.seg) + addr.ea;
652 case X86EMUL_MODE_PROT64:
653 if (is_noncanonical_address(la))
656 *max_size = min_t(u64, ~0u, (1ull << 48) - la);
657 if (size > *max_size)
661 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
665 /* code segment in protected mode or read-only data segment */
666 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
667 || !(desc.type & 2)) && write)
669 /* unreadable code segment */
670 if (!fetch && (desc.type & 8) && !(desc.type & 2))
672 lim = desc_limit_scaled(&desc);
673 if (!(desc.type & 8) && (desc.type & 4)) {
674 /* expand-down segment */
677 lim = desc.d ? 0xffffffff : 0xffff;
681 *max_size = min_t(u64, ~0u, (u64)lim + 1 - addr.ea);
682 if (size > *max_size)
687 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
688 return emulate_gp(ctxt, 0);
690 return X86EMUL_CONTINUE;
692 if (addr.seg == VCPU_SREG_SS)
693 return emulate_ss(ctxt, 0);
695 return emulate_gp(ctxt, 0);
698 static int linearize(struct x86_emulate_ctxt *ctxt,
699 struct segmented_address addr,
700 unsigned size, bool write,
704 return __linearize(ctxt, addr, &max_size, size, write, false,
708 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
709 enum x86emul_mode mode)
714 struct segmented_address addr = { .seg = VCPU_SREG_CS,
717 if (ctxt->op_bytes != sizeof(unsigned long))
718 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
719 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
720 if (rc == X86EMUL_CONTINUE)
721 ctxt->_eip = addr.ea;
725 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
727 return assign_eip(ctxt, dst, ctxt->mode);
730 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
731 const struct desc_struct *cs_desc)
733 enum x86emul_mode mode = ctxt->mode;
736 if (ctxt->mode >= X86EMUL_MODE_PROT32 && cs_desc->l) {
739 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
741 mode = X86EMUL_MODE_PROT64;
744 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
745 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
746 return assign_eip(ctxt, dst, mode);
749 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
751 return assign_eip_near(ctxt, ctxt->_eip + rel);
754 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
755 struct segmented_address addr,
762 rc = linearize(ctxt, addr, size, false, &linear);
763 if (rc != X86EMUL_CONTINUE)
765 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
769 * Prefetch the remaining bytes of the instruction without crossing page
770 * boundary if they are not in fetch_cache yet.
772 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
775 unsigned size, max_size;
776 unsigned long linear;
777 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
778 struct segmented_address addr = { .seg = VCPU_SREG_CS,
779 .ea = ctxt->eip + cur_size };
782 * We do not know exactly how many bytes will be needed, and
783 * __linearize is expensive, so fetch as much as possible. We
784 * just have to avoid going beyond the 15 byte limit, the end
785 * of the segment, or the end of the page.
787 * __linearize is called with size 0 so that it does not do any
788 * boundary check itself. Instead, we use max_size to check
791 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
793 if (unlikely(rc != X86EMUL_CONTINUE))
796 size = min_t(unsigned, 15UL ^ cur_size, max_size);
797 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
800 * One instruction can only straddle two pages,
801 * and one has been loaded at the beginning of
802 * x86_decode_insn. So, if not enough bytes
803 * still, we must have hit the 15-byte boundary.
805 if (unlikely(size < op_size))
806 return emulate_gp(ctxt, 0);
808 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
809 size, &ctxt->exception);
810 if (unlikely(rc != X86EMUL_CONTINUE))
812 ctxt->fetch.end += size;
813 return X86EMUL_CONTINUE;
816 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
819 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
821 if (unlikely(done_size < size))
822 return __do_insn_fetch_bytes(ctxt, size - done_size);
824 return X86EMUL_CONTINUE;
827 /* Fetch next part of the instruction being emulated. */
828 #define insn_fetch(_type, _ctxt) \
831 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
832 if (rc != X86EMUL_CONTINUE) \
834 ctxt->_eip += sizeof(_type); \
835 _x = *(_type __aligned(1) *) ctxt->fetch.ptr; \
836 ctxt->fetch.ptr += sizeof(_type); \
840 #define insn_fetch_arr(_arr, _size, _ctxt) \
842 rc = do_insn_fetch_bytes(_ctxt, _size); \
843 if (rc != X86EMUL_CONTINUE) \
845 ctxt->_eip += (_size); \
846 memcpy(_arr, ctxt->fetch.ptr, _size); \
847 ctxt->fetch.ptr += (_size); \
851 * Given the 'reg' portion of a ModRM byte, and a register block, return a
852 * pointer into the block that addresses the relevant register.
853 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
855 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
859 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
861 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
862 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
864 p = reg_rmw(ctxt, modrm_reg);
868 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
869 struct segmented_address addr,
870 u16 *size, unsigned long *address, int op_bytes)
877 rc = segmented_read_std(ctxt, addr, size, 2);
878 if (rc != X86EMUL_CONTINUE)
881 rc = segmented_read_std(ctxt, addr, address, op_bytes);
895 FASTOP1SRC2(mul, mul_ex);
896 FASTOP1SRC2(imul, imul_ex);
897 FASTOP1SRC2EX(div, div_ex);
898 FASTOP1SRC2EX(idiv, idiv_ex);
927 FASTOP2R(cmp, cmp_r);
929 static u8 test_cc(unsigned int condition, unsigned long flags)
932 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
934 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
935 asm("push %[flags]; popf; call *%[fastop]"
936 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
940 static void fetch_register_operand(struct operand *op)
944 op->val = *(u8 *)op->addr.reg;
947 op->val = *(u16 *)op->addr.reg;
950 op->val = *(u32 *)op->addr.reg;
953 op->val = *(u64 *)op->addr.reg;
958 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
960 ctxt->ops->get_fpu(ctxt);
962 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
963 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
964 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
965 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
966 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
967 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
968 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
969 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
971 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
972 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
973 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
974 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
975 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
976 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
977 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
978 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
982 ctxt->ops->put_fpu(ctxt);
985 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
988 ctxt->ops->get_fpu(ctxt);
990 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
991 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
992 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
993 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
994 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
995 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
996 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
997 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
999 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1000 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1001 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1002 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1003 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1004 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1005 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1006 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1010 ctxt->ops->put_fpu(ctxt);
1013 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1015 ctxt->ops->get_fpu(ctxt);
1017 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1018 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1019 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1020 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1021 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1022 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1023 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1024 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1027 ctxt->ops->put_fpu(ctxt);
1030 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1032 ctxt->ops->get_fpu(ctxt);
1034 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1035 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1036 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1037 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1038 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1039 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1040 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1041 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1044 ctxt->ops->put_fpu(ctxt);
1047 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1049 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1050 return emulate_nm(ctxt);
1052 ctxt->ops->get_fpu(ctxt);
1053 asm volatile("fninit");
1054 ctxt->ops->put_fpu(ctxt);
1055 return X86EMUL_CONTINUE;
1058 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1062 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1063 return emulate_nm(ctxt);
1065 ctxt->ops->get_fpu(ctxt);
1066 asm volatile("fnstcw %0": "+m"(fcw));
1067 ctxt->ops->put_fpu(ctxt);
1069 ctxt->dst.val = fcw;
1071 return X86EMUL_CONTINUE;
1074 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1078 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1079 return emulate_nm(ctxt);
1081 ctxt->ops->get_fpu(ctxt);
1082 asm volatile("fnstsw %0": "+m"(fsw));
1083 ctxt->ops->put_fpu(ctxt);
1085 ctxt->dst.val = fsw;
1087 return X86EMUL_CONTINUE;
1090 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1093 unsigned reg = ctxt->modrm_reg;
1095 if (!(ctxt->d & ModRM))
1096 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1098 if (ctxt->d & Sse) {
1102 read_sse_reg(ctxt, &op->vec_val, reg);
1105 if (ctxt->d & Mmx) {
1114 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1115 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1117 fetch_register_operand(op);
1118 op->orig_val = op->val;
1121 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1123 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1124 ctxt->modrm_seg = VCPU_SREG_SS;
1127 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1131 int index_reg, base_reg, scale;
1132 int rc = X86EMUL_CONTINUE;
1135 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1136 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1137 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1139 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1140 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1141 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1142 ctxt->modrm_seg = VCPU_SREG_DS;
1144 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1146 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1147 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1149 if (ctxt->d & Sse) {
1152 op->addr.xmm = ctxt->modrm_rm;
1153 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1156 if (ctxt->d & Mmx) {
1159 op->addr.mm = ctxt->modrm_rm & 7;
1162 fetch_register_operand(op);
1168 if (ctxt->ad_bytes == 2) {
1169 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1170 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1171 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1172 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1174 /* 16-bit ModR/M decode. */
1175 switch (ctxt->modrm_mod) {
1177 if (ctxt->modrm_rm == 6)
1178 modrm_ea += insn_fetch(u16, ctxt);
1181 modrm_ea += insn_fetch(s8, ctxt);
1184 modrm_ea += insn_fetch(u16, ctxt);
1187 switch (ctxt->modrm_rm) {
1189 modrm_ea += bx + si;
1192 modrm_ea += bx + di;
1195 modrm_ea += bp + si;
1198 modrm_ea += bp + di;
1207 if (ctxt->modrm_mod != 0)
1214 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1215 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1216 ctxt->modrm_seg = VCPU_SREG_SS;
1217 modrm_ea = (u16)modrm_ea;
1219 /* 32/64-bit ModR/M decode. */
1220 if ((ctxt->modrm_rm & 7) == 4) {
1221 sib = insn_fetch(u8, ctxt);
1222 index_reg |= (sib >> 3) & 7;
1223 base_reg |= sib & 7;
1226 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1227 modrm_ea += insn_fetch(s32, ctxt);
1229 modrm_ea += reg_read(ctxt, base_reg);
1230 adjust_modrm_seg(ctxt, base_reg);
1231 /* Increment ESP on POP [ESP] */
1232 if ((ctxt->d & IncSP) &&
1233 base_reg == VCPU_REGS_RSP)
1234 modrm_ea += ctxt->op_bytes;
1237 modrm_ea += reg_read(ctxt, index_reg) << scale;
1238 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1239 modrm_ea += insn_fetch(s32, ctxt);
1240 if (ctxt->mode == X86EMUL_MODE_PROT64)
1241 ctxt->rip_relative = 1;
1243 base_reg = ctxt->modrm_rm;
1244 modrm_ea += reg_read(ctxt, base_reg);
1245 adjust_modrm_seg(ctxt, base_reg);
1247 switch (ctxt->modrm_mod) {
1249 modrm_ea += insn_fetch(s8, ctxt);
1252 modrm_ea += insn_fetch(s32, ctxt);
1256 op->addr.mem.ea = modrm_ea;
1257 if (ctxt->ad_bytes != 8)
1258 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1264 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1267 int rc = X86EMUL_CONTINUE;
1270 switch (ctxt->ad_bytes) {
1272 op->addr.mem.ea = insn_fetch(u16, ctxt);
1275 op->addr.mem.ea = insn_fetch(u32, ctxt);
1278 op->addr.mem.ea = insn_fetch(u64, ctxt);
1285 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1289 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1290 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1292 if (ctxt->src.bytes == 2)
1293 sv = (s16)ctxt->src.val & (s16)mask;
1294 else if (ctxt->src.bytes == 4)
1295 sv = (s32)ctxt->src.val & (s32)mask;
1297 sv = (s64)ctxt->src.val & (s64)mask;
1299 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1300 ctxt->dst.addr.mem.ea + (sv >> 3));
1303 /* only subword offset */
1304 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1307 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1308 unsigned long addr, void *dest, unsigned size)
1311 struct read_cache *mc = &ctxt->mem_read;
1313 if (mc->pos < mc->end)
1316 WARN_ON((mc->end + size) >= sizeof(mc->data));
1318 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1320 if (rc != X86EMUL_CONTINUE)
1326 memcpy(dest, mc->data + mc->pos, size);
1328 return X86EMUL_CONTINUE;
1331 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1332 struct segmented_address addr,
1339 rc = linearize(ctxt, addr, size, false, &linear);
1340 if (rc != X86EMUL_CONTINUE)
1342 return read_emulated(ctxt, linear, data, size);
1345 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1346 struct segmented_address addr,
1353 rc = linearize(ctxt, addr, size, true, &linear);
1354 if (rc != X86EMUL_CONTINUE)
1356 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1360 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1361 struct segmented_address addr,
1362 const void *orig_data, const void *data,
1368 rc = linearize(ctxt, addr, size, true, &linear);
1369 if (rc != X86EMUL_CONTINUE)
1371 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1372 size, &ctxt->exception);
1375 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1376 unsigned int size, unsigned short port,
1379 struct read_cache *rc = &ctxt->io_read;
1381 if (rc->pos == rc->end) { /* refill pio read ahead */
1382 unsigned int in_page, n;
1383 unsigned int count = ctxt->rep_prefix ?
1384 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1385 in_page = (ctxt->eflags & EFLG_DF) ?
1386 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1387 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1388 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1391 rc->pos = rc->end = 0;
1392 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1397 if (ctxt->rep_prefix && (ctxt->d & String) &&
1398 !(ctxt->eflags & EFLG_DF)) {
1399 ctxt->dst.data = rc->data + rc->pos;
1400 ctxt->dst.type = OP_MEM_STR;
1401 ctxt->dst.count = (rc->end - rc->pos) / size;
1404 memcpy(dest, rc->data + rc->pos, size);
1410 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1411 u16 index, struct desc_struct *desc)
1416 ctxt->ops->get_idt(ctxt, &dt);
1418 if (dt.size < index * 8 + 7)
1419 return emulate_gp(ctxt, index << 3 | 0x2);
1421 addr = dt.address + index * 8;
1422 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1426 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1427 u16 selector, struct desc_ptr *dt)
1429 const struct x86_emulate_ops *ops = ctxt->ops;
1432 if (selector & 1 << 2) {
1433 struct desc_struct desc;
1436 memset (dt, 0, sizeof *dt);
1437 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1441 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1442 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1444 ops->get_gdt(ctxt, dt);
1447 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1448 u16 selector, ulong *desc_addr_p)
1451 u16 index = selector >> 3;
1454 get_descriptor_table_ptr(ctxt, selector, &dt);
1456 if (dt.size < index * 8 + 7)
1457 return emulate_gp(ctxt, selector & 0xfffc);
1459 addr = dt.address + index * 8;
1461 #ifdef CONFIG_X86_64
1462 if (addr >> 32 != 0) {
1465 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1466 if (!(efer & EFER_LMA))
1471 *desc_addr_p = addr;
1472 return X86EMUL_CONTINUE;
1475 /* allowed just for 8 bytes segments */
1476 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1477 u16 selector, struct desc_struct *desc,
1482 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1483 if (rc != X86EMUL_CONTINUE)
1486 return ctxt->ops->read_std(ctxt, *desc_addr_p, desc, sizeof(*desc),
1490 /* allowed just for 8 bytes segments */
1491 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1492 u16 selector, struct desc_struct *desc)
1497 rc = get_descriptor_ptr(ctxt, selector, &addr);
1498 if (rc != X86EMUL_CONTINUE)
1501 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1505 /* Does not support long mode */
1506 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1507 u16 selector, int seg, u8 cpl,
1508 enum x86_transfer_type transfer,
1509 struct desc_struct *desc)
1511 struct desc_struct seg_desc, old_desc;
1513 unsigned err_vec = GP_VECTOR;
1515 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1521 memset(&seg_desc, 0, sizeof seg_desc);
1523 if (ctxt->mode == X86EMUL_MODE_REAL) {
1524 /* set real mode segment descriptor (keep limit etc. for
1526 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1527 set_desc_base(&seg_desc, selector << 4);
1529 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1530 /* VM86 needs a clean new segment descriptor */
1531 set_desc_base(&seg_desc, selector << 4);
1532 set_desc_limit(&seg_desc, 0xffff);
1542 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1543 if ((seg == VCPU_SREG_CS
1544 || (seg == VCPU_SREG_SS
1545 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1546 || seg == VCPU_SREG_TR)
1550 /* TR should be in GDT only */
1551 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1554 if (null_selector) /* for NULL selector skip all following checks */
1557 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1558 if (ret != X86EMUL_CONTINUE)
1561 err_code = selector & 0xfffc;
1562 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1565 /* can't load system descriptor into segment selector */
1566 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1567 if (transfer == X86_TRANSFER_CALL_JMP)
1568 return X86EMUL_UNHANDLEABLE;
1573 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1582 * segment is not a writable data segment or segment
1583 * selector's RPL != CPL or segment selector's RPL != CPL
1585 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1589 if (!(seg_desc.type & 8))
1592 if (seg_desc.type & 4) {
1598 if (rpl > cpl || dpl != cpl)
1601 /* in long-mode d/b must be clear if l is set */
1602 if (seg_desc.d && seg_desc.l) {
1605 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1606 if (efer & EFER_LMA)
1610 /* CS(RPL) <- CPL */
1611 selector = (selector & 0xfffc) | cpl;
1614 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1616 old_desc = seg_desc;
1617 seg_desc.type |= 2; /* busy */
1618 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1619 sizeof(seg_desc), &ctxt->exception);
1620 if (ret != X86EMUL_CONTINUE)
1623 case VCPU_SREG_LDTR:
1624 if (seg_desc.s || seg_desc.type != 2)
1627 default: /* DS, ES, FS, or GS */
1629 * segment is not a data or readable code segment or
1630 * ((segment is a data or nonconforming code segment)
1631 * and (both RPL and CPL > DPL))
1633 if ((seg_desc.type & 0xa) == 0x8 ||
1634 (((seg_desc.type & 0xc) != 0xc) &&
1635 (rpl > dpl && cpl > dpl)))
1641 /* mark segment as accessed */
1642 if (!(seg_desc.type & 1)) {
1644 ret = write_segment_descriptor(ctxt, selector,
1646 if (ret != X86EMUL_CONTINUE)
1649 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1650 ret = ctxt->ops->read_std(ctxt, desc_addr+8, &base3,
1651 sizeof(base3), &ctxt->exception);
1652 if (ret != X86EMUL_CONTINUE)
1654 if (is_noncanonical_address(get_desc_base(&seg_desc) |
1655 ((u64)base3 << 32)))
1656 return emulate_gp(ctxt, 0);
1659 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1662 return X86EMUL_CONTINUE;
1664 return emulate_exception(ctxt, err_vec, err_code, true);
1667 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1668 u16 selector, int seg)
1670 u8 cpl = ctxt->ops->cpl(ctxt);
1671 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1672 X86_TRANSFER_NONE, NULL);
1675 static void write_register_operand(struct operand *op)
1677 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1678 switch (op->bytes) {
1680 *(u8 *)op->addr.reg = (u8)op->val;
1683 *(u16 *)op->addr.reg = (u16)op->val;
1686 *op->addr.reg = (u32)op->val;
1687 break; /* 64b: zero-extend */
1689 *op->addr.reg = op->val;
1694 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1698 write_register_operand(op);
1701 if (ctxt->lock_prefix)
1702 return segmented_cmpxchg(ctxt,
1708 return segmented_write(ctxt,
1714 return segmented_write(ctxt,
1717 op->bytes * op->count);
1720 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1723 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1731 return X86EMUL_CONTINUE;
1734 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1736 struct segmented_address addr;
1738 rsp_increment(ctxt, -bytes);
1739 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1740 addr.seg = VCPU_SREG_SS;
1742 return segmented_write(ctxt, addr, data, bytes);
1745 static int em_push(struct x86_emulate_ctxt *ctxt)
1747 /* Disable writeback. */
1748 ctxt->dst.type = OP_NONE;
1749 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1752 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1753 void *dest, int len)
1756 struct segmented_address addr;
1758 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1759 addr.seg = VCPU_SREG_SS;
1760 rc = segmented_read(ctxt, addr, dest, len);
1761 if (rc != X86EMUL_CONTINUE)
1764 rsp_increment(ctxt, len);
1768 static int em_pop(struct x86_emulate_ctxt *ctxt)
1770 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1773 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1774 void *dest, int len)
1777 unsigned long val, change_mask;
1778 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1779 int cpl = ctxt->ops->cpl(ctxt);
1781 rc = emulate_pop(ctxt, &val, len);
1782 if (rc != X86EMUL_CONTINUE)
1785 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1786 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_AC | EFLG_ID;
1788 switch(ctxt->mode) {
1789 case X86EMUL_MODE_PROT64:
1790 case X86EMUL_MODE_PROT32:
1791 case X86EMUL_MODE_PROT16:
1793 change_mask |= EFLG_IOPL;
1795 change_mask |= EFLG_IF;
1797 case X86EMUL_MODE_VM86:
1799 return emulate_gp(ctxt, 0);
1800 change_mask |= EFLG_IF;
1802 default: /* real mode */
1803 change_mask |= (EFLG_IOPL | EFLG_IF);
1807 *(unsigned long *)dest =
1808 (ctxt->eflags & ~change_mask) | (val & change_mask);
1813 static int em_popf(struct x86_emulate_ctxt *ctxt)
1815 ctxt->dst.type = OP_REG;
1816 ctxt->dst.addr.reg = &ctxt->eflags;
1817 ctxt->dst.bytes = ctxt->op_bytes;
1818 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1821 static int em_enter(struct x86_emulate_ctxt *ctxt)
1824 unsigned frame_size = ctxt->src.val;
1825 unsigned nesting_level = ctxt->src2.val & 31;
1829 return X86EMUL_UNHANDLEABLE;
1831 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1832 rc = push(ctxt, &rbp, stack_size(ctxt));
1833 if (rc != X86EMUL_CONTINUE)
1835 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1837 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1838 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1840 return X86EMUL_CONTINUE;
1843 static int em_leave(struct x86_emulate_ctxt *ctxt)
1845 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1847 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1850 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1852 int seg = ctxt->src2.val;
1854 ctxt->src.val = get_segment_selector(ctxt, seg);
1855 if (ctxt->op_bytes == 4) {
1856 rsp_increment(ctxt, -2);
1860 return em_push(ctxt);
1863 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1865 int seg = ctxt->src2.val;
1866 unsigned long selector;
1869 rc = emulate_pop(ctxt, &selector, 2);
1870 if (rc != X86EMUL_CONTINUE)
1873 if (ctxt->modrm_reg == VCPU_SREG_SS)
1874 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1875 if (ctxt->op_bytes > 2)
1876 rsp_increment(ctxt, ctxt->op_bytes - 2);
1878 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1882 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1884 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1885 int rc = X86EMUL_CONTINUE;
1886 int reg = VCPU_REGS_RAX;
1888 while (reg <= VCPU_REGS_RDI) {
1889 (reg == VCPU_REGS_RSP) ?
1890 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1893 if (rc != X86EMUL_CONTINUE)
1902 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1904 ctxt->src.val = (unsigned long)ctxt->eflags & ~EFLG_VM;
1905 return em_push(ctxt);
1908 static int em_popa(struct x86_emulate_ctxt *ctxt)
1910 int rc = X86EMUL_CONTINUE;
1911 int reg = VCPU_REGS_RDI;
1913 while (reg >= VCPU_REGS_RAX) {
1914 if (reg == VCPU_REGS_RSP) {
1915 rsp_increment(ctxt, ctxt->op_bytes);
1919 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1920 if (rc != X86EMUL_CONTINUE)
1927 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1929 const struct x86_emulate_ops *ops = ctxt->ops;
1936 /* TODO: Add limit checks */
1937 ctxt->src.val = ctxt->eflags;
1939 if (rc != X86EMUL_CONTINUE)
1942 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1944 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1946 if (rc != X86EMUL_CONTINUE)
1949 ctxt->src.val = ctxt->_eip;
1951 if (rc != X86EMUL_CONTINUE)
1954 ops->get_idt(ctxt, &dt);
1956 eip_addr = dt.address + (irq << 2);
1957 cs_addr = dt.address + (irq << 2) + 2;
1959 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1960 if (rc != X86EMUL_CONTINUE)
1963 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1964 if (rc != X86EMUL_CONTINUE)
1967 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1968 if (rc != X86EMUL_CONTINUE)
1976 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1980 invalidate_registers(ctxt);
1981 rc = __emulate_int_real(ctxt, irq);
1982 if (rc == X86EMUL_CONTINUE)
1983 writeback_registers(ctxt);
1987 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1989 switch(ctxt->mode) {
1990 case X86EMUL_MODE_REAL:
1991 return __emulate_int_real(ctxt, irq);
1992 case X86EMUL_MODE_VM86:
1993 case X86EMUL_MODE_PROT16:
1994 case X86EMUL_MODE_PROT32:
1995 case X86EMUL_MODE_PROT64:
1997 /* Protected mode interrupts unimplemented yet */
1998 return X86EMUL_UNHANDLEABLE;
2002 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2004 int rc = X86EMUL_CONTINUE;
2005 unsigned long temp_eip = 0;
2006 unsigned long temp_eflags = 0;
2007 unsigned long cs = 0;
2008 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
2009 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
2010 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
2011 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
2013 /* TODO: Add stack limit check */
2015 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2017 if (rc != X86EMUL_CONTINUE)
2020 if (temp_eip & ~0xffff)
2021 return emulate_gp(ctxt, 0);
2023 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2025 if (rc != X86EMUL_CONTINUE)
2028 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2030 if (rc != X86EMUL_CONTINUE)
2033 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2035 if (rc != X86EMUL_CONTINUE)
2038 ctxt->_eip = temp_eip;
2041 if (ctxt->op_bytes == 4)
2042 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2043 else if (ctxt->op_bytes == 2) {
2044 ctxt->eflags &= ~0xffff;
2045 ctxt->eflags |= temp_eflags;
2048 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2049 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
2050 ctxt->ops->set_nmi_mask(ctxt, false);
2055 static int em_iret(struct x86_emulate_ctxt *ctxt)
2057 switch(ctxt->mode) {
2058 case X86EMUL_MODE_REAL:
2059 return emulate_iret_real(ctxt);
2060 case X86EMUL_MODE_VM86:
2061 case X86EMUL_MODE_PROT16:
2062 case X86EMUL_MODE_PROT32:
2063 case X86EMUL_MODE_PROT64:
2065 /* iret from protected mode unimplemented yet */
2066 return X86EMUL_UNHANDLEABLE;
2070 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2073 unsigned short sel, old_sel;
2074 struct desc_struct old_desc, new_desc;
2075 const struct x86_emulate_ops *ops = ctxt->ops;
2076 u8 cpl = ctxt->ops->cpl(ctxt);
2078 /* Assignment of RIP may only fail in 64-bit mode */
2079 if (ctxt->mode == X86EMUL_MODE_PROT64)
2080 ops->get_segment(ctxt, &old_sel, &old_desc, NULL,
2083 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2085 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2086 X86_TRANSFER_CALL_JMP,
2088 if (rc != X86EMUL_CONTINUE)
2091 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2092 if (rc != X86EMUL_CONTINUE) {
2093 WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64);
2094 /* assigning eip failed; restore the old cs */
2095 ops->set_segment(ctxt, old_sel, &old_desc, 0, VCPU_SREG_CS);
2101 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2103 return assign_eip_near(ctxt, ctxt->src.val);
2106 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2111 old_eip = ctxt->_eip;
2112 rc = assign_eip_near(ctxt, ctxt->src.val);
2113 if (rc != X86EMUL_CONTINUE)
2115 ctxt->src.val = old_eip;
2120 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2122 u64 old = ctxt->dst.orig_val64;
2124 if (ctxt->dst.bytes == 16)
2125 return X86EMUL_UNHANDLEABLE;
2127 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2128 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2129 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2130 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2131 ctxt->eflags &= ~EFLG_ZF;
2133 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2134 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2136 ctxt->eflags |= EFLG_ZF;
2138 return X86EMUL_CONTINUE;
2141 static int em_ret(struct x86_emulate_ctxt *ctxt)
2146 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2147 if (rc != X86EMUL_CONTINUE)
2150 return assign_eip_near(ctxt, eip);
2153 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2156 unsigned long eip, cs;
2158 int cpl = ctxt->ops->cpl(ctxt);
2159 struct desc_struct old_desc, new_desc;
2160 const struct x86_emulate_ops *ops = ctxt->ops;
2162 if (ctxt->mode == X86EMUL_MODE_PROT64)
2163 ops->get_segment(ctxt, &old_cs, &old_desc, NULL,
2166 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2167 if (rc != X86EMUL_CONTINUE)
2169 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2170 if (rc != X86EMUL_CONTINUE)
2172 /* Outer-privilege level return is not implemented */
2173 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2174 return X86EMUL_UNHANDLEABLE;
2175 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2178 if (rc != X86EMUL_CONTINUE)
2180 rc = assign_eip_far(ctxt, eip, &new_desc);
2181 if (rc != X86EMUL_CONTINUE) {
2182 WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64);
2183 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
2188 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2192 rc = em_ret_far(ctxt);
2193 if (rc != X86EMUL_CONTINUE)
2195 rsp_increment(ctxt, ctxt->src.val);
2196 return X86EMUL_CONTINUE;
2199 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2201 /* Save real source value, then compare EAX against destination. */
2202 ctxt->dst.orig_val = ctxt->dst.val;
2203 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2204 ctxt->src.orig_val = ctxt->src.val;
2205 ctxt->src.val = ctxt->dst.orig_val;
2206 fastop(ctxt, em_cmp);
2208 if (ctxt->eflags & EFLG_ZF) {
2209 /* Success: write back to memory; no update of EAX */
2210 ctxt->src.type = OP_NONE;
2211 ctxt->dst.val = ctxt->src.orig_val;
2213 /* Failure: write the value we saw to EAX. */
2214 ctxt->src.type = OP_REG;
2215 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2216 ctxt->src.val = ctxt->dst.orig_val;
2217 /* Create write-cycle to dest by writing the same value */
2218 ctxt->dst.val = ctxt->dst.orig_val;
2220 return X86EMUL_CONTINUE;
2223 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2225 int seg = ctxt->src2.val;
2229 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2231 rc = load_segment_descriptor(ctxt, sel, seg);
2232 if (rc != X86EMUL_CONTINUE)
2235 ctxt->dst.val = ctxt->src.val;
2240 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2241 struct desc_struct *cs, struct desc_struct *ss)
2243 cs->l = 0; /* will be adjusted later */
2244 set_desc_base(cs, 0); /* flat segment */
2245 cs->g = 1; /* 4kb granularity */
2246 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2247 cs->type = 0x0b; /* Read, Execute, Accessed */
2249 cs->dpl = 0; /* will be adjusted later */
2254 set_desc_base(ss, 0); /* flat segment */
2255 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2256 ss->g = 1; /* 4kb granularity */
2258 ss->type = 0x03; /* Read/Write, Accessed */
2259 ss->d = 1; /* 32bit stack segment */
2266 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2268 u32 eax, ebx, ecx, edx;
2271 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2272 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2273 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2274 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2277 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2279 const struct x86_emulate_ops *ops = ctxt->ops;
2280 u32 eax, ebx, ecx, edx;
2283 * syscall should always be enabled in longmode - so only become
2284 * vendor specific (cpuid) if other modes are active...
2286 if (ctxt->mode == X86EMUL_MODE_PROT64)
2291 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2293 * Intel ("GenuineIntel")
2294 * remark: Intel CPUs only support "syscall" in 64bit
2295 * longmode. Also an 64bit guest with a
2296 * 32bit compat-app running will #UD !! While this
2297 * behaviour can be fixed (by emulating) into AMD
2298 * response - CPUs of AMD can't behave like Intel.
2300 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2301 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2302 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2305 /* AMD ("AuthenticAMD") */
2306 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2307 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2308 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2311 /* AMD ("AMDisbetter!") */
2312 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2313 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2314 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2317 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2321 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2323 const struct x86_emulate_ops *ops = ctxt->ops;
2324 struct desc_struct cs, ss;
2329 /* syscall is not available in real mode */
2330 if (ctxt->mode == X86EMUL_MODE_REAL ||
2331 ctxt->mode == X86EMUL_MODE_VM86)
2332 return emulate_ud(ctxt);
2334 if (!(em_syscall_is_enabled(ctxt)))
2335 return emulate_ud(ctxt);
2337 ops->get_msr(ctxt, MSR_EFER, &efer);
2338 setup_syscalls_segments(ctxt, &cs, &ss);
2340 if (!(efer & EFER_SCE))
2341 return emulate_ud(ctxt);
2343 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2345 cs_sel = (u16)(msr_data & 0xfffc);
2346 ss_sel = (u16)(msr_data + 8);
2348 if (efer & EFER_LMA) {
2352 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2353 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2355 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2356 if (efer & EFER_LMA) {
2357 #ifdef CONFIG_X86_64
2358 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2361 ctxt->mode == X86EMUL_MODE_PROT64 ?
2362 MSR_LSTAR : MSR_CSTAR, &msr_data);
2363 ctxt->_eip = msr_data;
2365 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2366 ctxt->eflags &= ~msr_data;
2367 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
2371 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2372 ctxt->_eip = (u32)msr_data;
2374 ctxt->eflags &= ~(EFLG_VM | EFLG_IF);
2377 return X86EMUL_CONTINUE;
2380 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2382 const struct x86_emulate_ops *ops = ctxt->ops;
2383 struct desc_struct cs, ss;
2388 ops->get_msr(ctxt, MSR_EFER, &efer);
2389 /* inject #GP if in real mode */
2390 if (ctxt->mode == X86EMUL_MODE_REAL)
2391 return emulate_gp(ctxt, 0);
2394 * Not recognized on AMD in compat mode (but is recognized in legacy
2397 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2398 && !vendor_intel(ctxt))
2399 return emulate_ud(ctxt);
2401 /* sysenter/sysexit have not been tested in 64bit mode. */
2402 if (ctxt->mode == X86EMUL_MODE_PROT64)
2403 return X86EMUL_UNHANDLEABLE;
2405 setup_syscalls_segments(ctxt, &cs, &ss);
2407 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2408 switch (ctxt->mode) {
2409 case X86EMUL_MODE_PROT32:
2410 if ((msr_data & 0xfffc) == 0x0)
2411 return emulate_gp(ctxt, 0);
2413 case X86EMUL_MODE_PROT64:
2414 if (msr_data == 0x0)
2415 return emulate_gp(ctxt, 0);
2421 ctxt->eflags &= ~(EFLG_VM | EFLG_IF);
2422 cs_sel = (u16)msr_data;
2423 cs_sel &= ~SELECTOR_RPL_MASK;
2424 ss_sel = cs_sel + 8;
2425 ss_sel &= ~SELECTOR_RPL_MASK;
2426 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2431 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2432 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2434 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2435 ctxt->_eip = msr_data;
2437 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2438 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2440 return X86EMUL_CONTINUE;
2443 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2445 const struct x86_emulate_ops *ops = ctxt->ops;
2446 struct desc_struct cs, ss;
2447 u64 msr_data, rcx, rdx;
2449 u16 cs_sel = 0, ss_sel = 0;
2451 /* inject #GP if in real mode or Virtual 8086 mode */
2452 if (ctxt->mode == X86EMUL_MODE_REAL ||
2453 ctxt->mode == X86EMUL_MODE_VM86)
2454 return emulate_gp(ctxt, 0);
2456 setup_syscalls_segments(ctxt, &cs, &ss);
2458 if ((ctxt->rex_prefix & 0x8) != 0x0)
2459 usermode = X86EMUL_MODE_PROT64;
2461 usermode = X86EMUL_MODE_PROT32;
2463 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2464 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2468 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2470 case X86EMUL_MODE_PROT32:
2471 cs_sel = (u16)(msr_data + 16);
2472 if ((msr_data & 0xfffc) == 0x0)
2473 return emulate_gp(ctxt, 0);
2474 ss_sel = (u16)(msr_data + 24);
2478 case X86EMUL_MODE_PROT64:
2479 cs_sel = (u16)(msr_data + 32);
2480 if (msr_data == 0x0)
2481 return emulate_gp(ctxt, 0);
2482 ss_sel = cs_sel + 8;
2485 if (is_noncanonical_address(rcx) ||
2486 is_noncanonical_address(rdx))
2487 return emulate_gp(ctxt, 0);
2490 cs_sel |= SELECTOR_RPL_MASK;
2491 ss_sel |= SELECTOR_RPL_MASK;
2493 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2494 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2497 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2499 return X86EMUL_CONTINUE;
2502 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2505 if (ctxt->mode == X86EMUL_MODE_REAL)
2507 if (ctxt->mode == X86EMUL_MODE_VM86)
2509 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2510 return ctxt->ops->cpl(ctxt) > iopl;
2513 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2516 const struct x86_emulate_ops *ops = ctxt->ops;
2517 struct desc_struct tr_seg;
2520 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2521 unsigned mask = (1 << len) - 1;
2524 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2527 if (desc_limit_scaled(&tr_seg) < 103)
2529 base = get_desc_base(&tr_seg);
2530 #ifdef CONFIG_X86_64
2531 base |= ((u64)base3) << 32;
2533 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2534 if (r != X86EMUL_CONTINUE)
2536 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2538 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2539 if (r != X86EMUL_CONTINUE)
2541 if ((perm >> bit_idx) & mask)
2546 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2552 if (emulator_bad_iopl(ctxt))
2553 if (!emulator_io_port_access_allowed(ctxt, port, len))
2556 ctxt->perm_ok = true;
2561 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2562 struct tss_segment_16 *tss)
2564 tss->ip = ctxt->_eip;
2565 tss->flag = ctxt->eflags;
2566 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2567 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2568 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2569 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2570 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2571 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2572 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2573 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2575 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2576 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2577 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2578 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2579 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2582 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2583 struct tss_segment_16 *tss)
2588 ctxt->_eip = tss->ip;
2589 ctxt->eflags = tss->flag | 2;
2590 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2591 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2592 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2593 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2594 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2595 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2596 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2597 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2600 * SDM says that segment selectors are loaded before segment
2603 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2604 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2605 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2606 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2607 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2612 * Now load segment descriptors. If fault happens at this stage
2613 * it is handled in a context of new task
2615 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
2616 X86_TRANSFER_TASK_SWITCH, NULL);
2617 if (ret != X86EMUL_CONTINUE)
2619 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
2620 X86_TRANSFER_TASK_SWITCH, NULL);
2621 if (ret != X86EMUL_CONTINUE)
2623 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
2624 X86_TRANSFER_TASK_SWITCH, NULL);
2625 if (ret != X86EMUL_CONTINUE)
2627 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
2628 X86_TRANSFER_TASK_SWITCH, NULL);
2629 if (ret != X86EMUL_CONTINUE)
2631 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
2632 X86_TRANSFER_TASK_SWITCH, NULL);
2633 if (ret != X86EMUL_CONTINUE)
2636 return X86EMUL_CONTINUE;
2639 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2640 u16 tss_selector, u16 old_tss_sel,
2641 ulong old_tss_base, struct desc_struct *new_desc)
2643 const struct x86_emulate_ops *ops = ctxt->ops;
2644 struct tss_segment_16 tss_seg;
2646 u32 new_tss_base = get_desc_base(new_desc);
2648 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2650 if (ret != X86EMUL_CONTINUE)
2653 save_state_to_tss16(ctxt, &tss_seg);
2655 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2657 if (ret != X86EMUL_CONTINUE)
2660 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2662 if (ret != X86EMUL_CONTINUE)
2665 if (old_tss_sel != 0xffff) {
2666 tss_seg.prev_task_link = old_tss_sel;
2668 ret = ops->write_std(ctxt, new_tss_base,
2669 &tss_seg.prev_task_link,
2670 sizeof tss_seg.prev_task_link,
2672 if (ret != X86EMUL_CONTINUE)
2676 return load_state_from_tss16(ctxt, &tss_seg);
2679 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2680 struct tss_segment_32 *tss)
2682 /* CR3 and ldt selector are not saved intentionally */
2683 tss->eip = ctxt->_eip;
2684 tss->eflags = ctxt->eflags;
2685 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2686 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2687 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2688 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2689 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2690 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2691 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2692 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2694 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2695 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2696 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2697 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2698 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2699 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2702 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2703 struct tss_segment_32 *tss)
2708 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2709 return emulate_gp(ctxt, 0);
2710 ctxt->_eip = tss->eip;
2711 ctxt->eflags = tss->eflags | 2;
2713 /* General purpose registers */
2714 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2715 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2716 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2717 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2718 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2719 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2720 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2721 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2724 * SDM says that segment selectors are loaded before segment
2725 * descriptors. This is important because CPL checks will
2728 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2729 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2730 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2731 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2732 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2733 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2734 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2737 * If we're switching between Protected Mode and VM86, we need to make
2738 * sure to update the mode before loading the segment descriptors so
2739 * that the selectors are interpreted correctly.
2741 if (ctxt->eflags & X86_EFLAGS_VM) {
2742 ctxt->mode = X86EMUL_MODE_VM86;
2745 ctxt->mode = X86EMUL_MODE_PROT32;
2750 * Now load segment descriptors. If fault happenes at this stage
2751 * it is handled in a context of new task
2753 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
2754 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
2755 if (ret != X86EMUL_CONTINUE)
2757 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
2758 X86_TRANSFER_TASK_SWITCH, NULL);
2759 if (ret != X86EMUL_CONTINUE)
2761 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
2762 X86_TRANSFER_TASK_SWITCH, NULL);
2763 if (ret != X86EMUL_CONTINUE)
2765 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
2766 X86_TRANSFER_TASK_SWITCH, NULL);
2767 if (ret != X86EMUL_CONTINUE)
2769 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
2770 X86_TRANSFER_TASK_SWITCH, NULL);
2771 if (ret != X86EMUL_CONTINUE)
2773 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
2774 X86_TRANSFER_TASK_SWITCH, NULL);
2775 if (ret != X86EMUL_CONTINUE)
2777 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
2778 X86_TRANSFER_TASK_SWITCH, NULL);
2779 if (ret != X86EMUL_CONTINUE)
2782 return X86EMUL_CONTINUE;
2785 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2786 u16 tss_selector, u16 old_tss_sel,
2787 ulong old_tss_base, struct desc_struct *new_desc)
2789 const struct x86_emulate_ops *ops = ctxt->ops;
2790 struct tss_segment_32 tss_seg;
2792 u32 new_tss_base = get_desc_base(new_desc);
2793 u32 eip_offset = offsetof(struct tss_segment_32, eip);
2794 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
2796 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2798 if (ret != X86EMUL_CONTINUE)
2801 save_state_to_tss32(ctxt, &tss_seg);
2803 /* Only GP registers and segment selectors are saved */
2804 ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
2805 ldt_sel_offset - eip_offset, &ctxt->exception);
2806 if (ret != X86EMUL_CONTINUE)
2809 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2811 if (ret != X86EMUL_CONTINUE)
2814 if (old_tss_sel != 0xffff) {
2815 tss_seg.prev_task_link = old_tss_sel;
2817 ret = ops->write_std(ctxt, new_tss_base,
2818 &tss_seg.prev_task_link,
2819 sizeof tss_seg.prev_task_link,
2821 if (ret != X86EMUL_CONTINUE)
2825 return load_state_from_tss32(ctxt, &tss_seg);
2828 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2829 u16 tss_selector, int idt_index, int reason,
2830 bool has_error_code, u32 error_code)
2832 const struct x86_emulate_ops *ops = ctxt->ops;
2833 struct desc_struct curr_tss_desc, next_tss_desc;
2835 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2836 ulong old_tss_base =
2837 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2841 /* FIXME: old_tss_base == ~0 ? */
2843 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2844 if (ret != X86EMUL_CONTINUE)
2846 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2847 if (ret != X86EMUL_CONTINUE)
2850 /* FIXME: check that next_tss_desc is tss */
2853 * Check privileges. The three cases are task switch caused by...
2855 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2856 * 2. Exception/IRQ/iret: No check is performed
2857 * 3. jmp/call to TSS/task-gate: No check is performed since the
2858 * hardware checks it before exiting.
2860 if (reason == TASK_SWITCH_GATE) {
2861 if (idt_index != -1) {
2862 /* Software interrupts */
2863 struct desc_struct task_gate_desc;
2866 ret = read_interrupt_descriptor(ctxt, idt_index,
2868 if (ret != X86EMUL_CONTINUE)
2871 dpl = task_gate_desc.dpl;
2872 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2873 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2877 desc_limit = desc_limit_scaled(&next_tss_desc);
2878 if (!next_tss_desc.p ||
2879 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2880 desc_limit < 0x2b)) {
2881 return emulate_ts(ctxt, tss_selector & 0xfffc);
2884 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2885 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2886 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2889 if (reason == TASK_SWITCH_IRET)
2890 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2892 /* set back link to prev task only if NT bit is set in eflags
2893 note that old_tss_sel is not used after this point */
2894 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2895 old_tss_sel = 0xffff;
2897 if (next_tss_desc.type & 8)
2898 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2899 old_tss_base, &next_tss_desc);
2901 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2902 old_tss_base, &next_tss_desc);
2903 if (ret != X86EMUL_CONTINUE)
2906 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2907 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2909 if (reason != TASK_SWITCH_IRET) {
2910 next_tss_desc.type |= (1 << 1); /* set busy flag */
2911 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2914 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2915 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2917 if (has_error_code) {
2918 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2919 ctxt->lock_prefix = 0;
2920 ctxt->src.val = (unsigned long) error_code;
2921 ret = em_push(ctxt);
2927 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2928 u16 tss_selector, int idt_index, int reason,
2929 bool has_error_code, u32 error_code)
2933 invalidate_registers(ctxt);
2934 ctxt->_eip = ctxt->eip;
2935 ctxt->dst.type = OP_NONE;
2937 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2938 has_error_code, error_code);
2940 if (rc == X86EMUL_CONTINUE) {
2941 ctxt->eip = ctxt->_eip;
2942 writeback_registers(ctxt);
2945 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2948 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2951 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2953 register_address_increment(ctxt, reg, df * op->bytes);
2954 op->addr.mem.ea = register_address(ctxt, reg);
2957 static int em_das(struct x86_emulate_ctxt *ctxt)
2960 bool af, cf, old_cf;
2962 cf = ctxt->eflags & X86_EFLAGS_CF;
2968 af = ctxt->eflags & X86_EFLAGS_AF;
2969 if ((al & 0x0f) > 9 || af) {
2971 cf = old_cf | (al >= 250);
2976 if (old_al > 0x99 || old_cf) {
2982 /* Set PF, ZF, SF */
2983 ctxt->src.type = OP_IMM;
2985 ctxt->src.bytes = 1;
2986 fastop(ctxt, em_or);
2987 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2989 ctxt->eflags |= X86_EFLAGS_CF;
2991 ctxt->eflags |= X86_EFLAGS_AF;
2992 return X86EMUL_CONTINUE;
2995 static int em_aam(struct x86_emulate_ctxt *ctxt)
2999 if (ctxt->src.val == 0)
3000 return emulate_de(ctxt);
3002 al = ctxt->dst.val & 0xff;
3003 ah = al / ctxt->src.val;
3004 al %= ctxt->src.val;
3006 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3008 /* Set PF, ZF, SF */
3009 ctxt->src.type = OP_IMM;
3011 ctxt->src.bytes = 1;
3012 fastop(ctxt, em_or);
3014 return X86EMUL_CONTINUE;
3017 static int em_aad(struct x86_emulate_ctxt *ctxt)
3019 u8 al = ctxt->dst.val & 0xff;
3020 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3022 al = (al + (ah * ctxt->src.val)) & 0xff;
3024 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3026 /* Set PF, ZF, SF */
3027 ctxt->src.type = OP_IMM;
3029 ctxt->src.bytes = 1;
3030 fastop(ctxt, em_or);
3032 return X86EMUL_CONTINUE;
3035 static int em_call(struct x86_emulate_ctxt *ctxt)
3038 long rel = ctxt->src.val;
3040 ctxt->src.val = (unsigned long)ctxt->_eip;
3041 rc = jmp_rel(ctxt, rel);
3042 if (rc != X86EMUL_CONTINUE)
3044 return em_push(ctxt);
3047 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3052 struct desc_struct old_desc, new_desc;
3053 const struct x86_emulate_ops *ops = ctxt->ops;
3054 int cpl = ctxt->ops->cpl(ctxt);
3056 old_eip = ctxt->_eip;
3057 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3059 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3060 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3061 X86_TRANSFER_CALL_JMP, &new_desc);
3062 if (rc != X86EMUL_CONTINUE)
3065 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3066 if (rc != X86EMUL_CONTINUE)
3069 ctxt->src.val = old_cs;
3071 if (rc != X86EMUL_CONTINUE)
3074 ctxt->src.val = old_eip;
3076 /* If we failed, we tainted the memory, but the very least we should
3078 if (rc != X86EMUL_CONTINUE)
3082 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3087 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3092 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3093 if (rc != X86EMUL_CONTINUE)
3095 rc = assign_eip_near(ctxt, eip);
3096 if (rc != X86EMUL_CONTINUE)
3098 rsp_increment(ctxt, ctxt->src.val);
3099 return X86EMUL_CONTINUE;
3102 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3104 /* Write back the register source. */
3105 ctxt->src.val = ctxt->dst.val;
3106 write_register_operand(&ctxt->src);
3108 /* Write back the memory destination with implicit LOCK prefix. */
3109 ctxt->dst.val = ctxt->src.orig_val;
3110 ctxt->lock_prefix = 1;
3111 return X86EMUL_CONTINUE;
3114 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3116 ctxt->dst.val = ctxt->src2.val;
3117 return fastop(ctxt, em_imul);
3120 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3122 ctxt->dst.type = OP_REG;
3123 ctxt->dst.bytes = ctxt->src.bytes;
3124 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3125 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3127 return X86EMUL_CONTINUE;
3130 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3134 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3135 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3136 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3137 return X86EMUL_CONTINUE;
3140 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3144 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3145 return emulate_gp(ctxt, 0);
3146 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3147 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3148 return X86EMUL_CONTINUE;
3151 static int em_mov(struct x86_emulate_ctxt *ctxt)
3153 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3154 return X86EMUL_CONTINUE;
3157 #define FFL(x) bit(X86_FEATURE_##x)
3159 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3161 u32 ebx, ecx, edx, eax = 1;
3165 * Check MOVBE is set in the guest-visible CPUID leaf.
3167 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3168 if (!(ecx & FFL(MOVBE)))
3169 return emulate_ud(ctxt);
3171 switch (ctxt->op_bytes) {
3174 * From MOVBE definition: "...When the operand size is 16 bits,
3175 * the upper word of the destination register remains unchanged
3178 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3179 * rules so we have to do the operation almost per hand.
3181 tmp = (u16)ctxt->src.val;
3182 ctxt->dst.val &= ~0xffffUL;
3183 ctxt->dst.val |= (unsigned long)swab16(tmp);
3186 ctxt->dst.val = swab32((u32)ctxt->src.val);
3189 ctxt->dst.val = swab64(ctxt->src.val);
3194 return X86EMUL_CONTINUE;
3197 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3199 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3200 return emulate_gp(ctxt, 0);
3202 /* Disable writeback. */
3203 ctxt->dst.type = OP_NONE;
3204 return X86EMUL_CONTINUE;
3207 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3211 if (ctxt->mode == X86EMUL_MODE_PROT64)
3212 val = ctxt->src.val & ~0ULL;
3214 val = ctxt->src.val & ~0U;
3216 /* #UD condition is already handled. */
3217 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3218 return emulate_gp(ctxt, 0);
3220 /* Disable writeback. */
3221 ctxt->dst.type = OP_NONE;
3222 return X86EMUL_CONTINUE;
3225 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3229 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3230 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3231 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3232 return emulate_gp(ctxt, 0);
3234 return X86EMUL_CONTINUE;
3237 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3241 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3242 return emulate_gp(ctxt, 0);
3244 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3245 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3246 return X86EMUL_CONTINUE;
3249 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3251 if (ctxt->modrm_reg > VCPU_SREG_GS)
3252 return emulate_ud(ctxt);
3254 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3255 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3256 ctxt->dst.bytes = 2;
3257 return X86EMUL_CONTINUE;
3260 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3262 u16 sel = ctxt->src.val;
3264 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3265 return emulate_ud(ctxt);
3267 if (ctxt->modrm_reg == VCPU_SREG_SS)
3268 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3270 /* Disable writeback. */
3271 ctxt->dst.type = OP_NONE;
3272 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3275 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3277 u16 sel = ctxt->src.val;
3279 /* Disable writeback. */
3280 ctxt->dst.type = OP_NONE;
3281 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3284 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3286 u16 sel = ctxt->src.val;
3288 /* Disable writeback. */
3289 ctxt->dst.type = OP_NONE;
3290 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3293 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3298 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3299 if (rc == X86EMUL_CONTINUE)
3300 ctxt->ops->invlpg(ctxt, linear);
3301 /* Disable writeback. */
3302 ctxt->dst.type = OP_NONE;
3303 return X86EMUL_CONTINUE;
3306 static int em_clts(struct x86_emulate_ctxt *ctxt)
3310 cr0 = ctxt->ops->get_cr(ctxt, 0);
3312 ctxt->ops->set_cr(ctxt, 0, cr0);
3313 return X86EMUL_CONTINUE;
3316 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3318 int rc = ctxt->ops->fix_hypercall(ctxt);
3320 if (rc != X86EMUL_CONTINUE)
3323 /* Let the processor re-execute the fixed hypercall */
3324 ctxt->_eip = ctxt->eip;
3325 /* Disable writeback. */
3326 ctxt->dst.type = OP_NONE;
3327 return X86EMUL_CONTINUE;
3330 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3331 void (*get)(struct x86_emulate_ctxt *ctxt,
3332 struct desc_ptr *ptr))
3334 struct desc_ptr desc_ptr;
3336 if (ctxt->mode == X86EMUL_MODE_PROT64)
3338 get(ctxt, &desc_ptr);
3339 if (ctxt->op_bytes == 2) {
3341 desc_ptr.address &= 0x00ffffff;
3343 /* Disable writeback. */
3344 ctxt->dst.type = OP_NONE;
3345 return segmented_write(ctxt, ctxt->dst.addr.mem,
3346 &desc_ptr, 2 + ctxt->op_bytes);
3349 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3351 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3354 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3356 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3359 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3361 struct desc_ptr desc_ptr;
3364 if (ctxt->mode == X86EMUL_MODE_PROT64)
3366 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3367 &desc_ptr.size, &desc_ptr.address,
3369 if (rc != X86EMUL_CONTINUE)
3371 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3372 is_noncanonical_address(desc_ptr.address))
3373 return emulate_gp(ctxt, 0);
3375 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3377 ctxt->ops->set_idt(ctxt, &desc_ptr);
3378 /* Disable writeback. */
3379 ctxt->dst.type = OP_NONE;
3380 return X86EMUL_CONTINUE;
3383 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3385 return em_lgdt_lidt(ctxt, true);
3388 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3392 rc = ctxt->ops->fix_hypercall(ctxt);
3394 /* Disable writeback. */
3395 ctxt->dst.type = OP_NONE;
3399 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3401 return em_lgdt_lidt(ctxt, false);
3404 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3406 if (ctxt->dst.type == OP_MEM)
3407 ctxt->dst.bytes = 2;
3408 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3409 return X86EMUL_CONTINUE;
3412 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3414 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3415 | (ctxt->src.val & 0x0f));
3416 ctxt->dst.type = OP_NONE;
3417 return X86EMUL_CONTINUE;
3420 static int em_loop(struct x86_emulate_ctxt *ctxt)
3422 int rc = X86EMUL_CONTINUE;
3424 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3425 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3426 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3427 rc = jmp_rel(ctxt, ctxt->src.val);
3432 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3434 int rc = X86EMUL_CONTINUE;
3436 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3437 rc = jmp_rel(ctxt, ctxt->src.val);
3442 static int em_in(struct x86_emulate_ctxt *ctxt)
3444 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3446 return X86EMUL_IO_NEEDED;
3448 return X86EMUL_CONTINUE;
3451 static int em_out(struct x86_emulate_ctxt *ctxt)
3453 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3455 /* Disable writeback. */
3456 ctxt->dst.type = OP_NONE;
3457 return X86EMUL_CONTINUE;
3460 static int em_cli(struct x86_emulate_ctxt *ctxt)
3462 if (emulator_bad_iopl(ctxt))
3463 return emulate_gp(ctxt, 0);
3465 ctxt->eflags &= ~X86_EFLAGS_IF;
3466 return X86EMUL_CONTINUE;
3469 static int em_sti(struct x86_emulate_ctxt *ctxt)
3471 if (emulator_bad_iopl(ctxt))
3472 return emulate_gp(ctxt, 0);
3474 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3475 ctxt->eflags |= X86_EFLAGS_IF;
3476 return X86EMUL_CONTINUE;
3479 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3481 u32 eax, ebx, ecx, edx;
3483 eax = reg_read(ctxt, VCPU_REGS_RAX);
3484 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3485 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3486 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3487 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3488 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3489 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3490 return X86EMUL_CONTINUE;
3493 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3497 flags = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF;
3498 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3500 ctxt->eflags &= ~0xffUL;
3501 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3502 return X86EMUL_CONTINUE;
3505 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3507 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3508 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3509 return X86EMUL_CONTINUE;
3512 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3514 switch (ctxt->op_bytes) {
3515 #ifdef CONFIG_X86_64
3517 asm("bswap %0" : "+r"(ctxt->dst.val));
3521 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3524 return X86EMUL_CONTINUE;
3527 static int em_clflush(struct x86_emulate_ctxt *ctxt)
3529 /* emulating clflush regardless of cpuid */
3530 return X86EMUL_CONTINUE;
3533 static bool valid_cr(int nr)
3545 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3547 if (!valid_cr(ctxt->modrm_reg))
3548 return emulate_ud(ctxt);
3550 return X86EMUL_CONTINUE;
3553 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3555 u64 new_val = ctxt->src.val64;
3556 int cr = ctxt->modrm_reg;
3559 static u64 cr_reserved_bits[] = {
3560 0xffffffff00000000ULL,
3561 0, 0, 0, /* CR3 checked later */
3568 return emulate_ud(ctxt);
3570 if (new_val & cr_reserved_bits[cr])
3571 return emulate_gp(ctxt, 0);
3576 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3577 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3578 return emulate_gp(ctxt, 0);
3580 cr4 = ctxt->ops->get_cr(ctxt, 4);
3581 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3583 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3584 !(cr4 & X86_CR4_PAE))
3585 return emulate_gp(ctxt, 0);
3592 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3593 if (efer & EFER_LMA)
3594 rsvd = CR3_L_MODE_RESERVED_BITS & ~CR3_PCID_INVD;
3597 return emulate_gp(ctxt, 0);
3602 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3604 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3605 return emulate_gp(ctxt, 0);
3611 return X86EMUL_CONTINUE;
3614 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3618 ctxt->ops->get_dr(ctxt, 7, &dr7);
3620 /* Check if DR7.Global_Enable is set */
3621 return dr7 & (1 << 13);
3624 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3626 int dr = ctxt->modrm_reg;
3630 return emulate_ud(ctxt);
3632 cr4 = ctxt->ops->get_cr(ctxt, 4);
3633 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3634 return emulate_ud(ctxt);
3636 if (check_dr7_gd(ctxt)) {
3639 ctxt->ops->get_dr(ctxt, 6, &dr6);
3641 dr6 |= DR6_BD | DR6_RTM;
3642 ctxt->ops->set_dr(ctxt, 6, dr6);
3643 return emulate_db(ctxt);
3646 return X86EMUL_CONTINUE;
3649 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3651 u64 new_val = ctxt->src.val64;
3652 int dr = ctxt->modrm_reg;
3654 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3655 return emulate_gp(ctxt, 0);
3657 return check_dr_read(ctxt);
3660 static int check_svme(struct x86_emulate_ctxt *ctxt)
3664 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3666 if (!(efer & EFER_SVME))
3667 return emulate_ud(ctxt);
3669 return X86EMUL_CONTINUE;
3672 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3674 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3676 /* Valid physical address? */
3677 if (rax & 0xffff000000000000ULL)
3678 return emulate_gp(ctxt, 0);
3680 return check_svme(ctxt);
3683 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3685 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3687 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3688 return emulate_ud(ctxt);
3690 return X86EMUL_CONTINUE;
3693 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3695 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3696 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3698 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3699 ctxt->ops->check_pmc(ctxt, rcx))
3700 return emulate_gp(ctxt, 0);
3702 return X86EMUL_CONTINUE;
3705 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3707 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3708 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3709 return emulate_gp(ctxt, 0);
3711 return X86EMUL_CONTINUE;
3714 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3716 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3717 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3718 return emulate_gp(ctxt, 0);
3720 return X86EMUL_CONTINUE;
3723 #define D(_y) { .flags = (_y) }
3724 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
3725 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
3726 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3727 #define N D(NotImpl)
3728 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3729 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3730 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3731 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
3732 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3733 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3734 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3735 #define II(_f, _e, _i) \
3736 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
3737 #define IIP(_f, _e, _i, _p) \
3738 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
3739 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3740 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3742 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3743 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3744 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3745 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3746 #define I2bvIP(_f, _e, _i, _p) \
3747 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3749 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3750 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3751 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3753 static const struct opcode group7_rm0[] = {
3755 I(SrcNone | Priv | EmulateOnUD, em_vmcall),
3759 static const struct opcode group7_rm1[] = {
3760 DI(SrcNone | Priv, monitor),
3761 DI(SrcNone | Priv, mwait),
3765 static const struct opcode group7_rm3[] = {
3766 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3767 II(SrcNone | Prot | EmulateOnUD, em_vmmcall, vmmcall),
3768 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3769 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3770 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3771 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3772 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3773 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3776 static const struct opcode group7_rm7[] = {
3778 DIP(SrcNone, rdtscp, check_rdtsc),
3782 static const struct opcode group1[] = {
3784 F(Lock | PageTable, em_or),
3787 F(Lock | PageTable, em_and),
3793 static const struct opcode group1A[] = {
3794 I(DstMem | SrcNone | Mov | Stack | IncSP, em_pop), N, N, N, N, N, N, N,
3797 static const struct opcode group2[] = {
3798 F(DstMem | ModRM, em_rol),
3799 F(DstMem | ModRM, em_ror),
3800 F(DstMem | ModRM, em_rcl),
3801 F(DstMem | ModRM, em_rcr),
3802 F(DstMem | ModRM, em_shl),
3803 F(DstMem | ModRM, em_shr),
3804 F(DstMem | ModRM, em_shl),
3805 F(DstMem | ModRM, em_sar),
3808 static const struct opcode group3[] = {
3809 F(DstMem | SrcImm | NoWrite, em_test),
3810 F(DstMem | SrcImm | NoWrite, em_test),
3811 F(DstMem | SrcNone | Lock, em_not),
3812 F(DstMem | SrcNone | Lock, em_neg),
3813 F(DstXacc | Src2Mem, em_mul_ex),
3814 F(DstXacc | Src2Mem, em_imul_ex),
3815 F(DstXacc | Src2Mem, em_div_ex),
3816 F(DstXacc | Src2Mem, em_idiv_ex),
3819 static const struct opcode group4[] = {
3820 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3821 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3825 static const struct opcode group5[] = {
3826 F(DstMem | SrcNone | Lock, em_inc),
3827 F(DstMem | SrcNone | Lock, em_dec),
3828 I(SrcMem | NearBranch, em_call_near_abs),
3829 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3830 I(SrcMem | NearBranch, em_jmp_abs),
3831 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
3832 I(SrcMem | Stack, em_push), D(Undefined),
3835 static const struct opcode group6[] = {
3838 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3839 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3843 static const struct group_dual group7 = { {
3844 II(Mov | DstMem, em_sgdt, sgdt),
3845 II(Mov | DstMem, em_sidt, sidt),
3846 II(SrcMem | Priv, em_lgdt, lgdt),
3847 II(SrcMem | Priv, em_lidt, lidt),
3848 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3849 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3850 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3854 N, EXT(0, group7_rm3),
3855 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3856 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3860 static const struct opcode group8[] = {
3862 F(DstMem | SrcImmByte | NoWrite, em_bt),
3863 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3864 F(DstMem | SrcImmByte | Lock, em_btr),
3865 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3868 static const struct group_dual group9 = { {
3869 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3871 N, N, N, N, N, N, N, N,
3874 static const struct opcode group11[] = {
3875 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3879 static const struct gprefix pfx_0f_ae_7 = {
3880 I(SrcMem | ByteOp, em_clflush), N, N, N,
3883 static const struct group_dual group15 = { {
3884 N, N, N, N, N, N, N, GP(0, &pfx_0f_ae_7),
3886 N, N, N, N, N, N, N, N,
3889 static const struct gprefix pfx_0f_6f_0f_7f = {
3890 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3893 static const struct instr_dual instr_dual_0f_2b = {
3897 static const struct gprefix pfx_0f_2b = {
3898 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
3901 static const struct gprefix pfx_0f_28_0f_29 = {
3902 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
3905 static const struct gprefix pfx_0f_e7 = {
3906 N, I(Sse, em_mov), N, N,
3909 static const struct escape escape_d9 = { {
3910 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
3913 N, N, N, N, N, N, N, N,
3915 N, N, N, N, N, N, N, N,
3917 N, N, N, N, N, N, N, N,
3919 N, N, N, N, N, N, N, N,
3921 N, N, N, N, N, N, N, N,
3923 N, N, N, N, N, N, N, N,
3925 N, N, N, N, N, N, N, N,
3927 N, N, N, N, N, N, N, N,
3930 static const struct escape escape_db = { {
3931 N, N, N, N, N, N, N, N,
3934 N, N, N, N, N, N, N, N,
3936 N, N, N, N, N, N, N, N,
3938 N, N, N, N, N, N, N, N,
3940 N, N, N, N, N, N, N, N,
3942 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3944 N, N, N, N, N, N, N, N,
3946 N, N, N, N, N, N, N, N,
3948 N, N, N, N, N, N, N, N,
3951 static const struct escape escape_dd = { {
3952 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
3955 N, N, N, N, N, N, N, N,
3957 N, N, N, N, N, N, N, N,
3959 N, N, N, N, N, N, N, N,
3961 N, N, N, N, N, N, N, N,
3963 N, N, N, N, N, N, N, N,
3965 N, N, N, N, N, N, N, N,
3967 N, N, N, N, N, N, N, N,
3969 N, N, N, N, N, N, N, N,
3972 static const struct instr_dual instr_dual_0f_c3 = {
3973 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
3976 static const struct opcode opcode_table[256] = {
3978 F6ALU(Lock, em_add),
3979 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3980 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3982 F6ALU(Lock | PageTable, em_or),
3983 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3986 F6ALU(Lock, em_adc),
3987 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3988 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3990 F6ALU(Lock, em_sbb),
3991 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3992 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3994 F6ALU(Lock | PageTable, em_and), N, N,
3996 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3998 F6ALU(Lock, em_xor), N, N,
4000 F6ALU(NoWrite, em_cmp), N, N,
4002 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4004 X8(I(SrcReg | Stack, em_push)),
4006 X8(I(DstReg | Stack, em_pop)),
4008 I(ImplicitOps | Stack | No64, em_pusha),
4009 I(ImplicitOps | Stack | No64, em_popa),
4010 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
4013 I(SrcImm | Mov | Stack, em_push),
4014 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4015 I(SrcImmByte | Mov | Stack, em_push),
4016 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4017 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4018 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4020 X16(D(SrcImmByte | NearBranch)),
4022 G(ByteOp | DstMem | SrcImm, group1),
4023 G(DstMem | SrcImm, group1),
4024 G(ByteOp | DstMem | SrcImm | No64, group1),
4025 G(DstMem | SrcImmByte, group1),
4026 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4027 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4029 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4030 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4031 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4032 D(ModRM | SrcMem | NoAccess | DstReg),
4033 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4036 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4038 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4039 I(SrcImmFAddr | No64, em_call_far), N,
4040 II(ImplicitOps | Stack, em_pushf, pushf),
4041 II(ImplicitOps | Stack, em_popf, popf),
4042 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4044 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4045 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4046 I2bv(SrcSI | DstDI | Mov | String, em_mov),
4047 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp_r),
4049 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4050 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4051 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4052 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4054 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4056 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4058 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4059 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4060 I(ImplicitOps | NearBranch, em_ret),
4061 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4062 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4063 G(ByteOp, group11), G(0, group11),
4065 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4066 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4067 I(ImplicitOps, em_ret_far),
4068 D(ImplicitOps), DI(SrcImmByte, intn),
4069 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4071 G(Src2One | ByteOp, group2), G(Src2One, group2),
4072 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4073 I(DstAcc | SrcImmUByte | No64, em_aam),
4074 I(DstAcc | SrcImmUByte | No64, em_aad),
4075 F(DstAcc | ByteOp | No64, em_salc),
4076 I(DstAcc | SrcXLat | ByteOp, em_mov),
4078 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4080 X3(I(SrcImmByte | NearBranch, em_loop)),
4081 I(SrcImmByte | NearBranch, em_jcxz),
4082 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4083 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4085 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4086 I(SrcImmFAddr | No64, em_jmp_far),
4087 D(SrcImmByte | ImplicitOps | NearBranch),
4088 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4089 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4091 N, DI(ImplicitOps, icebp), N, N,
4092 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4093 G(ByteOp, group3), G(0, group3),
4095 D(ImplicitOps), D(ImplicitOps),
4096 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4097 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4100 static const struct opcode twobyte_table[256] = {
4102 G(0, group6), GD(0, &group7), N, N,
4103 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4104 II(ImplicitOps | Priv, em_clts, clts), N,
4105 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4106 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4108 N, N, N, N, N, N, N, N,
4109 D(ImplicitOps | ModRM | SrcMem | NoAccess),
4110 N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4112 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4113 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4114 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4116 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4119 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4120 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4121 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4124 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4125 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4126 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4127 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4128 I(ImplicitOps | EmulateOnUD, em_sysenter),
4129 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4131 N, N, N, N, N, N, N, N,
4133 X16(D(DstReg | SrcMem | ModRM)),
4135 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4140 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4145 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4147 X16(D(SrcImm | NearBranch)),
4149 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4151 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4152 II(ImplicitOps, em_cpuid, cpuid),
4153 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4154 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4155 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4157 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4158 DI(ImplicitOps, rsm),
4159 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4160 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4161 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4162 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4164 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4165 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4166 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4167 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4168 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4169 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4173 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4174 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
4175 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4177 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4178 N, ID(0, &instr_dual_0f_c3),
4179 N, N, N, GD(0, &group9),
4181 X8(I(DstReg, em_bswap)),
4183 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4185 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4186 N, N, N, N, N, N, N, N,
4188 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4191 static const struct instr_dual instr_dual_0f_38_f0 = {
4192 I(DstReg | SrcMem | Mov, em_movbe), N
4195 static const struct instr_dual instr_dual_0f_38_f1 = {
4196 I(DstMem | SrcReg | Mov, em_movbe), N
4199 static const struct gprefix three_byte_0f_38_f0 = {
4200 ID(0, &instr_dual_0f_38_f0), N, N, N
4203 static const struct gprefix three_byte_0f_38_f1 = {
4204 ID(0, &instr_dual_0f_38_f1), N, N, N
4208 * Insns below are selected by the prefix which indexed by the third opcode
4211 static const struct opcode opcode_map_0f_38[256] = {
4213 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4215 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4217 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4218 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4237 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4241 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4247 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4248 unsigned size, bool sign_extension)
4250 int rc = X86EMUL_CONTINUE;
4254 op->addr.mem.ea = ctxt->_eip;
4255 /* NB. Immediates are sign-extended as necessary. */
4256 switch (op->bytes) {
4258 op->val = insn_fetch(s8, ctxt);
4261 op->val = insn_fetch(s16, ctxt);
4264 op->val = insn_fetch(s32, ctxt);
4267 op->val = insn_fetch(s64, ctxt);
4270 if (!sign_extension) {
4271 switch (op->bytes) {
4279 op->val &= 0xffffffff;
4287 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4290 int rc = X86EMUL_CONTINUE;
4294 decode_register_operand(ctxt, op);
4297 rc = decode_imm(ctxt, op, 1, false);
4300 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4304 if (ctxt->d & BitOp)
4305 fetch_bit_operand(ctxt);
4306 op->orig_val = op->val;
4309 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4313 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4314 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4315 fetch_register_operand(op);
4316 op->orig_val = op->val;
4320 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4321 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4322 fetch_register_operand(op);
4323 op->orig_val = op->val;
4326 if (ctxt->d & ByteOp) {
4331 op->bytes = ctxt->op_bytes;
4332 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4333 fetch_register_operand(op);
4334 op->orig_val = op->val;
4338 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4340 register_address(ctxt, VCPU_REGS_RDI);
4341 op->addr.mem.seg = VCPU_SREG_ES;
4348 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4349 fetch_register_operand(op);
4354 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4357 rc = decode_imm(ctxt, op, 1, true);
4365 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4368 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4371 ctxt->memop.bytes = 1;
4372 if (ctxt->memop.type == OP_REG) {
4373 ctxt->memop.addr.reg = decode_register(ctxt,
4374 ctxt->modrm_rm, true);
4375 fetch_register_operand(&ctxt->memop);
4379 ctxt->memop.bytes = 2;
4382 ctxt->memop.bytes = 4;
4385 rc = decode_imm(ctxt, op, 2, false);
4388 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4392 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4394 register_address(ctxt, VCPU_REGS_RSI);
4395 op->addr.mem.seg = ctxt->seg_override;
4401 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4404 reg_read(ctxt, VCPU_REGS_RBX) +
4405 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4406 op->addr.mem.seg = ctxt->seg_override;
4411 op->addr.mem.ea = ctxt->_eip;
4412 op->bytes = ctxt->op_bytes + 2;
4413 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4416 ctxt->memop.bytes = ctxt->op_bytes + 2;
4420 op->val = VCPU_SREG_ES;
4424 op->val = VCPU_SREG_CS;
4428 op->val = VCPU_SREG_SS;
4432 op->val = VCPU_SREG_DS;
4436 op->val = VCPU_SREG_FS;
4440 op->val = VCPU_SREG_GS;
4443 /* Special instructions do their own operand decoding. */
4445 op->type = OP_NONE; /* Disable writeback. */
4453 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4455 int rc = X86EMUL_CONTINUE;
4456 int mode = ctxt->mode;
4457 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4458 bool op_prefix = false;
4459 bool has_seg_override = false;
4460 struct opcode opcode;
4462 ctxt->memop.type = OP_NONE;
4463 ctxt->memopp = NULL;
4464 ctxt->_eip = ctxt->eip;
4465 ctxt->fetch.ptr = ctxt->fetch.data;
4466 ctxt->fetch.end = ctxt->fetch.data + insn_len;
4467 ctxt->opcode_len = 1;
4469 memcpy(ctxt->fetch.data, insn, insn_len);
4471 rc = __do_insn_fetch_bytes(ctxt, 1);
4472 if (rc != X86EMUL_CONTINUE)
4477 case X86EMUL_MODE_REAL:
4478 case X86EMUL_MODE_VM86:
4479 case X86EMUL_MODE_PROT16:
4480 def_op_bytes = def_ad_bytes = 2;
4482 case X86EMUL_MODE_PROT32:
4483 def_op_bytes = def_ad_bytes = 4;
4485 #ifdef CONFIG_X86_64
4486 case X86EMUL_MODE_PROT64:
4492 return EMULATION_FAILED;
4495 ctxt->op_bytes = def_op_bytes;
4496 ctxt->ad_bytes = def_ad_bytes;
4498 /* Legacy prefixes. */
4500 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4501 case 0x66: /* operand-size override */
4503 /* switch between 2/4 bytes */
4504 ctxt->op_bytes = def_op_bytes ^ 6;
4506 case 0x67: /* address-size override */
4507 if (mode == X86EMUL_MODE_PROT64)
4508 /* switch between 4/8 bytes */
4509 ctxt->ad_bytes = def_ad_bytes ^ 12;
4511 /* switch between 2/4 bytes */
4512 ctxt->ad_bytes = def_ad_bytes ^ 6;
4514 case 0x26: /* ES override */
4515 case 0x2e: /* CS override */
4516 case 0x36: /* SS override */
4517 case 0x3e: /* DS override */
4518 has_seg_override = true;
4519 ctxt->seg_override = (ctxt->b >> 3) & 3;
4521 case 0x64: /* FS override */
4522 case 0x65: /* GS override */
4523 has_seg_override = true;
4524 ctxt->seg_override = ctxt->b & 7;
4526 case 0x40 ... 0x4f: /* REX */
4527 if (mode != X86EMUL_MODE_PROT64)
4529 ctxt->rex_prefix = ctxt->b;
4531 case 0xf0: /* LOCK */
4532 ctxt->lock_prefix = 1;
4534 case 0xf2: /* REPNE/REPNZ */
4535 case 0xf3: /* REP/REPE/REPZ */
4536 ctxt->rep_prefix = ctxt->b;
4542 /* Any legacy prefix after a REX prefix nullifies its effect. */
4544 ctxt->rex_prefix = 0;
4550 if (ctxt->rex_prefix & 8)
4551 ctxt->op_bytes = 8; /* REX.W */
4553 /* Opcode byte(s). */
4554 opcode = opcode_table[ctxt->b];
4555 /* Two-byte opcode? */
4556 if (ctxt->b == 0x0f) {
4557 ctxt->opcode_len = 2;
4558 ctxt->b = insn_fetch(u8, ctxt);
4559 opcode = twobyte_table[ctxt->b];
4561 /* 0F_38 opcode map */
4562 if (ctxt->b == 0x38) {
4563 ctxt->opcode_len = 3;
4564 ctxt->b = insn_fetch(u8, ctxt);
4565 opcode = opcode_map_0f_38[ctxt->b];
4568 ctxt->d = opcode.flags;
4570 if (ctxt->d & ModRM)
4571 ctxt->modrm = insn_fetch(u8, ctxt);
4573 /* vex-prefix instructions are not implemented */
4574 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
4575 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
4579 while (ctxt->d & GroupMask) {
4580 switch (ctxt->d & GroupMask) {
4582 goffset = (ctxt->modrm >> 3) & 7;
4583 opcode = opcode.u.group[goffset];
4586 goffset = (ctxt->modrm >> 3) & 7;
4587 if ((ctxt->modrm >> 6) == 3)
4588 opcode = opcode.u.gdual->mod3[goffset];
4590 opcode = opcode.u.gdual->mod012[goffset];
4593 goffset = ctxt->modrm & 7;
4594 opcode = opcode.u.group[goffset];
4597 if (ctxt->rep_prefix && op_prefix)
4598 return EMULATION_FAILED;
4599 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4600 switch (simd_prefix) {
4601 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4602 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4603 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4604 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4608 if (ctxt->modrm > 0xbf)
4609 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4611 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4614 if ((ctxt->modrm >> 6) == 3)
4615 opcode = opcode.u.idual->mod3;
4617 opcode = opcode.u.idual->mod012;
4620 return EMULATION_FAILED;
4623 ctxt->d &= ~(u64)GroupMask;
4624 ctxt->d |= opcode.flags;
4629 return EMULATION_FAILED;
4631 ctxt->execute = opcode.u.execute;
4633 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
4634 return EMULATION_FAILED;
4636 if (unlikely(ctxt->d &
4637 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
4640 * These are copied unconditionally here, and checked unconditionally
4641 * in x86_emulate_insn.
4643 ctxt->check_perm = opcode.check_perm;
4644 ctxt->intercept = opcode.intercept;
4646 if (ctxt->d & NotImpl)
4647 return EMULATION_FAILED;
4649 if (mode == X86EMUL_MODE_PROT64) {
4650 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
4652 else if (ctxt->d & NearBranch)
4656 if (ctxt->d & Op3264) {
4657 if (mode == X86EMUL_MODE_PROT64)
4663 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
4667 ctxt->op_bytes = 16;
4668 else if (ctxt->d & Mmx)
4672 /* ModRM and SIB bytes. */
4673 if (ctxt->d & ModRM) {
4674 rc = decode_modrm(ctxt, &ctxt->memop);
4675 if (!has_seg_override) {
4676 has_seg_override = true;
4677 ctxt->seg_override = ctxt->modrm_seg;
4679 } else if (ctxt->d & MemAbs)
4680 rc = decode_abs(ctxt, &ctxt->memop);
4681 if (rc != X86EMUL_CONTINUE)
4684 if (!has_seg_override)
4685 ctxt->seg_override = VCPU_SREG_DS;
4687 ctxt->memop.addr.mem.seg = ctxt->seg_override;
4690 * Decode and fetch the source operand: register, memory
4693 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4694 if (rc != X86EMUL_CONTINUE)
4698 * Decode and fetch the second source operand: register, memory
4701 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4702 if (rc != X86EMUL_CONTINUE)
4705 /* Decode and fetch the destination operand: register or memory. */
4706 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4708 if (ctxt->rip_relative)
4709 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
4710 ctxt->memopp->addr.mem.ea + ctxt->_eip);
4713 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4716 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4718 return ctxt->d & PageTable;
4721 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4723 /* The second termination condition only applies for REPE
4724 * and REPNE. Test if the repeat string operation prefix is
4725 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4726 * corresponding termination condition according to:
4727 * - if REPE/REPZ and ZF = 0 then done
4728 * - if REPNE/REPNZ and ZF = 1 then done
4730 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4731 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4732 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4733 ((ctxt->eflags & EFLG_ZF) == 0))
4734 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4735 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4741 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4745 ctxt->ops->get_fpu(ctxt);
4746 asm volatile("1: fwait \n\t"
4748 ".pushsection .fixup,\"ax\" \n\t"
4750 "movb $1, %[fault] \n\t"
4753 _ASM_EXTABLE(1b, 3b)
4754 : [fault]"+qm"(fault));
4755 ctxt->ops->put_fpu(ctxt);
4757 if (unlikely(fault))
4758 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4760 return X86EMUL_CONTINUE;
4763 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4766 if (op->type == OP_MM)
4767 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4770 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4772 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4773 if (!(ctxt->d & ByteOp))
4774 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4775 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4776 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
4778 : "c"(ctxt->src2.val));
4779 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4780 if (!fop) /* exception is returned in fop variable */
4781 return emulate_de(ctxt);
4782 return X86EMUL_CONTINUE;
4785 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
4787 memset(&ctxt->rip_relative, 0,
4788 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
4790 ctxt->io_read.pos = 0;
4791 ctxt->io_read.end = 0;
4792 ctxt->mem_read.end = 0;
4795 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4797 const struct x86_emulate_ops *ops = ctxt->ops;
4798 int rc = X86EMUL_CONTINUE;
4799 int saved_dst_type = ctxt->dst.type;
4801 ctxt->mem_read.pos = 0;
4803 /* LOCK prefix is allowed only with some instructions */
4804 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4805 rc = emulate_ud(ctxt);
4809 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4810 rc = emulate_ud(ctxt);
4814 if (unlikely(ctxt->d &
4815 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
4816 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4817 (ctxt->d & Undefined)) {
4818 rc = emulate_ud(ctxt);
4822 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4823 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4824 rc = emulate_ud(ctxt);
4828 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4829 rc = emulate_nm(ctxt);
4833 if (ctxt->d & Mmx) {
4834 rc = flush_pending_x87_faults(ctxt);
4835 if (rc != X86EMUL_CONTINUE)
4838 * Now that we know the fpu is exception safe, we can fetch
4841 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4842 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4843 if (!(ctxt->d & Mov))
4844 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4847 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4848 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4849 X86_ICPT_PRE_EXCEPT);
4850 if (rc != X86EMUL_CONTINUE)
4854 /* Instruction can only be executed in protected mode */
4855 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4856 rc = emulate_ud(ctxt);
4860 /* Privileged instruction can be executed only in CPL=0 */
4861 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4862 if (ctxt->d & PrivUD)
4863 rc = emulate_ud(ctxt);
4865 rc = emulate_gp(ctxt, 0);
4869 /* Do instruction specific permission checks */
4870 if (ctxt->d & CheckPerm) {
4871 rc = ctxt->check_perm(ctxt);
4872 if (rc != X86EMUL_CONTINUE)
4876 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4877 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4878 X86_ICPT_POST_EXCEPT);
4879 if (rc != X86EMUL_CONTINUE)
4883 if (ctxt->rep_prefix && (ctxt->d & String)) {
4884 /* All REP prefixes have the same first termination condition */
4885 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4886 ctxt->eip = ctxt->_eip;
4887 ctxt->eflags &= ~EFLG_RF;
4893 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4894 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4895 ctxt->src.valptr, ctxt->src.bytes);
4896 if (rc != X86EMUL_CONTINUE)
4898 ctxt->src.orig_val64 = ctxt->src.val64;
4901 if (ctxt->src2.type == OP_MEM) {
4902 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4903 &ctxt->src2.val, ctxt->src2.bytes);
4904 if (rc != X86EMUL_CONTINUE)
4908 if ((ctxt->d & DstMask) == ImplicitOps)
4912 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4913 /* optimisation - avoid slow emulated read if Mov */
4914 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4915 &ctxt->dst.val, ctxt->dst.bytes);
4916 if (rc != X86EMUL_CONTINUE) {
4917 if (rc == X86EMUL_PROPAGATE_FAULT &&
4918 ctxt->exception.vector == PF_VECTOR)
4919 ctxt->exception.error_code |= PFERR_WRITE_MASK;
4923 ctxt->dst.orig_val = ctxt->dst.val;
4927 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4928 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4929 X86_ICPT_POST_MEMACCESS);
4930 if (rc != X86EMUL_CONTINUE)
4934 if (ctxt->rep_prefix && (ctxt->d & String))
4935 ctxt->eflags |= EFLG_RF;
4937 ctxt->eflags &= ~EFLG_RF;
4939 if (ctxt->execute) {
4940 if (ctxt->d & Fastop) {
4941 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4942 rc = fastop(ctxt, fop);
4943 if (rc != X86EMUL_CONTINUE)
4947 rc = ctxt->execute(ctxt);
4948 if (rc != X86EMUL_CONTINUE)
4953 if (ctxt->opcode_len == 2)
4955 else if (ctxt->opcode_len == 3)
4956 goto threebyte_insn;
4959 case 0x63: /* movsxd */
4960 if (ctxt->mode != X86EMUL_MODE_PROT64)
4961 goto cannot_emulate;
4962 ctxt->dst.val = (s32) ctxt->src.val;
4964 case 0x70 ... 0x7f: /* jcc (short) */
4965 if (test_cc(ctxt->b, ctxt->eflags))
4966 rc = jmp_rel(ctxt, ctxt->src.val);
4968 case 0x8d: /* lea r16/r32, m */
4969 ctxt->dst.val = ctxt->src.addr.mem.ea;
4971 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4972 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4973 ctxt->dst.type = OP_NONE;
4977 case 0x98: /* cbw/cwde/cdqe */
4978 switch (ctxt->op_bytes) {
4979 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4980 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4981 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4984 case 0xcc: /* int3 */
4985 rc = emulate_int(ctxt, 3);
4987 case 0xcd: /* int n */
4988 rc = emulate_int(ctxt, ctxt->src.val);
4990 case 0xce: /* into */
4991 if (ctxt->eflags & EFLG_OF)
4992 rc = emulate_int(ctxt, 4);
4994 case 0xe9: /* jmp rel */
4995 case 0xeb: /* jmp rel short */
4996 rc = jmp_rel(ctxt, ctxt->src.val);
4997 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4999 case 0xf4: /* hlt */
5000 ctxt->ops->halt(ctxt);
5002 case 0xf5: /* cmc */
5003 /* complement carry flag from eflags reg */
5004 ctxt->eflags ^= EFLG_CF;
5006 case 0xf8: /* clc */
5007 ctxt->eflags &= ~EFLG_CF;
5009 case 0xf9: /* stc */
5010 ctxt->eflags |= EFLG_CF;
5012 case 0xfc: /* cld */
5013 ctxt->eflags &= ~EFLG_DF;
5015 case 0xfd: /* std */
5016 ctxt->eflags |= EFLG_DF;
5019 goto cannot_emulate;
5022 if (rc != X86EMUL_CONTINUE)
5026 if (ctxt->d & SrcWrite) {
5027 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5028 rc = writeback(ctxt, &ctxt->src);
5029 if (rc != X86EMUL_CONTINUE)
5032 if (!(ctxt->d & NoWrite)) {
5033 rc = writeback(ctxt, &ctxt->dst);
5034 if (rc != X86EMUL_CONTINUE)
5039 * restore dst type in case the decoding will be reused
5040 * (happens for string instruction )
5042 ctxt->dst.type = saved_dst_type;
5044 if ((ctxt->d & SrcMask) == SrcSI)
5045 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5047 if ((ctxt->d & DstMask) == DstDI)
5048 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5050 if (ctxt->rep_prefix && (ctxt->d & String)) {
5052 struct read_cache *r = &ctxt->io_read;
5053 if ((ctxt->d & SrcMask) == SrcSI)
5054 count = ctxt->src.count;
5056 count = ctxt->dst.count;
5057 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5059 if (!string_insn_completed(ctxt)) {
5061 * Re-enter guest when pio read ahead buffer is empty
5062 * or, if it is not used, after each 1024 iteration.
5064 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5065 (r->end == 0 || r->end != r->pos)) {
5067 * Reset read cache. Usually happens before
5068 * decode, but since instruction is restarted
5069 * we have to do it here.
5071 ctxt->mem_read.end = 0;
5072 writeback_registers(ctxt);
5073 return EMULATION_RESTART;
5075 goto done; /* skip rip writeback */
5077 ctxt->eflags &= ~EFLG_RF;
5080 ctxt->eip = ctxt->_eip;
5083 if (rc == X86EMUL_PROPAGATE_FAULT) {
5084 WARN_ON(ctxt->exception.vector > 0x1f);
5085 ctxt->have_exception = true;
5087 if (rc == X86EMUL_INTERCEPTED)
5088 return EMULATION_INTERCEPTED;
5090 if (rc == X86EMUL_CONTINUE)
5091 writeback_registers(ctxt);
5093 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5097 case 0x09: /* wbinvd */
5098 (ctxt->ops->wbinvd)(ctxt);
5100 case 0x08: /* invd */
5101 case 0x0d: /* GrpP (prefetch) */
5102 case 0x18: /* Grp16 (prefetch/nop) */
5103 case 0x1f: /* nop */
5105 case 0x20: /* mov cr, reg */
5106 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5108 case 0x21: /* mov from dr to reg */
5109 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5111 case 0x40 ... 0x4f: /* cmov */
5112 if (test_cc(ctxt->b, ctxt->eflags))
5113 ctxt->dst.val = ctxt->src.val;
5114 else if (ctxt->mode != X86EMUL_MODE_PROT64 ||
5115 ctxt->op_bytes != 4)
5116 ctxt->dst.type = OP_NONE; /* no writeback */
5118 case 0x80 ... 0x8f: /* jnz rel, etc*/
5119 if (test_cc(ctxt->b, ctxt->eflags))
5120 rc = jmp_rel(ctxt, ctxt->src.val);
5122 case 0x90 ... 0x9f: /* setcc r/m8 */
5123 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5125 case 0xb6 ... 0xb7: /* movzx */
5126 ctxt->dst.bytes = ctxt->op_bytes;
5127 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5128 : (u16) ctxt->src.val;
5130 case 0xbe ... 0xbf: /* movsx */
5131 ctxt->dst.bytes = ctxt->op_bytes;
5132 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5133 (s16) ctxt->src.val;
5136 goto cannot_emulate;
5141 if (rc != X86EMUL_CONTINUE)
5147 return EMULATION_FAILED;
5150 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5152 invalidate_registers(ctxt);
5155 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5157 writeback_registers(ctxt);