1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstMem16 (OpMem16 << DstShift)
90 #define DstImmUByte (OpImmUByte << DstShift)
91 #define DstDX (OpDX << DstShift)
92 #define DstAccLo (OpAccLo << DstShift)
93 #define DstMask (OpMask << DstShift)
94 /* Source operand type. */
96 #define SrcNone (OpNone << SrcShift)
97 #define SrcReg (OpReg << SrcShift)
98 #define SrcMem (OpMem << SrcShift)
99 #define SrcMem16 (OpMem16 << SrcShift)
100 #define SrcMem32 (OpMem32 << SrcShift)
101 #define SrcImm (OpImm << SrcShift)
102 #define SrcImmByte (OpImmByte << SrcShift)
103 #define SrcOne (OpOne << SrcShift)
104 #define SrcImmUByte (OpImmUByte << SrcShift)
105 #define SrcImmU (OpImmU << SrcShift)
106 #define SrcSI (OpSI << SrcShift)
107 #define SrcXLat (OpXLat << SrcShift)
108 #define SrcImmFAddr (OpImmFAddr << SrcShift)
109 #define SrcMemFAddr (OpMemFAddr << SrcShift)
110 #define SrcAcc (OpAcc << SrcShift)
111 #define SrcImmU16 (OpImmU16 << SrcShift)
112 #define SrcImm64 (OpImm64 << SrcShift)
113 #define SrcDX (OpDX << SrcShift)
114 #define SrcMem8 (OpMem8 << SrcShift)
115 #define SrcAccHi (OpAccHi << SrcShift)
116 #define SrcMask (OpMask << SrcShift)
117 #define BitOp (1<<11)
118 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
119 #define String (1<<13) /* String instruction (rep capable) */
120 #define Stack (1<<14) /* Stack instruction (push/pop) */
121 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
122 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
123 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
124 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
125 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
126 #define Escape (5<<15) /* Escape to coprocessor instruction */
127 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
128 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
129 #define Sse (1<<18) /* SSE Vector instruction */
130 /* Generic ModRM decode. */
131 #define ModRM (1<<19)
132 /* Destination is only written; never read. */
135 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
136 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
137 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
138 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
139 #define Undefined (1<<25) /* No Such Instruction */
140 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
141 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
143 #define PageTable (1 << 29) /* instruction used to write page table */
144 #define NotImpl (1 << 30) /* instruction is not implemented */
145 /* Source 2 operand type */
146 #define Src2Shift (31)
147 #define Src2None (OpNone << Src2Shift)
148 #define Src2Mem (OpMem << Src2Shift)
149 #define Src2CL (OpCL << Src2Shift)
150 #define Src2ImmByte (OpImmByte << Src2Shift)
151 #define Src2One (OpOne << Src2Shift)
152 #define Src2Imm (OpImm << Src2Shift)
153 #define Src2ES (OpES << Src2Shift)
154 #define Src2CS (OpCS << Src2Shift)
155 #define Src2SS (OpSS << Src2Shift)
156 #define Src2DS (OpDS << Src2Shift)
157 #define Src2FS (OpFS << Src2Shift)
158 #define Src2GS (OpGS << Src2Shift)
159 #define Src2Mask (OpMask << Src2Shift)
160 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
161 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
162 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
163 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
164 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
165 #define NoWrite ((u64)1 << 45) /* No writeback */
166 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
167 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
168 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
169 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
170 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
171 #define NearBranch ((u64)1 << 52) /* Near branches */
172 #define No16 ((u64)1 << 53) /* No 16 bit operand */
173 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
175 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
177 #define X2(x...) x, x
178 #define X3(x...) X2(x), x
179 #define X4(x...) X2(x), X2(x)
180 #define X5(x...) X4(x), x
181 #define X6(x...) X4(x), X2(x)
182 #define X7(x...) X4(x), X3(x)
183 #define X8(x...) X4(x), X4(x)
184 #define X16(x...) X8(x), X8(x)
186 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
187 #define FASTOP_SIZE 8
190 * fastop functions have a special calling convention:
195 * flags: rflags (in/out)
196 * ex: rsi (in:fastop pointer, out:zero if exception)
198 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
199 * different operand sizes can be reached by calculation, rather than a jump
200 * table (which would be bigger than the code).
202 * fastop functions are declared as taking a never-defined fastop parameter,
203 * so they can't be called from C directly.
212 int (*execute)(struct x86_emulate_ctxt *ctxt);
213 const struct opcode *group;
214 const struct group_dual *gdual;
215 const struct gprefix *gprefix;
216 const struct escape *esc;
217 const struct instr_dual *idual;
218 const struct mode_dual *mdual;
219 void (*fastop)(struct fastop *fake);
221 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
225 struct opcode mod012[8];
226 struct opcode mod3[8];
230 struct opcode pfx_no;
231 struct opcode pfx_66;
232 struct opcode pfx_f2;
233 struct opcode pfx_f3;
238 struct opcode high[64];
242 struct opcode mod012;
247 struct opcode mode32;
248 struct opcode mode64;
251 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
253 enum x86_transfer_type {
255 X86_TRANSFER_CALL_JMP,
257 X86_TRANSFER_TASK_SWITCH,
260 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
262 if (!(ctxt->regs_valid & (1 << nr))) {
263 ctxt->regs_valid |= 1 << nr;
264 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
266 return ctxt->_regs[nr];
269 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
271 ctxt->regs_valid |= 1 << nr;
272 ctxt->regs_dirty |= 1 << nr;
273 return &ctxt->_regs[nr];
276 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
279 return reg_write(ctxt, nr);
282 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
286 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
287 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
290 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
292 ctxt->regs_dirty = 0;
293 ctxt->regs_valid = 0;
297 * These EFLAGS bits are restored from saved value during emulation, and
298 * any changes are written back to the saved value after emulation.
300 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
301 X86_EFLAGS_PF|X86_EFLAGS_CF)
309 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
311 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
312 #define FOP_RET "ret \n\t"
314 #define FOP_START(op) \
315 extern void em_##op(struct fastop *fake); \
316 asm(".pushsection .text, \"ax\" \n\t" \
317 ".global em_" #op " \n\t" \
324 #define FOPNOP() FOP_ALIGN FOP_RET
326 #define FOP1E(op, dst) \
327 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
329 #define FOP1EEX(op, dst) \
330 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
332 #define FASTOP1(op) \
337 ON64(FOP1E(op##q, rax)) \
340 /* 1-operand, using src2 (for MUL/DIV r/m) */
341 #define FASTOP1SRC2(op, name) \
346 ON64(FOP1E(op, rcx)) \
349 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
350 #define FASTOP1SRC2EX(op, name) \
355 ON64(FOP1EEX(op, rcx)) \
358 #define FOP2E(op, dst, src) \
359 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
361 #define FASTOP2(op) \
363 FOP2E(op##b, al, dl) \
364 FOP2E(op##w, ax, dx) \
365 FOP2E(op##l, eax, edx) \
366 ON64(FOP2E(op##q, rax, rdx)) \
369 /* 2 operand, word only */
370 #define FASTOP2W(op) \
373 FOP2E(op##w, ax, dx) \
374 FOP2E(op##l, eax, edx) \
375 ON64(FOP2E(op##q, rax, rdx)) \
378 /* 2 operand, src is CL */
379 #define FASTOP2CL(op) \
381 FOP2E(op##b, al, cl) \
382 FOP2E(op##w, ax, cl) \
383 FOP2E(op##l, eax, cl) \
384 ON64(FOP2E(op##q, rax, cl)) \
387 /* 2 operand, src and dest are reversed */
388 #define FASTOP2R(op, name) \
390 FOP2E(op##b, dl, al) \
391 FOP2E(op##w, dx, ax) \
392 FOP2E(op##l, edx, eax) \
393 ON64(FOP2E(op##q, rdx, rax)) \
396 #define FOP3E(op, dst, src, src2) \
397 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
399 /* 3-operand, word-only, src2=cl */
400 #define FASTOP3WCL(op) \
403 FOP3E(op##w, ax, dx, cl) \
404 FOP3E(op##l, eax, edx, cl) \
405 ON64(FOP3E(op##q, rax, rdx, cl)) \
408 /* Special case for SETcc - 1 instruction per cc */
409 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
411 asm(".global kvm_fastop_exception \n"
412 "kvm_fastop_exception: xor %esi, %esi; ret");
433 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
436 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
437 enum x86_intercept intercept,
438 enum x86_intercept_stage stage)
440 struct x86_instruction_info info = {
441 .intercept = intercept,
442 .rep_prefix = ctxt->rep_prefix,
443 .modrm_mod = ctxt->modrm_mod,
444 .modrm_reg = ctxt->modrm_reg,
445 .modrm_rm = ctxt->modrm_rm,
446 .src_val = ctxt->src.val64,
447 .dst_val = ctxt->dst.val64,
448 .src_bytes = ctxt->src.bytes,
449 .dst_bytes = ctxt->dst.bytes,
450 .ad_bytes = ctxt->ad_bytes,
451 .next_rip = ctxt->eip,
454 return ctxt->ops->intercept(ctxt, &info, stage);
457 static void assign_masked(ulong *dest, ulong src, ulong mask)
459 *dest = (*dest & ~mask) | (src & mask);
462 static void assign_register(unsigned long *reg, u64 val, int bytes)
464 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
467 *(u8 *)reg = (u8)val;
470 *(u16 *)reg = (u16)val;
474 break; /* 64b: zero-extend */
481 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
483 return (1UL << (ctxt->ad_bytes << 3)) - 1;
486 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
489 struct desc_struct ss;
491 if (ctxt->mode == X86EMUL_MODE_PROT64)
493 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
494 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
497 static int stack_size(struct x86_emulate_ctxt *ctxt)
499 return (__fls(stack_mask(ctxt)) + 1) >> 3;
502 /* Access/update address held in a register, based on addressing mode. */
503 static inline unsigned long
504 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
506 if (ctxt->ad_bytes == sizeof(unsigned long))
509 return reg & ad_mask(ctxt);
512 static inline unsigned long
513 register_address(struct x86_emulate_ctxt *ctxt, int reg)
515 return address_mask(ctxt, reg_read(ctxt, reg));
518 static void masked_increment(ulong *reg, ulong mask, int inc)
520 assign_masked(reg, *reg + inc, mask);
524 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
528 if (ctxt->ad_bytes == sizeof(unsigned long))
531 mask = ad_mask(ctxt);
532 masked_increment(reg_rmw(ctxt, reg), mask, inc);
535 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
537 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
540 static u32 desc_limit_scaled(struct desc_struct *desc)
542 u32 limit = get_desc_limit(desc);
544 return desc->g ? (limit << 12) | 0xfff : limit;
547 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
549 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
552 return ctxt->ops->get_cached_segment_base(ctxt, seg);
555 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
556 u32 error, bool valid)
559 ctxt->exception.vector = vec;
560 ctxt->exception.error_code = error;
561 ctxt->exception.error_code_valid = valid;
562 return X86EMUL_PROPAGATE_FAULT;
565 static int emulate_db(struct x86_emulate_ctxt *ctxt)
567 return emulate_exception(ctxt, DB_VECTOR, 0, false);
570 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
572 return emulate_exception(ctxt, GP_VECTOR, err, true);
575 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
577 return emulate_exception(ctxt, SS_VECTOR, err, true);
580 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
582 return emulate_exception(ctxt, UD_VECTOR, 0, false);
585 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
587 return emulate_exception(ctxt, TS_VECTOR, err, true);
590 static int emulate_de(struct x86_emulate_ctxt *ctxt)
592 return emulate_exception(ctxt, DE_VECTOR, 0, false);
595 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
597 return emulate_exception(ctxt, NM_VECTOR, 0, false);
600 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
603 struct desc_struct desc;
605 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
609 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
614 struct desc_struct desc;
616 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
617 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
621 * x86 defines three classes of vector instructions: explicitly
622 * aligned, explicitly unaligned, and the rest, which change behaviour
623 * depending on whether they're AVX encoded or not.
625 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
626 * subject to the same check.
628 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
630 if (likely(size < 16))
633 if (ctxt->d & Aligned)
635 else if (ctxt->d & Unaligned)
637 else if (ctxt->d & Avx)
643 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
644 struct segmented_address addr,
645 unsigned *max_size, unsigned size,
646 bool write, bool fetch,
647 enum x86emul_mode mode, ulong *linear)
649 struct desc_struct desc;
655 la = seg_base(ctxt, addr.seg) + addr.ea;
658 case X86EMUL_MODE_PROT64:
659 if (is_noncanonical_address(la))
662 *max_size = min_t(u64, ~0u, (1ull << 48) - la);
663 if (size > *max_size)
667 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
671 /* code segment in protected mode or read-only data segment */
672 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
673 || !(desc.type & 2)) && write)
675 /* unreadable code segment */
676 if (!fetch && (desc.type & 8) && !(desc.type & 2))
678 lim = desc_limit_scaled(&desc);
679 if (!(desc.type & 8) && (desc.type & 4)) {
680 /* expand-down segment */
683 lim = desc.d ? 0xffffffff : 0xffff;
687 if (lim == 0xffffffff)
690 *max_size = (u64)lim + 1 - addr.ea;
691 if (size > *max_size)
697 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
698 return emulate_gp(ctxt, 0);
700 return X86EMUL_CONTINUE;
702 if (addr.seg == VCPU_SREG_SS)
703 return emulate_ss(ctxt, 0);
705 return emulate_gp(ctxt, 0);
708 static int linearize(struct x86_emulate_ctxt *ctxt,
709 struct segmented_address addr,
710 unsigned size, bool write,
714 return __linearize(ctxt, addr, &max_size, size, write, false,
718 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
719 enum x86emul_mode mode)
724 struct segmented_address addr = { .seg = VCPU_SREG_CS,
727 if (ctxt->op_bytes != sizeof(unsigned long))
728 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
729 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
730 if (rc == X86EMUL_CONTINUE)
731 ctxt->_eip = addr.ea;
735 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
737 return assign_eip(ctxt, dst, ctxt->mode);
740 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
741 const struct desc_struct *cs_desc)
743 enum x86emul_mode mode = ctxt->mode;
747 if (ctxt->mode >= X86EMUL_MODE_PROT16) {
751 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
753 mode = X86EMUL_MODE_PROT64;
755 mode = X86EMUL_MODE_PROT32; /* temporary value */
758 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
759 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
760 rc = assign_eip(ctxt, dst, mode);
761 if (rc == X86EMUL_CONTINUE)
766 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
768 return assign_eip_near(ctxt, ctxt->_eip + rel);
771 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
772 struct segmented_address addr,
779 rc = linearize(ctxt, addr, size, false, &linear);
780 if (rc != X86EMUL_CONTINUE)
782 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
786 * Prefetch the remaining bytes of the instruction without crossing page
787 * boundary if they are not in fetch_cache yet.
789 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
792 unsigned size, max_size;
793 unsigned long linear;
794 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
795 struct segmented_address addr = { .seg = VCPU_SREG_CS,
796 .ea = ctxt->eip + cur_size };
799 * We do not know exactly how many bytes will be needed, and
800 * __linearize is expensive, so fetch as much as possible. We
801 * just have to avoid going beyond the 15 byte limit, the end
802 * of the segment, or the end of the page.
804 * __linearize is called with size 0 so that it does not do any
805 * boundary check itself. Instead, we use max_size to check
808 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
810 if (unlikely(rc != X86EMUL_CONTINUE))
813 size = min_t(unsigned, 15UL ^ cur_size, max_size);
814 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
817 * One instruction can only straddle two pages,
818 * and one has been loaded at the beginning of
819 * x86_decode_insn. So, if not enough bytes
820 * still, we must have hit the 15-byte boundary.
822 if (unlikely(size < op_size))
823 return emulate_gp(ctxt, 0);
825 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
826 size, &ctxt->exception);
827 if (unlikely(rc != X86EMUL_CONTINUE))
829 ctxt->fetch.end += size;
830 return X86EMUL_CONTINUE;
833 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
836 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
838 if (unlikely(done_size < size))
839 return __do_insn_fetch_bytes(ctxt, size - done_size);
841 return X86EMUL_CONTINUE;
844 /* Fetch next part of the instruction being emulated. */
845 #define insn_fetch(_type, _ctxt) \
848 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
849 if (rc != X86EMUL_CONTINUE) \
851 ctxt->_eip += sizeof(_type); \
852 _x = *(_type __aligned(1) *) ctxt->fetch.ptr; \
853 ctxt->fetch.ptr += sizeof(_type); \
857 #define insn_fetch_arr(_arr, _size, _ctxt) \
859 rc = do_insn_fetch_bytes(_ctxt, _size); \
860 if (rc != X86EMUL_CONTINUE) \
862 ctxt->_eip += (_size); \
863 memcpy(_arr, ctxt->fetch.ptr, _size); \
864 ctxt->fetch.ptr += (_size); \
868 * Given the 'reg' portion of a ModRM byte, and a register block, return a
869 * pointer into the block that addresses the relevant register.
870 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
872 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
876 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
878 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
879 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
881 p = reg_rmw(ctxt, modrm_reg);
885 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
886 struct segmented_address addr,
887 u16 *size, unsigned long *address, int op_bytes)
894 rc = segmented_read_std(ctxt, addr, size, 2);
895 if (rc != X86EMUL_CONTINUE)
898 rc = segmented_read_std(ctxt, addr, address, op_bytes);
912 FASTOP1SRC2(mul, mul_ex);
913 FASTOP1SRC2(imul, imul_ex);
914 FASTOP1SRC2EX(div, div_ex);
915 FASTOP1SRC2EX(idiv, idiv_ex);
944 FASTOP2R(cmp, cmp_r);
946 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
948 /* If src is zero, do not writeback, but update flags */
949 if (ctxt->src.val == 0)
950 ctxt->dst.type = OP_NONE;
951 return fastop(ctxt, em_bsf);
954 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
956 /* If src is zero, do not writeback, but update flags */
957 if (ctxt->src.val == 0)
958 ctxt->dst.type = OP_NONE;
959 return fastop(ctxt, em_bsr);
962 static u8 test_cc(unsigned int condition, unsigned long flags)
965 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
967 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
968 asm("push %[flags]; popf; call *%[fastop]"
969 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
973 static void fetch_register_operand(struct operand *op)
977 op->val = *(u8 *)op->addr.reg;
980 op->val = *(u16 *)op->addr.reg;
983 op->val = *(u32 *)op->addr.reg;
986 op->val = *(u64 *)op->addr.reg;
991 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
993 ctxt->ops->get_fpu(ctxt);
995 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
996 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
997 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
998 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
999 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
1000 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
1001 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
1002 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1003 #ifdef CONFIG_X86_64
1004 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1005 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1006 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1007 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1008 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1009 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1010 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1011 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1015 ctxt->ops->put_fpu(ctxt);
1018 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1021 ctxt->ops->get_fpu(ctxt);
1023 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1024 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1025 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1026 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1027 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1028 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1029 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1030 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1031 #ifdef CONFIG_X86_64
1032 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1033 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1034 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1035 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1036 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1037 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1038 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1039 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1043 ctxt->ops->put_fpu(ctxt);
1046 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1048 ctxt->ops->get_fpu(ctxt);
1050 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1051 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1052 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1053 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1054 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1055 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1056 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1057 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1060 ctxt->ops->put_fpu(ctxt);
1063 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1065 ctxt->ops->get_fpu(ctxt);
1067 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1068 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1069 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1070 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1071 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1072 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1073 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1074 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1077 ctxt->ops->put_fpu(ctxt);
1080 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1082 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1083 return emulate_nm(ctxt);
1085 ctxt->ops->get_fpu(ctxt);
1086 asm volatile("fninit");
1087 ctxt->ops->put_fpu(ctxt);
1088 return X86EMUL_CONTINUE;
1091 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1095 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1096 return emulate_nm(ctxt);
1098 ctxt->ops->get_fpu(ctxt);
1099 asm volatile("fnstcw %0": "+m"(fcw));
1100 ctxt->ops->put_fpu(ctxt);
1102 ctxt->dst.val = fcw;
1104 return X86EMUL_CONTINUE;
1107 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1111 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1112 return emulate_nm(ctxt);
1114 ctxt->ops->get_fpu(ctxt);
1115 asm volatile("fnstsw %0": "+m"(fsw));
1116 ctxt->ops->put_fpu(ctxt);
1118 ctxt->dst.val = fsw;
1120 return X86EMUL_CONTINUE;
1123 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1126 unsigned reg = ctxt->modrm_reg;
1128 if (!(ctxt->d & ModRM))
1129 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1131 if (ctxt->d & Sse) {
1135 read_sse_reg(ctxt, &op->vec_val, reg);
1138 if (ctxt->d & Mmx) {
1147 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1148 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1150 fetch_register_operand(op);
1151 op->orig_val = op->val;
1154 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1156 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1157 ctxt->modrm_seg = VCPU_SREG_SS;
1160 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1164 int index_reg, base_reg, scale;
1165 int rc = X86EMUL_CONTINUE;
1168 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1169 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1170 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1172 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1173 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1174 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1175 ctxt->modrm_seg = VCPU_SREG_DS;
1177 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1179 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1180 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1182 if (ctxt->d & Sse) {
1185 op->addr.xmm = ctxt->modrm_rm;
1186 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1189 if (ctxt->d & Mmx) {
1192 op->addr.mm = ctxt->modrm_rm & 7;
1195 fetch_register_operand(op);
1201 if (ctxt->ad_bytes == 2) {
1202 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1203 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1204 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1205 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1207 /* 16-bit ModR/M decode. */
1208 switch (ctxt->modrm_mod) {
1210 if (ctxt->modrm_rm == 6)
1211 modrm_ea += insn_fetch(u16, ctxt);
1214 modrm_ea += insn_fetch(s8, ctxt);
1217 modrm_ea += insn_fetch(u16, ctxt);
1220 switch (ctxt->modrm_rm) {
1222 modrm_ea += bx + si;
1225 modrm_ea += bx + di;
1228 modrm_ea += bp + si;
1231 modrm_ea += bp + di;
1240 if (ctxt->modrm_mod != 0)
1247 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1248 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1249 ctxt->modrm_seg = VCPU_SREG_SS;
1250 modrm_ea = (u16)modrm_ea;
1252 /* 32/64-bit ModR/M decode. */
1253 if ((ctxt->modrm_rm & 7) == 4) {
1254 sib = insn_fetch(u8, ctxt);
1255 index_reg |= (sib >> 3) & 7;
1256 base_reg |= sib & 7;
1259 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1260 modrm_ea += insn_fetch(s32, ctxt);
1262 modrm_ea += reg_read(ctxt, base_reg);
1263 adjust_modrm_seg(ctxt, base_reg);
1264 /* Increment ESP on POP [ESP] */
1265 if ((ctxt->d & IncSP) &&
1266 base_reg == VCPU_REGS_RSP)
1267 modrm_ea += ctxt->op_bytes;
1270 modrm_ea += reg_read(ctxt, index_reg) << scale;
1271 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1272 modrm_ea += insn_fetch(s32, ctxt);
1273 if (ctxt->mode == X86EMUL_MODE_PROT64)
1274 ctxt->rip_relative = 1;
1276 base_reg = ctxt->modrm_rm;
1277 modrm_ea += reg_read(ctxt, base_reg);
1278 adjust_modrm_seg(ctxt, base_reg);
1280 switch (ctxt->modrm_mod) {
1282 modrm_ea += insn_fetch(s8, ctxt);
1285 modrm_ea += insn_fetch(s32, ctxt);
1289 op->addr.mem.ea = modrm_ea;
1290 if (ctxt->ad_bytes != 8)
1291 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1297 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1300 int rc = X86EMUL_CONTINUE;
1303 switch (ctxt->ad_bytes) {
1305 op->addr.mem.ea = insn_fetch(u16, ctxt);
1308 op->addr.mem.ea = insn_fetch(u32, ctxt);
1311 op->addr.mem.ea = insn_fetch(u64, ctxt);
1318 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1322 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1323 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1325 if (ctxt->src.bytes == 2)
1326 sv = (s16)ctxt->src.val & (s16)mask;
1327 else if (ctxt->src.bytes == 4)
1328 sv = (s32)ctxt->src.val & (s32)mask;
1330 sv = (s64)ctxt->src.val & (s64)mask;
1332 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1333 ctxt->dst.addr.mem.ea + (sv >> 3));
1336 /* only subword offset */
1337 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1340 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1341 unsigned long addr, void *dest, unsigned size)
1344 struct read_cache *mc = &ctxt->mem_read;
1346 if (mc->pos < mc->end)
1349 WARN_ON((mc->end + size) >= sizeof(mc->data));
1351 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1353 if (rc != X86EMUL_CONTINUE)
1359 memcpy(dest, mc->data + mc->pos, size);
1361 return X86EMUL_CONTINUE;
1364 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1365 struct segmented_address addr,
1372 rc = linearize(ctxt, addr, size, false, &linear);
1373 if (rc != X86EMUL_CONTINUE)
1375 return read_emulated(ctxt, linear, data, size);
1378 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1379 struct segmented_address addr,
1386 rc = linearize(ctxt, addr, size, true, &linear);
1387 if (rc != X86EMUL_CONTINUE)
1389 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1393 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1394 struct segmented_address addr,
1395 const void *orig_data, const void *data,
1401 rc = linearize(ctxt, addr, size, true, &linear);
1402 if (rc != X86EMUL_CONTINUE)
1404 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1405 size, &ctxt->exception);
1408 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1409 unsigned int size, unsigned short port,
1412 struct read_cache *rc = &ctxt->io_read;
1414 if (rc->pos == rc->end) { /* refill pio read ahead */
1415 unsigned int in_page, n;
1416 unsigned int count = ctxt->rep_prefix ?
1417 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1418 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1419 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1420 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1421 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1424 rc->pos = rc->end = 0;
1425 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1430 if (ctxt->rep_prefix && (ctxt->d & String) &&
1431 !(ctxt->eflags & X86_EFLAGS_DF)) {
1432 ctxt->dst.data = rc->data + rc->pos;
1433 ctxt->dst.type = OP_MEM_STR;
1434 ctxt->dst.count = (rc->end - rc->pos) / size;
1437 memcpy(dest, rc->data + rc->pos, size);
1443 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1444 u16 index, struct desc_struct *desc)
1449 ctxt->ops->get_idt(ctxt, &dt);
1451 if (dt.size < index * 8 + 7)
1452 return emulate_gp(ctxt, index << 3 | 0x2);
1454 addr = dt.address + index * 8;
1455 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1459 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1460 u16 selector, struct desc_ptr *dt)
1462 const struct x86_emulate_ops *ops = ctxt->ops;
1465 if (selector & 1 << 2) {
1466 struct desc_struct desc;
1469 memset (dt, 0, sizeof *dt);
1470 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1474 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1475 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1477 ops->get_gdt(ctxt, dt);
1480 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1481 u16 selector, ulong *desc_addr_p)
1484 u16 index = selector >> 3;
1487 get_descriptor_table_ptr(ctxt, selector, &dt);
1489 if (dt.size < index * 8 + 7)
1490 return emulate_gp(ctxt, selector & 0xfffc);
1492 addr = dt.address + index * 8;
1494 #ifdef CONFIG_X86_64
1495 if (addr >> 32 != 0) {
1498 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1499 if (!(efer & EFER_LMA))
1504 *desc_addr_p = addr;
1505 return X86EMUL_CONTINUE;
1508 /* allowed just for 8 bytes segments */
1509 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1510 u16 selector, struct desc_struct *desc,
1515 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1516 if (rc != X86EMUL_CONTINUE)
1519 return ctxt->ops->read_std(ctxt, *desc_addr_p, desc, sizeof(*desc),
1523 /* allowed just for 8 bytes segments */
1524 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1525 u16 selector, struct desc_struct *desc)
1530 rc = get_descriptor_ptr(ctxt, selector, &addr);
1531 if (rc != X86EMUL_CONTINUE)
1534 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1538 /* Does not support long mode */
1539 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1540 u16 selector, int seg, u8 cpl,
1541 enum x86_transfer_type transfer,
1542 struct desc_struct *desc)
1544 struct desc_struct seg_desc, old_desc;
1546 unsigned err_vec = GP_VECTOR;
1548 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1554 memset(&seg_desc, 0, sizeof seg_desc);
1556 if (ctxt->mode == X86EMUL_MODE_REAL) {
1557 /* set real mode segment descriptor (keep limit etc. for
1559 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1560 set_desc_base(&seg_desc, selector << 4);
1562 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1563 /* VM86 needs a clean new segment descriptor */
1564 set_desc_base(&seg_desc, selector << 4);
1565 set_desc_limit(&seg_desc, 0xffff);
1575 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1576 if ((seg == VCPU_SREG_CS
1577 || (seg == VCPU_SREG_SS
1578 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1579 || seg == VCPU_SREG_TR)
1583 /* TR should be in GDT only */
1584 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1587 if (null_selector) /* for NULL selector skip all following checks */
1590 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1591 if (ret != X86EMUL_CONTINUE)
1594 err_code = selector & 0xfffc;
1595 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1598 /* can't load system descriptor into segment selector */
1599 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1600 if (transfer == X86_TRANSFER_CALL_JMP)
1601 return X86EMUL_UNHANDLEABLE;
1606 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1615 * segment is not a writable data segment or segment
1616 * selector's RPL != CPL or segment selector's RPL != CPL
1618 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1622 if (!(seg_desc.type & 8))
1625 if (seg_desc.type & 4) {
1631 if (rpl > cpl || dpl != cpl)
1634 /* in long-mode d/b must be clear if l is set */
1635 if (seg_desc.d && seg_desc.l) {
1638 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1639 if (efer & EFER_LMA)
1643 /* CS(RPL) <- CPL */
1644 selector = (selector & 0xfffc) | cpl;
1647 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1649 old_desc = seg_desc;
1650 seg_desc.type |= 2; /* busy */
1651 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1652 sizeof(seg_desc), &ctxt->exception);
1653 if (ret != X86EMUL_CONTINUE)
1656 case VCPU_SREG_LDTR:
1657 if (seg_desc.s || seg_desc.type != 2)
1660 default: /* DS, ES, FS, or GS */
1662 * segment is not a data or readable code segment or
1663 * ((segment is a data or nonconforming code segment)
1664 * and (both RPL and CPL > DPL))
1666 if ((seg_desc.type & 0xa) == 0x8 ||
1667 (((seg_desc.type & 0xc) != 0xc) &&
1668 (rpl > dpl && cpl > dpl)))
1674 /* mark segment as accessed */
1675 if (!(seg_desc.type & 1)) {
1677 ret = write_segment_descriptor(ctxt, selector,
1679 if (ret != X86EMUL_CONTINUE)
1682 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1683 ret = ctxt->ops->read_std(ctxt, desc_addr+8, &base3,
1684 sizeof(base3), &ctxt->exception);
1685 if (ret != X86EMUL_CONTINUE)
1687 if (is_noncanonical_address(get_desc_base(&seg_desc) |
1688 ((u64)base3 << 32)))
1689 return emulate_gp(ctxt, 0);
1692 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1695 return X86EMUL_CONTINUE;
1697 return emulate_exception(ctxt, err_vec, err_code, true);
1700 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1701 u16 selector, int seg)
1703 u8 cpl = ctxt->ops->cpl(ctxt);
1704 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1705 X86_TRANSFER_NONE, NULL);
1708 static void write_register_operand(struct operand *op)
1710 return assign_register(op->addr.reg, op->val, op->bytes);
1713 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1717 write_register_operand(op);
1720 if (ctxt->lock_prefix)
1721 return segmented_cmpxchg(ctxt,
1727 return segmented_write(ctxt,
1733 return segmented_write(ctxt,
1736 op->bytes * op->count);
1739 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1742 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1750 return X86EMUL_CONTINUE;
1753 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1755 struct segmented_address addr;
1757 rsp_increment(ctxt, -bytes);
1758 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1759 addr.seg = VCPU_SREG_SS;
1761 return segmented_write(ctxt, addr, data, bytes);
1764 static int em_push(struct x86_emulate_ctxt *ctxt)
1766 /* Disable writeback. */
1767 ctxt->dst.type = OP_NONE;
1768 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1771 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1772 void *dest, int len)
1775 struct segmented_address addr;
1777 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1778 addr.seg = VCPU_SREG_SS;
1779 rc = segmented_read(ctxt, addr, dest, len);
1780 if (rc != X86EMUL_CONTINUE)
1783 rsp_increment(ctxt, len);
1787 static int em_pop(struct x86_emulate_ctxt *ctxt)
1789 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1792 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1793 void *dest, int len)
1796 unsigned long val, change_mask;
1797 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1798 int cpl = ctxt->ops->cpl(ctxt);
1800 rc = emulate_pop(ctxt, &val, len);
1801 if (rc != X86EMUL_CONTINUE)
1804 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1805 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1806 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1807 X86_EFLAGS_AC | X86_EFLAGS_ID;
1809 switch(ctxt->mode) {
1810 case X86EMUL_MODE_PROT64:
1811 case X86EMUL_MODE_PROT32:
1812 case X86EMUL_MODE_PROT16:
1814 change_mask |= X86_EFLAGS_IOPL;
1816 change_mask |= X86_EFLAGS_IF;
1818 case X86EMUL_MODE_VM86:
1820 return emulate_gp(ctxt, 0);
1821 change_mask |= X86_EFLAGS_IF;
1823 default: /* real mode */
1824 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1828 *(unsigned long *)dest =
1829 (ctxt->eflags & ~change_mask) | (val & change_mask);
1834 static int em_popf(struct x86_emulate_ctxt *ctxt)
1836 ctxt->dst.type = OP_REG;
1837 ctxt->dst.addr.reg = &ctxt->eflags;
1838 ctxt->dst.bytes = ctxt->op_bytes;
1839 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1842 static int em_enter(struct x86_emulate_ctxt *ctxt)
1845 unsigned frame_size = ctxt->src.val;
1846 unsigned nesting_level = ctxt->src2.val & 31;
1850 return X86EMUL_UNHANDLEABLE;
1852 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1853 rc = push(ctxt, &rbp, stack_size(ctxt));
1854 if (rc != X86EMUL_CONTINUE)
1856 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1858 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1859 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1861 return X86EMUL_CONTINUE;
1864 static int em_leave(struct x86_emulate_ctxt *ctxt)
1866 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1868 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1871 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1873 int seg = ctxt->src2.val;
1875 ctxt->src.val = get_segment_selector(ctxt, seg);
1876 if (ctxt->op_bytes == 4) {
1877 rsp_increment(ctxt, -2);
1881 return em_push(ctxt);
1884 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1886 int seg = ctxt->src2.val;
1887 unsigned long selector;
1890 rc = emulate_pop(ctxt, &selector, 2);
1891 if (rc != X86EMUL_CONTINUE)
1894 if (ctxt->modrm_reg == VCPU_SREG_SS)
1895 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1896 if (ctxt->op_bytes > 2)
1897 rsp_increment(ctxt, ctxt->op_bytes - 2);
1899 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1903 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1905 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1906 int rc = X86EMUL_CONTINUE;
1907 int reg = VCPU_REGS_RAX;
1909 while (reg <= VCPU_REGS_RDI) {
1910 (reg == VCPU_REGS_RSP) ?
1911 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1914 if (rc != X86EMUL_CONTINUE)
1923 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1925 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
1926 return em_push(ctxt);
1929 static int em_popa(struct x86_emulate_ctxt *ctxt)
1931 int rc = X86EMUL_CONTINUE;
1932 int reg = VCPU_REGS_RDI;
1935 while (reg >= VCPU_REGS_RAX) {
1936 if (reg == VCPU_REGS_RSP) {
1937 rsp_increment(ctxt, ctxt->op_bytes);
1941 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
1942 if (rc != X86EMUL_CONTINUE)
1944 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
1950 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1952 const struct x86_emulate_ops *ops = ctxt->ops;
1959 /* TODO: Add limit checks */
1960 ctxt->src.val = ctxt->eflags;
1962 if (rc != X86EMUL_CONTINUE)
1965 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
1967 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1969 if (rc != X86EMUL_CONTINUE)
1972 ctxt->src.val = ctxt->_eip;
1974 if (rc != X86EMUL_CONTINUE)
1977 ops->get_idt(ctxt, &dt);
1979 eip_addr = dt.address + (irq << 2);
1980 cs_addr = dt.address + (irq << 2) + 2;
1982 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1983 if (rc != X86EMUL_CONTINUE)
1986 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1987 if (rc != X86EMUL_CONTINUE)
1990 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1991 if (rc != X86EMUL_CONTINUE)
1999 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2003 invalidate_registers(ctxt);
2004 rc = __emulate_int_real(ctxt, irq);
2005 if (rc == X86EMUL_CONTINUE)
2006 writeback_registers(ctxt);
2010 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2012 switch(ctxt->mode) {
2013 case X86EMUL_MODE_REAL:
2014 return __emulate_int_real(ctxt, irq);
2015 case X86EMUL_MODE_VM86:
2016 case X86EMUL_MODE_PROT16:
2017 case X86EMUL_MODE_PROT32:
2018 case X86EMUL_MODE_PROT64:
2020 /* Protected mode interrupts unimplemented yet */
2021 return X86EMUL_UNHANDLEABLE;
2025 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2027 int rc = X86EMUL_CONTINUE;
2028 unsigned long temp_eip = 0;
2029 unsigned long temp_eflags = 0;
2030 unsigned long cs = 0;
2031 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2032 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2033 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2034 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2035 X86_EFLAGS_AC | X86_EFLAGS_ID |
2036 X86_EFLAGS_FIXED_BIT;
2037 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2040 /* TODO: Add stack limit check */
2042 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2044 if (rc != X86EMUL_CONTINUE)
2047 if (temp_eip & ~0xffff)
2048 return emulate_gp(ctxt, 0);
2050 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2052 if (rc != X86EMUL_CONTINUE)
2055 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2057 if (rc != X86EMUL_CONTINUE)
2060 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2062 if (rc != X86EMUL_CONTINUE)
2065 ctxt->_eip = temp_eip;
2067 if (ctxt->op_bytes == 4)
2068 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2069 else if (ctxt->op_bytes == 2) {
2070 ctxt->eflags &= ~0xffff;
2071 ctxt->eflags |= temp_eflags;
2074 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2075 ctxt->eflags |= X86_EFLAGS_FIXED_BIT;
2076 ctxt->ops->set_nmi_mask(ctxt, false);
2081 static int em_iret(struct x86_emulate_ctxt *ctxt)
2083 switch(ctxt->mode) {
2084 case X86EMUL_MODE_REAL:
2085 return emulate_iret_real(ctxt);
2086 case X86EMUL_MODE_VM86:
2087 case X86EMUL_MODE_PROT16:
2088 case X86EMUL_MODE_PROT32:
2089 case X86EMUL_MODE_PROT64:
2091 /* iret from protected mode unimplemented yet */
2092 return X86EMUL_UNHANDLEABLE;
2096 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2099 unsigned short sel, old_sel;
2100 struct desc_struct old_desc, new_desc;
2101 const struct x86_emulate_ops *ops = ctxt->ops;
2102 u8 cpl = ctxt->ops->cpl(ctxt);
2104 /* Assignment of RIP may only fail in 64-bit mode */
2105 if (ctxt->mode == X86EMUL_MODE_PROT64)
2106 ops->get_segment(ctxt, &old_sel, &old_desc, NULL,
2109 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2111 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2112 X86_TRANSFER_CALL_JMP,
2114 if (rc != X86EMUL_CONTINUE)
2117 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2118 if (rc != X86EMUL_CONTINUE) {
2119 WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64);
2120 /* assigning eip failed; restore the old cs */
2121 ops->set_segment(ctxt, old_sel, &old_desc, 0, VCPU_SREG_CS);
2127 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2129 return assign_eip_near(ctxt, ctxt->src.val);
2132 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2137 old_eip = ctxt->_eip;
2138 rc = assign_eip_near(ctxt, ctxt->src.val);
2139 if (rc != X86EMUL_CONTINUE)
2141 ctxt->src.val = old_eip;
2146 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2148 u64 old = ctxt->dst.orig_val64;
2150 if (ctxt->dst.bytes == 16)
2151 return X86EMUL_UNHANDLEABLE;
2153 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2154 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2155 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2156 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2157 ctxt->eflags &= ~X86_EFLAGS_ZF;
2159 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2160 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2162 ctxt->eflags |= X86_EFLAGS_ZF;
2164 return X86EMUL_CONTINUE;
2167 static int em_ret(struct x86_emulate_ctxt *ctxt)
2172 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2173 if (rc != X86EMUL_CONTINUE)
2176 return assign_eip_near(ctxt, eip);
2179 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2182 unsigned long eip, cs;
2184 int cpl = ctxt->ops->cpl(ctxt);
2185 struct desc_struct old_desc, new_desc;
2186 const struct x86_emulate_ops *ops = ctxt->ops;
2188 if (ctxt->mode == X86EMUL_MODE_PROT64)
2189 ops->get_segment(ctxt, &old_cs, &old_desc, NULL,
2192 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2193 if (rc != X86EMUL_CONTINUE)
2195 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2196 if (rc != X86EMUL_CONTINUE)
2198 /* Outer-privilege level return is not implemented */
2199 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2200 return X86EMUL_UNHANDLEABLE;
2201 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2204 if (rc != X86EMUL_CONTINUE)
2206 rc = assign_eip_far(ctxt, eip, &new_desc);
2207 if (rc != X86EMUL_CONTINUE) {
2208 WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64);
2209 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
2214 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2218 rc = em_ret_far(ctxt);
2219 if (rc != X86EMUL_CONTINUE)
2221 rsp_increment(ctxt, ctxt->src.val);
2222 return X86EMUL_CONTINUE;
2225 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2227 /* Save real source value, then compare EAX against destination. */
2228 ctxt->dst.orig_val = ctxt->dst.val;
2229 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2230 ctxt->src.orig_val = ctxt->src.val;
2231 ctxt->src.val = ctxt->dst.orig_val;
2232 fastop(ctxt, em_cmp);
2234 if (ctxt->eflags & X86_EFLAGS_ZF) {
2235 /* Success: write back to memory; no update of EAX */
2236 ctxt->src.type = OP_NONE;
2237 ctxt->dst.val = ctxt->src.orig_val;
2239 /* Failure: write the value we saw to EAX. */
2240 ctxt->src.type = OP_REG;
2241 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2242 ctxt->src.val = ctxt->dst.orig_val;
2243 /* Create write-cycle to dest by writing the same value */
2244 ctxt->dst.val = ctxt->dst.orig_val;
2246 return X86EMUL_CONTINUE;
2249 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2251 int seg = ctxt->src2.val;
2255 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2257 rc = load_segment_descriptor(ctxt, sel, seg);
2258 if (rc != X86EMUL_CONTINUE)
2261 ctxt->dst.val = ctxt->src.val;
2266 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2267 struct desc_struct *cs, struct desc_struct *ss)
2269 cs->l = 0; /* will be adjusted later */
2270 set_desc_base(cs, 0); /* flat segment */
2271 cs->g = 1; /* 4kb granularity */
2272 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2273 cs->type = 0x0b; /* Read, Execute, Accessed */
2275 cs->dpl = 0; /* will be adjusted later */
2280 set_desc_base(ss, 0); /* flat segment */
2281 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2282 ss->g = 1; /* 4kb granularity */
2284 ss->type = 0x03; /* Read/Write, Accessed */
2285 ss->d = 1; /* 32bit stack segment */
2292 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2294 u32 eax, ebx, ecx, edx;
2297 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2298 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2299 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2300 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2303 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2305 const struct x86_emulate_ops *ops = ctxt->ops;
2306 u32 eax, ebx, ecx, edx;
2309 * syscall should always be enabled in longmode - so only become
2310 * vendor specific (cpuid) if other modes are active...
2312 if (ctxt->mode == X86EMUL_MODE_PROT64)
2317 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2319 * Intel ("GenuineIntel")
2320 * remark: Intel CPUs only support "syscall" in 64bit
2321 * longmode. Also an 64bit guest with a
2322 * 32bit compat-app running will #UD !! While this
2323 * behaviour can be fixed (by emulating) into AMD
2324 * response - CPUs of AMD can't behave like Intel.
2326 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2327 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2328 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2331 /* AMD ("AuthenticAMD") */
2332 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2333 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2334 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2337 /* AMD ("AMDisbetter!") */
2338 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2339 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2340 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2343 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2347 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2349 const struct x86_emulate_ops *ops = ctxt->ops;
2350 struct desc_struct cs, ss;
2355 /* syscall is not available in real mode */
2356 if (ctxt->mode == X86EMUL_MODE_REAL ||
2357 ctxt->mode == X86EMUL_MODE_VM86)
2358 return emulate_ud(ctxt);
2360 if (!(em_syscall_is_enabled(ctxt)))
2361 return emulate_ud(ctxt);
2363 ops->get_msr(ctxt, MSR_EFER, &efer);
2364 setup_syscalls_segments(ctxt, &cs, &ss);
2366 if (!(efer & EFER_SCE))
2367 return emulate_ud(ctxt);
2369 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2371 cs_sel = (u16)(msr_data & 0xfffc);
2372 ss_sel = (u16)(msr_data + 8);
2374 if (efer & EFER_LMA) {
2378 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2379 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2381 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2382 if (efer & EFER_LMA) {
2383 #ifdef CONFIG_X86_64
2384 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2387 ctxt->mode == X86EMUL_MODE_PROT64 ?
2388 MSR_LSTAR : MSR_CSTAR, &msr_data);
2389 ctxt->_eip = msr_data;
2391 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2392 ctxt->eflags &= ~msr_data;
2393 ctxt->eflags |= X86_EFLAGS_FIXED_BIT;
2397 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2398 ctxt->_eip = (u32)msr_data;
2400 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2403 return X86EMUL_CONTINUE;
2406 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2408 const struct x86_emulate_ops *ops = ctxt->ops;
2409 struct desc_struct cs, ss;
2414 ops->get_msr(ctxt, MSR_EFER, &efer);
2415 /* inject #GP if in real mode */
2416 if (ctxt->mode == X86EMUL_MODE_REAL)
2417 return emulate_gp(ctxt, 0);
2420 * Not recognized on AMD in compat mode (but is recognized in legacy
2423 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2424 && !vendor_intel(ctxt))
2425 return emulate_ud(ctxt);
2427 /* sysenter/sysexit have not been tested in 64bit mode. */
2428 if (ctxt->mode == X86EMUL_MODE_PROT64)
2429 return X86EMUL_UNHANDLEABLE;
2431 setup_syscalls_segments(ctxt, &cs, &ss);
2433 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2434 if ((msr_data & 0xfffc) == 0x0)
2435 return emulate_gp(ctxt, 0);
2437 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2438 cs_sel = (u16)msr_data & ~SELECTOR_RPL_MASK;
2439 ss_sel = cs_sel + 8;
2440 if (efer & EFER_LMA) {
2445 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2446 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2448 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2449 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2451 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2452 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2455 return X86EMUL_CONTINUE;
2458 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2460 const struct x86_emulate_ops *ops = ctxt->ops;
2461 struct desc_struct cs, ss;
2462 u64 msr_data, rcx, rdx;
2464 u16 cs_sel = 0, ss_sel = 0;
2466 /* inject #GP if in real mode or Virtual 8086 mode */
2467 if (ctxt->mode == X86EMUL_MODE_REAL ||
2468 ctxt->mode == X86EMUL_MODE_VM86)
2469 return emulate_gp(ctxt, 0);
2471 setup_syscalls_segments(ctxt, &cs, &ss);
2473 if ((ctxt->rex_prefix & 0x8) != 0x0)
2474 usermode = X86EMUL_MODE_PROT64;
2476 usermode = X86EMUL_MODE_PROT32;
2478 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2479 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2483 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2485 case X86EMUL_MODE_PROT32:
2486 cs_sel = (u16)(msr_data + 16);
2487 if ((msr_data & 0xfffc) == 0x0)
2488 return emulate_gp(ctxt, 0);
2489 ss_sel = (u16)(msr_data + 24);
2493 case X86EMUL_MODE_PROT64:
2494 cs_sel = (u16)(msr_data + 32);
2495 if (msr_data == 0x0)
2496 return emulate_gp(ctxt, 0);
2497 ss_sel = cs_sel + 8;
2500 if (is_noncanonical_address(rcx) ||
2501 is_noncanonical_address(rdx))
2502 return emulate_gp(ctxt, 0);
2505 cs_sel |= SELECTOR_RPL_MASK;
2506 ss_sel |= SELECTOR_RPL_MASK;
2508 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2509 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2512 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2514 return X86EMUL_CONTINUE;
2517 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2520 if (ctxt->mode == X86EMUL_MODE_REAL)
2522 if (ctxt->mode == X86EMUL_MODE_VM86)
2524 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2525 return ctxt->ops->cpl(ctxt) > iopl;
2528 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2531 const struct x86_emulate_ops *ops = ctxt->ops;
2532 struct desc_struct tr_seg;
2535 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2536 unsigned mask = (1 << len) - 1;
2539 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2542 if (desc_limit_scaled(&tr_seg) < 103)
2544 base = get_desc_base(&tr_seg);
2545 #ifdef CONFIG_X86_64
2546 base |= ((u64)base3) << 32;
2548 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2549 if (r != X86EMUL_CONTINUE)
2551 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2553 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2554 if (r != X86EMUL_CONTINUE)
2556 if ((perm >> bit_idx) & mask)
2561 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2567 if (emulator_bad_iopl(ctxt))
2568 if (!emulator_io_port_access_allowed(ctxt, port, len))
2571 ctxt->perm_ok = true;
2576 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2577 struct tss_segment_16 *tss)
2579 tss->ip = ctxt->_eip;
2580 tss->flag = ctxt->eflags;
2581 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2582 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2583 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2584 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2585 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2586 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2587 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2588 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2590 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2591 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2592 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2593 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2594 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2597 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2598 struct tss_segment_16 *tss)
2603 ctxt->_eip = tss->ip;
2604 ctxt->eflags = tss->flag | 2;
2605 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2606 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2607 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2608 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2609 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2610 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2611 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2612 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2615 * SDM says that segment selectors are loaded before segment
2618 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2619 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2620 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2621 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2622 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2627 * Now load segment descriptors. If fault happens at this stage
2628 * it is handled in a context of new task
2630 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
2631 X86_TRANSFER_TASK_SWITCH, NULL);
2632 if (ret != X86EMUL_CONTINUE)
2634 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
2635 X86_TRANSFER_TASK_SWITCH, NULL);
2636 if (ret != X86EMUL_CONTINUE)
2638 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
2639 X86_TRANSFER_TASK_SWITCH, NULL);
2640 if (ret != X86EMUL_CONTINUE)
2642 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
2643 X86_TRANSFER_TASK_SWITCH, NULL);
2644 if (ret != X86EMUL_CONTINUE)
2646 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
2647 X86_TRANSFER_TASK_SWITCH, NULL);
2648 if (ret != X86EMUL_CONTINUE)
2651 return X86EMUL_CONTINUE;
2654 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2655 u16 tss_selector, u16 old_tss_sel,
2656 ulong old_tss_base, struct desc_struct *new_desc)
2658 const struct x86_emulate_ops *ops = ctxt->ops;
2659 struct tss_segment_16 tss_seg;
2661 u32 new_tss_base = get_desc_base(new_desc);
2663 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2665 if (ret != X86EMUL_CONTINUE)
2668 save_state_to_tss16(ctxt, &tss_seg);
2670 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2672 if (ret != X86EMUL_CONTINUE)
2675 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2677 if (ret != X86EMUL_CONTINUE)
2680 if (old_tss_sel != 0xffff) {
2681 tss_seg.prev_task_link = old_tss_sel;
2683 ret = ops->write_std(ctxt, new_tss_base,
2684 &tss_seg.prev_task_link,
2685 sizeof tss_seg.prev_task_link,
2687 if (ret != X86EMUL_CONTINUE)
2691 return load_state_from_tss16(ctxt, &tss_seg);
2694 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2695 struct tss_segment_32 *tss)
2697 /* CR3 and ldt selector are not saved intentionally */
2698 tss->eip = ctxt->_eip;
2699 tss->eflags = ctxt->eflags;
2700 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2701 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2702 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2703 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2704 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2705 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2706 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2707 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2709 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2710 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2711 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2712 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2713 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2714 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2717 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2718 struct tss_segment_32 *tss)
2723 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2724 return emulate_gp(ctxt, 0);
2725 ctxt->_eip = tss->eip;
2726 ctxt->eflags = tss->eflags | 2;
2728 /* General purpose registers */
2729 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2730 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2731 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2732 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2733 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2734 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2735 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2736 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2739 * SDM says that segment selectors are loaded before segment
2740 * descriptors. This is important because CPL checks will
2743 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2744 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2745 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2746 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2747 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2748 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2749 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2752 * If we're switching between Protected Mode and VM86, we need to make
2753 * sure to update the mode before loading the segment descriptors so
2754 * that the selectors are interpreted correctly.
2756 if (ctxt->eflags & X86_EFLAGS_VM) {
2757 ctxt->mode = X86EMUL_MODE_VM86;
2760 ctxt->mode = X86EMUL_MODE_PROT32;
2765 * Now load segment descriptors. If fault happenes at this stage
2766 * it is handled in a context of new task
2768 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
2769 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
2770 if (ret != X86EMUL_CONTINUE)
2772 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
2773 X86_TRANSFER_TASK_SWITCH, NULL);
2774 if (ret != X86EMUL_CONTINUE)
2776 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
2777 X86_TRANSFER_TASK_SWITCH, NULL);
2778 if (ret != X86EMUL_CONTINUE)
2780 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
2781 X86_TRANSFER_TASK_SWITCH, NULL);
2782 if (ret != X86EMUL_CONTINUE)
2784 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
2785 X86_TRANSFER_TASK_SWITCH, NULL);
2786 if (ret != X86EMUL_CONTINUE)
2788 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
2789 X86_TRANSFER_TASK_SWITCH, NULL);
2790 if (ret != X86EMUL_CONTINUE)
2792 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
2793 X86_TRANSFER_TASK_SWITCH, NULL);
2794 if (ret != X86EMUL_CONTINUE)
2797 return X86EMUL_CONTINUE;
2800 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2801 u16 tss_selector, u16 old_tss_sel,
2802 ulong old_tss_base, struct desc_struct *new_desc)
2804 const struct x86_emulate_ops *ops = ctxt->ops;
2805 struct tss_segment_32 tss_seg;
2807 u32 new_tss_base = get_desc_base(new_desc);
2808 u32 eip_offset = offsetof(struct tss_segment_32, eip);
2809 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
2811 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2813 if (ret != X86EMUL_CONTINUE)
2816 save_state_to_tss32(ctxt, &tss_seg);
2818 /* Only GP registers and segment selectors are saved */
2819 ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
2820 ldt_sel_offset - eip_offset, &ctxt->exception);
2821 if (ret != X86EMUL_CONTINUE)
2824 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2826 if (ret != X86EMUL_CONTINUE)
2829 if (old_tss_sel != 0xffff) {
2830 tss_seg.prev_task_link = old_tss_sel;
2832 ret = ops->write_std(ctxt, new_tss_base,
2833 &tss_seg.prev_task_link,
2834 sizeof tss_seg.prev_task_link,
2836 if (ret != X86EMUL_CONTINUE)
2840 return load_state_from_tss32(ctxt, &tss_seg);
2843 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2844 u16 tss_selector, int idt_index, int reason,
2845 bool has_error_code, u32 error_code)
2847 const struct x86_emulate_ops *ops = ctxt->ops;
2848 struct desc_struct curr_tss_desc, next_tss_desc;
2850 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2851 ulong old_tss_base =
2852 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2856 /* FIXME: old_tss_base == ~0 ? */
2858 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2859 if (ret != X86EMUL_CONTINUE)
2861 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2862 if (ret != X86EMUL_CONTINUE)
2865 /* FIXME: check that next_tss_desc is tss */
2868 * Check privileges. The three cases are task switch caused by...
2870 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2871 * 2. Exception/IRQ/iret: No check is performed
2872 * 3. jmp/call to TSS/task-gate: No check is performed since the
2873 * hardware checks it before exiting.
2875 if (reason == TASK_SWITCH_GATE) {
2876 if (idt_index != -1) {
2877 /* Software interrupts */
2878 struct desc_struct task_gate_desc;
2881 ret = read_interrupt_descriptor(ctxt, idt_index,
2883 if (ret != X86EMUL_CONTINUE)
2886 dpl = task_gate_desc.dpl;
2887 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2888 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2892 desc_limit = desc_limit_scaled(&next_tss_desc);
2893 if (!next_tss_desc.p ||
2894 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2895 desc_limit < 0x2b)) {
2896 return emulate_ts(ctxt, tss_selector & 0xfffc);
2899 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2900 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2901 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2904 if (reason == TASK_SWITCH_IRET)
2905 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2907 /* set back link to prev task only if NT bit is set in eflags
2908 note that old_tss_sel is not used after this point */
2909 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2910 old_tss_sel = 0xffff;
2912 if (next_tss_desc.type & 8)
2913 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2914 old_tss_base, &next_tss_desc);
2916 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2917 old_tss_base, &next_tss_desc);
2918 if (ret != X86EMUL_CONTINUE)
2921 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2922 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2924 if (reason != TASK_SWITCH_IRET) {
2925 next_tss_desc.type |= (1 << 1); /* set busy flag */
2926 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2929 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2930 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2932 if (has_error_code) {
2933 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2934 ctxt->lock_prefix = 0;
2935 ctxt->src.val = (unsigned long) error_code;
2936 ret = em_push(ctxt);
2942 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2943 u16 tss_selector, int idt_index, int reason,
2944 bool has_error_code, u32 error_code)
2948 invalidate_registers(ctxt);
2949 ctxt->_eip = ctxt->eip;
2950 ctxt->dst.type = OP_NONE;
2952 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2953 has_error_code, error_code);
2955 if (rc == X86EMUL_CONTINUE) {
2956 ctxt->eip = ctxt->_eip;
2957 writeback_registers(ctxt);
2960 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2963 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2966 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
2968 register_address_increment(ctxt, reg, df * op->bytes);
2969 op->addr.mem.ea = register_address(ctxt, reg);
2972 static int em_das(struct x86_emulate_ctxt *ctxt)
2975 bool af, cf, old_cf;
2977 cf = ctxt->eflags & X86_EFLAGS_CF;
2983 af = ctxt->eflags & X86_EFLAGS_AF;
2984 if ((al & 0x0f) > 9 || af) {
2986 cf = old_cf | (al >= 250);
2991 if (old_al > 0x99 || old_cf) {
2997 /* Set PF, ZF, SF */
2998 ctxt->src.type = OP_IMM;
3000 ctxt->src.bytes = 1;
3001 fastop(ctxt, em_or);
3002 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3004 ctxt->eflags |= X86_EFLAGS_CF;
3006 ctxt->eflags |= X86_EFLAGS_AF;
3007 return X86EMUL_CONTINUE;
3010 static int em_aam(struct x86_emulate_ctxt *ctxt)
3014 if (ctxt->src.val == 0)
3015 return emulate_de(ctxt);
3017 al = ctxt->dst.val & 0xff;
3018 ah = al / ctxt->src.val;
3019 al %= ctxt->src.val;
3021 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3023 /* Set PF, ZF, SF */
3024 ctxt->src.type = OP_IMM;
3026 ctxt->src.bytes = 1;
3027 fastop(ctxt, em_or);
3029 return X86EMUL_CONTINUE;
3032 static int em_aad(struct x86_emulate_ctxt *ctxt)
3034 u8 al = ctxt->dst.val & 0xff;
3035 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3037 al = (al + (ah * ctxt->src.val)) & 0xff;
3039 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3041 /* Set PF, ZF, SF */
3042 ctxt->src.type = OP_IMM;
3044 ctxt->src.bytes = 1;
3045 fastop(ctxt, em_or);
3047 return X86EMUL_CONTINUE;
3050 static int em_call(struct x86_emulate_ctxt *ctxt)
3053 long rel = ctxt->src.val;
3055 ctxt->src.val = (unsigned long)ctxt->_eip;
3056 rc = jmp_rel(ctxt, rel);
3057 if (rc != X86EMUL_CONTINUE)
3059 return em_push(ctxt);
3062 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3067 struct desc_struct old_desc, new_desc;
3068 const struct x86_emulate_ops *ops = ctxt->ops;
3069 int cpl = ctxt->ops->cpl(ctxt);
3070 enum x86emul_mode prev_mode = ctxt->mode;
3072 old_eip = ctxt->_eip;
3073 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3075 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3076 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3077 X86_TRANSFER_CALL_JMP, &new_desc);
3078 if (rc != X86EMUL_CONTINUE)
3081 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3082 if (rc != X86EMUL_CONTINUE)
3085 ctxt->src.val = old_cs;
3087 if (rc != X86EMUL_CONTINUE)
3090 ctxt->src.val = old_eip;
3092 /* If we failed, we tainted the memory, but the very least we should
3094 if (rc != X86EMUL_CONTINUE) {
3095 pr_warn_once("faulting far call emulation tainted memory\n");
3100 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3101 ctxt->mode = prev_mode;
3106 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3111 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3112 if (rc != X86EMUL_CONTINUE)
3114 rc = assign_eip_near(ctxt, eip);
3115 if (rc != X86EMUL_CONTINUE)
3117 rsp_increment(ctxt, ctxt->src.val);
3118 return X86EMUL_CONTINUE;
3121 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3123 /* Write back the register source. */
3124 ctxt->src.val = ctxt->dst.val;
3125 write_register_operand(&ctxt->src);
3127 /* Write back the memory destination with implicit LOCK prefix. */
3128 ctxt->dst.val = ctxt->src.orig_val;
3129 ctxt->lock_prefix = 1;
3130 return X86EMUL_CONTINUE;
3133 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3135 ctxt->dst.val = ctxt->src2.val;
3136 return fastop(ctxt, em_imul);
3139 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3141 ctxt->dst.type = OP_REG;
3142 ctxt->dst.bytes = ctxt->src.bytes;
3143 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3144 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3146 return X86EMUL_CONTINUE;
3149 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3153 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3154 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3155 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3156 return X86EMUL_CONTINUE;
3159 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3163 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3164 return emulate_gp(ctxt, 0);
3165 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3166 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3167 return X86EMUL_CONTINUE;
3170 static int em_mov(struct x86_emulate_ctxt *ctxt)
3172 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3173 return X86EMUL_CONTINUE;
3176 #define FFL(x) bit(X86_FEATURE_##x)
3178 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3180 u32 ebx, ecx, edx, eax = 1;
3184 * Check MOVBE is set in the guest-visible CPUID leaf.
3186 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3187 if (!(ecx & FFL(MOVBE)))
3188 return emulate_ud(ctxt);
3190 switch (ctxt->op_bytes) {
3193 * From MOVBE definition: "...When the operand size is 16 bits,
3194 * the upper word of the destination register remains unchanged
3197 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3198 * rules so we have to do the operation almost per hand.
3200 tmp = (u16)ctxt->src.val;
3201 ctxt->dst.val &= ~0xffffUL;
3202 ctxt->dst.val |= (unsigned long)swab16(tmp);
3205 ctxt->dst.val = swab32((u32)ctxt->src.val);
3208 ctxt->dst.val = swab64(ctxt->src.val);
3213 return X86EMUL_CONTINUE;
3216 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3218 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3219 return emulate_gp(ctxt, 0);
3221 /* Disable writeback. */
3222 ctxt->dst.type = OP_NONE;
3223 return X86EMUL_CONTINUE;
3226 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3230 if (ctxt->mode == X86EMUL_MODE_PROT64)
3231 val = ctxt->src.val & ~0ULL;
3233 val = ctxt->src.val & ~0U;
3235 /* #UD condition is already handled. */
3236 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3237 return emulate_gp(ctxt, 0);
3239 /* Disable writeback. */
3240 ctxt->dst.type = OP_NONE;
3241 return X86EMUL_CONTINUE;
3244 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3248 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3249 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3250 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3251 return emulate_gp(ctxt, 0);
3253 return X86EMUL_CONTINUE;
3256 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3260 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3261 return emulate_gp(ctxt, 0);
3263 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3264 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3265 return X86EMUL_CONTINUE;
3268 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3270 if (ctxt->modrm_reg > VCPU_SREG_GS)
3271 return emulate_ud(ctxt);
3273 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3274 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3275 ctxt->dst.bytes = 2;
3276 return X86EMUL_CONTINUE;
3279 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3281 u16 sel = ctxt->src.val;
3283 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3284 return emulate_ud(ctxt);
3286 if (ctxt->modrm_reg == VCPU_SREG_SS)
3287 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3289 /* Disable writeback. */
3290 ctxt->dst.type = OP_NONE;
3291 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3294 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3296 u16 sel = ctxt->src.val;
3298 /* Disable writeback. */
3299 ctxt->dst.type = OP_NONE;
3300 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3303 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3305 u16 sel = ctxt->src.val;
3307 /* Disable writeback. */
3308 ctxt->dst.type = OP_NONE;
3309 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3312 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3317 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3318 if (rc == X86EMUL_CONTINUE)
3319 ctxt->ops->invlpg(ctxt, linear);
3320 /* Disable writeback. */
3321 ctxt->dst.type = OP_NONE;
3322 return X86EMUL_CONTINUE;
3325 static int em_clts(struct x86_emulate_ctxt *ctxt)
3329 cr0 = ctxt->ops->get_cr(ctxt, 0);
3331 ctxt->ops->set_cr(ctxt, 0, cr0);
3332 return X86EMUL_CONTINUE;
3335 static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3337 int rc = ctxt->ops->fix_hypercall(ctxt);
3339 if (rc != X86EMUL_CONTINUE)
3342 /* Let the processor re-execute the fixed hypercall */
3343 ctxt->_eip = ctxt->eip;
3344 /* Disable writeback. */
3345 ctxt->dst.type = OP_NONE;
3346 return X86EMUL_CONTINUE;
3349 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3350 void (*get)(struct x86_emulate_ctxt *ctxt,
3351 struct desc_ptr *ptr))
3353 struct desc_ptr desc_ptr;
3355 if (ctxt->mode == X86EMUL_MODE_PROT64)
3357 get(ctxt, &desc_ptr);
3358 if (ctxt->op_bytes == 2) {
3360 desc_ptr.address &= 0x00ffffff;
3362 /* Disable writeback. */
3363 ctxt->dst.type = OP_NONE;
3364 return segmented_write(ctxt, ctxt->dst.addr.mem,
3365 &desc_ptr, 2 + ctxt->op_bytes);
3368 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3370 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3373 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3375 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3378 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3380 struct desc_ptr desc_ptr;
3383 if (ctxt->mode == X86EMUL_MODE_PROT64)
3385 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3386 &desc_ptr.size, &desc_ptr.address,
3388 if (rc != X86EMUL_CONTINUE)
3390 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3391 is_noncanonical_address(desc_ptr.address))
3392 return emulate_gp(ctxt, 0);
3394 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3396 ctxt->ops->set_idt(ctxt, &desc_ptr);
3397 /* Disable writeback. */
3398 ctxt->dst.type = OP_NONE;
3399 return X86EMUL_CONTINUE;
3402 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3404 return em_lgdt_lidt(ctxt, true);
3407 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3409 return em_lgdt_lidt(ctxt, false);
3412 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3414 if (ctxt->dst.type == OP_MEM)
3415 ctxt->dst.bytes = 2;
3416 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3417 return X86EMUL_CONTINUE;
3420 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3422 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3423 | (ctxt->src.val & 0x0f));
3424 ctxt->dst.type = OP_NONE;
3425 return X86EMUL_CONTINUE;
3428 static int em_loop(struct x86_emulate_ctxt *ctxt)
3430 int rc = X86EMUL_CONTINUE;
3432 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3433 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3434 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3435 rc = jmp_rel(ctxt, ctxt->src.val);
3440 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3442 int rc = X86EMUL_CONTINUE;
3444 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3445 rc = jmp_rel(ctxt, ctxt->src.val);
3450 static int em_in(struct x86_emulate_ctxt *ctxt)
3452 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3454 return X86EMUL_IO_NEEDED;
3456 return X86EMUL_CONTINUE;
3459 static int em_out(struct x86_emulate_ctxt *ctxt)
3461 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3463 /* Disable writeback. */
3464 ctxt->dst.type = OP_NONE;
3465 return X86EMUL_CONTINUE;
3468 static int em_cli(struct x86_emulate_ctxt *ctxt)
3470 if (emulator_bad_iopl(ctxt))
3471 return emulate_gp(ctxt, 0);
3473 ctxt->eflags &= ~X86_EFLAGS_IF;
3474 return X86EMUL_CONTINUE;
3477 static int em_sti(struct x86_emulate_ctxt *ctxt)
3479 if (emulator_bad_iopl(ctxt))
3480 return emulate_gp(ctxt, 0);
3482 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3483 ctxt->eflags |= X86_EFLAGS_IF;
3484 return X86EMUL_CONTINUE;
3487 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3489 u32 eax, ebx, ecx, edx;
3491 eax = reg_read(ctxt, VCPU_REGS_RAX);
3492 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3493 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3494 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3495 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3496 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3497 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3498 return X86EMUL_CONTINUE;
3501 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3505 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
3507 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3509 ctxt->eflags &= ~0xffUL;
3510 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3511 return X86EMUL_CONTINUE;
3514 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3516 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3517 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3518 return X86EMUL_CONTINUE;
3521 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3523 switch (ctxt->op_bytes) {
3524 #ifdef CONFIG_X86_64
3526 asm("bswap %0" : "+r"(ctxt->dst.val));
3530 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3533 return X86EMUL_CONTINUE;
3536 static int em_clflush(struct x86_emulate_ctxt *ctxt)
3538 /* emulating clflush regardless of cpuid */
3539 return X86EMUL_CONTINUE;
3542 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
3544 ctxt->dst.val = (s32) ctxt->src.val;
3545 return X86EMUL_CONTINUE;
3548 static bool valid_cr(int nr)
3560 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3562 if (!valid_cr(ctxt->modrm_reg))
3563 return emulate_ud(ctxt);
3565 return X86EMUL_CONTINUE;
3568 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3570 u64 new_val = ctxt->src.val64;
3571 int cr = ctxt->modrm_reg;
3574 static u64 cr_reserved_bits[] = {
3575 0xffffffff00000000ULL,
3576 0, 0, 0, /* CR3 checked later */
3583 return emulate_ud(ctxt);
3585 if (new_val & cr_reserved_bits[cr])
3586 return emulate_gp(ctxt, 0);
3591 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3592 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3593 return emulate_gp(ctxt, 0);
3595 cr4 = ctxt->ops->get_cr(ctxt, 4);
3596 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3598 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3599 !(cr4 & X86_CR4_PAE))
3600 return emulate_gp(ctxt, 0);
3607 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3608 if (efer & EFER_LMA)
3609 rsvd = CR3_L_MODE_RESERVED_BITS & ~CR3_PCID_INVD;
3612 return emulate_gp(ctxt, 0);
3617 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3619 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3620 return emulate_gp(ctxt, 0);
3626 return X86EMUL_CONTINUE;
3629 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3633 ctxt->ops->get_dr(ctxt, 7, &dr7);
3635 /* Check if DR7.Global_Enable is set */
3636 return dr7 & (1 << 13);
3639 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3641 int dr = ctxt->modrm_reg;
3645 return emulate_ud(ctxt);
3647 cr4 = ctxt->ops->get_cr(ctxt, 4);
3648 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3649 return emulate_ud(ctxt);
3651 if (check_dr7_gd(ctxt)) {
3654 ctxt->ops->get_dr(ctxt, 6, &dr6);
3656 dr6 |= DR6_BD | DR6_RTM;
3657 ctxt->ops->set_dr(ctxt, 6, dr6);
3658 return emulate_db(ctxt);
3661 return X86EMUL_CONTINUE;
3664 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3666 u64 new_val = ctxt->src.val64;
3667 int dr = ctxt->modrm_reg;
3669 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3670 return emulate_gp(ctxt, 0);
3672 return check_dr_read(ctxt);
3675 static int check_svme(struct x86_emulate_ctxt *ctxt)
3679 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3681 if (!(efer & EFER_SVME))
3682 return emulate_ud(ctxt);
3684 return X86EMUL_CONTINUE;
3687 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3689 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3691 /* Valid physical address? */
3692 if (rax & 0xffff000000000000ULL)
3693 return emulate_gp(ctxt, 0);
3695 return check_svme(ctxt);
3698 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3700 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3702 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3703 return emulate_ud(ctxt);
3705 return X86EMUL_CONTINUE;
3708 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3710 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3711 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3713 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3714 ctxt->ops->check_pmc(ctxt, rcx))
3715 return emulate_gp(ctxt, 0);
3717 return X86EMUL_CONTINUE;
3720 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3722 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3723 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3724 return emulate_gp(ctxt, 0);
3726 return X86EMUL_CONTINUE;
3729 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3731 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3732 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3733 return emulate_gp(ctxt, 0);
3735 return X86EMUL_CONTINUE;
3738 #define D(_y) { .flags = (_y) }
3739 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
3740 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
3741 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3742 #define N D(NotImpl)
3743 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3744 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3745 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3746 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
3747 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
3748 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3749 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3750 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3751 #define II(_f, _e, _i) \
3752 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
3753 #define IIP(_f, _e, _i, _p) \
3754 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
3755 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3756 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3758 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3759 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3760 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3761 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3762 #define I2bvIP(_f, _e, _i, _p) \
3763 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3765 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3766 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3767 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3769 static const struct opcode group7_rm0[] = {
3771 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
3775 static const struct opcode group7_rm1[] = {
3776 DI(SrcNone | Priv, monitor),
3777 DI(SrcNone | Priv, mwait),
3781 static const struct opcode group7_rm3[] = {
3782 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3783 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
3784 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3785 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3786 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3787 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3788 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3789 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3792 static const struct opcode group7_rm7[] = {
3794 DIP(SrcNone, rdtscp, check_rdtsc),
3798 static const struct opcode group1[] = {
3800 F(Lock | PageTable, em_or),
3803 F(Lock | PageTable, em_and),
3809 static const struct opcode group1A[] = {
3810 I(DstMem | SrcNone | Mov | Stack | IncSP, em_pop), N, N, N, N, N, N, N,
3813 static const struct opcode group2[] = {
3814 F(DstMem | ModRM, em_rol),
3815 F(DstMem | ModRM, em_ror),
3816 F(DstMem | ModRM, em_rcl),
3817 F(DstMem | ModRM, em_rcr),
3818 F(DstMem | ModRM, em_shl),
3819 F(DstMem | ModRM, em_shr),
3820 F(DstMem | ModRM, em_shl),
3821 F(DstMem | ModRM, em_sar),
3824 static const struct opcode group3[] = {
3825 F(DstMem | SrcImm | NoWrite, em_test),
3826 F(DstMem | SrcImm | NoWrite, em_test),
3827 F(DstMem | SrcNone | Lock, em_not),
3828 F(DstMem | SrcNone | Lock, em_neg),
3829 F(DstXacc | Src2Mem, em_mul_ex),
3830 F(DstXacc | Src2Mem, em_imul_ex),
3831 F(DstXacc | Src2Mem, em_div_ex),
3832 F(DstXacc | Src2Mem, em_idiv_ex),
3835 static const struct opcode group4[] = {
3836 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3837 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3841 static const struct opcode group5[] = {
3842 F(DstMem | SrcNone | Lock, em_inc),
3843 F(DstMem | SrcNone | Lock, em_dec),
3844 I(SrcMem | NearBranch, em_call_near_abs),
3845 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3846 I(SrcMem | NearBranch, em_jmp_abs),
3847 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
3848 I(SrcMem | Stack, em_push), D(Undefined),
3851 static const struct opcode group6[] = {
3852 DI(Prot | DstMem, sldt),
3853 DI(Prot | DstMem, str),
3854 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3855 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3859 static const struct group_dual group7 = { {
3860 II(Mov | DstMem, em_sgdt, sgdt),
3861 II(Mov | DstMem, em_sidt, sidt),
3862 II(SrcMem | Priv, em_lgdt, lgdt),
3863 II(SrcMem | Priv, em_lidt, lidt),
3864 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3865 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3866 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3870 N, EXT(0, group7_rm3),
3871 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3872 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3876 static const struct opcode group8[] = {
3878 F(DstMem | SrcImmByte | NoWrite, em_bt),
3879 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3880 F(DstMem | SrcImmByte | Lock, em_btr),
3881 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3884 static const struct group_dual group9 = { {
3885 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3887 N, N, N, N, N, N, N, N,
3890 static const struct opcode group11[] = {
3891 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3895 static const struct gprefix pfx_0f_ae_7 = {
3896 I(SrcMem | ByteOp, em_clflush), N, N, N,
3899 static const struct group_dual group15 = { {
3900 N, N, N, N, N, N, N, GP(0, &pfx_0f_ae_7),
3902 N, N, N, N, N, N, N, N,
3905 static const struct gprefix pfx_0f_6f_0f_7f = {
3906 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3909 static const struct instr_dual instr_dual_0f_2b = {
3913 static const struct gprefix pfx_0f_2b = {
3914 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
3917 static const struct gprefix pfx_0f_28_0f_29 = {
3918 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
3921 static const struct gprefix pfx_0f_e7 = {
3922 N, I(Sse, em_mov), N, N,
3925 static const struct escape escape_d9 = { {
3926 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
3929 N, N, N, N, N, N, N, N,
3931 N, N, N, N, N, N, N, N,
3933 N, N, N, N, N, N, N, N,
3935 N, N, N, N, N, N, N, N,
3937 N, N, N, N, N, N, N, N,
3939 N, N, N, N, N, N, N, N,
3941 N, N, N, N, N, N, N, N,
3943 N, N, N, N, N, N, N, N,
3946 static const struct escape escape_db = { {
3947 N, N, N, N, N, N, N, N,
3950 N, N, N, N, N, N, N, N,
3952 N, N, N, N, N, N, N, N,
3954 N, N, N, N, N, N, N, N,
3956 N, N, N, N, N, N, N, N,
3958 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3960 N, N, N, N, N, N, N, N,
3962 N, N, N, N, N, N, N, N,
3964 N, N, N, N, N, N, N, N,
3967 static const struct escape escape_dd = { {
3968 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
3971 N, N, N, N, N, N, N, N,
3973 N, N, N, N, N, N, N, N,
3975 N, N, N, N, N, N, N, N,
3977 N, N, N, N, N, N, N, N,
3979 N, N, N, N, N, N, N, N,
3981 N, N, N, N, N, N, N, N,
3983 N, N, N, N, N, N, N, N,
3985 N, N, N, N, N, N, N, N,
3988 static const struct instr_dual instr_dual_0f_c3 = {
3989 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
3992 static const struct mode_dual mode_dual_63 = {
3993 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
3996 static const struct opcode opcode_table[256] = {
3998 F6ALU(Lock, em_add),
3999 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4000 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4002 F6ALU(Lock | PageTable, em_or),
4003 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4006 F6ALU(Lock, em_adc),
4007 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4008 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4010 F6ALU(Lock, em_sbb),
4011 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4012 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4014 F6ALU(Lock | PageTable, em_and), N, N,
4016 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4018 F6ALU(Lock, em_xor), N, N,
4020 F6ALU(NoWrite, em_cmp), N, N,
4022 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4024 X8(I(SrcReg | Stack, em_push)),
4026 X8(I(DstReg | Stack, em_pop)),
4028 I(ImplicitOps | Stack | No64, em_pusha),
4029 I(ImplicitOps | Stack | No64, em_popa),
4030 N, MD(ModRM, &mode_dual_63),
4033 I(SrcImm | Mov | Stack, em_push),
4034 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4035 I(SrcImmByte | Mov | Stack, em_push),
4036 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4037 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4038 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4040 X16(D(SrcImmByte | NearBranch)),
4042 G(ByteOp | DstMem | SrcImm, group1),
4043 G(DstMem | SrcImm, group1),
4044 G(ByteOp | DstMem | SrcImm | No64, group1),
4045 G(DstMem | SrcImmByte, group1),
4046 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4047 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4049 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4050 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4051 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4052 D(ModRM | SrcMem | NoAccess | DstReg),
4053 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4056 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4058 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4059 I(SrcImmFAddr | No64, em_call_far), N,
4060 II(ImplicitOps | Stack, em_pushf, pushf),
4061 II(ImplicitOps | Stack, em_popf, popf),
4062 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4064 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4065 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4066 I2bv(SrcSI | DstDI | Mov | String, em_mov),
4067 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp_r),
4069 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4070 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4071 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4072 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4074 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4076 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4078 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4079 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4080 I(ImplicitOps | NearBranch, em_ret),
4081 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4082 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4083 G(ByteOp, group11), G(0, group11),
4085 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4086 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4087 I(ImplicitOps, em_ret_far),
4088 D(ImplicitOps), DI(SrcImmByte, intn),
4089 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4091 G(Src2One | ByteOp, group2), G(Src2One, group2),
4092 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4093 I(DstAcc | SrcImmUByte | No64, em_aam),
4094 I(DstAcc | SrcImmUByte | No64, em_aad),
4095 F(DstAcc | ByteOp | No64, em_salc),
4096 I(DstAcc | SrcXLat | ByteOp, em_mov),
4098 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4100 X3(I(SrcImmByte | NearBranch, em_loop)),
4101 I(SrcImmByte | NearBranch, em_jcxz),
4102 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4103 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4105 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4106 I(SrcImmFAddr | No64, em_jmp_far),
4107 D(SrcImmByte | ImplicitOps | NearBranch),
4108 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4109 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4111 N, DI(ImplicitOps, icebp), N, N,
4112 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4113 G(ByteOp, group3), G(0, group3),
4115 D(ImplicitOps), D(ImplicitOps),
4116 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4117 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4120 static const struct opcode twobyte_table[256] = {
4122 G(0, group6), GD(0, &group7), N, N,
4123 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4124 II(ImplicitOps | Priv, em_clts, clts), N,
4125 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4126 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4128 N, N, N, N, N, N, N, N,
4129 D(ImplicitOps | ModRM | SrcMem | NoAccess),
4130 N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4132 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4133 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4134 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4136 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4139 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4140 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4141 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4144 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4145 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4146 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4147 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4148 I(ImplicitOps | EmulateOnUD, em_sysenter),
4149 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4151 N, N, N, N, N, N, N, N,
4153 X16(D(DstReg | SrcMem | ModRM)),
4155 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4160 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4165 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4167 X16(D(SrcImm | NearBranch)),
4169 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4171 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4172 II(ImplicitOps, em_cpuid, cpuid),
4173 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4174 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4175 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4177 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4178 DI(ImplicitOps, rsm),
4179 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4180 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4181 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4182 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4184 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4185 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4186 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4187 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4188 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4189 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4193 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4194 I(DstReg | SrcMem | ModRM, em_bsf_c),
4195 I(DstReg | SrcMem | ModRM, em_bsr_c),
4196 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4198 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4199 N, ID(0, &instr_dual_0f_c3),
4200 N, N, N, GD(0, &group9),
4202 X8(I(DstReg, em_bswap)),
4204 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4206 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4207 N, N, N, N, N, N, N, N,
4209 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4212 static const struct instr_dual instr_dual_0f_38_f0 = {
4213 I(DstReg | SrcMem | Mov, em_movbe), N
4216 static const struct instr_dual instr_dual_0f_38_f1 = {
4217 I(DstMem | SrcReg | Mov, em_movbe), N
4220 static const struct gprefix three_byte_0f_38_f0 = {
4221 ID(0, &instr_dual_0f_38_f0), N, N, N
4224 static const struct gprefix three_byte_0f_38_f1 = {
4225 ID(0, &instr_dual_0f_38_f1), N, N, N
4229 * Insns below are selected by the prefix which indexed by the third opcode
4232 static const struct opcode opcode_map_0f_38[256] = {
4234 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4236 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4238 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4239 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4260 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4264 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4270 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4271 unsigned size, bool sign_extension)
4273 int rc = X86EMUL_CONTINUE;
4277 op->addr.mem.ea = ctxt->_eip;
4278 /* NB. Immediates are sign-extended as necessary. */
4279 switch (op->bytes) {
4281 op->val = insn_fetch(s8, ctxt);
4284 op->val = insn_fetch(s16, ctxt);
4287 op->val = insn_fetch(s32, ctxt);
4290 op->val = insn_fetch(s64, ctxt);
4293 if (!sign_extension) {
4294 switch (op->bytes) {
4302 op->val &= 0xffffffff;
4310 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4313 int rc = X86EMUL_CONTINUE;
4317 decode_register_operand(ctxt, op);
4320 rc = decode_imm(ctxt, op, 1, false);
4323 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4327 if (ctxt->d & BitOp)
4328 fetch_bit_operand(ctxt);
4329 op->orig_val = op->val;
4332 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4336 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4337 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4338 fetch_register_operand(op);
4339 op->orig_val = op->val;
4343 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4344 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4345 fetch_register_operand(op);
4346 op->orig_val = op->val;
4349 if (ctxt->d & ByteOp) {
4354 op->bytes = ctxt->op_bytes;
4355 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4356 fetch_register_operand(op);
4357 op->orig_val = op->val;
4361 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4363 register_address(ctxt, VCPU_REGS_RDI);
4364 op->addr.mem.seg = VCPU_SREG_ES;
4371 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4372 fetch_register_operand(op);
4377 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4380 rc = decode_imm(ctxt, op, 1, true);
4388 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4391 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4394 ctxt->memop.bytes = 1;
4395 if (ctxt->memop.type == OP_REG) {
4396 ctxt->memop.addr.reg = decode_register(ctxt,
4397 ctxt->modrm_rm, true);
4398 fetch_register_operand(&ctxt->memop);
4402 ctxt->memop.bytes = 2;
4405 ctxt->memop.bytes = 4;
4408 rc = decode_imm(ctxt, op, 2, false);
4411 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4415 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4417 register_address(ctxt, VCPU_REGS_RSI);
4418 op->addr.mem.seg = ctxt->seg_override;
4424 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4427 reg_read(ctxt, VCPU_REGS_RBX) +
4428 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4429 op->addr.mem.seg = ctxt->seg_override;
4434 op->addr.mem.ea = ctxt->_eip;
4435 op->bytes = ctxt->op_bytes + 2;
4436 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4439 ctxt->memop.bytes = ctxt->op_bytes + 2;
4443 op->val = VCPU_SREG_ES;
4447 op->val = VCPU_SREG_CS;
4451 op->val = VCPU_SREG_SS;
4455 op->val = VCPU_SREG_DS;
4459 op->val = VCPU_SREG_FS;
4463 op->val = VCPU_SREG_GS;
4466 /* Special instructions do their own operand decoding. */
4468 op->type = OP_NONE; /* Disable writeback. */
4476 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4478 int rc = X86EMUL_CONTINUE;
4479 int mode = ctxt->mode;
4480 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4481 bool op_prefix = false;
4482 bool has_seg_override = false;
4483 struct opcode opcode;
4485 ctxt->memop.type = OP_NONE;
4486 ctxt->memopp = NULL;
4487 ctxt->_eip = ctxt->eip;
4488 ctxt->fetch.ptr = ctxt->fetch.data;
4489 ctxt->fetch.end = ctxt->fetch.data + insn_len;
4490 ctxt->opcode_len = 1;
4492 memcpy(ctxt->fetch.data, insn, insn_len);
4494 rc = __do_insn_fetch_bytes(ctxt, 1);
4495 if (rc != X86EMUL_CONTINUE)
4500 case X86EMUL_MODE_REAL:
4501 case X86EMUL_MODE_VM86:
4502 case X86EMUL_MODE_PROT16:
4503 def_op_bytes = def_ad_bytes = 2;
4505 case X86EMUL_MODE_PROT32:
4506 def_op_bytes = def_ad_bytes = 4;
4508 #ifdef CONFIG_X86_64
4509 case X86EMUL_MODE_PROT64:
4515 return EMULATION_FAILED;
4518 ctxt->op_bytes = def_op_bytes;
4519 ctxt->ad_bytes = def_ad_bytes;
4521 /* Legacy prefixes. */
4523 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4524 case 0x66: /* operand-size override */
4526 /* switch between 2/4 bytes */
4527 ctxt->op_bytes = def_op_bytes ^ 6;
4529 case 0x67: /* address-size override */
4530 if (mode == X86EMUL_MODE_PROT64)
4531 /* switch between 4/8 bytes */
4532 ctxt->ad_bytes = def_ad_bytes ^ 12;
4534 /* switch between 2/4 bytes */
4535 ctxt->ad_bytes = def_ad_bytes ^ 6;
4537 case 0x26: /* ES override */
4538 case 0x2e: /* CS override */
4539 case 0x36: /* SS override */
4540 case 0x3e: /* DS override */
4541 has_seg_override = true;
4542 ctxt->seg_override = (ctxt->b >> 3) & 3;
4544 case 0x64: /* FS override */
4545 case 0x65: /* GS override */
4546 has_seg_override = true;
4547 ctxt->seg_override = ctxt->b & 7;
4549 case 0x40 ... 0x4f: /* REX */
4550 if (mode != X86EMUL_MODE_PROT64)
4552 ctxt->rex_prefix = ctxt->b;
4554 case 0xf0: /* LOCK */
4555 ctxt->lock_prefix = 1;
4557 case 0xf2: /* REPNE/REPNZ */
4558 case 0xf3: /* REP/REPE/REPZ */
4559 ctxt->rep_prefix = ctxt->b;
4565 /* Any legacy prefix after a REX prefix nullifies its effect. */
4567 ctxt->rex_prefix = 0;
4573 if (ctxt->rex_prefix & 8)
4574 ctxt->op_bytes = 8; /* REX.W */
4576 /* Opcode byte(s). */
4577 opcode = opcode_table[ctxt->b];
4578 /* Two-byte opcode? */
4579 if (ctxt->b == 0x0f) {
4580 ctxt->opcode_len = 2;
4581 ctxt->b = insn_fetch(u8, ctxt);
4582 opcode = twobyte_table[ctxt->b];
4584 /* 0F_38 opcode map */
4585 if (ctxt->b == 0x38) {
4586 ctxt->opcode_len = 3;
4587 ctxt->b = insn_fetch(u8, ctxt);
4588 opcode = opcode_map_0f_38[ctxt->b];
4591 ctxt->d = opcode.flags;
4593 if (ctxt->d & ModRM)
4594 ctxt->modrm = insn_fetch(u8, ctxt);
4596 /* vex-prefix instructions are not implemented */
4597 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
4598 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
4602 while (ctxt->d & GroupMask) {
4603 switch (ctxt->d & GroupMask) {
4605 goffset = (ctxt->modrm >> 3) & 7;
4606 opcode = opcode.u.group[goffset];
4609 goffset = (ctxt->modrm >> 3) & 7;
4610 if ((ctxt->modrm >> 6) == 3)
4611 opcode = opcode.u.gdual->mod3[goffset];
4613 opcode = opcode.u.gdual->mod012[goffset];
4616 goffset = ctxt->modrm & 7;
4617 opcode = opcode.u.group[goffset];
4620 if (ctxt->rep_prefix && op_prefix)
4621 return EMULATION_FAILED;
4622 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4623 switch (simd_prefix) {
4624 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4625 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4626 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4627 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4631 if (ctxt->modrm > 0xbf)
4632 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4634 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4637 if ((ctxt->modrm >> 6) == 3)
4638 opcode = opcode.u.idual->mod3;
4640 opcode = opcode.u.idual->mod012;
4643 if (ctxt->mode == X86EMUL_MODE_PROT64)
4644 opcode = opcode.u.mdual->mode64;
4646 opcode = opcode.u.mdual->mode32;
4649 return EMULATION_FAILED;
4652 ctxt->d &= ~(u64)GroupMask;
4653 ctxt->d |= opcode.flags;
4658 return EMULATION_FAILED;
4660 ctxt->execute = opcode.u.execute;
4662 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
4663 return EMULATION_FAILED;
4665 if (unlikely(ctxt->d &
4666 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
4669 * These are copied unconditionally here, and checked unconditionally
4670 * in x86_emulate_insn.
4672 ctxt->check_perm = opcode.check_perm;
4673 ctxt->intercept = opcode.intercept;
4675 if (ctxt->d & NotImpl)
4676 return EMULATION_FAILED;
4678 if (mode == X86EMUL_MODE_PROT64) {
4679 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
4681 else if (ctxt->d & NearBranch)
4685 if (ctxt->d & Op3264) {
4686 if (mode == X86EMUL_MODE_PROT64)
4692 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
4696 ctxt->op_bytes = 16;
4697 else if (ctxt->d & Mmx)
4701 /* ModRM and SIB bytes. */
4702 if (ctxt->d & ModRM) {
4703 rc = decode_modrm(ctxt, &ctxt->memop);
4704 if (!has_seg_override) {
4705 has_seg_override = true;
4706 ctxt->seg_override = ctxt->modrm_seg;
4708 } else if (ctxt->d & MemAbs)
4709 rc = decode_abs(ctxt, &ctxt->memop);
4710 if (rc != X86EMUL_CONTINUE)
4713 if (!has_seg_override)
4714 ctxt->seg_override = VCPU_SREG_DS;
4716 ctxt->memop.addr.mem.seg = ctxt->seg_override;
4719 * Decode and fetch the source operand: register, memory
4722 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4723 if (rc != X86EMUL_CONTINUE)
4727 * Decode and fetch the second source operand: register, memory
4730 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4731 if (rc != X86EMUL_CONTINUE)
4734 /* Decode and fetch the destination operand: register or memory. */
4735 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4737 if (ctxt->rip_relative)
4738 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
4739 ctxt->memopp->addr.mem.ea + ctxt->_eip);
4742 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4745 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4747 return ctxt->d & PageTable;
4750 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4752 /* The second termination condition only applies for REPE
4753 * and REPNE. Test if the repeat string operation prefix is
4754 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4755 * corresponding termination condition according to:
4756 * - if REPE/REPZ and ZF = 0 then done
4757 * - if REPNE/REPNZ and ZF = 1 then done
4759 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4760 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4761 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4762 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
4763 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4764 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
4770 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4774 ctxt->ops->get_fpu(ctxt);
4775 asm volatile("1: fwait \n\t"
4777 ".pushsection .fixup,\"ax\" \n\t"
4779 "movb $1, %[fault] \n\t"
4782 _ASM_EXTABLE(1b, 3b)
4783 : [fault]"+qm"(fault));
4784 ctxt->ops->put_fpu(ctxt);
4786 if (unlikely(fault))
4787 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4789 return X86EMUL_CONTINUE;
4792 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4795 if (op->type == OP_MM)
4796 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4799 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4801 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4802 if (!(ctxt->d & ByteOp))
4803 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4804 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4805 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
4807 : "c"(ctxt->src2.val));
4808 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4809 if (!fop) /* exception is returned in fop variable */
4810 return emulate_de(ctxt);
4811 return X86EMUL_CONTINUE;
4814 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
4816 memset(&ctxt->rip_relative, 0,
4817 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
4819 ctxt->io_read.pos = 0;
4820 ctxt->io_read.end = 0;
4821 ctxt->mem_read.end = 0;
4824 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4826 const struct x86_emulate_ops *ops = ctxt->ops;
4827 int rc = X86EMUL_CONTINUE;
4828 int saved_dst_type = ctxt->dst.type;
4830 ctxt->mem_read.pos = 0;
4832 /* LOCK prefix is allowed only with some instructions */
4833 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4834 rc = emulate_ud(ctxt);
4838 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4839 rc = emulate_ud(ctxt);
4843 if (unlikely(ctxt->d &
4844 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
4845 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4846 (ctxt->d & Undefined)) {
4847 rc = emulate_ud(ctxt);
4851 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4852 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4853 rc = emulate_ud(ctxt);
4857 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4858 rc = emulate_nm(ctxt);
4862 if (ctxt->d & Mmx) {
4863 rc = flush_pending_x87_faults(ctxt);
4864 if (rc != X86EMUL_CONTINUE)
4867 * Now that we know the fpu is exception safe, we can fetch
4870 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4871 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4872 if (!(ctxt->d & Mov))
4873 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4876 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4877 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4878 X86_ICPT_PRE_EXCEPT);
4879 if (rc != X86EMUL_CONTINUE)
4883 /* Instruction can only be executed in protected mode */
4884 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4885 rc = emulate_ud(ctxt);
4889 /* Privileged instruction can be executed only in CPL=0 */
4890 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4891 if (ctxt->d & PrivUD)
4892 rc = emulate_ud(ctxt);
4894 rc = emulate_gp(ctxt, 0);
4898 /* Do instruction specific permission checks */
4899 if (ctxt->d & CheckPerm) {
4900 rc = ctxt->check_perm(ctxt);
4901 if (rc != X86EMUL_CONTINUE)
4905 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4906 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4907 X86_ICPT_POST_EXCEPT);
4908 if (rc != X86EMUL_CONTINUE)
4912 if (ctxt->rep_prefix && (ctxt->d & String)) {
4913 /* All REP prefixes have the same first termination condition */
4914 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4915 ctxt->eip = ctxt->_eip;
4916 ctxt->eflags &= ~X86_EFLAGS_RF;
4922 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4923 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4924 ctxt->src.valptr, ctxt->src.bytes);
4925 if (rc != X86EMUL_CONTINUE)
4927 ctxt->src.orig_val64 = ctxt->src.val64;
4930 if (ctxt->src2.type == OP_MEM) {
4931 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4932 &ctxt->src2.val, ctxt->src2.bytes);
4933 if (rc != X86EMUL_CONTINUE)
4937 if ((ctxt->d & DstMask) == ImplicitOps)
4941 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4942 /* optimisation - avoid slow emulated read if Mov */
4943 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4944 &ctxt->dst.val, ctxt->dst.bytes);
4945 if (rc != X86EMUL_CONTINUE) {
4946 if (!(ctxt->d & NoWrite) &&
4947 rc == X86EMUL_PROPAGATE_FAULT &&
4948 ctxt->exception.vector == PF_VECTOR)
4949 ctxt->exception.error_code |= PFERR_WRITE_MASK;
4953 /* Copy full 64-bit value for CMPXCHG8B. */
4954 ctxt->dst.orig_val64 = ctxt->dst.val64;
4958 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4959 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4960 X86_ICPT_POST_MEMACCESS);
4961 if (rc != X86EMUL_CONTINUE)
4965 if (ctxt->rep_prefix && (ctxt->d & String))
4966 ctxt->eflags |= X86_EFLAGS_RF;
4968 ctxt->eflags &= ~X86_EFLAGS_RF;
4970 if (ctxt->execute) {
4971 if (ctxt->d & Fastop) {
4972 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4973 rc = fastop(ctxt, fop);
4974 if (rc != X86EMUL_CONTINUE)
4978 rc = ctxt->execute(ctxt);
4979 if (rc != X86EMUL_CONTINUE)
4984 if (ctxt->opcode_len == 2)
4986 else if (ctxt->opcode_len == 3)
4987 goto threebyte_insn;
4990 case 0x70 ... 0x7f: /* jcc (short) */
4991 if (test_cc(ctxt->b, ctxt->eflags))
4992 rc = jmp_rel(ctxt, ctxt->src.val);
4994 case 0x8d: /* lea r16/r32, m */
4995 ctxt->dst.val = ctxt->src.addr.mem.ea;
4997 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4998 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4999 ctxt->dst.type = OP_NONE;
5003 case 0x98: /* cbw/cwde/cdqe */
5004 switch (ctxt->op_bytes) {
5005 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5006 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5007 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5010 case 0xcc: /* int3 */
5011 rc = emulate_int(ctxt, 3);
5013 case 0xcd: /* int n */
5014 rc = emulate_int(ctxt, ctxt->src.val);
5016 case 0xce: /* into */
5017 if (ctxt->eflags & X86_EFLAGS_OF)
5018 rc = emulate_int(ctxt, 4);
5020 case 0xe9: /* jmp rel */
5021 case 0xeb: /* jmp rel short */
5022 rc = jmp_rel(ctxt, ctxt->src.val);
5023 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5025 case 0xf4: /* hlt */
5026 ctxt->ops->halt(ctxt);
5028 case 0xf5: /* cmc */
5029 /* complement carry flag from eflags reg */
5030 ctxt->eflags ^= X86_EFLAGS_CF;
5032 case 0xf8: /* clc */
5033 ctxt->eflags &= ~X86_EFLAGS_CF;
5035 case 0xf9: /* stc */
5036 ctxt->eflags |= X86_EFLAGS_CF;
5038 case 0xfc: /* cld */
5039 ctxt->eflags &= ~X86_EFLAGS_DF;
5041 case 0xfd: /* std */
5042 ctxt->eflags |= X86_EFLAGS_DF;
5045 goto cannot_emulate;
5048 if (rc != X86EMUL_CONTINUE)
5052 if (ctxt->d & SrcWrite) {
5053 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5054 rc = writeback(ctxt, &ctxt->src);
5055 if (rc != X86EMUL_CONTINUE)
5058 if (!(ctxt->d & NoWrite)) {
5059 rc = writeback(ctxt, &ctxt->dst);
5060 if (rc != X86EMUL_CONTINUE)
5065 * restore dst type in case the decoding will be reused
5066 * (happens for string instruction )
5068 ctxt->dst.type = saved_dst_type;
5070 if ((ctxt->d & SrcMask) == SrcSI)
5071 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5073 if ((ctxt->d & DstMask) == DstDI)
5074 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5076 if (ctxt->rep_prefix && (ctxt->d & String)) {
5078 struct read_cache *r = &ctxt->io_read;
5079 if ((ctxt->d & SrcMask) == SrcSI)
5080 count = ctxt->src.count;
5082 count = ctxt->dst.count;
5083 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5085 if (!string_insn_completed(ctxt)) {
5087 * Re-enter guest when pio read ahead buffer is empty
5088 * or, if it is not used, after each 1024 iteration.
5090 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5091 (r->end == 0 || r->end != r->pos)) {
5093 * Reset read cache. Usually happens before
5094 * decode, but since instruction is restarted
5095 * we have to do it here.
5097 ctxt->mem_read.end = 0;
5098 writeback_registers(ctxt);
5099 return EMULATION_RESTART;
5101 goto done; /* skip rip writeback */
5103 ctxt->eflags &= ~X86_EFLAGS_RF;
5106 ctxt->eip = ctxt->_eip;
5109 if (rc == X86EMUL_PROPAGATE_FAULT) {
5110 WARN_ON(ctxt->exception.vector > 0x1f);
5111 ctxt->have_exception = true;
5113 if (rc == X86EMUL_INTERCEPTED)
5114 return EMULATION_INTERCEPTED;
5116 if (rc == X86EMUL_CONTINUE)
5117 writeback_registers(ctxt);
5119 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5123 case 0x09: /* wbinvd */
5124 (ctxt->ops->wbinvd)(ctxt);
5126 case 0x08: /* invd */
5127 case 0x0d: /* GrpP (prefetch) */
5128 case 0x18: /* Grp16 (prefetch/nop) */
5129 case 0x1f: /* nop */
5131 case 0x20: /* mov cr, reg */
5132 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5134 case 0x21: /* mov from dr to reg */
5135 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5137 case 0x40 ... 0x4f: /* cmov */
5138 if (test_cc(ctxt->b, ctxt->eflags))
5139 ctxt->dst.val = ctxt->src.val;
5140 else if (ctxt->op_bytes != 4)
5141 ctxt->dst.type = OP_NONE; /* no writeback */
5143 case 0x80 ... 0x8f: /* jnz rel, etc*/
5144 if (test_cc(ctxt->b, ctxt->eflags))
5145 rc = jmp_rel(ctxt, ctxt->src.val);
5147 case 0x90 ... 0x9f: /* setcc r/m8 */
5148 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5150 case 0xb6 ... 0xb7: /* movzx */
5151 ctxt->dst.bytes = ctxt->op_bytes;
5152 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5153 : (u16) ctxt->src.val;
5155 case 0xbe ... 0xbf: /* movsx */
5156 ctxt->dst.bytes = ctxt->op_bytes;
5157 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5158 (s16) ctxt->src.val;
5161 goto cannot_emulate;
5166 if (rc != X86EMUL_CONTINUE)
5172 return EMULATION_FAILED;
5175 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5177 invalidate_registers(ctxt);
5180 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5182 writeback_registers(ctxt);