1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstImmUByte (OpImmUByte << DstShift)
90 #define DstDX (OpDX << DstShift)
91 #define DstAccLo (OpAccLo << DstShift)
92 #define DstMask (OpMask << DstShift)
93 /* Source operand type. */
95 #define SrcNone (OpNone << SrcShift)
96 #define SrcReg (OpReg << SrcShift)
97 #define SrcMem (OpMem << SrcShift)
98 #define SrcMem16 (OpMem16 << SrcShift)
99 #define SrcMem32 (OpMem32 << SrcShift)
100 #define SrcImm (OpImm << SrcShift)
101 #define SrcImmByte (OpImmByte << SrcShift)
102 #define SrcOne (OpOne << SrcShift)
103 #define SrcImmUByte (OpImmUByte << SrcShift)
104 #define SrcImmU (OpImmU << SrcShift)
105 #define SrcSI (OpSI << SrcShift)
106 #define SrcXLat (OpXLat << SrcShift)
107 #define SrcImmFAddr (OpImmFAddr << SrcShift)
108 #define SrcMemFAddr (OpMemFAddr << SrcShift)
109 #define SrcAcc (OpAcc << SrcShift)
110 #define SrcImmU16 (OpImmU16 << SrcShift)
111 #define SrcImm64 (OpImm64 << SrcShift)
112 #define SrcDX (OpDX << SrcShift)
113 #define SrcMem8 (OpMem8 << SrcShift)
114 #define SrcAccHi (OpAccHi << SrcShift)
115 #define SrcMask (OpMask << SrcShift)
116 #define BitOp (1<<11)
117 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
118 #define String (1<<13) /* String instruction (rep capable) */
119 #define Stack (1<<14) /* Stack instruction (push/pop) */
120 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
121 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
122 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
123 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
124 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
125 #define Escape (5<<15) /* Escape to coprocessor instruction */
126 #define Sse (1<<18) /* SSE Vector instruction */
127 /* Generic ModRM decode. */
128 #define ModRM (1<<19)
129 /* Destination is only written; never read. */
132 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
133 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
134 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
135 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
136 #define Undefined (1<<25) /* No Such Instruction */
137 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
138 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
140 #define PageTable (1 << 29) /* instruction used to write page table */
141 #define NotImpl (1 << 30) /* instruction is not implemented */
142 /* Source 2 operand type */
143 #define Src2Shift (31)
144 #define Src2None (OpNone << Src2Shift)
145 #define Src2Mem (OpMem << Src2Shift)
146 #define Src2CL (OpCL << Src2Shift)
147 #define Src2ImmByte (OpImmByte << Src2Shift)
148 #define Src2One (OpOne << Src2Shift)
149 #define Src2Imm (OpImm << Src2Shift)
150 #define Src2ES (OpES << Src2Shift)
151 #define Src2CS (OpCS << Src2Shift)
152 #define Src2SS (OpSS << Src2Shift)
153 #define Src2DS (OpDS << Src2Shift)
154 #define Src2FS (OpFS << Src2Shift)
155 #define Src2GS (OpGS << Src2Shift)
156 #define Src2Mask (OpMask << Src2Shift)
157 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
158 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
159 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
160 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
161 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
162 #define NoWrite ((u64)1 << 45) /* No writeback */
163 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
165 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
167 #define X2(x...) x, x
168 #define X3(x...) X2(x), x
169 #define X4(x...) X2(x), X2(x)
170 #define X5(x...) X4(x), x
171 #define X6(x...) X4(x), X2(x)
172 #define X7(x...) X4(x), X3(x)
173 #define X8(x...) X4(x), X4(x)
174 #define X16(x...) X8(x), X8(x)
176 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
177 #define FASTOP_SIZE 8
180 * fastop functions have a special calling convention:
185 * flags: rflags (in/out)
186 * ex: rsi (in:fastop pointer, out:zero if exception)
188 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
189 * different operand sizes can be reached by calculation, rather than a jump
190 * table (which would be bigger than the code).
192 * fastop functions are declared as taking a never-defined fastop parameter,
193 * so they can't be called from C directly.
202 int (*execute)(struct x86_emulate_ctxt *ctxt);
203 const struct opcode *group;
204 const struct group_dual *gdual;
205 const struct gprefix *gprefix;
206 const struct escape *esc;
207 void (*fastop)(struct fastop *fake);
209 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
213 struct opcode mod012[8];
214 struct opcode mod3[8];
218 struct opcode pfx_no;
219 struct opcode pfx_66;
220 struct opcode pfx_f2;
221 struct opcode pfx_f3;
226 struct opcode high[64];
229 /* EFLAGS bit definitions. */
230 #define EFLG_ID (1<<21)
231 #define EFLG_VIP (1<<20)
232 #define EFLG_VIF (1<<19)
233 #define EFLG_AC (1<<18)
234 #define EFLG_VM (1<<17)
235 #define EFLG_RF (1<<16)
236 #define EFLG_IOPL (3<<12)
237 #define EFLG_NT (1<<14)
238 #define EFLG_OF (1<<11)
239 #define EFLG_DF (1<<10)
240 #define EFLG_IF (1<<9)
241 #define EFLG_TF (1<<8)
242 #define EFLG_SF (1<<7)
243 #define EFLG_ZF (1<<6)
244 #define EFLG_AF (1<<4)
245 #define EFLG_PF (1<<2)
246 #define EFLG_CF (1<<0)
248 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
249 #define EFLG_RESERVED_ONE_MASK 2
251 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
253 if (!(ctxt->regs_valid & (1 << nr))) {
254 ctxt->regs_valid |= 1 << nr;
255 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
257 return ctxt->_regs[nr];
260 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
262 ctxt->regs_valid |= 1 << nr;
263 ctxt->regs_dirty |= 1 << nr;
264 return &ctxt->_regs[nr];
267 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
270 return reg_write(ctxt, nr);
273 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
277 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
278 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
281 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
283 ctxt->regs_dirty = 0;
284 ctxt->regs_valid = 0;
288 * These EFLAGS bits are restored from saved value during emulation, and
289 * any changes are written back to the saved value after emulation.
291 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
299 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
301 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
302 #define FOP_RET "ret \n\t"
304 #define FOP_START(op) \
305 extern void em_##op(struct fastop *fake); \
306 asm(".pushsection .text, \"ax\" \n\t" \
307 ".global em_" #op " \n\t" \
314 #define FOPNOP() FOP_ALIGN FOP_RET
316 #define FOP1E(op, dst) \
317 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
319 #define FOP1EEX(op, dst) \
320 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
322 #define FASTOP1(op) \
327 ON64(FOP1E(op##q, rax)) \
330 /* 1-operand, using src2 (for MUL/DIV r/m) */
331 #define FASTOP1SRC2(op, name) \
336 ON64(FOP1E(op, rcx)) \
339 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
340 #define FASTOP1SRC2EX(op, name) \
345 ON64(FOP1EEX(op, rcx)) \
348 #define FOP2E(op, dst, src) \
349 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
351 #define FASTOP2(op) \
353 FOP2E(op##b, al, dl) \
354 FOP2E(op##w, ax, dx) \
355 FOP2E(op##l, eax, edx) \
356 ON64(FOP2E(op##q, rax, rdx)) \
359 /* 2 operand, word only */
360 #define FASTOP2W(op) \
363 FOP2E(op##w, ax, dx) \
364 FOP2E(op##l, eax, edx) \
365 ON64(FOP2E(op##q, rax, rdx)) \
368 /* 2 operand, src is CL */
369 #define FASTOP2CL(op) \
371 FOP2E(op##b, al, cl) \
372 FOP2E(op##w, ax, cl) \
373 FOP2E(op##l, eax, cl) \
374 ON64(FOP2E(op##q, rax, cl)) \
377 #define FOP3E(op, dst, src, src2) \
378 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
380 /* 3-operand, word-only, src2=cl */
381 #define FASTOP3WCL(op) \
384 FOP3E(op##w, ax, dx, cl) \
385 FOP3E(op##l, eax, edx, cl) \
386 ON64(FOP3E(op##q, rax, rdx, cl)) \
389 /* Special case for SETcc - 1 instruction per cc */
390 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
392 asm(".global kvm_fastop_exception \n"
393 "kvm_fastop_exception: xor %esi, %esi; ret");
414 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
417 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
418 enum x86_intercept intercept,
419 enum x86_intercept_stage stage)
421 struct x86_instruction_info info = {
422 .intercept = intercept,
423 .rep_prefix = ctxt->rep_prefix,
424 .modrm_mod = ctxt->modrm_mod,
425 .modrm_reg = ctxt->modrm_reg,
426 .modrm_rm = ctxt->modrm_rm,
427 .src_val = ctxt->src.val64,
428 .src_bytes = ctxt->src.bytes,
429 .dst_bytes = ctxt->dst.bytes,
430 .ad_bytes = ctxt->ad_bytes,
431 .next_rip = ctxt->eip,
434 return ctxt->ops->intercept(ctxt, &info, stage);
437 static void assign_masked(ulong *dest, ulong src, ulong mask)
439 *dest = (*dest & ~mask) | (src & mask);
442 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
444 return (1UL << (ctxt->ad_bytes << 3)) - 1;
447 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
450 struct desc_struct ss;
452 if (ctxt->mode == X86EMUL_MODE_PROT64)
454 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
455 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
458 static int stack_size(struct x86_emulate_ctxt *ctxt)
460 return (__fls(stack_mask(ctxt)) + 1) >> 3;
463 /* Access/update address held in a register, based on addressing mode. */
464 static inline unsigned long
465 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
467 if (ctxt->ad_bytes == sizeof(unsigned long))
470 return reg & ad_mask(ctxt);
473 static inline unsigned long
474 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
476 return address_mask(ctxt, reg);
479 static void masked_increment(ulong *reg, ulong mask, int inc)
481 assign_masked(reg, *reg + inc, mask);
485 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
489 if (ctxt->ad_bytes == sizeof(unsigned long))
492 mask = ad_mask(ctxt);
493 masked_increment(reg, mask, inc);
496 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
498 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
501 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
503 register_address_increment(ctxt, &ctxt->_eip, rel);
506 static u32 desc_limit_scaled(struct desc_struct *desc)
508 u32 limit = get_desc_limit(desc);
510 return desc->g ? (limit << 12) | 0xfff : limit;
513 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
515 ctxt->has_seg_override = true;
516 ctxt->seg_override = seg;
519 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
521 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
524 return ctxt->ops->get_cached_segment_base(ctxt, seg);
527 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
529 if (!ctxt->has_seg_override)
532 return ctxt->seg_override;
535 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
536 u32 error, bool valid)
538 ctxt->exception.vector = vec;
539 ctxt->exception.error_code = error;
540 ctxt->exception.error_code_valid = valid;
541 return X86EMUL_PROPAGATE_FAULT;
544 static int emulate_db(struct x86_emulate_ctxt *ctxt)
546 return emulate_exception(ctxt, DB_VECTOR, 0, false);
549 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
551 return emulate_exception(ctxt, GP_VECTOR, err, true);
554 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
556 return emulate_exception(ctxt, SS_VECTOR, err, true);
559 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
561 return emulate_exception(ctxt, UD_VECTOR, 0, false);
564 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
566 return emulate_exception(ctxt, TS_VECTOR, err, true);
569 static int emulate_de(struct x86_emulate_ctxt *ctxt)
571 return emulate_exception(ctxt, DE_VECTOR, 0, false);
574 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
576 return emulate_exception(ctxt, NM_VECTOR, 0, false);
579 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
582 struct desc_struct desc;
584 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
588 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
593 struct desc_struct desc;
595 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
596 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
600 * x86 defines three classes of vector instructions: explicitly
601 * aligned, explicitly unaligned, and the rest, which change behaviour
602 * depending on whether they're AVX encoded or not.
604 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
605 * subject to the same check.
607 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
609 if (likely(size < 16))
612 if (ctxt->d & Aligned)
614 else if (ctxt->d & Unaligned)
616 else if (ctxt->d & Avx)
622 static int __linearize(struct x86_emulate_ctxt *ctxt,
623 struct segmented_address addr,
624 unsigned size, bool write, bool fetch,
627 struct desc_struct desc;
634 la = seg_base(ctxt, addr.seg) + addr.ea;
635 switch (ctxt->mode) {
636 case X86EMUL_MODE_PROT64:
637 if (((signed long)la << 16) >> 16 != la)
638 return emulate_gp(ctxt, 0);
641 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
645 /* code segment in protected mode or read-only data segment */
646 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
647 || !(desc.type & 2)) && write)
649 /* unreadable code segment */
650 if (!fetch && (desc.type & 8) && !(desc.type & 2))
652 lim = desc_limit_scaled(&desc);
653 if ((desc.type & 8) || !(desc.type & 4)) {
654 /* expand-up segment */
655 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
658 /* expand-down segment */
659 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
661 lim = desc.d ? 0xffffffff : 0xffff;
662 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
665 cpl = ctxt->ops->cpl(ctxt);
666 if (!(desc.type & 8)) {
670 } else if ((desc.type & 8) && !(desc.type & 4)) {
671 /* nonconforming code segment */
674 } else if ((desc.type & 8) && (desc.type & 4)) {
675 /* conforming code segment */
681 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
683 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
684 return emulate_gp(ctxt, 0);
686 return X86EMUL_CONTINUE;
688 if (addr.seg == VCPU_SREG_SS)
689 return emulate_ss(ctxt, sel);
691 return emulate_gp(ctxt, sel);
694 static int linearize(struct x86_emulate_ctxt *ctxt,
695 struct segmented_address addr,
696 unsigned size, bool write,
699 return __linearize(ctxt, addr, size, write, false, linear);
703 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
704 struct segmented_address addr,
711 rc = linearize(ctxt, addr, size, false, &linear);
712 if (rc != X86EMUL_CONTINUE)
714 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
718 * Fetch the next byte of the instruction being emulated which is pointed to
719 * by ctxt->_eip, then increment ctxt->_eip.
721 * Also prefetch the remaining bytes of the instruction without crossing page
722 * boundary if they are not in fetch_cache yet.
724 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
726 struct fetch_cache *fc = &ctxt->fetch;
730 if (ctxt->_eip == fc->end) {
731 unsigned long linear;
732 struct segmented_address addr = { .seg = VCPU_SREG_CS,
734 cur_size = fc->end - fc->start;
735 size = min(15UL - cur_size,
736 PAGE_SIZE - offset_in_page(ctxt->_eip));
737 rc = __linearize(ctxt, addr, size, false, true, &linear);
738 if (unlikely(rc != X86EMUL_CONTINUE))
740 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
741 size, &ctxt->exception);
742 if (unlikely(rc != X86EMUL_CONTINUE))
746 *dest = fc->data[ctxt->_eip - fc->start];
748 return X86EMUL_CONTINUE;
751 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
752 void *dest, unsigned size)
756 /* x86 instructions are limited to 15 bytes. */
757 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
758 return X86EMUL_UNHANDLEABLE;
760 rc = do_insn_fetch_byte(ctxt, dest++);
761 if (rc != X86EMUL_CONTINUE)
764 return X86EMUL_CONTINUE;
767 /* Fetch next part of the instruction being emulated. */
768 #define insn_fetch(_type, _ctxt) \
769 ({ unsigned long _x; \
770 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
771 if (rc != X86EMUL_CONTINUE) \
776 #define insn_fetch_arr(_arr, _size, _ctxt) \
777 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
778 if (rc != X86EMUL_CONTINUE) \
783 * Given the 'reg' portion of a ModRM byte, and a register block, return a
784 * pointer into the block that addresses the relevant register.
785 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
787 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
791 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
793 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
794 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
796 p = reg_rmw(ctxt, modrm_reg);
800 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
801 struct segmented_address addr,
802 u16 *size, unsigned long *address, int op_bytes)
809 rc = segmented_read_std(ctxt, addr, size, 2);
810 if (rc != X86EMUL_CONTINUE)
813 rc = segmented_read_std(ctxt, addr, address, op_bytes);
827 FASTOP1SRC2(mul, mul_ex);
828 FASTOP1SRC2(imul, imul_ex);
829 FASTOP1SRC2EX(div, div_ex);
830 FASTOP1SRC2EX(idiv, idiv_ex);
859 static u8 test_cc(unsigned int condition, unsigned long flags)
862 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
864 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
865 asm("push %[flags]; popf; call *%[fastop]"
866 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
870 static void fetch_register_operand(struct operand *op)
874 op->val = *(u8 *)op->addr.reg;
877 op->val = *(u16 *)op->addr.reg;
880 op->val = *(u32 *)op->addr.reg;
883 op->val = *(u64 *)op->addr.reg;
888 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
890 ctxt->ops->get_fpu(ctxt);
892 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
893 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
894 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
895 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
896 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
897 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
898 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
899 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
901 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
902 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
903 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
904 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
905 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
906 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
907 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
908 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
912 ctxt->ops->put_fpu(ctxt);
915 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
918 ctxt->ops->get_fpu(ctxt);
920 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
921 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
922 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
923 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
924 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
925 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
926 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
927 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
929 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
930 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
931 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
932 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
933 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
934 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
935 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
936 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
940 ctxt->ops->put_fpu(ctxt);
943 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
945 ctxt->ops->get_fpu(ctxt);
947 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
948 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
949 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
950 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
951 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
952 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
953 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
954 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
957 ctxt->ops->put_fpu(ctxt);
960 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
962 ctxt->ops->get_fpu(ctxt);
964 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
965 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
966 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
967 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
968 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
969 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
970 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
971 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
974 ctxt->ops->put_fpu(ctxt);
977 static int em_fninit(struct x86_emulate_ctxt *ctxt)
979 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
980 return emulate_nm(ctxt);
982 ctxt->ops->get_fpu(ctxt);
983 asm volatile("fninit");
984 ctxt->ops->put_fpu(ctxt);
985 return X86EMUL_CONTINUE;
988 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
992 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
993 return emulate_nm(ctxt);
995 ctxt->ops->get_fpu(ctxt);
996 asm volatile("fnstcw %0": "+m"(fcw));
997 ctxt->ops->put_fpu(ctxt);
999 /* force 2 byte destination */
1000 ctxt->dst.bytes = 2;
1001 ctxt->dst.val = fcw;
1003 return X86EMUL_CONTINUE;
1006 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1010 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1011 return emulate_nm(ctxt);
1013 ctxt->ops->get_fpu(ctxt);
1014 asm volatile("fnstsw %0": "+m"(fsw));
1015 ctxt->ops->put_fpu(ctxt);
1017 /* force 2 byte destination */
1018 ctxt->dst.bytes = 2;
1019 ctxt->dst.val = fsw;
1021 return X86EMUL_CONTINUE;
1024 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1027 unsigned reg = ctxt->modrm_reg;
1029 if (!(ctxt->d & ModRM))
1030 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1032 if (ctxt->d & Sse) {
1036 read_sse_reg(ctxt, &op->vec_val, reg);
1039 if (ctxt->d & Mmx) {
1048 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1049 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1051 fetch_register_operand(op);
1052 op->orig_val = op->val;
1055 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1057 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1058 ctxt->modrm_seg = VCPU_SREG_SS;
1061 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1065 int index_reg = 0, base_reg = 0, scale;
1066 int rc = X86EMUL_CONTINUE;
1069 if (ctxt->rex_prefix) {
1070 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
1071 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
1072 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1075 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1076 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1077 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1078 ctxt->modrm_seg = VCPU_SREG_DS;
1080 if (ctxt->modrm_mod == 3) {
1082 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1083 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1085 if (ctxt->d & Sse) {
1088 op->addr.xmm = ctxt->modrm_rm;
1089 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1092 if (ctxt->d & Mmx) {
1095 op->addr.xmm = ctxt->modrm_rm & 7;
1098 fetch_register_operand(op);
1104 if (ctxt->ad_bytes == 2) {
1105 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1106 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1107 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1108 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1110 /* 16-bit ModR/M decode. */
1111 switch (ctxt->modrm_mod) {
1113 if (ctxt->modrm_rm == 6)
1114 modrm_ea += insn_fetch(u16, ctxt);
1117 modrm_ea += insn_fetch(s8, ctxt);
1120 modrm_ea += insn_fetch(u16, ctxt);
1123 switch (ctxt->modrm_rm) {
1125 modrm_ea += bx + si;
1128 modrm_ea += bx + di;
1131 modrm_ea += bp + si;
1134 modrm_ea += bp + di;
1143 if (ctxt->modrm_mod != 0)
1150 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1151 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1152 ctxt->modrm_seg = VCPU_SREG_SS;
1153 modrm_ea = (u16)modrm_ea;
1155 /* 32/64-bit ModR/M decode. */
1156 if ((ctxt->modrm_rm & 7) == 4) {
1157 sib = insn_fetch(u8, ctxt);
1158 index_reg |= (sib >> 3) & 7;
1159 base_reg |= sib & 7;
1162 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1163 modrm_ea += insn_fetch(s32, ctxt);
1165 modrm_ea += reg_read(ctxt, base_reg);
1166 adjust_modrm_seg(ctxt, base_reg);
1169 modrm_ea += reg_read(ctxt, index_reg) << scale;
1170 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1171 if (ctxt->mode == X86EMUL_MODE_PROT64)
1172 ctxt->rip_relative = 1;
1174 base_reg = ctxt->modrm_rm;
1175 modrm_ea += reg_read(ctxt, base_reg);
1176 adjust_modrm_seg(ctxt, base_reg);
1178 switch (ctxt->modrm_mod) {
1180 if (ctxt->modrm_rm == 5)
1181 modrm_ea += insn_fetch(s32, ctxt);
1184 modrm_ea += insn_fetch(s8, ctxt);
1187 modrm_ea += insn_fetch(s32, ctxt);
1191 op->addr.mem.ea = modrm_ea;
1196 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1199 int rc = X86EMUL_CONTINUE;
1202 switch (ctxt->ad_bytes) {
1204 op->addr.mem.ea = insn_fetch(u16, ctxt);
1207 op->addr.mem.ea = insn_fetch(u32, ctxt);
1210 op->addr.mem.ea = insn_fetch(u64, ctxt);
1217 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1221 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1222 mask = ~(ctxt->dst.bytes * 8 - 1);
1224 if (ctxt->src.bytes == 2)
1225 sv = (s16)ctxt->src.val & (s16)mask;
1226 else if (ctxt->src.bytes == 4)
1227 sv = (s32)ctxt->src.val & (s32)mask;
1229 ctxt->dst.addr.mem.ea += (sv >> 3);
1232 /* only subword offset */
1233 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1236 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1237 unsigned long addr, void *dest, unsigned size)
1240 struct read_cache *mc = &ctxt->mem_read;
1242 if (mc->pos < mc->end)
1245 WARN_ON((mc->end + size) >= sizeof(mc->data));
1247 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1249 if (rc != X86EMUL_CONTINUE)
1255 memcpy(dest, mc->data + mc->pos, size);
1257 return X86EMUL_CONTINUE;
1260 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1261 struct segmented_address addr,
1268 rc = linearize(ctxt, addr, size, false, &linear);
1269 if (rc != X86EMUL_CONTINUE)
1271 return read_emulated(ctxt, linear, data, size);
1274 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1275 struct segmented_address addr,
1282 rc = linearize(ctxt, addr, size, true, &linear);
1283 if (rc != X86EMUL_CONTINUE)
1285 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1289 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1290 struct segmented_address addr,
1291 const void *orig_data, const void *data,
1297 rc = linearize(ctxt, addr, size, true, &linear);
1298 if (rc != X86EMUL_CONTINUE)
1300 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1301 size, &ctxt->exception);
1304 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1305 unsigned int size, unsigned short port,
1308 struct read_cache *rc = &ctxt->io_read;
1310 if (rc->pos == rc->end) { /* refill pio read ahead */
1311 unsigned int in_page, n;
1312 unsigned int count = ctxt->rep_prefix ?
1313 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1314 in_page = (ctxt->eflags & EFLG_DF) ?
1315 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1316 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1317 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1321 rc->pos = rc->end = 0;
1322 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1327 if (ctxt->rep_prefix && (ctxt->d & String) &&
1328 !(ctxt->eflags & EFLG_DF)) {
1329 ctxt->dst.data = rc->data + rc->pos;
1330 ctxt->dst.type = OP_MEM_STR;
1331 ctxt->dst.count = (rc->end - rc->pos) / size;
1334 memcpy(dest, rc->data + rc->pos, size);
1340 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1341 u16 index, struct desc_struct *desc)
1346 ctxt->ops->get_idt(ctxt, &dt);
1348 if (dt.size < index * 8 + 7)
1349 return emulate_gp(ctxt, index << 3 | 0x2);
1351 addr = dt.address + index * 8;
1352 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1356 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1357 u16 selector, struct desc_ptr *dt)
1359 const struct x86_emulate_ops *ops = ctxt->ops;
1361 if (selector & 1 << 2) {
1362 struct desc_struct desc;
1365 memset (dt, 0, sizeof *dt);
1366 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1369 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1370 dt->address = get_desc_base(&desc);
1372 ops->get_gdt(ctxt, dt);
1375 /* allowed just for 8 bytes segments */
1376 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1377 u16 selector, struct desc_struct *desc,
1381 u16 index = selector >> 3;
1384 get_descriptor_table_ptr(ctxt, selector, &dt);
1386 if (dt.size < index * 8 + 7)
1387 return emulate_gp(ctxt, selector & 0xfffc);
1389 *desc_addr_p = addr = dt.address + index * 8;
1390 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1394 /* allowed just for 8 bytes segments */
1395 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1396 u16 selector, struct desc_struct *desc)
1399 u16 index = selector >> 3;
1402 get_descriptor_table_ptr(ctxt, selector, &dt);
1404 if (dt.size < index * 8 + 7)
1405 return emulate_gp(ctxt, selector & 0xfffc);
1407 addr = dt.address + index * 8;
1408 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1412 /* Does not support long mode */
1413 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1414 u16 selector, int seg)
1416 struct desc_struct seg_desc, old_desc;
1418 unsigned err_vec = GP_VECTOR;
1420 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1425 memset(&seg_desc, 0, sizeof seg_desc);
1427 if (ctxt->mode == X86EMUL_MODE_REAL) {
1428 /* set real mode segment descriptor (keep limit etc. for
1430 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1431 set_desc_base(&seg_desc, selector << 4);
1433 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1434 /* VM86 needs a clean new segment descriptor */
1435 set_desc_base(&seg_desc, selector << 4);
1436 set_desc_limit(&seg_desc, 0xffff);
1445 cpl = ctxt->ops->cpl(ctxt);
1447 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1448 if ((seg == VCPU_SREG_CS
1449 || (seg == VCPU_SREG_SS
1450 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1451 || seg == VCPU_SREG_TR)
1455 /* TR should be in GDT only */
1456 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1459 if (null_selector) /* for NULL selector skip all following checks */
1462 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1463 if (ret != X86EMUL_CONTINUE)
1466 err_code = selector & 0xfffc;
1467 err_vec = GP_VECTOR;
1469 /* can't load system descriptor into segment selector */
1470 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1474 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1483 * segment is not a writable data segment or segment
1484 * selector's RPL != CPL or segment selector's RPL != CPL
1486 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1490 if (!(seg_desc.type & 8))
1493 if (seg_desc.type & 4) {
1499 if (rpl > cpl || dpl != cpl)
1502 /* CS(RPL) <- CPL */
1503 selector = (selector & 0xfffc) | cpl;
1506 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1508 old_desc = seg_desc;
1509 seg_desc.type |= 2; /* busy */
1510 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1511 sizeof(seg_desc), &ctxt->exception);
1512 if (ret != X86EMUL_CONTINUE)
1515 case VCPU_SREG_LDTR:
1516 if (seg_desc.s || seg_desc.type != 2)
1519 default: /* DS, ES, FS, or GS */
1521 * segment is not a data or readable code segment or
1522 * ((segment is a data or nonconforming code segment)
1523 * and (both RPL and CPL > DPL))
1525 if ((seg_desc.type & 0xa) == 0x8 ||
1526 (((seg_desc.type & 0xc) != 0xc) &&
1527 (rpl > dpl && cpl > dpl)))
1533 /* mark segment as accessed */
1535 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1536 if (ret != X86EMUL_CONTINUE)
1540 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1541 return X86EMUL_CONTINUE;
1543 emulate_exception(ctxt, err_vec, err_code, true);
1544 return X86EMUL_PROPAGATE_FAULT;
1547 static void write_register_operand(struct operand *op)
1549 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1550 switch (op->bytes) {
1552 *(u8 *)op->addr.reg = (u8)op->val;
1555 *(u16 *)op->addr.reg = (u16)op->val;
1558 *op->addr.reg = (u32)op->val;
1559 break; /* 64b: zero-extend */
1561 *op->addr.reg = op->val;
1566 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1572 write_register_operand(op);
1575 if (ctxt->lock_prefix)
1576 rc = segmented_cmpxchg(ctxt,
1582 rc = segmented_write(ctxt,
1586 if (rc != X86EMUL_CONTINUE)
1590 rc = segmented_write(ctxt,
1593 op->bytes * op->count);
1594 if (rc != X86EMUL_CONTINUE)
1598 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1601 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1609 return X86EMUL_CONTINUE;
1612 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1614 struct segmented_address addr;
1616 rsp_increment(ctxt, -bytes);
1617 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1618 addr.seg = VCPU_SREG_SS;
1620 return segmented_write(ctxt, addr, data, bytes);
1623 static int em_push(struct x86_emulate_ctxt *ctxt)
1625 /* Disable writeback. */
1626 ctxt->dst.type = OP_NONE;
1627 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1630 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1631 void *dest, int len)
1634 struct segmented_address addr;
1636 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1637 addr.seg = VCPU_SREG_SS;
1638 rc = segmented_read(ctxt, addr, dest, len);
1639 if (rc != X86EMUL_CONTINUE)
1642 rsp_increment(ctxt, len);
1646 static int em_pop(struct x86_emulate_ctxt *ctxt)
1648 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1651 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1652 void *dest, int len)
1655 unsigned long val, change_mask;
1656 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1657 int cpl = ctxt->ops->cpl(ctxt);
1659 rc = emulate_pop(ctxt, &val, len);
1660 if (rc != X86EMUL_CONTINUE)
1663 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1664 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1666 switch(ctxt->mode) {
1667 case X86EMUL_MODE_PROT64:
1668 case X86EMUL_MODE_PROT32:
1669 case X86EMUL_MODE_PROT16:
1671 change_mask |= EFLG_IOPL;
1673 change_mask |= EFLG_IF;
1675 case X86EMUL_MODE_VM86:
1677 return emulate_gp(ctxt, 0);
1678 change_mask |= EFLG_IF;
1680 default: /* real mode */
1681 change_mask |= (EFLG_IOPL | EFLG_IF);
1685 *(unsigned long *)dest =
1686 (ctxt->eflags & ~change_mask) | (val & change_mask);
1691 static int em_popf(struct x86_emulate_ctxt *ctxt)
1693 ctxt->dst.type = OP_REG;
1694 ctxt->dst.addr.reg = &ctxt->eflags;
1695 ctxt->dst.bytes = ctxt->op_bytes;
1696 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1699 static int em_enter(struct x86_emulate_ctxt *ctxt)
1702 unsigned frame_size = ctxt->src.val;
1703 unsigned nesting_level = ctxt->src2.val & 31;
1707 return X86EMUL_UNHANDLEABLE;
1709 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1710 rc = push(ctxt, &rbp, stack_size(ctxt));
1711 if (rc != X86EMUL_CONTINUE)
1713 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1715 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1716 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1718 return X86EMUL_CONTINUE;
1721 static int em_leave(struct x86_emulate_ctxt *ctxt)
1723 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1725 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1728 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1730 int seg = ctxt->src2.val;
1732 ctxt->src.val = get_segment_selector(ctxt, seg);
1734 return em_push(ctxt);
1737 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1739 int seg = ctxt->src2.val;
1740 unsigned long selector;
1743 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1744 if (rc != X86EMUL_CONTINUE)
1747 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1751 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1753 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1754 int rc = X86EMUL_CONTINUE;
1755 int reg = VCPU_REGS_RAX;
1757 while (reg <= VCPU_REGS_RDI) {
1758 (reg == VCPU_REGS_RSP) ?
1759 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1762 if (rc != X86EMUL_CONTINUE)
1771 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1773 ctxt->src.val = (unsigned long)ctxt->eflags;
1774 return em_push(ctxt);
1777 static int em_popa(struct x86_emulate_ctxt *ctxt)
1779 int rc = X86EMUL_CONTINUE;
1780 int reg = VCPU_REGS_RDI;
1782 while (reg >= VCPU_REGS_RAX) {
1783 if (reg == VCPU_REGS_RSP) {
1784 rsp_increment(ctxt, ctxt->op_bytes);
1788 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1789 if (rc != X86EMUL_CONTINUE)
1796 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1798 const struct x86_emulate_ops *ops = ctxt->ops;
1805 /* TODO: Add limit checks */
1806 ctxt->src.val = ctxt->eflags;
1808 if (rc != X86EMUL_CONTINUE)
1811 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1813 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1815 if (rc != X86EMUL_CONTINUE)
1818 ctxt->src.val = ctxt->_eip;
1820 if (rc != X86EMUL_CONTINUE)
1823 ops->get_idt(ctxt, &dt);
1825 eip_addr = dt.address + (irq << 2);
1826 cs_addr = dt.address + (irq << 2) + 2;
1828 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1829 if (rc != X86EMUL_CONTINUE)
1832 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1833 if (rc != X86EMUL_CONTINUE)
1836 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1837 if (rc != X86EMUL_CONTINUE)
1845 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1849 invalidate_registers(ctxt);
1850 rc = __emulate_int_real(ctxt, irq);
1851 if (rc == X86EMUL_CONTINUE)
1852 writeback_registers(ctxt);
1856 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1858 switch(ctxt->mode) {
1859 case X86EMUL_MODE_REAL:
1860 return __emulate_int_real(ctxt, irq);
1861 case X86EMUL_MODE_VM86:
1862 case X86EMUL_MODE_PROT16:
1863 case X86EMUL_MODE_PROT32:
1864 case X86EMUL_MODE_PROT64:
1866 /* Protected mode interrupts unimplemented yet */
1867 return X86EMUL_UNHANDLEABLE;
1871 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1873 int rc = X86EMUL_CONTINUE;
1874 unsigned long temp_eip = 0;
1875 unsigned long temp_eflags = 0;
1876 unsigned long cs = 0;
1877 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1878 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1879 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1880 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1882 /* TODO: Add stack limit check */
1884 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1886 if (rc != X86EMUL_CONTINUE)
1889 if (temp_eip & ~0xffff)
1890 return emulate_gp(ctxt, 0);
1892 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1894 if (rc != X86EMUL_CONTINUE)
1897 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1899 if (rc != X86EMUL_CONTINUE)
1902 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1904 if (rc != X86EMUL_CONTINUE)
1907 ctxt->_eip = temp_eip;
1910 if (ctxt->op_bytes == 4)
1911 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1912 else if (ctxt->op_bytes == 2) {
1913 ctxt->eflags &= ~0xffff;
1914 ctxt->eflags |= temp_eflags;
1917 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1918 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1923 static int em_iret(struct x86_emulate_ctxt *ctxt)
1925 switch(ctxt->mode) {
1926 case X86EMUL_MODE_REAL:
1927 return emulate_iret_real(ctxt);
1928 case X86EMUL_MODE_VM86:
1929 case X86EMUL_MODE_PROT16:
1930 case X86EMUL_MODE_PROT32:
1931 case X86EMUL_MODE_PROT64:
1933 /* iret from protected mode unimplemented yet */
1934 return X86EMUL_UNHANDLEABLE;
1938 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1943 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1945 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1946 if (rc != X86EMUL_CONTINUE)
1950 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1951 return X86EMUL_CONTINUE;
1954 static int em_grp45(struct x86_emulate_ctxt *ctxt)
1956 int rc = X86EMUL_CONTINUE;
1958 switch (ctxt->modrm_reg) {
1959 case 2: /* call near abs */ {
1961 old_eip = ctxt->_eip;
1962 ctxt->_eip = ctxt->src.val;
1963 ctxt->src.val = old_eip;
1967 case 4: /* jmp abs */
1968 ctxt->_eip = ctxt->src.val;
1970 case 5: /* jmp far */
1971 rc = em_jmp_far(ctxt);
1980 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
1982 u64 old = ctxt->dst.orig_val64;
1984 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
1985 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
1986 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
1987 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
1988 ctxt->eflags &= ~EFLG_ZF;
1990 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
1991 (u32) reg_read(ctxt, VCPU_REGS_RBX);
1993 ctxt->eflags |= EFLG_ZF;
1995 return X86EMUL_CONTINUE;
1998 static int em_ret(struct x86_emulate_ctxt *ctxt)
2000 ctxt->dst.type = OP_REG;
2001 ctxt->dst.addr.reg = &ctxt->_eip;
2002 ctxt->dst.bytes = ctxt->op_bytes;
2003 return em_pop(ctxt);
2006 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2011 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2012 if (rc != X86EMUL_CONTINUE)
2014 if (ctxt->op_bytes == 4)
2015 ctxt->_eip = (u32)ctxt->_eip;
2016 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2017 if (rc != X86EMUL_CONTINUE)
2019 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2023 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2027 rc = em_ret_far(ctxt);
2028 if (rc != X86EMUL_CONTINUE)
2030 rsp_increment(ctxt, ctxt->src.val);
2031 return X86EMUL_CONTINUE;
2034 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2036 /* Save real source value, then compare EAX against destination. */
2037 ctxt->src.orig_val = ctxt->src.val;
2038 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
2039 fastop(ctxt, em_cmp);
2041 if (ctxt->eflags & EFLG_ZF) {
2042 /* Success: write back to memory. */
2043 ctxt->dst.val = ctxt->src.orig_val;
2045 /* Failure: write the value we saw to EAX. */
2046 ctxt->dst.type = OP_REG;
2047 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2049 return X86EMUL_CONTINUE;
2052 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2054 int seg = ctxt->src2.val;
2058 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2060 rc = load_segment_descriptor(ctxt, sel, seg);
2061 if (rc != X86EMUL_CONTINUE)
2064 ctxt->dst.val = ctxt->src.val;
2069 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2070 struct desc_struct *cs, struct desc_struct *ss)
2072 cs->l = 0; /* will be adjusted later */
2073 set_desc_base(cs, 0); /* flat segment */
2074 cs->g = 1; /* 4kb granularity */
2075 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2076 cs->type = 0x0b; /* Read, Execute, Accessed */
2078 cs->dpl = 0; /* will be adjusted later */
2083 set_desc_base(ss, 0); /* flat segment */
2084 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2085 ss->g = 1; /* 4kb granularity */
2087 ss->type = 0x03; /* Read/Write, Accessed */
2088 ss->d = 1; /* 32bit stack segment */
2095 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2097 u32 eax, ebx, ecx, edx;
2100 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2101 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2102 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2103 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2106 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2108 const struct x86_emulate_ops *ops = ctxt->ops;
2109 u32 eax, ebx, ecx, edx;
2112 * syscall should always be enabled in longmode - so only become
2113 * vendor specific (cpuid) if other modes are active...
2115 if (ctxt->mode == X86EMUL_MODE_PROT64)
2120 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2122 * Intel ("GenuineIntel")
2123 * remark: Intel CPUs only support "syscall" in 64bit
2124 * longmode. Also an 64bit guest with a
2125 * 32bit compat-app running will #UD !! While this
2126 * behaviour can be fixed (by emulating) into AMD
2127 * response - CPUs of AMD can't behave like Intel.
2129 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2130 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2131 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2134 /* AMD ("AuthenticAMD") */
2135 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2136 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2137 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2140 /* AMD ("AMDisbetter!") */
2141 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2142 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2143 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2146 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2150 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2152 const struct x86_emulate_ops *ops = ctxt->ops;
2153 struct desc_struct cs, ss;
2158 /* syscall is not available in real mode */
2159 if (ctxt->mode == X86EMUL_MODE_REAL ||
2160 ctxt->mode == X86EMUL_MODE_VM86)
2161 return emulate_ud(ctxt);
2163 if (!(em_syscall_is_enabled(ctxt)))
2164 return emulate_ud(ctxt);
2166 ops->get_msr(ctxt, MSR_EFER, &efer);
2167 setup_syscalls_segments(ctxt, &cs, &ss);
2169 if (!(efer & EFER_SCE))
2170 return emulate_ud(ctxt);
2172 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2174 cs_sel = (u16)(msr_data & 0xfffc);
2175 ss_sel = (u16)(msr_data + 8);
2177 if (efer & EFER_LMA) {
2181 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2182 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2184 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2185 if (efer & EFER_LMA) {
2186 #ifdef CONFIG_X86_64
2187 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2190 ctxt->mode == X86EMUL_MODE_PROT64 ?
2191 MSR_LSTAR : MSR_CSTAR, &msr_data);
2192 ctxt->_eip = msr_data;
2194 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2195 ctxt->eflags &= ~(msr_data | EFLG_RF);
2199 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2200 ctxt->_eip = (u32)msr_data;
2202 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2205 return X86EMUL_CONTINUE;
2208 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2210 const struct x86_emulate_ops *ops = ctxt->ops;
2211 struct desc_struct cs, ss;
2216 ops->get_msr(ctxt, MSR_EFER, &efer);
2217 /* inject #GP if in real mode */
2218 if (ctxt->mode == X86EMUL_MODE_REAL)
2219 return emulate_gp(ctxt, 0);
2222 * Not recognized on AMD in compat mode (but is recognized in legacy
2225 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2226 && !vendor_intel(ctxt))
2227 return emulate_ud(ctxt);
2229 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2230 * Therefore, we inject an #UD.
2232 if (ctxt->mode == X86EMUL_MODE_PROT64)
2233 return emulate_ud(ctxt);
2235 setup_syscalls_segments(ctxt, &cs, &ss);
2237 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2238 switch (ctxt->mode) {
2239 case X86EMUL_MODE_PROT32:
2240 if ((msr_data & 0xfffc) == 0x0)
2241 return emulate_gp(ctxt, 0);
2243 case X86EMUL_MODE_PROT64:
2244 if (msr_data == 0x0)
2245 return emulate_gp(ctxt, 0);
2251 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2252 cs_sel = (u16)msr_data;
2253 cs_sel &= ~SELECTOR_RPL_MASK;
2254 ss_sel = cs_sel + 8;
2255 ss_sel &= ~SELECTOR_RPL_MASK;
2256 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2261 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2262 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2264 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2265 ctxt->_eip = msr_data;
2267 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2268 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2270 return X86EMUL_CONTINUE;
2273 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2275 const struct x86_emulate_ops *ops = ctxt->ops;
2276 struct desc_struct cs, ss;
2279 u16 cs_sel = 0, ss_sel = 0;
2281 /* inject #GP if in real mode or Virtual 8086 mode */
2282 if (ctxt->mode == X86EMUL_MODE_REAL ||
2283 ctxt->mode == X86EMUL_MODE_VM86)
2284 return emulate_gp(ctxt, 0);
2286 setup_syscalls_segments(ctxt, &cs, &ss);
2288 if ((ctxt->rex_prefix & 0x8) != 0x0)
2289 usermode = X86EMUL_MODE_PROT64;
2291 usermode = X86EMUL_MODE_PROT32;
2295 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2297 case X86EMUL_MODE_PROT32:
2298 cs_sel = (u16)(msr_data + 16);
2299 if ((msr_data & 0xfffc) == 0x0)
2300 return emulate_gp(ctxt, 0);
2301 ss_sel = (u16)(msr_data + 24);
2303 case X86EMUL_MODE_PROT64:
2304 cs_sel = (u16)(msr_data + 32);
2305 if (msr_data == 0x0)
2306 return emulate_gp(ctxt, 0);
2307 ss_sel = cs_sel + 8;
2312 cs_sel |= SELECTOR_RPL_MASK;
2313 ss_sel |= SELECTOR_RPL_MASK;
2315 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2316 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2318 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2319 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2321 return X86EMUL_CONTINUE;
2324 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2327 if (ctxt->mode == X86EMUL_MODE_REAL)
2329 if (ctxt->mode == X86EMUL_MODE_VM86)
2331 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2332 return ctxt->ops->cpl(ctxt) > iopl;
2335 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2338 const struct x86_emulate_ops *ops = ctxt->ops;
2339 struct desc_struct tr_seg;
2342 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2343 unsigned mask = (1 << len) - 1;
2346 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2349 if (desc_limit_scaled(&tr_seg) < 103)
2351 base = get_desc_base(&tr_seg);
2352 #ifdef CONFIG_X86_64
2353 base |= ((u64)base3) << 32;
2355 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2356 if (r != X86EMUL_CONTINUE)
2358 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2360 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2361 if (r != X86EMUL_CONTINUE)
2363 if ((perm >> bit_idx) & mask)
2368 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2374 if (emulator_bad_iopl(ctxt))
2375 if (!emulator_io_port_access_allowed(ctxt, port, len))
2378 ctxt->perm_ok = true;
2383 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2384 struct tss_segment_16 *tss)
2386 tss->ip = ctxt->_eip;
2387 tss->flag = ctxt->eflags;
2388 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2389 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2390 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2391 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2392 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2393 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2394 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2395 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2397 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2398 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2399 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2400 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2401 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2404 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2405 struct tss_segment_16 *tss)
2409 ctxt->_eip = tss->ip;
2410 ctxt->eflags = tss->flag | 2;
2411 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2412 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2413 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2414 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2415 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2416 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2417 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2418 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2421 * SDM says that segment selectors are loaded before segment
2424 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2425 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2426 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2427 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2428 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2431 * Now load segment descriptors. If fault happens at this stage
2432 * it is handled in a context of new task
2434 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2435 if (ret != X86EMUL_CONTINUE)
2437 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2438 if (ret != X86EMUL_CONTINUE)
2440 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2441 if (ret != X86EMUL_CONTINUE)
2443 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2444 if (ret != X86EMUL_CONTINUE)
2446 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2447 if (ret != X86EMUL_CONTINUE)
2450 return X86EMUL_CONTINUE;
2453 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2454 u16 tss_selector, u16 old_tss_sel,
2455 ulong old_tss_base, struct desc_struct *new_desc)
2457 const struct x86_emulate_ops *ops = ctxt->ops;
2458 struct tss_segment_16 tss_seg;
2460 u32 new_tss_base = get_desc_base(new_desc);
2462 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2464 if (ret != X86EMUL_CONTINUE)
2465 /* FIXME: need to provide precise fault address */
2468 save_state_to_tss16(ctxt, &tss_seg);
2470 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2472 if (ret != X86EMUL_CONTINUE)
2473 /* FIXME: need to provide precise fault address */
2476 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2478 if (ret != X86EMUL_CONTINUE)
2479 /* FIXME: need to provide precise fault address */
2482 if (old_tss_sel != 0xffff) {
2483 tss_seg.prev_task_link = old_tss_sel;
2485 ret = ops->write_std(ctxt, new_tss_base,
2486 &tss_seg.prev_task_link,
2487 sizeof tss_seg.prev_task_link,
2489 if (ret != X86EMUL_CONTINUE)
2490 /* FIXME: need to provide precise fault address */
2494 return load_state_from_tss16(ctxt, &tss_seg);
2497 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2498 struct tss_segment_32 *tss)
2500 /* CR3 and ldt selector are not saved intentionally */
2501 tss->eip = ctxt->_eip;
2502 tss->eflags = ctxt->eflags;
2503 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2504 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2505 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2506 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2507 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2508 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2509 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2510 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2512 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2513 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2514 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2515 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2516 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2517 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2520 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2521 struct tss_segment_32 *tss)
2525 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2526 return emulate_gp(ctxt, 0);
2527 ctxt->_eip = tss->eip;
2528 ctxt->eflags = tss->eflags | 2;
2530 /* General purpose registers */
2531 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2532 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2533 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2534 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2535 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2536 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2537 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2538 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2541 * SDM says that segment selectors are loaded before segment
2544 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2545 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2546 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2547 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2548 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2549 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2550 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2553 * If we're switching between Protected Mode and VM86, we need to make
2554 * sure to update the mode before loading the segment descriptors so
2555 * that the selectors are interpreted correctly.
2557 * Need to get rflags to the vcpu struct immediately because it
2558 * influences the CPL which is checked at least when loading the segment
2559 * descriptors and when pushing an error code to the new kernel stack.
2561 * TODO Introduce a separate ctxt->ops->set_cpl callback
2563 if (ctxt->eflags & X86_EFLAGS_VM)
2564 ctxt->mode = X86EMUL_MODE_VM86;
2566 ctxt->mode = X86EMUL_MODE_PROT32;
2568 ctxt->ops->set_rflags(ctxt, ctxt->eflags);
2571 * Now load segment descriptors. If fault happenes at this stage
2572 * it is handled in a context of new task
2574 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2575 if (ret != X86EMUL_CONTINUE)
2577 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2578 if (ret != X86EMUL_CONTINUE)
2580 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2581 if (ret != X86EMUL_CONTINUE)
2583 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2584 if (ret != X86EMUL_CONTINUE)
2586 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2587 if (ret != X86EMUL_CONTINUE)
2589 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2590 if (ret != X86EMUL_CONTINUE)
2592 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2593 if (ret != X86EMUL_CONTINUE)
2596 return X86EMUL_CONTINUE;
2599 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2600 u16 tss_selector, u16 old_tss_sel,
2601 ulong old_tss_base, struct desc_struct *new_desc)
2603 const struct x86_emulate_ops *ops = ctxt->ops;
2604 struct tss_segment_32 tss_seg;
2606 u32 new_tss_base = get_desc_base(new_desc);
2607 u32 eip_offset = offsetof(struct tss_segment_32, eip);
2608 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
2610 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2612 if (ret != X86EMUL_CONTINUE)
2613 /* FIXME: need to provide precise fault address */
2616 save_state_to_tss32(ctxt, &tss_seg);
2618 /* Only GP registers and segment selectors are saved */
2619 ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
2620 ldt_sel_offset - eip_offset, &ctxt->exception);
2621 if (ret != X86EMUL_CONTINUE)
2622 /* FIXME: need to provide precise fault address */
2625 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2627 if (ret != X86EMUL_CONTINUE)
2628 /* FIXME: need to provide precise fault address */
2631 if (old_tss_sel != 0xffff) {
2632 tss_seg.prev_task_link = old_tss_sel;
2634 ret = ops->write_std(ctxt, new_tss_base,
2635 &tss_seg.prev_task_link,
2636 sizeof tss_seg.prev_task_link,
2638 if (ret != X86EMUL_CONTINUE)
2639 /* FIXME: need to provide precise fault address */
2643 return load_state_from_tss32(ctxt, &tss_seg);
2646 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2647 u16 tss_selector, int idt_index, int reason,
2648 bool has_error_code, u32 error_code)
2650 const struct x86_emulate_ops *ops = ctxt->ops;
2651 struct desc_struct curr_tss_desc, next_tss_desc;
2653 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2654 ulong old_tss_base =
2655 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2659 /* FIXME: old_tss_base == ~0 ? */
2661 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2662 if (ret != X86EMUL_CONTINUE)
2664 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2665 if (ret != X86EMUL_CONTINUE)
2668 /* FIXME: check that next_tss_desc is tss */
2671 * Check privileges. The three cases are task switch caused by...
2673 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2674 * 2. Exception/IRQ/iret: No check is performed
2675 * 3. jmp/call to TSS: Check against DPL of the TSS
2677 if (reason == TASK_SWITCH_GATE) {
2678 if (idt_index != -1) {
2679 /* Software interrupts */
2680 struct desc_struct task_gate_desc;
2683 ret = read_interrupt_descriptor(ctxt, idt_index,
2685 if (ret != X86EMUL_CONTINUE)
2688 dpl = task_gate_desc.dpl;
2689 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2690 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2692 } else if (reason != TASK_SWITCH_IRET) {
2693 int dpl = next_tss_desc.dpl;
2694 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2695 return emulate_gp(ctxt, tss_selector);
2699 desc_limit = desc_limit_scaled(&next_tss_desc);
2700 if (!next_tss_desc.p ||
2701 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2702 desc_limit < 0x2b)) {
2703 emulate_ts(ctxt, tss_selector & 0xfffc);
2704 return X86EMUL_PROPAGATE_FAULT;
2707 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2708 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2709 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2712 if (reason == TASK_SWITCH_IRET)
2713 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2715 /* set back link to prev task only if NT bit is set in eflags
2716 note that old_tss_sel is not used after this point */
2717 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2718 old_tss_sel = 0xffff;
2720 if (next_tss_desc.type & 8)
2721 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2722 old_tss_base, &next_tss_desc);
2724 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2725 old_tss_base, &next_tss_desc);
2726 if (ret != X86EMUL_CONTINUE)
2729 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2730 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2732 if (reason != TASK_SWITCH_IRET) {
2733 next_tss_desc.type |= (1 << 1); /* set busy flag */
2734 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2737 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2738 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2740 if (has_error_code) {
2741 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2742 ctxt->lock_prefix = 0;
2743 ctxt->src.val = (unsigned long) error_code;
2744 ret = em_push(ctxt);
2750 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2751 u16 tss_selector, int idt_index, int reason,
2752 bool has_error_code, u32 error_code)
2756 invalidate_registers(ctxt);
2757 ctxt->_eip = ctxt->eip;
2758 ctxt->dst.type = OP_NONE;
2760 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2761 has_error_code, error_code);
2763 if (rc == X86EMUL_CONTINUE) {
2764 ctxt->eip = ctxt->_eip;
2765 writeback_registers(ctxt);
2768 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2771 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2774 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2776 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2777 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2780 static int em_das(struct x86_emulate_ctxt *ctxt)
2783 bool af, cf, old_cf;
2785 cf = ctxt->eflags & X86_EFLAGS_CF;
2791 af = ctxt->eflags & X86_EFLAGS_AF;
2792 if ((al & 0x0f) > 9 || af) {
2794 cf = old_cf | (al >= 250);
2799 if (old_al > 0x99 || old_cf) {
2805 /* Set PF, ZF, SF */
2806 ctxt->src.type = OP_IMM;
2808 ctxt->src.bytes = 1;
2809 fastop(ctxt, em_or);
2810 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2812 ctxt->eflags |= X86_EFLAGS_CF;
2814 ctxt->eflags |= X86_EFLAGS_AF;
2815 return X86EMUL_CONTINUE;
2818 static int em_aam(struct x86_emulate_ctxt *ctxt)
2822 if (ctxt->src.val == 0)
2823 return emulate_de(ctxt);
2825 al = ctxt->dst.val & 0xff;
2826 ah = al / ctxt->src.val;
2827 al %= ctxt->src.val;
2829 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
2831 /* Set PF, ZF, SF */
2832 ctxt->src.type = OP_IMM;
2834 ctxt->src.bytes = 1;
2835 fastop(ctxt, em_or);
2837 return X86EMUL_CONTINUE;
2840 static int em_aad(struct x86_emulate_ctxt *ctxt)
2842 u8 al = ctxt->dst.val & 0xff;
2843 u8 ah = (ctxt->dst.val >> 8) & 0xff;
2845 al = (al + (ah * ctxt->src.val)) & 0xff;
2847 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
2849 /* Set PF, ZF, SF */
2850 ctxt->src.type = OP_IMM;
2852 ctxt->src.bytes = 1;
2853 fastop(ctxt, em_or);
2855 return X86EMUL_CONTINUE;
2858 static int em_call(struct x86_emulate_ctxt *ctxt)
2860 long rel = ctxt->src.val;
2862 ctxt->src.val = (unsigned long)ctxt->_eip;
2864 return em_push(ctxt);
2867 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2873 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2874 old_eip = ctxt->_eip;
2876 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2877 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2878 return X86EMUL_CONTINUE;
2881 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2883 ctxt->src.val = old_cs;
2885 if (rc != X86EMUL_CONTINUE)
2888 ctxt->src.val = old_eip;
2889 return em_push(ctxt);
2892 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2896 ctxt->dst.type = OP_REG;
2897 ctxt->dst.addr.reg = &ctxt->_eip;
2898 ctxt->dst.bytes = ctxt->op_bytes;
2899 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2900 if (rc != X86EMUL_CONTINUE)
2902 rsp_increment(ctxt, ctxt->src.val);
2903 return X86EMUL_CONTINUE;
2906 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2908 /* Write back the register source. */
2909 ctxt->src.val = ctxt->dst.val;
2910 write_register_operand(&ctxt->src);
2912 /* Write back the memory destination with implicit LOCK prefix. */
2913 ctxt->dst.val = ctxt->src.orig_val;
2914 ctxt->lock_prefix = 1;
2915 return X86EMUL_CONTINUE;
2918 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2920 ctxt->dst.val = ctxt->src2.val;
2921 return fastop(ctxt, em_imul);
2924 static int em_cwd(struct x86_emulate_ctxt *ctxt)
2926 ctxt->dst.type = OP_REG;
2927 ctxt->dst.bytes = ctxt->src.bytes;
2928 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
2929 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2931 return X86EMUL_CONTINUE;
2934 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2938 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2939 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
2940 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
2941 return X86EMUL_CONTINUE;
2944 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
2948 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
2949 return emulate_gp(ctxt, 0);
2950 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
2951 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
2952 return X86EMUL_CONTINUE;
2955 static int em_mov(struct x86_emulate_ctxt *ctxt)
2957 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
2958 return X86EMUL_CONTINUE;
2961 #define FFL(x) bit(X86_FEATURE_##x)
2963 static int em_movbe(struct x86_emulate_ctxt *ctxt)
2965 u32 ebx, ecx, edx, eax = 1;
2969 * Check MOVBE is set in the guest-visible CPUID leaf.
2971 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2972 if (!(ecx & FFL(MOVBE)))
2973 return emulate_ud(ctxt);
2975 switch (ctxt->op_bytes) {
2978 * From MOVBE definition: "...When the operand size is 16 bits,
2979 * the upper word of the destination register remains unchanged
2982 * Both casting ->valptr and ->val to u16 breaks strict aliasing
2983 * rules so we have to do the operation almost per hand.
2985 tmp = (u16)ctxt->src.val;
2986 ctxt->dst.val &= ~0xffffUL;
2987 ctxt->dst.val |= (unsigned long)swab16(tmp);
2990 ctxt->dst.val = swab32((u32)ctxt->src.val);
2993 ctxt->dst.val = swab64(ctxt->src.val);
2996 return X86EMUL_PROPAGATE_FAULT;
2998 return X86EMUL_CONTINUE;
3001 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3003 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3004 return emulate_gp(ctxt, 0);
3006 /* Disable writeback. */
3007 ctxt->dst.type = OP_NONE;
3008 return X86EMUL_CONTINUE;
3011 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3015 if (ctxt->mode == X86EMUL_MODE_PROT64)
3016 val = ctxt->src.val & ~0ULL;
3018 val = ctxt->src.val & ~0U;
3020 /* #UD condition is already handled. */
3021 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3022 return emulate_gp(ctxt, 0);
3024 /* Disable writeback. */
3025 ctxt->dst.type = OP_NONE;
3026 return X86EMUL_CONTINUE;
3029 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3033 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3034 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3035 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3036 return emulate_gp(ctxt, 0);
3038 return X86EMUL_CONTINUE;
3041 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3045 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3046 return emulate_gp(ctxt, 0);
3048 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3049 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3050 return X86EMUL_CONTINUE;
3053 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3055 if (ctxt->modrm_reg > VCPU_SREG_GS)
3056 return emulate_ud(ctxt);
3058 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3059 return X86EMUL_CONTINUE;
3062 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3064 u16 sel = ctxt->src.val;
3066 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3067 return emulate_ud(ctxt);
3069 if (ctxt->modrm_reg == VCPU_SREG_SS)
3070 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3072 /* Disable writeback. */
3073 ctxt->dst.type = OP_NONE;
3074 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3077 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3079 u16 sel = ctxt->src.val;
3081 /* Disable writeback. */
3082 ctxt->dst.type = OP_NONE;
3083 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3086 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3088 u16 sel = ctxt->src.val;
3090 /* Disable writeback. */
3091 ctxt->dst.type = OP_NONE;
3092 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3095 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3100 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3101 if (rc == X86EMUL_CONTINUE)
3102 ctxt->ops->invlpg(ctxt, linear);
3103 /* Disable writeback. */
3104 ctxt->dst.type = OP_NONE;
3105 return X86EMUL_CONTINUE;
3108 static int em_clts(struct x86_emulate_ctxt *ctxt)
3112 cr0 = ctxt->ops->get_cr(ctxt, 0);
3114 ctxt->ops->set_cr(ctxt, 0, cr0);
3115 return X86EMUL_CONTINUE;
3118 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3122 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3123 return X86EMUL_UNHANDLEABLE;
3125 rc = ctxt->ops->fix_hypercall(ctxt);
3126 if (rc != X86EMUL_CONTINUE)
3129 /* Let the processor re-execute the fixed hypercall */
3130 ctxt->_eip = ctxt->eip;
3131 /* Disable writeback. */
3132 ctxt->dst.type = OP_NONE;
3133 return X86EMUL_CONTINUE;
3136 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3137 void (*get)(struct x86_emulate_ctxt *ctxt,
3138 struct desc_ptr *ptr))
3140 struct desc_ptr desc_ptr;
3142 if (ctxt->mode == X86EMUL_MODE_PROT64)
3144 get(ctxt, &desc_ptr);
3145 if (ctxt->op_bytes == 2) {
3147 desc_ptr.address &= 0x00ffffff;
3149 /* Disable writeback. */
3150 ctxt->dst.type = OP_NONE;
3151 return segmented_write(ctxt, ctxt->dst.addr.mem,
3152 &desc_ptr, 2 + ctxt->op_bytes);
3155 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3157 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3160 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3162 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3165 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3167 struct desc_ptr desc_ptr;
3170 if (ctxt->mode == X86EMUL_MODE_PROT64)
3172 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3173 &desc_ptr.size, &desc_ptr.address,
3175 if (rc != X86EMUL_CONTINUE)
3177 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3178 /* Disable writeback. */
3179 ctxt->dst.type = OP_NONE;
3180 return X86EMUL_CONTINUE;
3183 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3187 rc = ctxt->ops->fix_hypercall(ctxt);
3189 /* Disable writeback. */
3190 ctxt->dst.type = OP_NONE;
3194 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3196 struct desc_ptr desc_ptr;
3199 if (ctxt->mode == X86EMUL_MODE_PROT64)
3201 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3202 &desc_ptr.size, &desc_ptr.address,
3204 if (rc != X86EMUL_CONTINUE)
3206 ctxt->ops->set_idt(ctxt, &desc_ptr);
3207 /* Disable writeback. */
3208 ctxt->dst.type = OP_NONE;
3209 return X86EMUL_CONTINUE;
3212 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3214 ctxt->dst.bytes = 2;
3215 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3216 return X86EMUL_CONTINUE;
3219 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3221 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3222 | (ctxt->src.val & 0x0f));
3223 ctxt->dst.type = OP_NONE;
3224 return X86EMUL_CONTINUE;
3227 static int em_loop(struct x86_emulate_ctxt *ctxt)
3229 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3230 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3231 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3232 jmp_rel(ctxt, ctxt->src.val);
3234 return X86EMUL_CONTINUE;
3237 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3239 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3240 jmp_rel(ctxt, ctxt->src.val);
3242 return X86EMUL_CONTINUE;
3245 static int em_in(struct x86_emulate_ctxt *ctxt)
3247 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3249 return X86EMUL_IO_NEEDED;
3251 return X86EMUL_CONTINUE;
3254 static int em_out(struct x86_emulate_ctxt *ctxt)
3256 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3258 /* Disable writeback. */
3259 ctxt->dst.type = OP_NONE;
3260 return X86EMUL_CONTINUE;
3263 static int em_cli(struct x86_emulate_ctxt *ctxt)
3265 if (emulator_bad_iopl(ctxt))
3266 return emulate_gp(ctxt, 0);
3268 ctxt->eflags &= ~X86_EFLAGS_IF;
3269 return X86EMUL_CONTINUE;
3272 static int em_sti(struct x86_emulate_ctxt *ctxt)
3274 if (emulator_bad_iopl(ctxt))
3275 return emulate_gp(ctxt, 0);
3277 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3278 ctxt->eflags |= X86_EFLAGS_IF;
3279 return X86EMUL_CONTINUE;
3282 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3284 u32 eax, ebx, ecx, edx;
3286 eax = reg_read(ctxt, VCPU_REGS_RAX);
3287 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3288 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3289 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3290 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3291 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3292 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3293 return X86EMUL_CONTINUE;
3296 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3300 flags = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF;
3301 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3303 ctxt->eflags &= ~0xffUL;
3304 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3305 return X86EMUL_CONTINUE;
3308 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3310 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3311 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3312 return X86EMUL_CONTINUE;
3315 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3317 switch (ctxt->op_bytes) {
3318 #ifdef CONFIG_X86_64
3320 asm("bswap %0" : "+r"(ctxt->dst.val));
3324 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3327 return X86EMUL_CONTINUE;
3330 static bool valid_cr(int nr)
3342 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3344 if (!valid_cr(ctxt->modrm_reg))
3345 return emulate_ud(ctxt);
3347 return X86EMUL_CONTINUE;
3350 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3352 u64 new_val = ctxt->src.val64;
3353 int cr = ctxt->modrm_reg;
3356 static u64 cr_reserved_bits[] = {
3357 0xffffffff00000000ULL,
3358 0, 0, 0, /* CR3 checked later */
3365 return emulate_ud(ctxt);
3367 if (new_val & cr_reserved_bits[cr])
3368 return emulate_gp(ctxt, 0);
3373 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3374 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3375 return emulate_gp(ctxt, 0);
3377 cr4 = ctxt->ops->get_cr(ctxt, 4);
3378 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3380 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3381 !(cr4 & X86_CR4_PAE))
3382 return emulate_gp(ctxt, 0);
3389 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3390 if (efer & EFER_LMA)
3391 rsvd = CR3_L_MODE_RESERVED_BITS;
3394 return emulate_gp(ctxt, 0);
3399 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3401 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3402 return emulate_gp(ctxt, 0);
3408 return X86EMUL_CONTINUE;
3411 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3415 ctxt->ops->get_dr(ctxt, 7, &dr7);
3417 /* Check if DR7.Global_Enable is set */
3418 return dr7 & (1 << 13);
3421 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3423 int dr = ctxt->modrm_reg;
3427 return emulate_ud(ctxt);
3429 cr4 = ctxt->ops->get_cr(ctxt, 4);
3430 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3431 return emulate_ud(ctxt);
3433 if (check_dr7_gd(ctxt))
3434 return emulate_db(ctxt);
3436 return X86EMUL_CONTINUE;
3439 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3441 u64 new_val = ctxt->src.val64;
3442 int dr = ctxt->modrm_reg;
3444 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3445 return emulate_gp(ctxt, 0);
3447 return check_dr_read(ctxt);
3450 static int check_svme(struct x86_emulate_ctxt *ctxt)
3454 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3456 if (!(efer & EFER_SVME))
3457 return emulate_ud(ctxt);
3459 return X86EMUL_CONTINUE;
3462 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3464 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3466 /* Valid physical address? */
3467 if (rax & 0xffff000000000000ULL)
3468 return emulate_gp(ctxt, 0);
3470 return check_svme(ctxt);
3473 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3475 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3477 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3478 return emulate_ud(ctxt);
3480 return X86EMUL_CONTINUE;
3483 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3485 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3486 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3488 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3490 return emulate_gp(ctxt, 0);
3492 return X86EMUL_CONTINUE;
3495 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3497 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3498 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3499 return emulate_gp(ctxt, 0);
3501 return X86EMUL_CONTINUE;
3504 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3506 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3507 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3508 return emulate_gp(ctxt, 0);
3510 return X86EMUL_CONTINUE;
3513 #define D(_y) { .flags = (_y) }
3514 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3515 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3516 .check_perm = (_p) }
3517 #define N D(NotImpl)
3518 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3519 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3520 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3521 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3522 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3523 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3524 #define II(_f, _e, _i) \
3525 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3526 #define IIP(_f, _e, _i, _p) \
3527 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3528 .check_perm = (_p) }
3529 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3531 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3532 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3533 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3534 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3535 #define I2bvIP(_f, _e, _i, _p) \
3536 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3538 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3539 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3540 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3542 static const struct opcode group7_rm1[] = {
3543 DI(SrcNone | Priv, monitor),
3544 DI(SrcNone | Priv, mwait),
3548 static const struct opcode group7_rm3[] = {
3549 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3550 II(SrcNone | Prot | EmulateOnUD, em_vmmcall, vmmcall),
3551 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3552 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3553 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3554 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3555 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3556 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3559 static const struct opcode group7_rm7[] = {
3561 DIP(SrcNone, rdtscp, check_rdtsc),
3565 static const struct opcode group1[] = {
3567 F(Lock | PageTable, em_or),
3570 F(Lock | PageTable, em_and),
3576 static const struct opcode group1A[] = {
3577 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3580 static const struct opcode group2[] = {
3581 F(DstMem | ModRM, em_rol),
3582 F(DstMem | ModRM, em_ror),
3583 F(DstMem | ModRM, em_rcl),
3584 F(DstMem | ModRM, em_rcr),
3585 F(DstMem | ModRM, em_shl),
3586 F(DstMem | ModRM, em_shr),
3587 F(DstMem | ModRM, em_shl),
3588 F(DstMem | ModRM, em_sar),
3591 static const struct opcode group3[] = {
3592 F(DstMem | SrcImm | NoWrite, em_test),
3593 F(DstMem | SrcImm | NoWrite, em_test),
3594 F(DstMem | SrcNone | Lock, em_not),
3595 F(DstMem | SrcNone | Lock, em_neg),
3596 F(DstXacc | Src2Mem, em_mul_ex),
3597 F(DstXacc | Src2Mem, em_imul_ex),
3598 F(DstXacc | Src2Mem, em_div_ex),
3599 F(DstXacc | Src2Mem, em_idiv_ex),
3602 static const struct opcode group4[] = {
3603 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3604 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3608 static const struct opcode group5[] = {
3609 F(DstMem | SrcNone | Lock, em_inc),
3610 F(DstMem | SrcNone | Lock, em_dec),
3611 I(SrcMem | Stack, em_grp45),
3612 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3613 I(SrcMem | Stack, em_grp45),
3614 I(SrcMemFAddr | ImplicitOps, em_grp45),
3615 I(SrcMem | Stack, em_grp45), D(Undefined),
3618 static const struct opcode group6[] = {
3621 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3622 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3626 static const struct group_dual group7 = { {
3627 II(Mov | DstMem | Priv, em_sgdt, sgdt),
3628 II(Mov | DstMem | Priv, em_sidt, sidt),
3629 II(SrcMem | Priv, em_lgdt, lgdt),
3630 II(SrcMem | Priv, em_lidt, lidt),
3631 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3632 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3633 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3635 I(SrcNone | Priv | EmulateOnUD, em_vmcall),
3637 N, EXT(0, group7_rm3),
3638 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3639 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3643 static const struct opcode group8[] = {
3645 F(DstMem | SrcImmByte | NoWrite, em_bt),
3646 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3647 F(DstMem | SrcImmByte | Lock, em_btr),
3648 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3651 static const struct group_dual group9 = { {
3652 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3654 N, N, N, N, N, N, N, N,
3657 static const struct opcode group11[] = {
3658 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3662 static const struct gprefix pfx_0f_6f_0f_7f = {
3663 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3666 static const struct gprefix pfx_vmovntpx = {
3667 I(0, em_mov), N, N, N,
3670 static const struct gprefix pfx_0f_28_0f_29 = {
3671 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
3674 static const struct escape escape_d9 = { {
3675 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3678 N, N, N, N, N, N, N, N,
3680 N, N, N, N, N, N, N, N,
3682 N, N, N, N, N, N, N, N,
3684 N, N, N, N, N, N, N, N,
3686 N, N, N, N, N, N, N, N,
3688 N, N, N, N, N, N, N, N,
3690 N, N, N, N, N, N, N, N,
3692 N, N, N, N, N, N, N, N,
3695 static const struct escape escape_db = { {
3696 N, N, N, N, N, N, N, N,
3699 N, N, N, N, N, N, N, N,
3701 N, N, N, N, N, N, N, N,
3703 N, N, N, N, N, N, N, N,
3705 N, N, N, N, N, N, N, N,
3707 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3709 N, N, N, N, N, N, N, N,
3711 N, N, N, N, N, N, N, N,
3713 N, N, N, N, N, N, N, N,
3716 static const struct escape escape_dd = { {
3717 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3720 N, N, N, N, N, N, N, N,
3722 N, N, N, N, N, N, N, N,
3724 N, N, N, N, N, N, N, N,
3726 N, N, N, N, N, N, N, N,
3728 N, N, N, N, N, N, N, N,
3730 N, N, N, N, N, N, N, N,
3732 N, N, N, N, N, N, N, N,
3734 N, N, N, N, N, N, N, N,
3737 static const struct opcode opcode_table[256] = {
3739 F6ALU(Lock, em_add),
3740 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3741 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3743 F6ALU(Lock | PageTable, em_or),
3744 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3747 F6ALU(Lock, em_adc),
3748 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3749 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3751 F6ALU(Lock, em_sbb),
3752 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3753 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3755 F6ALU(Lock | PageTable, em_and), N, N,
3757 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3759 F6ALU(Lock, em_xor), N, N,
3761 F6ALU(NoWrite, em_cmp), N, N,
3763 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
3765 X8(I(SrcReg | Stack, em_push)),
3767 X8(I(DstReg | Stack, em_pop)),
3769 I(ImplicitOps | Stack | No64, em_pusha),
3770 I(ImplicitOps | Stack | No64, em_popa),
3771 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3774 I(SrcImm | Mov | Stack, em_push),
3775 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3776 I(SrcImmByte | Mov | Stack, em_push),
3777 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3778 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3779 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3783 G(ByteOp | DstMem | SrcImm, group1),
3784 G(DstMem | SrcImm, group1),
3785 G(ByteOp | DstMem | SrcImm | No64, group1),
3786 G(DstMem | SrcImmByte, group1),
3787 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3788 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3790 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3791 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3792 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3793 D(ModRM | SrcMem | NoAccess | DstReg),
3794 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3797 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3799 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3800 I(SrcImmFAddr | No64, em_call_far), N,
3801 II(ImplicitOps | Stack, em_pushf, pushf),
3802 II(ImplicitOps | Stack, em_popf, popf),
3803 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
3805 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3806 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3807 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3808 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3810 F2bv(DstAcc | SrcImm | NoWrite, em_test),
3811 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3812 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3813 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3815 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3817 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3819 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
3820 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3821 I(ImplicitOps | Stack, em_ret),
3822 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3823 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3824 G(ByteOp, group11), G(0, group11),
3826 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3827 I(ImplicitOps | Stack | SrcImmU16, em_ret_far_imm),
3828 I(ImplicitOps | Stack, em_ret_far),
3829 D(ImplicitOps), DI(SrcImmByte, intn),
3830 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3832 G(Src2One | ByteOp, group2), G(Src2One, group2),
3833 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
3834 I(DstAcc | SrcImmUByte | No64, em_aam),
3835 I(DstAcc | SrcImmUByte | No64, em_aad),
3836 F(DstAcc | ByteOp | No64, em_salc),
3837 I(DstAcc | SrcXLat | ByteOp, em_mov),
3839 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3841 X3(I(SrcImmByte, em_loop)),
3842 I(SrcImmByte, em_jcxz),
3843 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3844 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3846 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3847 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3848 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3849 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3851 N, DI(ImplicitOps, icebp), N, N,
3852 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3853 G(ByteOp, group3), G(0, group3),
3855 D(ImplicitOps), D(ImplicitOps),
3856 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3857 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3860 static const struct opcode twobyte_table[256] = {
3862 G(0, group6), GD(0, &group7), N, N,
3863 N, I(ImplicitOps | EmulateOnUD, em_syscall),
3864 II(ImplicitOps | Priv, em_clts, clts), N,
3865 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3866 N, D(ImplicitOps | ModRM), N, N,
3868 N, N, N, N, N, N, N, N,
3869 D(ImplicitOps | ModRM), N, N, N, N, N, N, D(ImplicitOps | ModRM),
3871 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3872 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3873 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
3874 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
3876 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
3877 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
3878 N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
3881 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3882 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3883 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3884 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3885 I(ImplicitOps | EmulateOnUD, em_sysenter),
3886 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
3888 N, N, N, N, N, N, N, N,
3890 X16(D(DstReg | SrcMem | ModRM | Mov)),
3892 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3897 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3902 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3906 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3908 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3909 II(ImplicitOps, em_cpuid, cpuid),
3910 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
3911 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
3912 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
3914 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3915 DI(ImplicitOps, rsm),
3916 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3917 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
3918 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
3919 D(ModRM), F(DstReg | SrcMem | ModRM, em_imul),
3921 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3922 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3923 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3924 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3925 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3926 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3930 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3931 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
3932 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3934 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
3935 N, D(DstMem | SrcReg | ModRM | Mov),
3936 N, N, N, GD(0, &group9),
3938 X8(I(DstReg, em_bswap)),
3940 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3942 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3944 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3947 static const struct gprefix three_byte_0f_38_f0 = {
3948 I(DstReg | SrcMem | Mov, em_movbe), N, N, N
3951 static const struct gprefix three_byte_0f_38_f1 = {
3952 I(DstMem | SrcReg | Mov, em_movbe), N, N, N
3956 * Insns below are selected by the prefix which indexed by the third opcode
3959 static const struct opcode opcode_map_0f_38[256] = {
3961 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
3963 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
3965 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f0),
3966 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f1),
3985 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3989 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3995 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3996 unsigned size, bool sign_extension)
3998 int rc = X86EMUL_CONTINUE;
4002 op->addr.mem.ea = ctxt->_eip;
4003 /* NB. Immediates are sign-extended as necessary. */
4004 switch (op->bytes) {
4006 op->val = insn_fetch(s8, ctxt);
4009 op->val = insn_fetch(s16, ctxt);
4012 op->val = insn_fetch(s32, ctxt);
4015 op->val = insn_fetch(s64, ctxt);
4018 if (!sign_extension) {
4019 switch (op->bytes) {
4027 op->val &= 0xffffffff;
4035 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4038 int rc = X86EMUL_CONTINUE;
4042 decode_register_operand(ctxt, op);
4045 rc = decode_imm(ctxt, op, 1, false);
4048 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4052 if ((ctxt->d & BitOp) && op == &ctxt->dst)
4053 fetch_bit_operand(ctxt);
4054 op->orig_val = op->val;
4057 ctxt->memop.bytes = 8;
4061 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4062 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4063 fetch_register_operand(op);
4064 op->orig_val = op->val;
4068 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4069 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4070 fetch_register_operand(op);
4071 op->orig_val = op->val;
4074 if (ctxt->d & ByteOp) {
4079 op->bytes = ctxt->op_bytes;
4080 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4081 fetch_register_operand(op);
4082 op->orig_val = op->val;
4086 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4088 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4089 op->addr.mem.seg = VCPU_SREG_ES;
4096 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4097 fetch_register_operand(op);
4101 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4104 rc = decode_imm(ctxt, op, 1, true);
4111 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4114 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4117 ctxt->memop.bytes = 1;
4118 if (ctxt->memop.type == OP_REG) {
4119 ctxt->memop.addr.reg = decode_register(ctxt,
4120 ctxt->modrm_rm, true);
4121 fetch_register_operand(&ctxt->memop);
4125 ctxt->memop.bytes = 2;
4128 ctxt->memop.bytes = 4;
4131 rc = decode_imm(ctxt, op, 2, false);
4134 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4138 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4140 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4141 op->addr.mem.seg = seg_override(ctxt);
4147 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4149 register_address(ctxt,
4150 reg_read(ctxt, VCPU_REGS_RBX) +
4151 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4152 op->addr.mem.seg = seg_override(ctxt);
4157 op->addr.mem.ea = ctxt->_eip;
4158 op->bytes = ctxt->op_bytes + 2;
4159 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4162 ctxt->memop.bytes = ctxt->op_bytes + 2;
4165 op->val = VCPU_SREG_ES;
4168 op->val = VCPU_SREG_CS;
4171 op->val = VCPU_SREG_SS;
4174 op->val = VCPU_SREG_DS;
4177 op->val = VCPU_SREG_FS;
4180 op->val = VCPU_SREG_GS;
4183 /* Special instructions do their own operand decoding. */
4185 op->type = OP_NONE; /* Disable writeback. */
4193 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4195 int rc = X86EMUL_CONTINUE;
4196 int mode = ctxt->mode;
4197 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4198 bool op_prefix = false;
4199 struct opcode opcode;
4201 ctxt->memop.type = OP_NONE;
4202 ctxt->memopp = NULL;
4203 ctxt->_eip = ctxt->eip;
4204 ctxt->fetch.start = ctxt->_eip;
4205 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4206 ctxt->opcode_len = 1;
4208 memcpy(ctxt->fetch.data, insn, insn_len);
4211 case X86EMUL_MODE_REAL:
4212 case X86EMUL_MODE_VM86:
4213 case X86EMUL_MODE_PROT16:
4214 def_op_bytes = def_ad_bytes = 2;
4216 case X86EMUL_MODE_PROT32:
4217 def_op_bytes = def_ad_bytes = 4;
4219 #ifdef CONFIG_X86_64
4220 case X86EMUL_MODE_PROT64:
4226 return EMULATION_FAILED;
4229 ctxt->op_bytes = def_op_bytes;
4230 ctxt->ad_bytes = def_ad_bytes;
4232 /* Legacy prefixes. */
4234 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4235 case 0x66: /* operand-size override */
4237 /* switch between 2/4 bytes */
4238 ctxt->op_bytes = def_op_bytes ^ 6;
4240 case 0x67: /* address-size override */
4241 if (mode == X86EMUL_MODE_PROT64)
4242 /* switch between 4/8 bytes */
4243 ctxt->ad_bytes = def_ad_bytes ^ 12;
4245 /* switch between 2/4 bytes */
4246 ctxt->ad_bytes = def_ad_bytes ^ 6;
4248 case 0x26: /* ES override */
4249 case 0x2e: /* CS override */
4250 case 0x36: /* SS override */
4251 case 0x3e: /* DS override */
4252 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4254 case 0x64: /* FS override */
4255 case 0x65: /* GS override */
4256 set_seg_override(ctxt, ctxt->b & 7);
4258 case 0x40 ... 0x4f: /* REX */
4259 if (mode != X86EMUL_MODE_PROT64)
4261 ctxt->rex_prefix = ctxt->b;
4263 case 0xf0: /* LOCK */
4264 ctxt->lock_prefix = 1;
4266 case 0xf2: /* REPNE/REPNZ */
4267 case 0xf3: /* REP/REPE/REPZ */
4268 ctxt->rep_prefix = ctxt->b;
4274 /* Any legacy prefix after a REX prefix nullifies its effect. */
4276 ctxt->rex_prefix = 0;
4282 if (ctxt->rex_prefix & 8)
4283 ctxt->op_bytes = 8; /* REX.W */
4285 /* Opcode byte(s). */
4286 opcode = opcode_table[ctxt->b];
4287 /* Two-byte opcode? */
4288 if (ctxt->b == 0x0f) {
4289 ctxt->opcode_len = 2;
4290 ctxt->b = insn_fetch(u8, ctxt);
4291 opcode = twobyte_table[ctxt->b];
4293 /* 0F_38 opcode map */
4294 if (ctxt->b == 0x38) {
4295 ctxt->opcode_len = 3;
4296 ctxt->b = insn_fetch(u8, ctxt);
4297 opcode = opcode_map_0f_38[ctxt->b];
4300 ctxt->d = opcode.flags;
4302 if (ctxt->d & ModRM)
4303 ctxt->modrm = insn_fetch(u8, ctxt);
4305 while (ctxt->d & GroupMask) {
4306 switch (ctxt->d & GroupMask) {
4308 goffset = (ctxt->modrm >> 3) & 7;
4309 opcode = opcode.u.group[goffset];
4312 goffset = (ctxt->modrm >> 3) & 7;
4313 if ((ctxt->modrm >> 6) == 3)
4314 opcode = opcode.u.gdual->mod3[goffset];
4316 opcode = opcode.u.gdual->mod012[goffset];
4319 goffset = ctxt->modrm & 7;
4320 opcode = opcode.u.group[goffset];
4323 if (ctxt->rep_prefix && op_prefix)
4324 return EMULATION_FAILED;
4325 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4326 switch (simd_prefix) {
4327 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4328 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4329 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4330 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4334 if (ctxt->modrm > 0xbf)
4335 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4337 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4340 return EMULATION_FAILED;
4343 ctxt->d &= ~(u64)GroupMask;
4344 ctxt->d |= opcode.flags;
4347 ctxt->execute = opcode.u.execute;
4348 ctxt->check_perm = opcode.check_perm;
4349 ctxt->intercept = opcode.intercept;
4352 if (ctxt->d == 0 || (ctxt->d & NotImpl))
4353 return EMULATION_FAILED;
4355 if (!(ctxt->d & EmulateOnUD) && ctxt->ud)
4356 return EMULATION_FAILED;
4358 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4361 if (ctxt->d & Op3264) {
4362 if (mode == X86EMUL_MODE_PROT64)
4369 ctxt->op_bytes = 16;
4370 else if (ctxt->d & Mmx)
4373 /* ModRM and SIB bytes. */
4374 if (ctxt->d & ModRM) {
4375 rc = decode_modrm(ctxt, &ctxt->memop);
4376 if (!ctxt->has_seg_override)
4377 set_seg_override(ctxt, ctxt->modrm_seg);
4378 } else if (ctxt->d & MemAbs)
4379 rc = decode_abs(ctxt, &ctxt->memop);
4380 if (rc != X86EMUL_CONTINUE)
4383 if (!ctxt->has_seg_override)
4384 set_seg_override(ctxt, VCPU_SREG_DS);
4386 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4388 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4389 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4392 * Decode and fetch the source operand: register, memory
4395 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4396 if (rc != X86EMUL_CONTINUE)
4400 * Decode and fetch the second source operand: register, memory
4403 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4404 if (rc != X86EMUL_CONTINUE)
4407 /* Decode and fetch the destination operand: register or memory. */
4408 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4411 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4412 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4414 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4417 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4419 return ctxt->d & PageTable;
4422 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4424 /* The second termination condition only applies for REPE
4425 * and REPNE. Test if the repeat string operation prefix is
4426 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4427 * corresponding termination condition according to:
4428 * - if REPE/REPZ and ZF = 0 then done
4429 * - if REPNE/REPNZ and ZF = 1 then done
4431 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4432 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4433 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4434 ((ctxt->eflags & EFLG_ZF) == 0))
4435 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4436 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4442 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4446 ctxt->ops->get_fpu(ctxt);
4447 asm volatile("1: fwait \n\t"
4449 ".pushsection .fixup,\"ax\" \n\t"
4451 "movb $1, %[fault] \n\t"
4454 _ASM_EXTABLE(1b, 3b)
4455 : [fault]"+qm"(fault));
4456 ctxt->ops->put_fpu(ctxt);
4458 if (unlikely(fault))
4459 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4461 return X86EMUL_CONTINUE;
4464 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4467 if (op->type == OP_MM)
4468 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4471 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4473 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4474 if (!(ctxt->d & ByteOp))
4475 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4476 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4477 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
4479 : "c"(ctxt->src2.val));
4480 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4481 if (!fop) /* exception is returned in fop variable */
4482 return emulate_de(ctxt);
4483 return X86EMUL_CONTINUE;
4486 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4488 const struct x86_emulate_ops *ops = ctxt->ops;
4489 int rc = X86EMUL_CONTINUE;
4490 int saved_dst_type = ctxt->dst.type;
4492 ctxt->mem_read.pos = 0;
4494 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4495 (ctxt->d & Undefined)) {
4496 rc = emulate_ud(ctxt);
4500 /* LOCK prefix is allowed only with some instructions */
4501 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4502 rc = emulate_ud(ctxt);
4506 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4507 rc = emulate_ud(ctxt);
4511 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4512 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4513 rc = emulate_ud(ctxt);
4517 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4518 rc = emulate_nm(ctxt);
4522 if (ctxt->d & Mmx) {
4523 rc = flush_pending_x87_faults(ctxt);
4524 if (rc != X86EMUL_CONTINUE)
4527 * Now that we know the fpu is exception safe, we can fetch
4530 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4531 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4532 if (!(ctxt->d & Mov))
4533 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4536 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4537 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4538 X86_ICPT_PRE_EXCEPT);
4539 if (rc != X86EMUL_CONTINUE)
4543 /* Privileged instruction can be executed only in CPL=0 */
4544 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4545 rc = emulate_gp(ctxt, 0);
4549 /* Instruction can only be executed in protected mode */
4550 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4551 rc = emulate_ud(ctxt);
4555 /* Do instruction specific permission checks */
4556 if (ctxt->check_perm) {
4557 rc = ctxt->check_perm(ctxt);
4558 if (rc != X86EMUL_CONTINUE)
4562 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4563 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4564 X86_ICPT_POST_EXCEPT);
4565 if (rc != X86EMUL_CONTINUE)
4569 if (ctxt->rep_prefix && (ctxt->d & String)) {
4570 /* All REP prefixes have the same first termination condition */
4571 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4572 ctxt->eip = ctxt->_eip;
4577 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4578 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4579 ctxt->src.valptr, ctxt->src.bytes);
4580 if (rc != X86EMUL_CONTINUE)
4582 ctxt->src.orig_val64 = ctxt->src.val64;
4585 if (ctxt->src2.type == OP_MEM) {
4586 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4587 &ctxt->src2.val, ctxt->src2.bytes);
4588 if (rc != X86EMUL_CONTINUE)
4592 if ((ctxt->d & DstMask) == ImplicitOps)
4596 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4597 /* optimisation - avoid slow emulated read if Mov */
4598 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4599 &ctxt->dst.val, ctxt->dst.bytes);
4600 if (rc != X86EMUL_CONTINUE)
4603 ctxt->dst.orig_val = ctxt->dst.val;
4607 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4608 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4609 X86_ICPT_POST_MEMACCESS);
4610 if (rc != X86EMUL_CONTINUE)
4614 if (ctxt->execute) {
4615 if (ctxt->d & Fastop) {
4616 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4617 rc = fastop(ctxt, fop);
4618 if (rc != X86EMUL_CONTINUE)
4622 rc = ctxt->execute(ctxt);
4623 if (rc != X86EMUL_CONTINUE)
4628 if (ctxt->opcode_len == 2)
4630 else if (ctxt->opcode_len == 3)
4631 goto threebyte_insn;
4634 case 0x63: /* movsxd */
4635 if (ctxt->mode != X86EMUL_MODE_PROT64)
4636 goto cannot_emulate;
4637 ctxt->dst.val = (s32) ctxt->src.val;
4639 case 0x70 ... 0x7f: /* jcc (short) */
4640 if (test_cc(ctxt->b, ctxt->eflags))
4641 jmp_rel(ctxt, ctxt->src.val);
4643 case 0x8d: /* lea r16/r32, m */
4644 ctxt->dst.val = ctxt->src.addr.mem.ea;
4646 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4647 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4651 case 0x98: /* cbw/cwde/cdqe */
4652 switch (ctxt->op_bytes) {
4653 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4654 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4655 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4658 case 0xcc: /* int3 */
4659 rc = emulate_int(ctxt, 3);
4661 case 0xcd: /* int n */
4662 rc = emulate_int(ctxt, ctxt->src.val);
4664 case 0xce: /* into */
4665 if (ctxt->eflags & EFLG_OF)
4666 rc = emulate_int(ctxt, 4);
4668 case 0xe9: /* jmp rel */
4669 case 0xeb: /* jmp rel short */
4670 jmp_rel(ctxt, ctxt->src.val);
4671 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4673 case 0xf4: /* hlt */
4674 ctxt->ops->halt(ctxt);
4676 case 0xf5: /* cmc */
4677 /* complement carry flag from eflags reg */
4678 ctxt->eflags ^= EFLG_CF;
4680 case 0xf8: /* clc */
4681 ctxt->eflags &= ~EFLG_CF;
4683 case 0xf9: /* stc */
4684 ctxt->eflags |= EFLG_CF;
4686 case 0xfc: /* cld */
4687 ctxt->eflags &= ~EFLG_DF;
4689 case 0xfd: /* std */
4690 ctxt->eflags |= EFLG_DF;
4693 goto cannot_emulate;
4696 if (rc != X86EMUL_CONTINUE)
4700 if (!(ctxt->d & NoWrite)) {
4701 rc = writeback(ctxt, &ctxt->dst);
4702 if (rc != X86EMUL_CONTINUE)
4705 if (ctxt->d & SrcWrite) {
4706 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
4707 rc = writeback(ctxt, &ctxt->src);
4708 if (rc != X86EMUL_CONTINUE)
4713 * restore dst type in case the decoding will be reused
4714 * (happens for string instruction )
4716 ctxt->dst.type = saved_dst_type;
4718 if ((ctxt->d & SrcMask) == SrcSI)
4719 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4721 if ((ctxt->d & DstMask) == DstDI)
4722 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4724 if (ctxt->rep_prefix && (ctxt->d & String)) {
4726 struct read_cache *r = &ctxt->io_read;
4727 if ((ctxt->d & SrcMask) == SrcSI)
4728 count = ctxt->src.count;
4730 count = ctxt->dst.count;
4731 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4734 if (!string_insn_completed(ctxt)) {
4736 * Re-enter guest when pio read ahead buffer is empty
4737 * or, if it is not used, after each 1024 iteration.
4739 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4740 (r->end == 0 || r->end != r->pos)) {
4742 * Reset read cache. Usually happens before
4743 * decode, but since instruction is restarted
4744 * we have to do it here.
4746 ctxt->mem_read.end = 0;
4747 writeback_registers(ctxt);
4748 return EMULATION_RESTART;
4750 goto done; /* skip rip writeback */
4754 ctxt->eip = ctxt->_eip;
4757 if (rc == X86EMUL_PROPAGATE_FAULT)
4758 ctxt->have_exception = true;
4759 if (rc == X86EMUL_INTERCEPTED)
4760 return EMULATION_INTERCEPTED;
4762 if (rc == X86EMUL_CONTINUE)
4763 writeback_registers(ctxt);
4765 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4769 case 0x09: /* wbinvd */
4770 (ctxt->ops->wbinvd)(ctxt);
4772 case 0x08: /* invd */
4773 case 0x0d: /* GrpP (prefetch) */
4774 case 0x18: /* Grp16 (prefetch/nop) */
4775 case 0x1f: /* nop */
4777 case 0x20: /* mov cr, reg */
4778 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4780 case 0x21: /* mov from dr to reg */
4781 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4783 case 0x40 ... 0x4f: /* cmov */
4784 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4785 if (!test_cc(ctxt->b, ctxt->eflags))
4786 ctxt->dst.type = OP_NONE; /* no writeback */
4788 case 0x80 ... 0x8f: /* jnz rel, etc*/
4789 if (test_cc(ctxt->b, ctxt->eflags))
4790 jmp_rel(ctxt, ctxt->src.val);
4792 case 0x90 ... 0x9f: /* setcc r/m8 */
4793 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4795 case 0xae: /* clflush */
4797 case 0xb6 ... 0xb7: /* movzx */
4798 ctxt->dst.bytes = ctxt->op_bytes;
4799 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4800 : (u16) ctxt->src.val;
4802 case 0xbe ... 0xbf: /* movsx */
4803 ctxt->dst.bytes = ctxt->op_bytes;
4804 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4805 (s16) ctxt->src.val;
4807 case 0xc3: /* movnti */
4808 ctxt->dst.bytes = ctxt->op_bytes;
4809 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4810 (u64) ctxt->src.val;
4813 goto cannot_emulate;
4818 if (rc != X86EMUL_CONTINUE)
4824 return EMULATION_FAILED;
4827 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4829 invalidate_registers(ctxt);
4832 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4834 writeback_registers(ctxt);