1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#include <linux/kvm_host.h>
24#include "kvm_cache_regs.h"
25#include <linux/module.h>
26#include <asm/kvm_emulate.h>
27#include <linux/stringify.h>
28#include <asm/debugreg.h>
29
30#include "x86.h"
31#include "tss.h"
32
33
34
35
36#define OpNone 0ull
37#define OpImplicit 1ull
38#define OpReg 2ull
39#define OpMem 3ull
40#define OpAcc 4ull
41#define OpDI 5ull
42#define OpMem64 6ull
43#define OpImmUByte 7ull
44#define OpDX 8ull
45#define OpCL 9ull
46#define OpImmByte 10ull
47#define OpOne 11ull
48#define OpImm 12ull
49#define OpMem16 13ull
50#define OpMem32 14ull
51#define OpImmU 15ull
52#define OpSI 16ull
53#define OpImmFAddr 17ull
54#define OpMemFAddr 18ull
55#define OpImmU16 19ull
56#define OpES 20ull
57#define OpCS 21ull
58#define OpSS 22ull
59#define OpDS 23ull
60#define OpFS 24ull
61#define OpGS 25ull
62#define OpMem8 26ull
63#define OpImm64 27ull
64#define OpXLat 28ull
65#define OpAccLo 29ull
66#define OpAccHi 30ull
67
68#define OpBits 5
69#define OpMask ((1ull << OpBits) - 1)
70
71
72
73
74
75
76
77
78
79
80
81#define ByteOp (1<<0)
82
83#define DstShift 1
84#define ImplicitOps (OpImplicit << DstShift)
85#define DstReg (OpReg << DstShift)
86#define DstMem (OpMem << DstShift)
87#define DstAcc (OpAcc << DstShift)
88#define DstDI (OpDI << DstShift)
89#define DstMem64 (OpMem64 << DstShift)
90#define DstMem16 (OpMem16 << DstShift)
91#define DstImmUByte (OpImmUByte << DstShift)
92#define DstDX (OpDX << DstShift)
93#define DstAccLo (OpAccLo << DstShift)
94#define DstMask (OpMask << DstShift)
95
96#define SrcShift 6
97#define SrcNone (OpNone << SrcShift)
98#define SrcReg (OpReg << SrcShift)
99#define SrcMem (OpMem << SrcShift)
100#define SrcMem16 (OpMem16 << SrcShift)
101#define SrcMem32 (OpMem32 << SrcShift)
102#define SrcImm (OpImm << SrcShift)
103#define SrcImmByte (OpImmByte << SrcShift)
104#define SrcOne (OpOne << SrcShift)
105#define SrcImmUByte (OpImmUByte << SrcShift)
106#define SrcImmU (OpImmU << SrcShift)
107#define SrcSI (OpSI << SrcShift)
108#define SrcXLat (OpXLat << SrcShift)
109#define SrcImmFAddr (OpImmFAddr << SrcShift)
110#define SrcMemFAddr (OpMemFAddr << SrcShift)
111#define SrcAcc (OpAcc << SrcShift)
112#define SrcImmU16 (OpImmU16 << SrcShift)
113#define SrcImm64 (OpImm64 << SrcShift)
114#define SrcDX (OpDX << SrcShift)
115#define SrcMem8 (OpMem8 << SrcShift)
116#define SrcAccHi (OpAccHi << SrcShift)
117#define SrcMask (OpMask << SrcShift)
118#define BitOp (1<<11)
119#define MemAbs (1<<12)
120#define String (1<<13)
121#define Stack (1<<14)
122#define GroupMask (7<<15)
123#define Group (1<<15)
124#define GroupDual (2<<15)
125#define Prefix (3<<15)
126#define RMExt (4<<15)
127#define Escape (5<<15)
128#define InstrDual (6<<15)
129#define ModeDual (7<<15)
130#define Sse (1<<18)
131
132#define ModRM (1<<19)
133
134#define Mov (1<<20)
135
136#define Prot (1<<21)
137#define EmulateOnUD (1<<22)
138#define NoAccess (1<<23)
139#define Op3264 (1<<24)
140#define Undefined (1<<25)
141#define Lock (1<<26)
142#define Priv (1<<27)
143#define No64 (1<<28)
144#define PageTable (1 << 29)
145#define NotImpl (1 << 30)
146
147#define Src2Shift (31)
148#define Src2None (OpNone << Src2Shift)
149#define Src2Mem (OpMem << Src2Shift)
150#define Src2CL (OpCL << Src2Shift)
151#define Src2ImmByte (OpImmByte << Src2Shift)
152#define Src2One (OpOne << Src2Shift)
153#define Src2Imm (OpImm << Src2Shift)
154#define Src2ES (OpES << Src2Shift)
155#define Src2CS (OpCS << Src2Shift)
156#define Src2SS (OpSS << Src2Shift)
157#define Src2DS (OpDS << Src2Shift)
158#define Src2FS (OpFS << Src2Shift)
159#define Src2GS (OpGS << Src2Shift)
160#define Src2Mask (OpMask << Src2Shift)
161#define Mmx ((u64)1 << 40)
162#define Aligned ((u64)1 << 41)
163#define Unaligned ((u64)1 << 42)
164#define Avx ((u64)1 << 43)
165#define Fastop ((u64)1 << 44)
166#define NoWrite ((u64)1 << 45)
167#define SrcWrite ((u64)1 << 46)
168#define NoMod ((u64)1 << 47)
169#define Intercept ((u64)1 << 48)
170#define CheckPerm ((u64)1 << 49)
171#define PrivUD ((u64)1 << 51)
172#define NearBranch ((u64)1 << 52)
173#define No16 ((u64)1 << 53)
174#define IncSP ((u64)1 << 54)
175
176#define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
177
178#define X2(x...) x, x
179#define X3(x...) X2(x), x
180#define X4(x...) X2(x), X2(x)
181#define X5(x...) X4(x), x
182#define X6(x...) X4(x), X2(x)
183#define X7(x...) X4(x), X3(x)
184#define X8(x...) X4(x), X4(x)
185#define X16(x...) X8(x), X8(x)
186
187#define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
188#define FASTOP_SIZE 8
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207struct fastop;
208
209struct opcode {
210 u64 flags : 56;
211 u64 intercept : 8;
212 union {
213 int (*execute)(struct x86_emulate_ctxt *ctxt);
214 const struct opcode *group;
215 const struct group_dual *gdual;
216 const struct gprefix *gprefix;
217 const struct escape *esc;
218 const struct instr_dual *idual;
219 const struct mode_dual *mdual;
220 void (*fastop)(struct fastop *fake);
221 } u;
222 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
223};
224
225struct group_dual {
226 struct opcode mod012[8];
227 struct opcode mod3[8];
228};
229
230struct gprefix {
231 struct opcode pfx_no;
232 struct opcode pfx_66;
233 struct opcode pfx_f2;
234 struct opcode pfx_f3;
235};
236
237struct escape {
238 struct opcode op[8];
239 struct opcode high[64];
240};
241
242struct instr_dual {
243 struct opcode mod012;
244 struct opcode mod3;
245};
246
247struct mode_dual {
248 struct opcode mode32;
249 struct opcode mode64;
250};
251
252#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
253
254enum x86_transfer_type {
255 X86_TRANSFER_NONE,
256 X86_TRANSFER_CALL_JMP,
257 X86_TRANSFER_RET,
258 X86_TRANSFER_TASK_SWITCH,
259};
260
261static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
262{
263 if (!(ctxt->regs_valid & (1 << nr))) {
264 ctxt->regs_valid |= 1 << nr;
265 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
266 }
267 return ctxt->_regs[nr];
268}
269
270static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
271{
272 ctxt->regs_valid |= 1 << nr;
273 ctxt->regs_dirty |= 1 << nr;
274 return &ctxt->_regs[nr];
275}
276
277static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
278{
279 reg_read(ctxt, nr);
280 return reg_write(ctxt, nr);
281}
282
283static void writeback_registers(struct x86_emulate_ctxt *ctxt)
284{
285 unsigned reg;
286
287 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
288 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
289}
290
291static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
292{
293 ctxt->regs_dirty = 0;
294 ctxt->regs_valid = 0;
295}
296
297
298
299
300
301#define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
302 X86_EFLAGS_PF|X86_EFLAGS_CF)
303
304#ifdef CONFIG_X86_64
305#define ON64(x) x
306#else
307#define ON64(x)
308#endif
309
310static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
311
312#define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
313#define FOP_RET "ret \n\t"
314
315#define FOP_START(op) \
316 extern void em_##op(struct fastop *fake); \
317 asm(".pushsection .text, \"ax\" \n\t" \
318 ".global em_" #op " \n\t" \
319 FOP_ALIGN \
320 "em_" #op ": \n\t"
321
322#define FOP_END \
323 ".popsection")
324
325#define FOPNOP() FOP_ALIGN FOP_RET
326
327#define FOP1E(op, dst) \
328 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
329
330#define FOP1EEX(op, dst) \
331 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
332
333#define FASTOP1(op) \
334 FOP_START(op) \
335 FOP1E(op##b, al) \
336 FOP1E(op##w, ax) \
337 FOP1E(op##l, eax) \
338 ON64(FOP1E(op##q, rax)) \
339 FOP_END
340
341
342#define FASTOP1SRC2(op, name) \
343 FOP_START(name) \
344 FOP1E(op, cl) \
345 FOP1E(op, cx) \
346 FOP1E(op, ecx) \
347 ON64(FOP1E(op, rcx)) \
348 FOP_END
349
350
351#define FASTOP1SRC2EX(op, name) \
352 FOP_START(name) \
353 FOP1EEX(op, cl) \
354 FOP1EEX(op, cx) \
355 FOP1EEX(op, ecx) \
356 ON64(FOP1EEX(op, rcx)) \
357 FOP_END
358
359#define FOP2E(op, dst, src) \
360 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
361
362#define FASTOP2(op) \
363 FOP_START(op) \
364 FOP2E(op##b, al, dl) \
365 FOP2E(op##w, ax, dx) \
366 FOP2E(op##l, eax, edx) \
367 ON64(FOP2E(op##q, rax, rdx)) \
368 FOP_END
369
370
371#define FASTOP2W(op) \
372 FOP_START(op) \
373 FOPNOP() \
374 FOP2E(op##w, ax, dx) \
375 FOP2E(op##l, eax, edx) \
376 ON64(FOP2E(op##q, rax, rdx)) \
377 FOP_END
378
379
380#define FASTOP2CL(op) \
381 FOP_START(op) \
382 FOP2E(op##b, al, cl) \
383 FOP2E(op##w, ax, cl) \
384 FOP2E(op##l, eax, cl) \
385 ON64(FOP2E(op##q, rax, cl)) \
386 FOP_END
387
388
389#define FASTOP2R(op, name) \
390 FOP_START(name) \
391 FOP2E(op##b, dl, al) \
392 FOP2E(op##w, dx, ax) \
393 FOP2E(op##l, edx, eax) \
394 ON64(FOP2E(op##q, rdx, rax)) \
395 FOP_END
396
397#define FOP3E(op, dst, src, src2) \
398 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
399
400
401#define FASTOP3WCL(op) \
402 FOP_START(op) \
403 FOPNOP() \
404 FOP3E(op##w, ax, dx, cl) \
405 FOP3E(op##l, eax, edx, cl) \
406 ON64(FOP3E(op##q, rax, rdx, cl)) \
407 FOP_END
408
409
410#define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
411
412asm(".global kvm_fastop_exception \n"
413 "kvm_fastop_exception: xor %esi, %esi; ret");
414
415FOP_START(setcc)
416FOP_SETCC(seto)
417FOP_SETCC(setno)
418FOP_SETCC(setc)
419FOP_SETCC(setnc)
420FOP_SETCC(setz)
421FOP_SETCC(setnz)
422FOP_SETCC(setbe)
423FOP_SETCC(setnbe)
424FOP_SETCC(sets)
425FOP_SETCC(setns)
426FOP_SETCC(setp)
427FOP_SETCC(setnp)
428FOP_SETCC(setl)
429FOP_SETCC(setnl)
430FOP_SETCC(setle)
431FOP_SETCC(setnle)
432FOP_END;
433
434FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
435FOP_END;
436
437static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
438 enum x86_intercept intercept,
439 enum x86_intercept_stage stage)
440{
441 struct x86_instruction_info info = {
442 .intercept = intercept,
443 .rep_prefix = ctxt->rep_prefix,
444 .modrm_mod = ctxt->modrm_mod,
445 .modrm_reg = ctxt->modrm_reg,
446 .modrm_rm = ctxt->modrm_rm,
447 .src_val = ctxt->src.val64,
448 .dst_val = ctxt->dst.val64,
449 .src_bytes = ctxt->src.bytes,
450 .dst_bytes = ctxt->dst.bytes,
451 .ad_bytes = ctxt->ad_bytes,
452 .next_rip = ctxt->eip,
453 };
454
455 return ctxt->ops->intercept(ctxt, &info, stage);
456}
457
458static void assign_masked(ulong *dest, ulong src, ulong mask)
459{
460 *dest = (*dest & ~mask) | (src & mask);
461}
462
463static void assign_register(unsigned long *reg, u64 val, int bytes)
464{
465
466 switch (bytes) {
467 case 1:
468 *(u8 *)reg = (u8)val;
469 break;
470 case 2:
471 *(u16 *)reg = (u16)val;
472 break;
473 case 4:
474 *reg = (u32)val;
475 break;
476 case 8:
477 *reg = val;
478 break;
479 }
480}
481
482static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
483{
484 return (1UL << (ctxt->ad_bytes << 3)) - 1;
485}
486
487static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
488{
489 u16 sel;
490 struct desc_struct ss;
491
492 if (ctxt->mode == X86EMUL_MODE_PROT64)
493 return ~0UL;
494 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
495 return ~0U >> ((ss.d ^ 1) * 16);
496}
497
498static int stack_size(struct x86_emulate_ctxt *ctxt)
499{
500 return (__fls(stack_mask(ctxt)) + 1) >> 3;
501}
502
503
504static inline unsigned long
505address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
506{
507 if (ctxt->ad_bytes == sizeof(unsigned long))
508 return reg;
509 else
510 return reg & ad_mask(ctxt);
511}
512
513static inline unsigned long
514register_address(struct x86_emulate_ctxt *ctxt, int reg)
515{
516 return address_mask(ctxt, reg_read(ctxt, reg));
517}
518
519static void masked_increment(ulong *reg, ulong mask, int inc)
520{
521 assign_masked(reg, *reg + inc, mask);
522}
523
524static inline void
525register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
526{
527 ulong *preg = reg_rmw(ctxt, reg);
528
529 assign_register(preg, *preg + inc, ctxt->ad_bytes);
530}
531
532static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
533{
534 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
535}
536
537static u32 desc_limit_scaled(struct desc_struct *desc)
538{
539 u32 limit = get_desc_limit(desc);
540
541 return desc->g ? (limit << 12) | 0xfff : limit;
542}
543
544static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
545{
546 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
547 return 0;
548
549 return ctxt->ops->get_cached_segment_base(ctxt, seg);
550}
551
552static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
553 u32 error, bool valid)
554{
555 WARN_ON(vec > 0x1f);
556 ctxt->exception.vector = vec;
557 ctxt->exception.error_code = error;
558 ctxt->exception.error_code_valid = valid;
559 return X86EMUL_PROPAGATE_FAULT;
560}
561
562static int emulate_db(struct x86_emulate_ctxt *ctxt)
563{
564 return emulate_exception(ctxt, DB_VECTOR, 0, false);
565}
566
567static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
568{
569 return emulate_exception(ctxt, GP_VECTOR, err, true);
570}
571
572static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
573{
574 return emulate_exception(ctxt, SS_VECTOR, err, true);
575}
576
577static int emulate_ud(struct x86_emulate_ctxt *ctxt)
578{
579 return emulate_exception(ctxt, UD_VECTOR, 0, false);
580}
581
582static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
583{
584 return emulate_exception(ctxt, TS_VECTOR, err, true);
585}
586
587static int emulate_de(struct x86_emulate_ctxt *ctxt)
588{
589 return emulate_exception(ctxt, DE_VECTOR, 0, false);
590}
591
592static int emulate_nm(struct x86_emulate_ctxt *ctxt)
593{
594 return emulate_exception(ctxt, NM_VECTOR, 0, false);
595}
596
597static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
598{
599 u16 selector;
600 struct desc_struct desc;
601
602 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
603 return selector;
604}
605
606static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
607 unsigned seg)
608{
609 u16 dummy;
610 u32 base3;
611 struct desc_struct desc;
612
613 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
614 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
615}
616
617
618
619
620
621
622
623
624
625static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
626{
627 if (likely(size < 16))
628 return false;
629
630 if (ctxt->d & Aligned)
631 return true;
632 else if (ctxt->d & Unaligned)
633 return false;
634 else if (ctxt->d & Avx)
635 return false;
636 else
637 return true;
638}
639
640static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
641 struct segmented_address addr,
642 unsigned *max_size, unsigned size,
643 bool write, bool fetch,
644 enum x86emul_mode mode, ulong *linear)
645{
646 struct desc_struct desc;
647 bool usable;
648 ulong la;
649 u32 lim;
650 u16 sel;
651
652 la = seg_base(ctxt, addr.seg) + addr.ea;
653 *max_size = 0;
654 switch (mode) {
655 case X86EMUL_MODE_PROT64:
656 *linear = la;
657 if (is_noncanonical_address(la))
658 goto bad;
659
660 *max_size = min_t(u64, ~0u, (1ull << 48) - la);
661 if (size > *max_size)
662 goto bad;
663 break;
664 default:
665 *linear = la = (u32)la;
666 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
667 addr.seg);
668 if (!usable)
669 goto bad;
670
671 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
672 || !(desc.type & 2)) && write)
673 goto bad;
674
675 if (!fetch && (desc.type & 8) && !(desc.type & 2))
676 goto bad;
677 lim = desc_limit_scaled(&desc);
678 if (!(desc.type & 8) && (desc.type & 4)) {
679
680 if (addr.ea <= lim)
681 goto bad;
682 lim = desc.d ? 0xffffffff : 0xffff;
683 }
684 if (addr.ea > lim)
685 goto bad;
686 if (lim == 0xffffffff)
687 *max_size = ~0u;
688 else {
689 *max_size = (u64)lim + 1 - addr.ea;
690 if (size > *max_size)
691 goto bad;
692 }
693 break;
694 }
695 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
696 return emulate_gp(ctxt, 0);
697 return X86EMUL_CONTINUE;
698bad:
699 if (addr.seg == VCPU_SREG_SS)
700 return emulate_ss(ctxt, 0);
701 else
702 return emulate_gp(ctxt, 0);
703}
704
705static int linearize(struct x86_emulate_ctxt *ctxt,
706 struct segmented_address addr,
707 unsigned size, bool write,
708 ulong *linear)
709{
710 unsigned max_size;
711 return __linearize(ctxt, addr, &max_size, size, write, false,
712 ctxt->mode, linear);
713}
714
715static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
716 enum x86emul_mode mode)
717{
718 ulong linear;
719 int rc;
720 unsigned max_size;
721 struct segmented_address addr = { .seg = VCPU_SREG_CS,
722 .ea = dst };
723
724 if (ctxt->op_bytes != sizeof(unsigned long))
725 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
726 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
727 if (rc == X86EMUL_CONTINUE)
728 ctxt->_eip = addr.ea;
729 return rc;
730}
731
732static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
733{
734 return assign_eip(ctxt, dst, ctxt->mode);
735}
736
737static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
738 const struct desc_struct *cs_desc)
739{
740 enum x86emul_mode mode = ctxt->mode;
741 int rc;
742
743#ifdef CONFIG_X86_64
744 if (ctxt->mode >= X86EMUL_MODE_PROT16) {
745 if (cs_desc->l) {
746 u64 efer = 0;
747
748 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
749 if (efer & EFER_LMA)
750 mode = X86EMUL_MODE_PROT64;
751 } else
752 mode = X86EMUL_MODE_PROT32;
753 }
754#endif
755 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
756 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
757 rc = assign_eip(ctxt, dst, mode);
758 if (rc == X86EMUL_CONTINUE)
759 ctxt->mode = mode;
760 return rc;
761}
762
763static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
764{
765 return assign_eip_near(ctxt, ctxt->_eip + rel);
766}
767
768static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
769 struct segmented_address addr,
770 void *data,
771 unsigned size)
772{
773 int rc;
774 ulong linear;
775
776 rc = linearize(ctxt, addr, size, false, &linear);
777 if (rc != X86EMUL_CONTINUE)
778 return rc;
779 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
780}
781
782
783
784
785
786static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
787{
788 int rc;
789 unsigned size, max_size;
790 unsigned long linear;
791 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
792 struct segmented_address addr = { .seg = VCPU_SREG_CS,
793 .ea = ctxt->eip + cur_size };
794
795
796
797
798
799
800
801
802
803
804
805 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
806 &linear);
807 if (unlikely(rc != X86EMUL_CONTINUE))
808 return rc;
809
810 size = min_t(unsigned, 15UL ^ cur_size, max_size);
811 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
812
813
814
815
816
817
818
819 if (unlikely(size < op_size))
820 return emulate_gp(ctxt, 0);
821
822 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
823 size, &ctxt->exception);
824 if (unlikely(rc != X86EMUL_CONTINUE))
825 return rc;
826 ctxt->fetch.end += size;
827 return X86EMUL_CONTINUE;
828}
829
830static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
831 unsigned size)
832{
833 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
834
835 if (unlikely(done_size < size))
836 return __do_insn_fetch_bytes(ctxt, size - done_size);
837 else
838 return X86EMUL_CONTINUE;
839}
840
841
842#define insn_fetch(_type, _ctxt) \
843({ _type _x; \
844 \
845 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
846 if (rc != X86EMUL_CONTINUE) \
847 goto done; \
848 ctxt->_eip += sizeof(_type); \
849 _x = *(_type __aligned(1) *) ctxt->fetch.ptr; \
850 ctxt->fetch.ptr += sizeof(_type); \
851 _x; \
852})
853
854#define insn_fetch_arr(_arr, _size, _ctxt) \
855({ \
856 rc = do_insn_fetch_bytes(_ctxt, _size); \
857 if (rc != X86EMUL_CONTINUE) \
858 goto done; \
859 ctxt->_eip += (_size); \
860 memcpy(_arr, ctxt->fetch.ptr, _size); \
861 ctxt->fetch.ptr += (_size); \
862})
863
864
865
866
867
868
869static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
870 int byteop)
871{
872 void *p;
873 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
874
875 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
876 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
877 else
878 p = reg_rmw(ctxt, modrm_reg);
879 return p;
880}
881
882static int read_descriptor(struct x86_emulate_ctxt *ctxt,
883 struct segmented_address addr,
884 u16 *size, unsigned long *address, int op_bytes)
885{
886 int rc;
887
888 if (op_bytes == 2)
889 op_bytes = 3;
890 *address = 0;
891 rc = segmented_read_std(ctxt, addr, size, 2);
892 if (rc != X86EMUL_CONTINUE)
893 return rc;
894 addr.ea += 2;
895 rc = segmented_read_std(ctxt, addr, address, op_bytes);
896 return rc;
897}
898
899FASTOP2(add);
900FASTOP2(or);
901FASTOP2(adc);
902FASTOP2(sbb);
903FASTOP2(and);
904FASTOP2(sub);
905FASTOP2(xor);
906FASTOP2(cmp);
907FASTOP2(test);
908
909FASTOP1SRC2(mul, mul_ex);
910FASTOP1SRC2(imul, imul_ex);
911FASTOP1SRC2EX(div, div_ex);
912FASTOP1SRC2EX(idiv, idiv_ex);
913
914FASTOP3WCL(shld);
915FASTOP3WCL(shrd);
916
917FASTOP2W(imul);
918
919FASTOP1(not);
920FASTOP1(neg);
921FASTOP1(inc);
922FASTOP1(dec);
923
924FASTOP2CL(rol);
925FASTOP2CL(ror);
926FASTOP2CL(rcl);
927FASTOP2CL(rcr);
928FASTOP2CL(shl);
929FASTOP2CL(shr);
930FASTOP2CL(sar);
931
932FASTOP2W(bsf);
933FASTOP2W(bsr);
934FASTOP2W(bt);
935FASTOP2W(bts);
936FASTOP2W(btr);
937FASTOP2W(btc);
938
939FASTOP2(xadd);
940
941FASTOP2R(cmp, cmp_r);
942
943static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
944{
945
946 if (ctxt->src.val == 0)
947 ctxt->dst.type = OP_NONE;
948 return fastop(ctxt, em_bsf);
949}
950
951static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
952{
953
954 if (ctxt->src.val == 0)
955 ctxt->dst.type = OP_NONE;
956 return fastop(ctxt, em_bsr);
957}
958
959static u8 test_cc(unsigned int condition, unsigned long flags)
960{
961 u8 rc;
962 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
963
964 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
965 asm("push %[flags]; popf; call *%[fastop]"
966 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
967 return rc;
968}
969
970static void fetch_register_operand(struct operand *op)
971{
972 switch (op->bytes) {
973 case 1:
974 op->val = *(u8 *)op->addr.reg;
975 break;
976 case 2:
977 op->val = *(u16 *)op->addr.reg;
978 break;
979 case 4:
980 op->val = *(u32 *)op->addr.reg;
981 break;
982 case 8:
983 op->val = *(u64 *)op->addr.reg;
984 break;
985 }
986}
987
988static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
989{
990 ctxt->ops->get_fpu(ctxt);
991 switch (reg) {
992 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
993 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
994 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
995 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
996 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
997 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
998 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
999 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1000#ifdef CONFIG_X86_64
1001 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1002 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1003 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1004 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1005 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1006 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1007 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1008 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1009#endif
1010 default: BUG();
1011 }
1012 ctxt->ops->put_fpu(ctxt);
1013}
1014
1015static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1016 int reg)
1017{
1018 ctxt->ops->get_fpu(ctxt);
1019 switch (reg) {
1020 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1021 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1022 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1023 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1024 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1025 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1026 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1027 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1028#ifdef CONFIG_X86_64
1029 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1030 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1031 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1032 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1033 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1034 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1035 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1036 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1037#endif
1038 default: BUG();
1039 }
1040 ctxt->ops->put_fpu(ctxt);
1041}
1042
1043static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1044{
1045 ctxt->ops->get_fpu(ctxt);
1046 switch (reg) {
1047 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1048 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1049 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1050 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1051 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1052 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1053 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1054 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1055 default: BUG();
1056 }
1057 ctxt->ops->put_fpu(ctxt);
1058}
1059
1060static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1061{
1062 ctxt->ops->get_fpu(ctxt);
1063 switch (reg) {
1064 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1065 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1066 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1067 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1068 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1069 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1070 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1071 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1072 default: BUG();
1073 }
1074 ctxt->ops->put_fpu(ctxt);
1075}
1076
1077static int em_fninit(struct x86_emulate_ctxt *ctxt)
1078{
1079 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1080 return emulate_nm(ctxt);
1081
1082 ctxt->ops->get_fpu(ctxt);
1083 asm volatile("fninit");
1084 ctxt->ops->put_fpu(ctxt);
1085 return X86EMUL_CONTINUE;
1086}
1087
1088static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1089{
1090 u16 fcw;
1091
1092 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1093 return emulate_nm(ctxt);
1094
1095 ctxt->ops->get_fpu(ctxt);
1096 asm volatile("fnstcw %0": "+m"(fcw));
1097 ctxt->ops->put_fpu(ctxt);
1098
1099 ctxt->dst.val = fcw;
1100
1101 return X86EMUL_CONTINUE;
1102}
1103
1104static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1105{
1106 u16 fsw;
1107
1108 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1109 return emulate_nm(ctxt);
1110
1111 ctxt->ops->get_fpu(ctxt);
1112 asm volatile("fnstsw %0": "+m"(fsw));
1113 ctxt->ops->put_fpu(ctxt);
1114
1115 ctxt->dst.val = fsw;
1116
1117 return X86EMUL_CONTINUE;
1118}
1119
1120static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1121 struct operand *op)
1122{
1123 unsigned reg = ctxt->modrm_reg;
1124
1125 if (!(ctxt->d & ModRM))
1126 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1127
1128 if (ctxt->d & Sse) {
1129 op->type = OP_XMM;
1130 op->bytes = 16;
1131 op->addr.xmm = reg;
1132 read_sse_reg(ctxt, &op->vec_val, reg);
1133 return;
1134 }
1135 if (ctxt->d & Mmx) {
1136 reg &= 7;
1137 op->type = OP_MM;
1138 op->bytes = 8;
1139 op->addr.mm = reg;
1140 return;
1141 }
1142
1143 op->type = OP_REG;
1144 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1145 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1146
1147 fetch_register_operand(op);
1148 op->orig_val = op->val;
1149}
1150
1151static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1152{
1153 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1154 ctxt->modrm_seg = VCPU_SREG_SS;
1155}
1156
1157static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1158 struct operand *op)
1159{
1160 u8 sib;
1161 int index_reg, base_reg, scale;
1162 int rc = X86EMUL_CONTINUE;
1163 ulong modrm_ea = 0;
1164
1165 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8);
1166 index_reg = (ctxt->rex_prefix << 2) & 8;
1167 base_reg = (ctxt->rex_prefix << 3) & 8;
1168
1169 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1170 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1171 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1172 ctxt->modrm_seg = VCPU_SREG_DS;
1173
1174 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1175 op->type = OP_REG;
1176 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1177 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1178 ctxt->d & ByteOp);
1179 if (ctxt->d & Sse) {
1180 op->type = OP_XMM;
1181 op->bytes = 16;
1182 op->addr.xmm = ctxt->modrm_rm;
1183 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1184 return rc;
1185 }
1186 if (ctxt->d & Mmx) {
1187 op->type = OP_MM;
1188 op->bytes = 8;
1189 op->addr.mm = ctxt->modrm_rm & 7;
1190 return rc;
1191 }
1192 fetch_register_operand(op);
1193 return rc;
1194 }
1195
1196 op->type = OP_MEM;
1197
1198 if (ctxt->ad_bytes == 2) {
1199 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1200 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1201 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1202 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1203
1204
1205 switch (ctxt->modrm_mod) {
1206 case 0:
1207 if (ctxt->modrm_rm == 6)
1208 modrm_ea += insn_fetch(u16, ctxt);
1209 break;
1210 case 1:
1211 modrm_ea += insn_fetch(s8, ctxt);
1212 break;
1213 case 2:
1214 modrm_ea += insn_fetch(u16, ctxt);
1215 break;
1216 }
1217 switch (ctxt->modrm_rm) {
1218 case 0:
1219 modrm_ea += bx + si;
1220 break;
1221 case 1:
1222 modrm_ea += bx + di;
1223 break;
1224 case 2:
1225 modrm_ea += bp + si;
1226 break;
1227 case 3:
1228 modrm_ea += bp + di;
1229 break;
1230 case 4:
1231 modrm_ea += si;
1232 break;
1233 case 5:
1234 modrm_ea += di;
1235 break;
1236 case 6:
1237 if (ctxt->modrm_mod != 0)
1238 modrm_ea += bp;
1239 break;
1240 case 7:
1241 modrm_ea += bx;
1242 break;
1243 }
1244 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1245 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1246 ctxt->modrm_seg = VCPU_SREG_SS;
1247 modrm_ea = (u16)modrm_ea;
1248 } else {
1249
1250 if ((ctxt->modrm_rm & 7) == 4) {
1251 sib = insn_fetch(u8, ctxt);
1252 index_reg |= (sib >> 3) & 7;
1253 base_reg |= sib & 7;
1254 scale = sib >> 6;
1255
1256 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1257 modrm_ea += insn_fetch(s32, ctxt);
1258 else {
1259 modrm_ea += reg_read(ctxt, base_reg);
1260 adjust_modrm_seg(ctxt, base_reg);
1261
1262 if ((ctxt->d & IncSP) &&
1263 base_reg == VCPU_REGS_RSP)
1264 modrm_ea += ctxt->op_bytes;
1265 }
1266 if (index_reg != 4)
1267 modrm_ea += reg_read(ctxt, index_reg) << scale;
1268 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1269 modrm_ea += insn_fetch(s32, ctxt);
1270 if (ctxt->mode == X86EMUL_MODE_PROT64)
1271 ctxt->rip_relative = 1;
1272 } else {
1273 base_reg = ctxt->modrm_rm;
1274 modrm_ea += reg_read(ctxt, base_reg);
1275 adjust_modrm_seg(ctxt, base_reg);
1276 }
1277 switch (ctxt->modrm_mod) {
1278 case 1:
1279 modrm_ea += insn_fetch(s8, ctxt);
1280 break;
1281 case 2:
1282 modrm_ea += insn_fetch(s32, ctxt);
1283 break;
1284 }
1285 }
1286 op->addr.mem.ea = modrm_ea;
1287 if (ctxt->ad_bytes != 8)
1288 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1289
1290done:
1291 return rc;
1292}
1293
1294static int decode_abs(struct x86_emulate_ctxt *ctxt,
1295 struct operand *op)
1296{
1297 int rc = X86EMUL_CONTINUE;
1298
1299 op->type = OP_MEM;
1300 switch (ctxt->ad_bytes) {
1301 case 2:
1302 op->addr.mem.ea = insn_fetch(u16, ctxt);
1303 break;
1304 case 4:
1305 op->addr.mem.ea = insn_fetch(u32, ctxt);
1306 break;
1307 case 8:
1308 op->addr.mem.ea = insn_fetch(u64, ctxt);
1309 break;
1310 }
1311done:
1312 return rc;
1313}
1314
1315static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1316{
1317 long sv = 0, mask;
1318
1319 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1320 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1321
1322 if (ctxt->src.bytes == 2)
1323 sv = (s16)ctxt->src.val & (s16)mask;
1324 else if (ctxt->src.bytes == 4)
1325 sv = (s32)ctxt->src.val & (s32)mask;
1326 else
1327 sv = (s64)ctxt->src.val & (s64)mask;
1328
1329 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1330 ctxt->dst.addr.mem.ea + (sv >> 3));
1331 }
1332
1333
1334 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1335}
1336
1337static int read_emulated(struct x86_emulate_ctxt *ctxt,
1338 unsigned long addr, void *dest, unsigned size)
1339{
1340 int rc;
1341 struct read_cache *mc = &ctxt->mem_read;
1342
1343 if (mc->pos < mc->end)
1344 goto read_cached;
1345
1346 WARN_ON((mc->end + size) >= sizeof(mc->data));
1347
1348 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1349 &ctxt->exception);
1350 if (rc != X86EMUL_CONTINUE)
1351 return rc;
1352
1353 mc->end += size;
1354
1355read_cached:
1356 memcpy(dest, mc->data + mc->pos, size);
1357 mc->pos += size;
1358 return X86EMUL_CONTINUE;
1359}
1360
1361static int segmented_read(struct x86_emulate_ctxt *ctxt,
1362 struct segmented_address addr,
1363 void *data,
1364 unsigned size)
1365{
1366 int rc;
1367 ulong linear;
1368
1369 rc = linearize(ctxt, addr, size, false, &linear);
1370 if (rc != X86EMUL_CONTINUE)
1371 return rc;
1372 return read_emulated(ctxt, linear, data, size);
1373}
1374
1375static int segmented_write(struct x86_emulate_ctxt *ctxt,
1376 struct segmented_address addr,
1377 const void *data,
1378 unsigned size)
1379{
1380 int rc;
1381 ulong linear;
1382
1383 rc = linearize(ctxt, addr, size, true, &linear);
1384 if (rc != X86EMUL_CONTINUE)
1385 return rc;
1386 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1387 &ctxt->exception);
1388}
1389
1390static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1391 struct segmented_address addr,
1392 const void *orig_data, const void *data,
1393 unsigned size)
1394{
1395 int rc;
1396 ulong linear;
1397
1398 rc = linearize(ctxt, addr, size, true, &linear);
1399 if (rc != X86EMUL_CONTINUE)
1400 return rc;
1401 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1402 size, &ctxt->exception);
1403}
1404
1405static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1406 unsigned int size, unsigned short port,
1407 void *dest)
1408{
1409 struct read_cache *rc = &ctxt->io_read;
1410
1411 if (rc->pos == rc->end) {
1412 unsigned int in_page, n;
1413 unsigned int count = ctxt->rep_prefix ?
1414 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1415 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1416 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1417 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1418 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1419 if (n == 0)
1420 n = 1;
1421 rc->pos = rc->end = 0;
1422 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1423 return 0;
1424 rc->end = n * size;
1425 }
1426
1427 if (ctxt->rep_prefix && (ctxt->d & String) &&
1428 !(ctxt->eflags & X86_EFLAGS_DF)) {
1429 ctxt->dst.data = rc->data + rc->pos;
1430 ctxt->dst.type = OP_MEM_STR;
1431 ctxt->dst.count = (rc->end - rc->pos) / size;
1432 rc->pos = rc->end;
1433 } else {
1434 memcpy(dest, rc->data + rc->pos, size);
1435 rc->pos += size;
1436 }
1437 return 1;
1438}
1439
1440static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1441 u16 index, struct desc_struct *desc)
1442{
1443 struct desc_ptr dt;
1444 ulong addr;
1445
1446 ctxt->ops->get_idt(ctxt, &dt);
1447
1448 if (dt.size < index * 8 + 7)
1449 return emulate_gp(ctxt, index << 3 | 0x2);
1450
1451 addr = dt.address + index * 8;
1452 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1453 &ctxt->exception);
1454}
1455
1456static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1457 u16 selector, struct desc_ptr *dt)
1458{
1459 const struct x86_emulate_ops *ops = ctxt->ops;
1460 u32 base3 = 0;
1461
1462 if (selector & 1 << 2) {
1463 struct desc_struct desc;
1464 u16 sel;
1465
1466 memset (dt, 0, sizeof *dt);
1467 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1468 VCPU_SREG_LDTR))
1469 return;
1470
1471 dt->size = desc_limit_scaled(&desc);
1472 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1473 } else
1474 ops->get_gdt(ctxt, dt);
1475}
1476
1477static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1478 u16 selector, ulong *desc_addr_p)
1479{
1480 struct desc_ptr dt;
1481 u16 index = selector >> 3;
1482 ulong addr;
1483
1484 get_descriptor_table_ptr(ctxt, selector, &dt);
1485
1486 if (dt.size < index * 8 + 7)
1487 return emulate_gp(ctxt, selector & 0xfffc);
1488
1489 addr = dt.address + index * 8;
1490
1491#ifdef CONFIG_X86_64
1492 if (addr >> 32 != 0) {
1493 u64 efer = 0;
1494
1495 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1496 if (!(efer & EFER_LMA))
1497 addr &= (u32)-1;
1498 }
1499#endif
1500
1501 *desc_addr_p = addr;
1502 return X86EMUL_CONTINUE;
1503}
1504
1505
1506static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1507 u16 selector, struct desc_struct *desc,
1508 ulong *desc_addr_p)
1509{
1510 int rc;
1511
1512 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1513 if (rc != X86EMUL_CONTINUE)
1514 return rc;
1515
1516 return ctxt->ops->read_std(ctxt, *desc_addr_p, desc, sizeof(*desc),
1517 &ctxt->exception);
1518}
1519
1520
1521static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1522 u16 selector, struct desc_struct *desc)
1523{
1524 int rc;
1525 ulong addr;
1526
1527 rc = get_descriptor_ptr(ctxt, selector, &addr);
1528 if (rc != X86EMUL_CONTINUE)
1529 return rc;
1530
1531 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1532 &ctxt->exception);
1533}
1534
1535
1536static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1537 u16 selector, int seg, u8 cpl,
1538 enum x86_transfer_type transfer,
1539 struct desc_struct *desc)
1540{
1541 struct desc_struct seg_desc, old_desc;
1542 u8 dpl, rpl;
1543 unsigned err_vec = GP_VECTOR;
1544 u32 err_code = 0;
1545 bool null_selector = !(selector & ~0x3);
1546 ulong desc_addr;
1547 int ret;
1548 u16 dummy;
1549 u32 base3 = 0;
1550
1551 memset(&seg_desc, 0, sizeof seg_desc);
1552
1553 if (ctxt->mode == X86EMUL_MODE_REAL) {
1554
1555
1556 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1557 set_desc_base(&seg_desc, selector << 4);
1558 goto load;
1559 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1560
1561 set_desc_base(&seg_desc, selector << 4);
1562 set_desc_limit(&seg_desc, 0xffff);
1563 seg_desc.type = 3;
1564 seg_desc.p = 1;
1565 seg_desc.s = 1;
1566 seg_desc.dpl = 3;
1567 goto load;
1568 }
1569
1570 rpl = selector & 3;
1571
1572
1573 if ((seg == VCPU_SREG_CS
1574 || (seg == VCPU_SREG_SS
1575 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1576 || seg == VCPU_SREG_TR)
1577 && null_selector)
1578 goto exception;
1579
1580
1581 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1582 goto exception;
1583
1584 if (null_selector)
1585 goto load;
1586
1587 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1588 if (ret != X86EMUL_CONTINUE)
1589 return ret;
1590
1591 err_code = selector & 0xfffc;
1592 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1593 GP_VECTOR;
1594
1595
1596 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1597 if (transfer == X86_TRANSFER_CALL_JMP)
1598 return X86EMUL_UNHANDLEABLE;
1599 goto exception;
1600 }
1601
1602 if (!seg_desc.p) {
1603 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1604 goto exception;
1605 }
1606
1607 dpl = seg_desc.dpl;
1608
1609 switch (seg) {
1610 case VCPU_SREG_SS:
1611
1612
1613
1614
1615 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1616 goto exception;
1617 break;
1618 case VCPU_SREG_CS:
1619 if (!(seg_desc.type & 8))
1620 goto exception;
1621
1622 if (seg_desc.type & 4) {
1623
1624 if (dpl > cpl)
1625 goto exception;
1626 } else {
1627
1628 if (rpl > cpl || dpl != cpl)
1629 goto exception;
1630 }
1631
1632 if (seg_desc.d && seg_desc.l) {
1633 u64 efer = 0;
1634
1635 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1636 if (efer & EFER_LMA)
1637 goto exception;
1638 }
1639
1640
1641 selector = (selector & 0xfffc) | cpl;
1642 break;
1643 case VCPU_SREG_TR:
1644 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1645 goto exception;
1646 old_desc = seg_desc;
1647 seg_desc.type |= 2;
1648 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1649 sizeof(seg_desc), &ctxt->exception);
1650 if (ret != X86EMUL_CONTINUE)
1651 return ret;
1652 break;
1653 case VCPU_SREG_LDTR:
1654 if (seg_desc.s || seg_desc.type != 2)
1655 goto exception;
1656 break;
1657 default:
1658
1659
1660
1661
1662
1663 if ((seg_desc.type & 0xa) == 0x8 ||
1664 (((seg_desc.type & 0xc) != 0xc) &&
1665 (rpl > dpl && cpl > dpl)))
1666 goto exception;
1667 break;
1668 }
1669
1670 if (seg_desc.s) {
1671
1672 if (!(seg_desc.type & 1)) {
1673 seg_desc.type |= 1;
1674 ret = write_segment_descriptor(ctxt, selector,
1675 &seg_desc);
1676 if (ret != X86EMUL_CONTINUE)
1677 return ret;
1678 }
1679 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1680 ret = ctxt->ops->read_std(ctxt, desc_addr+8, &base3,
1681 sizeof(base3), &ctxt->exception);
1682 if (ret != X86EMUL_CONTINUE)
1683 return ret;
1684 if (is_noncanonical_address(get_desc_base(&seg_desc) |
1685 ((u64)base3 << 32)))
1686 return emulate_gp(ctxt, 0);
1687 }
1688load:
1689 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1690 if (desc)
1691 *desc = seg_desc;
1692 return X86EMUL_CONTINUE;
1693exception:
1694 return emulate_exception(ctxt, err_vec, err_code, true);
1695}
1696
1697static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1698 u16 selector, int seg)
1699{
1700 u8 cpl = ctxt->ops->cpl(ctxt);
1701 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1702 X86_TRANSFER_NONE, NULL);
1703}
1704
1705static void write_register_operand(struct operand *op)
1706{
1707 return assign_register(op->addr.reg, op->val, op->bytes);
1708}
1709
1710static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1711{
1712 switch (op->type) {
1713 case OP_REG:
1714 write_register_operand(op);
1715 break;
1716 case OP_MEM:
1717 if (ctxt->lock_prefix)
1718 return segmented_cmpxchg(ctxt,
1719 op->addr.mem,
1720 &op->orig_val,
1721 &op->val,
1722 op->bytes);
1723 else
1724 return segmented_write(ctxt,
1725 op->addr.mem,
1726 &op->val,
1727 op->bytes);
1728 break;
1729 case OP_MEM_STR:
1730 return segmented_write(ctxt,
1731 op->addr.mem,
1732 op->data,
1733 op->bytes * op->count);
1734 break;
1735 case OP_XMM:
1736 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1737 break;
1738 case OP_MM:
1739 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1740 break;
1741 case OP_NONE:
1742
1743 break;
1744 default:
1745 break;
1746 }
1747 return X86EMUL_CONTINUE;
1748}
1749
1750static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1751{
1752 struct segmented_address addr;
1753
1754 rsp_increment(ctxt, -bytes);
1755 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1756 addr.seg = VCPU_SREG_SS;
1757
1758 return segmented_write(ctxt, addr, data, bytes);
1759}
1760
1761static int em_push(struct x86_emulate_ctxt *ctxt)
1762{
1763
1764 ctxt->dst.type = OP_NONE;
1765 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1766}
1767
1768static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1769 void *dest, int len)
1770{
1771 int rc;
1772 struct segmented_address addr;
1773
1774 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1775 addr.seg = VCPU_SREG_SS;
1776 rc = segmented_read(ctxt, addr, dest, len);
1777 if (rc != X86EMUL_CONTINUE)
1778 return rc;
1779
1780 rsp_increment(ctxt, len);
1781 return rc;
1782}
1783
1784static int em_pop(struct x86_emulate_ctxt *ctxt)
1785{
1786 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1787}
1788
1789static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1790 void *dest, int len)
1791{
1792 int rc;
1793 unsigned long val, change_mask;
1794 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1795 int cpl = ctxt->ops->cpl(ctxt);
1796
1797 rc = emulate_pop(ctxt, &val, len);
1798 if (rc != X86EMUL_CONTINUE)
1799 return rc;
1800
1801 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1802 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1803 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1804 X86_EFLAGS_AC | X86_EFLAGS_ID;
1805
1806 switch(ctxt->mode) {
1807 case X86EMUL_MODE_PROT64:
1808 case X86EMUL_MODE_PROT32:
1809 case X86EMUL_MODE_PROT16:
1810 if (cpl == 0)
1811 change_mask |= X86_EFLAGS_IOPL;
1812 if (cpl <= iopl)
1813 change_mask |= X86_EFLAGS_IF;
1814 break;
1815 case X86EMUL_MODE_VM86:
1816 if (iopl < 3)
1817 return emulate_gp(ctxt, 0);
1818 change_mask |= X86_EFLAGS_IF;
1819 break;
1820 default:
1821 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1822 break;
1823 }
1824
1825 *(unsigned long *)dest =
1826 (ctxt->eflags & ~change_mask) | (val & change_mask);
1827
1828 return rc;
1829}
1830
1831static int em_popf(struct x86_emulate_ctxt *ctxt)
1832{
1833 ctxt->dst.type = OP_REG;
1834 ctxt->dst.addr.reg = &ctxt->eflags;
1835 ctxt->dst.bytes = ctxt->op_bytes;
1836 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1837}
1838
1839static int em_enter(struct x86_emulate_ctxt *ctxt)
1840{
1841 int rc;
1842 unsigned frame_size = ctxt->src.val;
1843 unsigned nesting_level = ctxt->src2.val & 31;
1844 ulong rbp;
1845
1846 if (nesting_level)
1847 return X86EMUL_UNHANDLEABLE;
1848
1849 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1850 rc = push(ctxt, &rbp, stack_size(ctxt));
1851 if (rc != X86EMUL_CONTINUE)
1852 return rc;
1853 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1854 stack_mask(ctxt));
1855 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1856 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1857 stack_mask(ctxt));
1858 return X86EMUL_CONTINUE;
1859}
1860
1861static int em_leave(struct x86_emulate_ctxt *ctxt)
1862{
1863 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1864 stack_mask(ctxt));
1865 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1866}
1867
1868static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1869{
1870 int seg = ctxt->src2.val;
1871
1872 ctxt->src.val = get_segment_selector(ctxt, seg);
1873 if (ctxt->op_bytes == 4) {
1874 rsp_increment(ctxt, -2);
1875 ctxt->op_bytes = 2;
1876 }
1877
1878 return em_push(ctxt);
1879}
1880
1881static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1882{
1883 int seg = ctxt->src2.val;
1884 unsigned long selector;
1885 int rc;
1886
1887 rc = emulate_pop(ctxt, &selector, 2);
1888 if (rc != X86EMUL_CONTINUE)
1889 return rc;
1890
1891 if (ctxt->modrm_reg == VCPU_SREG_SS)
1892 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1893 if (ctxt->op_bytes > 2)
1894 rsp_increment(ctxt, ctxt->op_bytes - 2);
1895
1896 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1897 return rc;
1898}
1899
1900static int em_pusha(struct x86_emulate_ctxt *ctxt)
1901{
1902 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1903 int rc = X86EMUL_CONTINUE;
1904 int reg = VCPU_REGS_RAX;
1905
1906 while (reg <= VCPU_REGS_RDI) {
1907 (reg == VCPU_REGS_RSP) ?
1908 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1909
1910 rc = em_push(ctxt);
1911 if (rc != X86EMUL_CONTINUE)
1912 return rc;
1913
1914 ++reg;
1915 }
1916
1917 return rc;
1918}
1919
1920static int em_pushf(struct x86_emulate_ctxt *ctxt)
1921{
1922 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
1923 return em_push(ctxt);
1924}
1925
1926static int em_popa(struct x86_emulate_ctxt *ctxt)
1927{
1928 int rc = X86EMUL_CONTINUE;
1929 int reg = VCPU_REGS_RDI;
1930 u32 val;
1931
1932 while (reg >= VCPU_REGS_RAX) {
1933 if (reg == VCPU_REGS_RSP) {
1934 rsp_increment(ctxt, ctxt->op_bytes);
1935 --reg;
1936 }
1937
1938 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
1939 if (rc != X86EMUL_CONTINUE)
1940 break;
1941 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
1942 --reg;
1943 }
1944 return rc;
1945}
1946
1947static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1948{
1949 const struct x86_emulate_ops *ops = ctxt->ops;
1950 int rc;
1951 struct desc_ptr dt;
1952 gva_t cs_addr;
1953 gva_t eip_addr;
1954 u16 cs, eip;
1955
1956
1957 ctxt->src.val = ctxt->eflags;
1958 rc = em_push(ctxt);
1959 if (rc != X86EMUL_CONTINUE)
1960 return rc;
1961
1962 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
1963
1964 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1965 rc = em_push(ctxt);
1966 if (rc != X86EMUL_CONTINUE)
1967 return rc;
1968
1969 ctxt->src.val = ctxt->_eip;
1970 rc = em_push(ctxt);
1971 if (rc != X86EMUL_CONTINUE)
1972 return rc;
1973
1974 ops->get_idt(ctxt, &dt);
1975
1976 eip_addr = dt.address + (irq << 2);
1977 cs_addr = dt.address + (irq << 2) + 2;
1978
1979 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1980 if (rc != X86EMUL_CONTINUE)
1981 return rc;
1982
1983 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1984 if (rc != X86EMUL_CONTINUE)
1985 return rc;
1986
1987 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1988 if (rc != X86EMUL_CONTINUE)
1989 return rc;
1990
1991 ctxt->_eip = eip;
1992
1993 return rc;
1994}
1995
1996int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1997{
1998 int rc;
1999
2000 invalidate_registers(ctxt);
2001 rc = __emulate_int_real(ctxt, irq);
2002 if (rc == X86EMUL_CONTINUE)
2003 writeback_registers(ctxt);
2004 return rc;
2005}
2006
2007static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2008{
2009 switch(ctxt->mode) {
2010 case X86EMUL_MODE_REAL:
2011 return __emulate_int_real(ctxt, irq);
2012 case X86EMUL_MODE_VM86:
2013 case X86EMUL_MODE_PROT16:
2014 case X86EMUL_MODE_PROT32:
2015 case X86EMUL_MODE_PROT64:
2016 default:
2017
2018 return X86EMUL_UNHANDLEABLE;
2019 }
2020}
2021
2022static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2023{
2024 int rc = X86EMUL_CONTINUE;
2025 unsigned long temp_eip = 0;
2026 unsigned long temp_eflags = 0;
2027 unsigned long cs = 0;
2028 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2029 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2030 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2031 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2032 X86_EFLAGS_AC | X86_EFLAGS_ID |
2033 X86_EFLAGS_FIXED;
2034 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2035 X86_EFLAGS_VIP;
2036
2037
2038
2039 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2040
2041 if (rc != X86EMUL_CONTINUE)
2042 return rc;
2043
2044 if (temp_eip & ~0xffff)
2045 return emulate_gp(ctxt, 0);
2046
2047 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2048
2049 if (rc != X86EMUL_CONTINUE)
2050 return rc;
2051
2052 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2053
2054 if (rc != X86EMUL_CONTINUE)
2055 return rc;
2056
2057 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2058
2059 if (rc != X86EMUL_CONTINUE)
2060 return rc;
2061
2062 ctxt->_eip = temp_eip;
2063
2064 if (ctxt->op_bytes == 4)
2065 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2066 else if (ctxt->op_bytes == 2) {
2067 ctxt->eflags &= ~0xffff;
2068 ctxt->eflags |= temp_eflags;
2069 }
2070
2071 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK;
2072 ctxt->eflags |= X86_EFLAGS_FIXED;
2073 ctxt->ops->set_nmi_mask(ctxt, false);
2074
2075 return rc;
2076}
2077
2078static int em_iret(struct x86_emulate_ctxt *ctxt)
2079{
2080 switch(ctxt->mode) {
2081 case X86EMUL_MODE_REAL:
2082 return emulate_iret_real(ctxt);
2083 case X86EMUL_MODE_VM86:
2084 case X86EMUL_MODE_PROT16:
2085 case X86EMUL_MODE_PROT32:
2086 case X86EMUL_MODE_PROT64:
2087 default:
2088
2089 return X86EMUL_UNHANDLEABLE;
2090 }
2091}
2092
2093static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2094{
2095 int rc;
2096 unsigned short sel, old_sel;
2097 struct desc_struct old_desc, new_desc;
2098 const struct x86_emulate_ops *ops = ctxt->ops;
2099 u8 cpl = ctxt->ops->cpl(ctxt);
2100
2101
2102 if (ctxt->mode == X86EMUL_MODE_PROT64)
2103 ops->get_segment(ctxt, &old_sel, &old_desc, NULL,
2104 VCPU_SREG_CS);
2105
2106 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2107
2108 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2109 X86_TRANSFER_CALL_JMP,
2110 &new_desc);
2111 if (rc != X86EMUL_CONTINUE)
2112 return rc;
2113
2114 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2115 if (rc != X86EMUL_CONTINUE) {
2116 WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64);
2117
2118 ops->set_segment(ctxt, old_sel, &old_desc, 0, VCPU_SREG_CS);
2119 return rc;
2120 }
2121 return rc;
2122}
2123
2124static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2125{
2126 return assign_eip_near(ctxt, ctxt->src.val);
2127}
2128
2129static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2130{
2131 int rc;
2132 long int old_eip;
2133
2134 old_eip = ctxt->_eip;
2135 rc = assign_eip_near(ctxt, ctxt->src.val);
2136 if (rc != X86EMUL_CONTINUE)
2137 return rc;
2138 ctxt->src.val = old_eip;
2139 rc = em_push(ctxt);
2140 return rc;
2141}
2142
2143static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2144{
2145 u64 old = ctxt->dst.orig_val64;
2146
2147 if (ctxt->dst.bytes == 16)
2148 return X86EMUL_UNHANDLEABLE;
2149
2150 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2151 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2152 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2153 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2154 ctxt->eflags &= ~X86_EFLAGS_ZF;
2155 } else {
2156 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2157 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2158
2159 ctxt->eflags |= X86_EFLAGS_ZF;
2160 }
2161 return X86EMUL_CONTINUE;
2162}
2163
2164static int em_ret(struct x86_emulate_ctxt *ctxt)
2165{
2166 int rc;
2167 unsigned long eip;
2168
2169 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2170 if (rc != X86EMUL_CONTINUE)
2171 return rc;
2172
2173 return assign_eip_near(ctxt, eip);
2174}
2175
2176static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2177{
2178 int rc;
2179 unsigned long eip, cs;
2180 u16 old_cs;
2181 int cpl = ctxt->ops->cpl(ctxt);
2182 struct desc_struct old_desc, new_desc;
2183 const struct x86_emulate_ops *ops = ctxt->ops;
2184
2185 if (ctxt->mode == X86EMUL_MODE_PROT64)
2186 ops->get_segment(ctxt, &old_cs, &old_desc, NULL,
2187 VCPU_SREG_CS);
2188
2189 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2190 if (rc != X86EMUL_CONTINUE)
2191 return rc;
2192 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2193 if (rc != X86EMUL_CONTINUE)
2194 return rc;
2195
2196 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2197 return X86EMUL_UNHANDLEABLE;
2198 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2199 X86_TRANSFER_RET,
2200 &new_desc);
2201 if (rc != X86EMUL_CONTINUE)
2202 return rc;
2203 rc = assign_eip_far(ctxt, eip, &new_desc);
2204 if (rc != X86EMUL_CONTINUE) {
2205 WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64);
2206 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
2207 }
2208 return rc;
2209}
2210
2211static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2212{
2213 int rc;
2214
2215 rc = em_ret_far(ctxt);
2216 if (rc != X86EMUL_CONTINUE)
2217 return rc;
2218 rsp_increment(ctxt, ctxt->src.val);
2219 return X86EMUL_CONTINUE;
2220}
2221
2222static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2223{
2224
2225 ctxt->dst.orig_val = ctxt->dst.val;
2226 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2227 ctxt->src.orig_val = ctxt->src.val;
2228 ctxt->src.val = ctxt->dst.orig_val;
2229 fastop(ctxt, em_cmp);
2230
2231 if (ctxt->eflags & X86_EFLAGS_ZF) {
2232
2233 ctxt->src.type = OP_NONE;
2234 ctxt->dst.val = ctxt->src.orig_val;
2235 } else {
2236
2237 ctxt->src.type = OP_REG;
2238 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2239 ctxt->src.val = ctxt->dst.orig_val;
2240
2241 ctxt->dst.val = ctxt->dst.orig_val;
2242 }
2243 return X86EMUL_CONTINUE;
2244}
2245
2246static int em_lseg(struct x86_emulate_ctxt *ctxt)
2247{
2248 int seg = ctxt->src2.val;
2249 unsigned short sel;
2250 int rc;
2251
2252 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2253
2254 rc = load_segment_descriptor(ctxt, sel, seg);
2255 if (rc != X86EMUL_CONTINUE)
2256 return rc;
2257
2258 ctxt->dst.val = ctxt->src.val;
2259 return rc;
2260}
2261
2262static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
2263{
2264 u32 eax, ebx, ecx, edx;
2265
2266 eax = 0x80000001;
2267 ecx = 0;
2268 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2269 return edx & bit(X86_FEATURE_LM);
2270}
2271
2272#define GET_SMSTATE(type, smbase, offset) \
2273 ({ \
2274 type __val; \
2275 int r = ctxt->ops->read_phys(ctxt, smbase + offset, &__val, \
2276 sizeof(__val)); \
2277 if (r != X86EMUL_CONTINUE) \
2278 return X86EMUL_UNHANDLEABLE; \
2279 __val; \
2280 })
2281
2282static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
2283{
2284 desc->g = (flags >> 23) & 1;
2285 desc->d = (flags >> 22) & 1;
2286 desc->l = (flags >> 21) & 1;
2287 desc->avl = (flags >> 20) & 1;
2288 desc->p = (flags >> 15) & 1;
2289 desc->dpl = (flags >> 13) & 3;
2290 desc->s = (flags >> 12) & 1;
2291 desc->type = (flags >> 8) & 15;
2292}
2293
2294static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2295{
2296 struct desc_struct desc;
2297 int offset;
2298 u16 selector;
2299
2300 selector = GET_SMSTATE(u32, smbase, 0x7fa8 + n * 4);
2301
2302 if (n < 3)
2303 offset = 0x7f84 + n * 12;
2304 else
2305 offset = 0x7f2c + (n - 3) * 12;
2306
2307 set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2308 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2309 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, offset));
2310 ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
2311 return X86EMUL_CONTINUE;
2312}
2313
2314static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2315{
2316 struct desc_struct desc;
2317 int offset;
2318 u16 selector;
2319 u32 base3;
2320
2321 offset = 0x7e00 + n * 16;
2322
2323 selector = GET_SMSTATE(u16, smbase, offset);
2324 rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smbase, offset + 2) << 8);
2325 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2326 set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2327 base3 = GET_SMSTATE(u32, smbase, offset + 12);
2328
2329 ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
2330 return X86EMUL_CONTINUE;
2331}
2332
2333static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,
2334 u64 cr0, u64 cr4)
2335{
2336 int bad;
2337
2338
2339
2340
2341
2342
2343 bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2344 if (bad)
2345 return X86EMUL_UNHANDLEABLE;
2346
2347 bad = ctxt->ops->set_cr(ctxt, 0, cr0);
2348 if (bad)
2349 return X86EMUL_UNHANDLEABLE;
2350
2351 if (cr4 & X86_CR4_PCIDE) {
2352 bad = ctxt->ops->set_cr(ctxt, 4, cr4);
2353 if (bad)
2354 return X86EMUL_UNHANDLEABLE;
2355 }
2356
2357 return X86EMUL_CONTINUE;
2358}
2359
2360static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt, u64 smbase)
2361{
2362 struct desc_struct desc;
2363 struct desc_ptr dt;
2364 u16 selector;
2365 u32 val, cr0, cr4;
2366 int i;
2367
2368 cr0 = GET_SMSTATE(u32, smbase, 0x7ffc);
2369 ctxt->ops->set_cr(ctxt, 3, GET_SMSTATE(u32, smbase, 0x7ff8));
2370 ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7ff4) | X86_EFLAGS_FIXED;
2371 ctxt->_eip = GET_SMSTATE(u32, smbase, 0x7ff0);
2372
2373 for (i = 0; i < 8; i++)
2374 *reg_write(ctxt, i) = GET_SMSTATE(u32, smbase, 0x7fd0 + i * 4);
2375
2376 val = GET_SMSTATE(u32, smbase, 0x7fcc);
2377 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2378 val = GET_SMSTATE(u32, smbase, 0x7fc8);
2379 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2380
2381 selector = GET_SMSTATE(u32, smbase, 0x7fc4);
2382 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f64));
2383 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f60));
2384 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f5c));
2385 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);
2386
2387 selector = GET_SMSTATE(u32, smbase, 0x7fc0);
2388 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f80));
2389 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f7c));
2390 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f78));
2391 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);
2392
2393 dt.address = GET_SMSTATE(u32, smbase, 0x7f74);
2394 dt.size = GET_SMSTATE(u32, smbase, 0x7f70);
2395 ctxt->ops->set_gdt(ctxt, &dt);
2396
2397 dt.address = GET_SMSTATE(u32, smbase, 0x7f58);
2398 dt.size = GET_SMSTATE(u32, smbase, 0x7f54);
2399 ctxt->ops->set_idt(ctxt, &dt);
2400
2401 for (i = 0; i < 6; i++) {
2402 int r = rsm_load_seg_32(ctxt, smbase, i);
2403 if (r != X86EMUL_CONTINUE)
2404 return r;
2405 }
2406
2407 cr4 = GET_SMSTATE(u32, smbase, 0x7f14);
2408
2409 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7ef8));
2410
2411 return rsm_enter_protected_mode(ctxt, cr0, cr4);
2412}
2413
2414static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt, u64 smbase)
2415{
2416 struct desc_struct desc;
2417 struct desc_ptr dt;
2418 u64 val, cr0, cr4;
2419 u32 base3;
2420 u16 selector;
2421 int i, r;
2422
2423 for (i = 0; i < 16; i++)
2424 *reg_write(ctxt, i) = GET_SMSTATE(u64, smbase, 0x7ff8 - i * 8);
2425
2426 ctxt->_eip = GET_SMSTATE(u64, smbase, 0x7f78);
2427 ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7f70) | X86_EFLAGS_FIXED;
2428
2429 val = GET_SMSTATE(u32, smbase, 0x7f68);
2430 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2431 val = GET_SMSTATE(u32, smbase, 0x7f60);
2432 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2433
2434 cr0 = GET_SMSTATE(u64, smbase, 0x7f58);
2435 ctxt->ops->set_cr(ctxt, 3, GET_SMSTATE(u64, smbase, 0x7f50));
2436 cr4 = GET_SMSTATE(u64, smbase, 0x7f48);
2437 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7f00));
2438 val = GET_SMSTATE(u64, smbase, 0x7ed0);
2439 ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA);
2440
2441 selector = GET_SMSTATE(u32, smbase, 0x7e90);
2442 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e92) << 8);
2443 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e94));
2444 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e98));
2445 base3 = GET_SMSTATE(u32, smbase, 0x7e9c);
2446 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);
2447
2448 dt.size = GET_SMSTATE(u32, smbase, 0x7e84);
2449 dt.address = GET_SMSTATE(u64, smbase, 0x7e88);
2450 ctxt->ops->set_idt(ctxt, &dt);
2451
2452 selector = GET_SMSTATE(u32, smbase, 0x7e70);
2453 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e72) << 8);
2454 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e74));
2455 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e78));
2456 base3 = GET_SMSTATE(u32, smbase, 0x7e7c);
2457 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);
2458
2459 dt.size = GET_SMSTATE(u32, smbase, 0x7e64);
2460 dt.address = GET_SMSTATE(u64, smbase, 0x7e68);
2461 ctxt->ops->set_gdt(ctxt, &dt);
2462
2463 r = rsm_enter_protected_mode(ctxt, cr0, cr4);
2464 if (r != X86EMUL_CONTINUE)
2465 return r;
2466
2467 for (i = 0; i < 6; i++) {
2468 r = rsm_load_seg_64(ctxt, smbase, i);
2469 if (r != X86EMUL_CONTINUE)
2470 return r;
2471 }
2472
2473 return X86EMUL_CONTINUE;
2474}
2475
2476static int em_rsm(struct x86_emulate_ctxt *ctxt)
2477{
2478 unsigned long cr0, cr4, efer;
2479 u64 smbase;
2480 int ret;
2481
2482 if ((ctxt->emul_flags & X86EMUL_SMM_MASK) == 0)
2483 return emulate_ud(ctxt);
2484
2485
2486
2487
2488
2489
2490 cr4 = ctxt->ops->get_cr(ctxt, 4);
2491 if (emulator_has_longmode(ctxt)) {
2492 struct desc_struct cs_desc;
2493
2494
2495 if (cr4 & X86_CR4_PCIDE) {
2496 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2497 cr4 &= ~X86_CR4_PCIDE;
2498 }
2499
2500
2501 memset(&cs_desc, 0, sizeof(cs_desc));
2502 cs_desc.type = 0xb;
2503 cs_desc.s = cs_desc.g = cs_desc.p = 1;
2504 ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);
2505 }
2506
2507
2508 cr0 = ctxt->ops->get_cr(ctxt, 0);
2509 if (cr0 & X86_CR0_PE)
2510 ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
2511
2512
2513 if (cr4 & X86_CR4_PAE)
2514 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);
2515
2516
2517 efer = 0;
2518 ctxt->ops->set_msr(ctxt, MSR_EFER, efer);
2519
2520 smbase = ctxt->ops->get_smbase(ctxt);
2521 if (emulator_has_longmode(ctxt))
2522 ret = rsm_load_state_64(ctxt, smbase + 0x8000);
2523 else
2524 ret = rsm_load_state_32(ctxt, smbase + 0x8000);
2525
2526 if (ret != X86EMUL_CONTINUE) {
2527
2528 return X86EMUL_UNHANDLEABLE;
2529 }
2530
2531 if ((ctxt->emul_flags & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
2532 ctxt->ops->set_nmi_mask(ctxt, false);
2533
2534 ctxt->emul_flags &= ~X86EMUL_SMM_INSIDE_NMI_MASK;
2535 ctxt->emul_flags &= ~X86EMUL_SMM_MASK;
2536 return X86EMUL_CONTINUE;
2537}
2538
2539static void
2540setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2541 struct desc_struct *cs, struct desc_struct *ss)
2542{
2543 cs->l = 0;
2544 set_desc_base(cs, 0);
2545 cs->g = 1;
2546 set_desc_limit(cs, 0xfffff);
2547 cs->type = 0x0b;
2548 cs->s = 1;
2549 cs->dpl = 0;
2550 cs->p = 1;
2551 cs->d = 1;
2552 cs->avl = 0;
2553
2554 set_desc_base(ss, 0);
2555 set_desc_limit(ss, 0xfffff);
2556 ss->g = 1;
2557 ss->s = 1;
2558 ss->type = 0x03;
2559 ss->d = 1;
2560 ss->dpl = 0;
2561 ss->p = 1;
2562 ss->l = 0;
2563 ss->avl = 0;
2564}
2565
2566static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2567{
2568 u32 eax, ebx, ecx, edx;
2569
2570 eax = ecx = 0;
2571 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2572 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2573 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2574 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2575}
2576
2577static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2578{
2579 const struct x86_emulate_ops *ops = ctxt->ops;
2580 u32 eax, ebx, ecx, edx;
2581
2582
2583
2584
2585
2586 if (ctxt->mode == X86EMUL_MODE_PROT64)
2587 return true;
2588
2589 eax = 0x00000000;
2590 ecx = 0x00000000;
2591 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2592
2593
2594
2595
2596
2597
2598
2599
2600 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2601 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2602 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2603 return false;
2604
2605
2606 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2607 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2608 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2609 return true;
2610
2611
2612 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2613 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2614 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2615 return true;
2616
2617
2618 return false;
2619}
2620
2621static int em_syscall(struct x86_emulate_ctxt *ctxt)
2622{
2623 const struct x86_emulate_ops *ops = ctxt->ops;
2624 struct desc_struct cs, ss;
2625 u64 msr_data;
2626 u16 cs_sel, ss_sel;
2627 u64 efer = 0;
2628
2629
2630 if (ctxt->mode == X86EMUL_MODE_REAL ||
2631 ctxt->mode == X86EMUL_MODE_VM86)
2632 return emulate_ud(ctxt);
2633
2634 if (!(em_syscall_is_enabled(ctxt)))
2635 return emulate_ud(ctxt);
2636
2637 ops->get_msr(ctxt, MSR_EFER, &efer);
2638 setup_syscalls_segments(ctxt, &cs, &ss);
2639
2640 if (!(efer & EFER_SCE))
2641 return emulate_ud(ctxt);
2642
2643 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2644 msr_data >>= 32;
2645 cs_sel = (u16)(msr_data & 0xfffc);
2646 ss_sel = (u16)(msr_data + 8);
2647
2648 if (efer & EFER_LMA) {
2649 cs.d = 0;
2650 cs.l = 1;
2651 }
2652 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2653 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2654
2655 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2656 if (efer & EFER_LMA) {
2657#ifdef CONFIG_X86_64
2658 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2659
2660 ops->get_msr(ctxt,
2661 ctxt->mode == X86EMUL_MODE_PROT64 ?
2662 MSR_LSTAR : MSR_CSTAR, &msr_data);
2663 ctxt->_eip = msr_data;
2664
2665 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2666 ctxt->eflags &= ~msr_data;
2667 ctxt->eflags |= X86_EFLAGS_FIXED;
2668#endif
2669 } else {
2670
2671 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2672 ctxt->_eip = (u32)msr_data;
2673
2674 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2675 }
2676
2677 return X86EMUL_CONTINUE;
2678}
2679
2680static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2681{
2682 const struct x86_emulate_ops *ops = ctxt->ops;
2683 struct desc_struct cs, ss;
2684 u64 msr_data;
2685 u16 cs_sel, ss_sel;
2686 u64 efer = 0;
2687
2688 ops->get_msr(ctxt, MSR_EFER, &efer);
2689
2690 if (ctxt->mode == X86EMUL_MODE_REAL)
2691 return emulate_gp(ctxt, 0);
2692
2693
2694
2695
2696
2697 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2698 && !vendor_intel(ctxt))
2699 return emulate_ud(ctxt);
2700
2701
2702 if (ctxt->mode == X86EMUL_MODE_PROT64)
2703 return X86EMUL_UNHANDLEABLE;
2704
2705 setup_syscalls_segments(ctxt, &cs, &ss);
2706
2707 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2708 if ((msr_data & 0xfffc) == 0x0)
2709 return emulate_gp(ctxt, 0);
2710
2711 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2712 cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2713 ss_sel = cs_sel + 8;
2714 if (efer & EFER_LMA) {
2715 cs.d = 0;
2716 cs.l = 1;
2717 }
2718
2719 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2720 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2721
2722 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2723 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2724
2725 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2726 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2727 (u32)msr_data;
2728
2729 return X86EMUL_CONTINUE;
2730}
2731
2732static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2733{
2734 const struct x86_emulate_ops *ops = ctxt->ops;
2735 struct desc_struct cs, ss;
2736 u64 msr_data, rcx, rdx;
2737 int usermode;
2738 u16 cs_sel = 0, ss_sel = 0;
2739
2740
2741 if (ctxt->mode == X86EMUL_MODE_REAL ||
2742 ctxt->mode == X86EMUL_MODE_VM86)
2743 return emulate_gp(ctxt, 0);
2744
2745 setup_syscalls_segments(ctxt, &cs, &ss);
2746
2747 if ((ctxt->rex_prefix & 0x8) != 0x0)
2748 usermode = X86EMUL_MODE_PROT64;
2749 else
2750 usermode = X86EMUL_MODE_PROT32;
2751
2752 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2753 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2754
2755 cs.dpl = 3;
2756 ss.dpl = 3;
2757 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2758 switch (usermode) {
2759 case X86EMUL_MODE_PROT32:
2760 cs_sel = (u16)(msr_data + 16);
2761 if ((msr_data & 0xfffc) == 0x0)
2762 return emulate_gp(ctxt, 0);
2763 ss_sel = (u16)(msr_data + 24);
2764 rcx = (u32)rcx;
2765 rdx = (u32)rdx;
2766 break;
2767 case X86EMUL_MODE_PROT64:
2768 cs_sel = (u16)(msr_data + 32);
2769 if (msr_data == 0x0)
2770 return emulate_gp(ctxt, 0);
2771 ss_sel = cs_sel + 8;
2772 cs.d = 0;
2773 cs.l = 1;
2774 if (is_noncanonical_address(rcx) ||
2775 is_noncanonical_address(rdx))
2776 return emulate_gp(ctxt, 0);
2777 break;
2778 }
2779 cs_sel |= SEGMENT_RPL_MASK;
2780 ss_sel |= SEGMENT_RPL_MASK;
2781
2782 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2783 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2784
2785 ctxt->_eip = rdx;
2786 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2787
2788 return X86EMUL_CONTINUE;
2789}
2790
2791static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2792{
2793 int iopl;
2794 if (ctxt->mode == X86EMUL_MODE_REAL)
2795 return false;
2796 if (ctxt->mode == X86EMUL_MODE_VM86)
2797 return true;
2798 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2799 return ctxt->ops->cpl(ctxt) > iopl;
2800}
2801
2802static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2803 u16 port, u16 len)
2804{
2805 const struct x86_emulate_ops *ops = ctxt->ops;
2806 struct desc_struct tr_seg;
2807 u32 base3;
2808 int r;
2809 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2810 unsigned mask = (1 << len) - 1;
2811 unsigned long base;
2812
2813 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2814 if (!tr_seg.p)
2815 return false;
2816 if (desc_limit_scaled(&tr_seg) < 103)
2817 return false;
2818 base = get_desc_base(&tr_seg);
2819#ifdef CONFIG_X86_64
2820 base |= ((u64)base3) << 32;
2821#endif
2822 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2823 if (r != X86EMUL_CONTINUE)
2824 return false;
2825 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2826 return false;
2827 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2828 if (r != X86EMUL_CONTINUE)
2829 return false;
2830 if ((perm >> bit_idx) & mask)
2831 return false;
2832 return true;
2833}
2834
2835static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2836 u16 port, u16 len)
2837{
2838 if (ctxt->perm_ok)
2839 return true;
2840
2841 if (emulator_bad_iopl(ctxt))
2842 if (!emulator_io_port_access_allowed(ctxt, port, len))
2843 return false;
2844
2845 ctxt->perm_ok = true;
2846
2847 return true;
2848}
2849
2850static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
2851{
2852
2853
2854
2855
2856#ifdef CONFIG_X86_64
2857 if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
2858 return;
2859
2860 *reg_write(ctxt, VCPU_REGS_RCX) = 0;
2861
2862 switch (ctxt->b) {
2863 case 0xa4:
2864 case 0xa5:
2865 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
2866
2867 case 0xaa:
2868 case 0xab:
2869 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
2870 }
2871#endif
2872}
2873
2874static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2875 struct tss_segment_16 *tss)
2876{
2877 tss->ip = ctxt->_eip;
2878 tss->flag = ctxt->eflags;
2879 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2880 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2881 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2882 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2883 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2884 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2885 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2886 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2887
2888 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2889 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2890 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2891 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2892 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2893}
2894
2895static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2896 struct tss_segment_16 *tss)
2897{
2898 int ret;
2899 u8 cpl;
2900
2901 ctxt->_eip = tss->ip;
2902 ctxt->eflags = tss->flag | 2;
2903 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2904 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2905 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2906 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2907 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2908 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2909 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2910 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2911
2912
2913
2914
2915
2916 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2917 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2918 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2919 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2920 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2921
2922 cpl = tss->cs & 3;
2923
2924
2925
2926
2927
2928 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
2929 X86_TRANSFER_TASK_SWITCH, NULL);
2930 if (ret != X86EMUL_CONTINUE)
2931 return ret;
2932 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
2933 X86_TRANSFER_TASK_SWITCH, NULL);
2934 if (ret != X86EMUL_CONTINUE)
2935 return ret;
2936 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
2937 X86_TRANSFER_TASK_SWITCH, NULL);
2938 if (ret != X86EMUL_CONTINUE)
2939 return ret;
2940 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
2941 X86_TRANSFER_TASK_SWITCH, NULL);
2942 if (ret != X86EMUL_CONTINUE)
2943 return ret;
2944 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
2945 X86_TRANSFER_TASK_SWITCH, NULL);
2946 if (ret != X86EMUL_CONTINUE)
2947 return ret;
2948
2949 return X86EMUL_CONTINUE;
2950}
2951
2952static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2953 u16 tss_selector, u16 old_tss_sel,
2954 ulong old_tss_base, struct desc_struct *new_desc)
2955{
2956 const struct x86_emulate_ops *ops = ctxt->ops;
2957 struct tss_segment_16 tss_seg;
2958 int ret;
2959 u32 new_tss_base = get_desc_base(new_desc);
2960
2961 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2962 &ctxt->exception);
2963 if (ret != X86EMUL_CONTINUE)
2964 return ret;
2965
2966 save_state_to_tss16(ctxt, &tss_seg);
2967
2968 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2969 &ctxt->exception);
2970 if (ret != X86EMUL_CONTINUE)
2971 return ret;
2972
2973 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2974 &ctxt->exception);
2975 if (ret != X86EMUL_CONTINUE)
2976 return ret;
2977
2978 if (old_tss_sel != 0xffff) {
2979 tss_seg.prev_task_link = old_tss_sel;
2980
2981 ret = ops->write_std(ctxt, new_tss_base,
2982 &tss_seg.prev_task_link,
2983 sizeof tss_seg.prev_task_link,
2984 &ctxt->exception);
2985 if (ret != X86EMUL_CONTINUE)
2986 return ret;
2987 }
2988
2989 return load_state_from_tss16(ctxt, &tss_seg);
2990}
2991
2992static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2993 struct tss_segment_32 *tss)
2994{
2995
2996 tss->eip = ctxt->_eip;
2997 tss->eflags = ctxt->eflags;
2998 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2999 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
3000 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
3001 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
3002 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
3003 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
3004 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
3005 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
3006
3007 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3008 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3009 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3010 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3011 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
3012 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
3013}
3014
3015static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
3016 struct tss_segment_32 *tss)
3017{
3018 int ret;
3019 u8 cpl;
3020
3021 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
3022 return emulate_gp(ctxt, 0);
3023 ctxt->_eip = tss->eip;
3024 ctxt->eflags = tss->eflags | 2;
3025
3026
3027 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
3028 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
3029 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
3030 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
3031 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
3032 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
3033 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
3034 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
3035
3036
3037
3038
3039
3040
3041 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
3042 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3043 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3044 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3045 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3046 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
3047 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
3048
3049
3050
3051
3052
3053
3054 if (ctxt->eflags & X86_EFLAGS_VM) {
3055 ctxt->mode = X86EMUL_MODE_VM86;
3056 cpl = 3;
3057 } else {
3058 ctxt->mode = X86EMUL_MODE_PROT32;
3059 cpl = tss->cs & 3;
3060 }
3061
3062
3063
3064
3065
3066 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
3067 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
3068 if (ret != X86EMUL_CONTINUE)
3069 return ret;
3070 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3071 X86_TRANSFER_TASK_SWITCH, NULL);
3072 if (ret != X86EMUL_CONTINUE)
3073 return ret;
3074 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3075 X86_TRANSFER_TASK_SWITCH, NULL);
3076 if (ret != X86EMUL_CONTINUE)
3077 return ret;
3078 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3079 X86_TRANSFER_TASK_SWITCH, NULL);
3080 if (ret != X86EMUL_CONTINUE)
3081 return ret;
3082 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3083 X86_TRANSFER_TASK_SWITCH, NULL);
3084 if (ret != X86EMUL_CONTINUE)
3085 return ret;
3086 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
3087 X86_TRANSFER_TASK_SWITCH, NULL);
3088 if (ret != X86EMUL_CONTINUE)
3089 return ret;
3090 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
3091 X86_TRANSFER_TASK_SWITCH, NULL);
3092
3093 return ret;
3094}
3095
3096static int task_switch_32(struct x86_emulate_ctxt *ctxt,
3097 u16 tss_selector, u16 old_tss_sel,
3098 ulong old_tss_base, struct desc_struct *new_desc)
3099{
3100 const struct x86_emulate_ops *ops = ctxt->ops;
3101 struct tss_segment_32 tss_seg;
3102 int ret;
3103 u32 new_tss_base = get_desc_base(new_desc);
3104 u32 eip_offset = offsetof(struct tss_segment_32, eip);
3105 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3106
3107 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
3108 &ctxt->exception);
3109 if (ret != X86EMUL_CONTINUE)
3110 return ret;
3111
3112 save_state_to_tss32(ctxt, &tss_seg);
3113
3114
3115 ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
3116 ldt_sel_offset - eip_offset, &ctxt->exception);
3117 if (ret != X86EMUL_CONTINUE)
3118 return ret;
3119
3120 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
3121 &ctxt->exception);
3122 if (ret != X86EMUL_CONTINUE)
3123 return ret;
3124
3125 if (old_tss_sel != 0xffff) {
3126 tss_seg.prev_task_link = old_tss_sel;
3127
3128 ret = ops->write_std(ctxt, new_tss_base,
3129 &tss_seg.prev_task_link,
3130 sizeof tss_seg.prev_task_link,
3131 &ctxt->exception);
3132 if (ret != X86EMUL_CONTINUE)
3133 return ret;
3134 }
3135
3136 return load_state_from_tss32(ctxt, &tss_seg);
3137}
3138
3139static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
3140 u16 tss_selector, int idt_index, int reason,
3141 bool has_error_code, u32 error_code)
3142{
3143 const struct x86_emulate_ops *ops = ctxt->ops;
3144 struct desc_struct curr_tss_desc, next_tss_desc;
3145 int ret;
3146 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
3147 ulong old_tss_base =
3148 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
3149 u32 desc_limit;
3150 ulong desc_addr, dr7;
3151
3152
3153
3154 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
3155 if (ret != X86EMUL_CONTINUE)
3156 return ret;
3157 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
3158 if (ret != X86EMUL_CONTINUE)
3159 return ret;
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171 if (reason == TASK_SWITCH_GATE) {
3172 if (idt_index != -1) {
3173
3174 struct desc_struct task_gate_desc;
3175 int dpl;
3176
3177 ret = read_interrupt_descriptor(ctxt, idt_index,
3178 &task_gate_desc);
3179 if (ret != X86EMUL_CONTINUE)
3180 return ret;
3181
3182 dpl = task_gate_desc.dpl;
3183 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
3184 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
3185 }
3186 }
3187
3188 desc_limit = desc_limit_scaled(&next_tss_desc);
3189 if (!next_tss_desc.p ||
3190 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
3191 desc_limit < 0x2b)) {
3192 return emulate_ts(ctxt, tss_selector & 0xfffc);
3193 }
3194
3195 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
3196 curr_tss_desc.type &= ~(1 << 1);
3197 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
3198 }
3199
3200 if (reason == TASK_SWITCH_IRET)
3201 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
3202
3203
3204
3205 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
3206 old_tss_sel = 0xffff;
3207
3208 if (next_tss_desc.type & 8)
3209 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
3210 old_tss_base, &next_tss_desc);
3211 else
3212 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
3213 old_tss_base, &next_tss_desc);
3214 if (ret != X86EMUL_CONTINUE)
3215 return ret;
3216
3217 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
3218 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
3219
3220 if (reason != TASK_SWITCH_IRET) {
3221 next_tss_desc.type |= (1 << 1);
3222 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
3223 }
3224
3225 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
3226 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
3227
3228 if (has_error_code) {
3229 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
3230 ctxt->lock_prefix = 0;
3231 ctxt->src.val = (unsigned long) error_code;
3232 ret = em_push(ctxt);
3233 }
3234
3235 ops->get_dr(ctxt, 7, &dr7);
3236 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));
3237
3238 return ret;
3239}
3240
3241int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
3242 u16 tss_selector, int idt_index, int reason,
3243 bool has_error_code, u32 error_code)
3244{
3245 int rc;
3246
3247 invalidate_registers(ctxt);
3248 ctxt->_eip = ctxt->eip;
3249 ctxt->dst.type = OP_NONE;
3250
3251 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
3252 has_error_code, error_code);
3253
3254 if (rc == X86EMUL_CONTINUE) {
3255 ctxt->eip = ctxt->_eip;
3256 writeback_registers(ctxt);
3257 }
3258
3259 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3260}
3261
3262static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
3263 struct operand *op)
3264{
3265 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
3266
3267 register_address_increment(ctxt, reg, df * op->bytes);
3268 op->addr.mem.ea = register_address(ctxt, reg);
3269}
3270
3271static int em_das(struct x86_emulate_ctxt *ctxt)
3272{
3273 u8 al, old_al;
3274 bool af, cf, old_cf;
3275
3276 cf = ctxt->eflags & X86_EFLAGS_CF;
3277 al = ctxt->dst.val;
3278
3279 old_al = al;
3280 old_cf = cf;
3281 cf = false;
3282 af = ctxt->eflags & X86_EFLAGS_AF;
3283 if ((al & 0x0f) > 9 || af) {
3284 al -= 6;
3285 cf = old_cf | (al >= 250);
3286 af = true;
3287 } else {
3288 af = false;
3289 }
3290 if (old_al > 0x99 || old_cf) {
3291 al -= 0x60;
3292 cf = true;
3293 }
3294
3295 ctxt->dst.val = al;
3296
3297 ctxt->src.type = OP_IMM;
3298 ctxt->src.val = 0;
3299 ctxt->src.bytes = 1;
3300 fastop(ctxt, em_or);
3301 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3302 if (cf)
3303 ctxt->eflags |= X86_EFLAGS_CF;
3304 if (af)
3305 ctxt->eflags |= X86_EFLAGS_AF;
3306 return X86EMUL_CONTINUE;
3307}
3308
3309static int em_aam(struct x86_emulate_ctxt *ctxt)
3310{
3311 u8 al, ah;
3312
3313 if (ctxt->src.val == 0)
3314 return emulate_de(ctxt);
3315
3316 al = ctxt->dst.val & 0xff;
3317 ah = al / ctxt->src.val;
3318 al %= ctxt->src.val;
3319
3320 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3321
3322
3323 ctxt->src.type = OP_IMM;
3324 ctxt->src.val = 0;
3325 ctxt->src.bytes = 1;
3326 fastop(ctxt, em_or);
3327
3328 return X86EMUL_CONTINUE;
3329}
3330
3331static int em_aad(struct x86_emulate_ctxt *ctxt)
3332{
3333 u8 al = ctxt->dst.val & 0xff;
3334 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3335
3336 al = (al + (ah * ctxt->src.val)) & 0xff;
3337
3338 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3339
3340
3341 ctxt->src.type = OP_IMM;
3342 ctxt->src.val = 0;
3343 ctxt->src.bytes = 1;
3344 fastop(ctxt, em_or);
3345
3346 return X86EMUL_CONTINUE;
3347}
3348
3349static int em_call(struct x86_emulate_ctxt *ctxt)
3350{
3351 int rc;
3352 long rel = ctxt->src.val;
3353
3354 ctxt->src.val = (unsigned long)ctxt->_eip;
3355 rc = jmp_rel(ctxt, rel);
3356 if (rc != X86EMUL_CONTINUE)
3357 return rc;
3358 return em_push(ctxt);
3359}
3360
3361static int em_call_far(struct x86_emulate_ctxt *ctxt)
3362{
3363 u16 sel, old_cs;
3364 ulong old_eip;
3365 int rc;
3366 struct desc_struct old_desc, new_desc;
3367 const struct x86_emulate_ops *ops = ctxt->ops;
3368 int cpl = ctxt->ops->cpl(ctxt);
3369 enum x86emul_mode prev_mode = ctxt->mode;
3370
3371 old_eip = ctxt->_eip;
3372 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3373
3374 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3375 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3376 X86_TRANSFER_CALL_JMP, &new_desc);
3377 if (rc != X86EMUL_CONTINUE)
3378 return rc;
3379
3380 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3381 if (rc != X86EMUL_CONTINUE)
3382 goto fail;
3383
3384 ctxt->src.val = old_cs;
3385 rc = em_push(ctxt);
3386 if (rc != X86EMUL_CONTINUE)
3387 goto fail;
3388
3389 ctxt->src.val = old_eip;
3390 rc = em_push(ctxt);
3391
3392
3393 if (rc != X86EMUL_CONTINUE) {
3394 pr_warn_once("faulting far call emulation tainted memory\n");
3395 goto fail;
3396 }
3397 return rc;
3398fail:
3399 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3400 ctxt->mode = prev_mode;
3401 return rc;
3402
3403}
3404
3405static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3406{
3407 int rc;
3408 unsigned long eip;
3409
3410 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3411 if (rc != X86EMUL_CONTINUE)
3412 return rc;
3413 rc = assign_eip_near(ctxt, eip);
3414 if (rc != X86EMUL_CONTINUE)
3415 return rc;
3416 rsp_increment(ctxt, ctxt->src.val);
3417 return X86EMUL_CONTINUE;
3418}
3419
3420static int em_xchg(struct x86_emulate_ctxt *ctxt)
3421{
3422
3423 ctxt->src.val = ctxt->dst.val;
3424 write_register_operand(&ctxt->src);
3425
3426
3427 ctxt->dst.val = ctxt->src.orig_val;
3428 ctxt->lock_prefix = 1;
3429 return X86EMUL_CONTINUE;
3430}
3431
3432static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3433{
3434 ctxt->dst.val = ctxt->src2.val;
3435 return fastop(ctxt, em_imul);
3436}
3437
3438static int em_cwd(struct x86_emulate_ctxt *ctxt)
3439{
3440 ctxt->dst.type = OP_REG;
3441 ctxt->dst.bytes = ctxt->src.bytes;
3442 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3443 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3444
3445 return X86EMUL_CONTINUE;
3446}
3447
3448static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3449{
3450 u64 tsc = 0;
3451
3452 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3453 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3454 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3455 return X86EMUL_CONTINUE;
3456}
3457
3458static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3459{
3460 u64 pmc;
3461
3462 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3463 return emulate_gp(ctxt, 0);
3464 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3465 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3466 return X86EMUL_CONTINUE;
3467}
3468
3469static int em_mov(struct x86_emulate_ctxt *ctxt)
3470{
3471 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3472 return X86EMUL_CONTINUE;
3473}
3474
3475#define FFL(x) bit(X86_FEATURE_##x)
3476
3477static int em_movbe(struct x86_emulate_ctxt *ctxt)
3478{
3479 u32 ebx, ecx, edx, eax = 1;
3480 u16 tmp;
3481
3482
3483
3484
3485 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3486 if (!(ecx & FFL(MOVBE)))
3487 return emulate_ud(ctxt);
3488
3489 switch (ctxt->op_bytes) {
3490 case 2:
3491
3492
3493
3494
3495
3496
3497
3498
3499 tmp = (u16)ctxt->src.val;
3500 ctxt->dst.val &= ~0xffffUL;
3501 ctxt->dst.val |= (unsigned long)swab16(tmp);
3502 break;
3503 case 4:
3504 ctxt->dst.val = swab32((u32)ctxt->src.val);
3505 break;
3506 case 8:
3507 ctxt->dst.val = swab64(ctxt->src.val);
3508 break;
3509 default:
3510 BUG();
3511 }
3512 return X86EMUL_CONTINUE;
3513}
3514
3515static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3516{
3517 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3518 return emulate_gp(ctxt, 0);
3519
3520
3521 ctxt->dst.type = OP_NONE;
3522 return X86EMUL_CONTINUE;
3523}
3524
3525static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3526{
3527 unsigned long val;
3528
3529 if (ctxt->mode == X86EMUL_MODE_PROT64)
3530 val = ctxt->src.val & ~0ULL;
3531 else
3532 val = ctxt->src.val & ~0U;
3533
3534
3535 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3536 return emulate_gp(ctxt, 0);
3537
3538
3539 ctxt->dst.type = OP_NONE;
3540 return X86EMUL_CONTINUE;
3541}
3542
3543static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3544{
3545 u64 msr_data;
3546
3547 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3548 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3549 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3550 return emulate_gp(ctxt, 0);
3551
3552 return X86EMUL_CONTINUE;
3553}
3554
3555static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3556{
3557 u64 msr_data;
3558
3559 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3560 return emulate_gp(ctxt, 0);
3561
3562 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3563 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3564 return X86EMUL_CONTINUE;
3565}
3566
3567static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3568{
3569 if (ctxt->modrm_reg > VCPU_SREG_GS)
3570 return emulate_ud(ctxt);
3571
3572 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3573 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3574 ctxt->dst.bytes = 2;
3575 return X86EMUL_CONTINUE;
3576}
3577
3578static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3579{
3580 u16 sel = ctxt->src.val;
3581
3582 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3583 return emulate_ud(ctxt);
3584
3585 if (ctxt->modrm_reg == VCPU_SREG_SS)
3586 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3587
3588
3589 ctxt->dst.type = OP_NONE;
3590 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3591}
3592
3593static int em_lldt(struct x86_emulate_ctxt *ctxt)
3594{
3595 u16 sel = ctxt->src.val;
3596
3597
3598 ctxt->dst.type = OP_NONE;
3599 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3600}
3601
3602static int em_ltr(struct x86_emulate_ctxt *ctxt)
3603{
3604 u16 sel = ctxt->src.val;
3605
3606
3607 ctxt->dst.type = OP_NONE;
3608 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3609}
3610
3611static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3612{
3613 int rc;
3614 ulong linear;
3615
3616 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3617 if (rc == X86EMUL_CONTINUE)
3618 ctxt->ops->invlpg(ctxt, linear);
3619
3620 ctxt->dst.type = OP_NONE;
3621 return X86EMUL_CONTINUE;
3622}
3623
3624static int em_clts(struct x86_emulate_ctxt *ctxt)
3625{
3626 ulong cr0;
3627
3628 cr0 = ctxt->ops->get_cr(ctxt, 0);
3629 cr0 &= ~X86_CR0_TS;
3630 ctxt->ops->set_cr(ctxt, 0, cr0);
3631 return X86EMUL_CONTINUE;
3632}
3633
3634static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3635{
3636 int rc = ctxt->ops->fix_hypercall(ctxt);
3637
3638 if (rc != X86EMUL_CONTINUE)
3639 return rc;
3640
3641
3642 ctxt->_eip = ctxt->eip;
3643
3644 ctxt->dst.type = OP_NONE;
3645 return X86EMUL_CONTINUE;
3646}
3647
3648static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3649 void (*get)(struct x86_emulate_ctxt *ctxt,
3650 struct desc_ptr *ptr))
3651{
3652 struct desc_ptr desc_ptr;
3653
3654 if (ctxt->mode == X86EMUL_MODE_PROT64)
3655 ctxt->op_bytes = 8;
3656 get(ctxt, &desc_ptr);
3657 if (ctxt->op_bytes == 2) {
3658 ctxt->op_bytes = 4;
3659 desc_ptr.address &= 0x00ffffff;
3660 }
3661
3662 ctxt->dst.type = OP_NONE;
3663 return segmented_write(ctxt, ctxt->dst.addr.mem,
3664 &desc_ptr, 2 + ctxt->op_bytes);
3665}
3666
3667static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3668{
3669 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3670}
3671
3672static int em_sidt(struct x86_emulate_ctxt *ctxt)
3673{
3674 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3675}
3676
3677static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3678{
3679 struct desc_ptr desc_ptr;
3680 int rc;
3681
3682 if (ctxt->mode == X86EMUL_MODE_PROT64)
3683 ctxt->op_bytes = 8;
3684 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3685 &desc_ptr.size, &desc_ptr.address,
3686 ctxt->op_bytes);
3687 if (rc != X86EMUL_CONTINUE)
3688 return rc;
3689 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3690 is_noncanonical_address(desc_ptr.address))
3691 return emulate_gp(ctxt, 0);
3692 if (lgdt)
3693 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3694 else
3695 ctxt->ops->set_idt(ctxt, &desc_ptr);
3696
3697 ctxt->dst.type = OP_NONE;
3698 return X86EMUL_CONTINUE;
3699}
3700
3701static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3702{
3703 return em_lgdt_lidt(ctxt, true);
3704}
3705
3706static int em_lidt(struct x86_emulate_ctxt *ctxt)
3707{
3708 return em_lgdt_lidt(ctxt, false);
3709}
3710
3711static int em_smsw(struct x86_emulate_ctxt *ctxt)
3712{
3713 if (ctxt->dst.type == OP_MEM)
3714 ctxt->dst.bytes = 2;
3715 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3716 return X86EMUL_CONTINUE;
3717}
3718
3719static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3720{
3721 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3722 | (ctxt->src.val & 0x0f));
3723 ctxt->dst.type = OP_NONE;
3724 return X86EMUL_CONTINUE;
3725}
3726
3727static int em_loop(struct x86_emulate_ctxt *ctxt)
3728{
3729 int rc = X86EMUL_CONTINUE;
3730
3731 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3732 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3733 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3734 rc = jmp_rel(ctxt, ctxt->src.val);
3735
3736 return rc;
3737}
3738
3739static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3740{
3741 int rc = X86EMUL_CONTINUE;
3742
3743 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3744 rc = jmp_rel(ctxt, ctxt->src.val);
3745
3746 return rc;
3747}
3748
3749static int em_in(struct x86_emulate_ctxt *ctxt)
3750{
3751 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3752 &ctxt->dst.val))
3753 return X86EMUL_IO_NEEDED;
3754
3755 return X86EMUL_CONTINUE;
3756}
3757
3758static int em_out(struct x86_emulate_ctxt *ctxt)
3759{
3760 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3761 &ctxt->src.val, 1);
3762
3763 ctxt->dst.type = OP_NONE;
3764 return X86EMUL_CONTINUE;
3765}
3766
3767static int em_cli(struct x86_emulate_ctxt *ctxt)
3768{
3769 if (emulator_bad_iopl(ctxt))
3770 return emulate_gp(ctxt, 0);
3771
3772 ctxt->eflags &= ~X86_EFLAGS_IF;
3773 return X86EMUL_CONTINUE;
3774}
3775
3776static int em_sti(struct x86_emulate_ctxt *ctxt)
3777{
3778 if (emulator_bad_iopl(ctxt))
3779 return emulate_gp(ctxt, 0);
3780
3781 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3782 ctxt->eflags |= X86_EFLAGS_IF;
3783 return X86EMUL_CONTINUE;
3784}
3785
3786static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3787{
3788 u32 eax, ebx, ecx, edx;
3789
3790 eax = reg_read(ctxt, VCPU_REGS_RAX);
3791 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3792 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3793 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3794 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3795 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3796 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3797 return X86EMUL_CONTINUE;
3798}
3799
3800static int em_sahf(struct x86_emulate_ctxt *ctxt)
3801{
3802 u32 flags;
3803
3804 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
3805 X86_EFLAGS_SF;
3806 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3807
3808 ctxt->eflags &= ~0xffUL;
3809 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3810 return X86EMUL_CONTINUE;
3811}
3812
3813static int em_lahf(struct x86_emulate_ctxt *ctxt)
3814{
3815 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3816 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3817 return X86EMUL_CONTINUE;
3818}
3819
3820static int em_bswap(struct x86_emulate_ctxt *ctxt)
3821{
3822 switch (ctxt->op_bytes) {
3823#ifdef CONFIG_X86_64
3824 case 8:
3825 asm("bswap %0" : "+r"(ctxt->dst.val));
3826 break;
3827#endif
3828 default:
3829 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3830 break;
3831 }
3832 return X86EMUL_CONTINUE;
3833}
3834
3835static int em_clflush(struct x86_emulate_ctxt *ctxt)
3836{
3837
3838 return X86EMUL_CONTINUE;
3839}
3840
3841static int em_movsxd(struct x86_emulate_ctxt *ctxt)
3842{
3843 ctxt->dst.val = (s32) ctxt->src.val;
3844 return X86EMUL_CONTINUE;
3845}
3846
3847static bool valid_cr(int nr)
3848{
3849 switch (nr) {
3850 case 0:
3851 case 2 ... 4:
3852 case 8:
3853 return true;
3854 default:
3855 return false;
3856 }
3857}
3858
3859static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3860{
3861 if (!valid_cr(ctxt->modrm_reg))
3862 return emulate_ud(ctxt);
3863
3864 return X86EMUL_CONTINUE;
3865}
3866
3867static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3868{
3869 u64 new_val = ctxt->src.val64;
3870 int cr = ctxt->modrm_reg;
3871 u64 efer = 0;
3872
3873 static u64 cr_reserved_bits[] = {
3874 0xffffffff00000000ULL,
3875 0, 0, 0,
3876 CR4_RESERVED_BITS,
3877 0, 0, 0,
3878 CR8_RESERVED_BITS,
3879 };
3880
3881 if (!valid_cr(cr))
3882 return emulate_ud(ctxt);
3883
3884 if (new_val & cr_reserved_bits[cr])
3885 return emulate_gp(ctxt, 0);
3886
3887 switch (cr) {
3888 case 0: {
3889 u64 cr4;
3890 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3891 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3892 return emulate_gp(ctxt, 0);
3893
3894 cr4 = ctxt->ops->get_cr(ctxt, 4);
3895 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3896
3897 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3898 !(cr4 & X86_CR4_PAE))
3899 return emulate_gp(ctxt, 0);
3900
3901 break;
3902 }
3903 case 3: {
3904 u64 rsvd = 0;
3905
3906 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3907 if (efer & EFER_LMA)
3908 rsvd = CR3_L_MODE_RESERVED_BITS & ~CR3_PCID_INVD;
3909
3910 if (new_val & rsvd)
3911 return emulate_gp(ctxt, 0);
3912
3913 break;
3914 }
3915 case 4: {
3916 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3917
3918 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3919 return emulate_gp(ctxt, 0);
3920
3921 break;
3922 }
3923 }
3924
3925 return X86EMUL_CONTINUE;
3926}
3927
3928static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3929{
3930 unsigned long dr7;
3931
3932 ctxt->ops->get_dr(ctxt, 7, &dr7);
3933
3934
3935 return dr7 & (1 << 13);
3936}
3937
3938static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3939{
3940 int dr = ctxt->modrm_reg;
3941 u64 cr4;
3942
3943 if (dr > 7)
3944 return emulate_ud(ctxt);
3945
3946 cr4 = ctxt->ops->get_cr(ctxt, 4);
3947 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3948 return emulate_ud(ctxt);
3949
3950 if (check_dr7_gd(ctxt)) {
3951 ulong dr6;
3952
3953 ctxt->ops->get_dr(ctxt, 6, &dr6);
3954 dr6 &= ~15;
3955 dr6 |= DR6_BD | DR6_RTM;
3956 ctxt->ops->set_dr(ctxt, 6, dr6);
3957 return emulate_db(ctxt);
3958 }
3959
3960 return X86EMUL_CONTINUE;
3961}
3962
3963static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3964{
3965 u64 new_val = ctxt->src.val64;
3966 int dr = ctxt->modrm_reg;
3967
3968 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3969 return emulate_gp(ctxt, 0);
3970
3971 return check_dr_read(ctxt);
3972}
3973
3974static int check_svme(struct x86_emulate_ctxt *ctxt)
3975{
3976 u64 efer;
3977
3978 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3979
3980 if (!(efer & EFER_SVME))
3981 return emulate_ud(ctxt);
3982
3983 return X86EMUL_CONTINUE;
3984}
3985
3986static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3987{
3988 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3989
3990
3991 if (rax & 0xffff000000000000ULL)
3992 return emulate_gp(ctxt, 0);
3993
3994 return check_svme(ctxt);
3995}
3996
3997static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3998{
3999 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4000
4001 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
4002 return emulate_ud(ctxt);
4003
4004 return X86EMUL_CONTINUE;
4005}
4006
4007static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
4008{
4009 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4010 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
4011
4012 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
4013 ctxt->ops->check_pmc(ctxt, rcx))
4014 return emulate_gp(ctxt, 0);
4015
4016 return X86EMUL_CONTINUE;
4017}
4018
4019static int check_perm_in(struct x86_emulate_ctxt *ctxt)
4020{
4021 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
4022 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
4023 return emulate_gp(ctxt, 0);
4024
4025 return X86EMUL_CONTINUE;
4026}
4027
4028static int check_perm_out(struct x86_emulate_ctxt *ctxt)
4029{
4030 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
4031 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
4032 return emulate_gp(ctxt, 0);
4033
4034 return X86EMUL_CONTINUE;
4035}
4036
4037#define D(_y) { .flags = (_y) }
4038#define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4039#define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4040 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4041#define N D(NotImpl)
4042#define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4043#define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4044#define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4045#define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4046#define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4047#define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4048#define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4049#define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4050#define II(_f, _e, _i) \
4051 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4052#define IIP(_f, _e, _i, _p) \
4053 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4054 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4055#define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4056
4057#define D2bv(_f) D((_f) | ByteOp), D(_f)
4058#define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4059#define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4060#define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4061#define I2bvIP(_f, _e, _i, _p) \
4062 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4063
4064#define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4065 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4066 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4067
4068static const struct opcode group7_rm0[] = {
4069 N,
4070 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
4071 N, N, N, N, N, N,
4072};
4073
4074static const struct opcode group7_rm1[] = {
4075 DI(SrcNone | Priv, monitor),
4076 DI(SrcNone | Priv, mwait),
4077 N, N, N, N, N, N,
4078};
4079
4080static const struct opcode group7_rm3[] = {
4081 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
4082 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
4083 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
4084 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
4085 DIP(SrcNone | Prot | Priv, stgi, check_svme),
4086 DIP(SrcNone | Prot | Priv, clgi, check_svme),
4087 DIP(SrcNone | Prot | Priv, skinit, check_svme),
4088 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
4089};
4090
4091static const struct opcode group7_rm7[] = {
4092 N,
4093 DIP(SrcNone, rdtscp, check_rdtsc),
4094 N, N, N, N, N, N,
4095};
4096
4097static const struct opcode group1[] = {
4098 F(Lock, em_add),
4099 F(Lock | PageTable, em_or),
4100 F(Lock, em_adc),
4101 F(Lock, em_sbb),
4102 F(Lock | PageTable, em_and),
4103 F(Lock, em_sub),
4104 F(Lock, em_xor),
4105 F(NoWrite, em_cmp),
4106};
4107
4108static const struct opcode group1A[] = {
4109 I(DstMem | SrcNone | Mov | Stack | IncSP, em_pop), N, N, N, N, N, N, N,
4110};
4111
4112static const struct opcode group2[] = {
4113 F(DstMem | ModRM, em_rol),
4114 F(DstMem | ModRM, em_ror),
4115 F(DstMem | ModRM, em_rcl),
4116 F(DstMem | ModRM, em_rcr),
4117 F(DstMem | ModRM, em_shl),
4118 F(DstMem | ModRM, em_shr),
4119 F(DstMem | ModRM, em_shl),
4120 F(DstMem | ModRM, em_sar),
4121};
4122
4123static const struct opcode group3[] = {
4124 F(DstMem | SrcImm | NoWrite, em_test),
4125 F(DstMem | SrcImm | NoWrite, em_test),
4126 F(DstMem | SrcNone | Lock, em_not),
4127 F(DstMem | SrcNone | Lock, em_neg),
4128 F(DstXacc | Src2Mem, em_mul_ex),
4129 F(DstXacc | Src2Mem, em_imul_ex),
4130 F(DstXacc | Src2Mem, em_div_ex),
4131 F(DstXacc | Src2Mem, em_idiv_ex),
4132};
4133
4134static const struct opcode group4[] = {
4135 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
4136 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
4137 N, N, N, N, N, N,
4138};
4139
4140static const struct opcode group5[] = {
4141 F(DstMem | SrcNone | Lock, em_inc),
4142 F(DstMem | SrcNone | Lock, em_dec),
4143 I(SrcMem | NearBranch, em_call_near_abs),
4144 I(SrcMemFAddr | ImplicitOps, em_call_far),
4145 I(SrcMem | NearBranch, em_jmp_abs),
4146 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
4147 I(SrcMem | Stack, em_push), D(Undefined),
4148};
4149
4150static const struct opcode group6[] = {
4151 DI(Prot | DstMem, sldt),
4152 DI(Prot | DstMem, str),
4153 II(Prot | Priv | SrcMem16, em_lldt, lldt),
4154 II(Prot | Priv | SrcMem16, em_ltr, ltr),
4155 N, N, N, N,
4156};
4157
4158static const struct group_dual group7 = { {
4159 II(Mov | DstMem, em_sgdt, sgdt),
4160 II(Mov | DstMem, em_sidt, sidt),
4161 II(SrcMem | Priv, em_lgdt, lgdt),
4162 II(SrcMem | Priv, em_lidt, lidt),
4163 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4164 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4165 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
4166}, {
4167 EXT(0, group7_rm0),
4168 EXT(0, group7_rm1),
4169 N, EXT(0, group7_rm3),
4170 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4171 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4172 EXT(0, group7_rm7),
4173} };
4174
4175static const struct opcode group8[] = {
4176 N, N, N, N,
4177 F(DstMem | SrcImmByte | NoWrite, em_bt),
4178 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
4179 F(DstMem | SrcImmByte | Lock, em_btr),
4180 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
4181};
4182
4183static const struct group_dual group9 = { {
4184 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
4185}, {
4186 N, N, N, N, N, N, N, N,
4187} };
4188
4189static const struct opcode group11[] = {
4190 I(DstMem | SrcImm | Mov | PageTable, em_mov),
4191 X7(D(Undefined)),
4192};
4193
4194static const struct gprefix pfx_0f_ae_7 = {
4195 I(SrcMem | ByteOp, em_clflush), N, N, N,
4196};
4197
4198static const struct group_dual group15 = { {
4199 N, N, N, N, N, N, N, GP(0, &pfx_0f_ae_7),
4200}, {
4201 N, N, N, N, N, N, N, N,
4202} };
4203
4204static const struct gprefix pfx_0f_6f_0f_7f = {
4205 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
4206};
4207
4208static const struct instr_dual instr_dual_0f_2b = {
4209 I(0, em_mov), N
4210};
4211
4212static const struct gprefix pfx_0f_2b = {
4213 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
4214};
4215
4216static const struct gprefix pfx_0f_28_0f_29 = {
4217 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4218};
4219
4220static const struct gprefix pfx_0f_e7 = {
4221 N, I(Sse, em_mov), N, N,
4222};
4223
4224static const struct escape escape_d9 = { {
4225 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
4226}, {
4227
4228 N, N, N, N, N, N, N, N,
4229
4230 N, N, N, N, N, N, N, N,
4231
4232 N, N, N, N, N, N, N, N,
4233
4234 N, N, N, N, N, N, N, N,
4235
4236 N, N, N, N, N, N, N, N,
4237
4238 N, N, N, N, N, N, N, N,
4239
4240 N, N, N, N, N, N, N, N,
4241
4242 N, N, N, N, N, N, N, N,
4243} };
4244
4245static const struct escape escape_db = { {
4246 N, N, N, N, N, N, N, N,
4247}, {
4248
4249 N, N, N, N, N, N, N, N,
4250
4251 N, N, N, N, N, N, N, N,
4252
4253 N, N, N, N, N, N, N, N,
4254
4255 N, N, N, N, N, N, N, N,
4256
4257 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
4258
4259 N, N, N, N, N, N, N, N,
4260
4261 N, N, N, N, N, N, N, N,
4262
4263 N, N, N, N, N, N, N, N,
4264} };
4265
4266static const struct escape escape_dd = { {
4267 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4268}, {
4269
4270 N, N, N, N, N, N, N, N,
4271
4272 N, N, N, N, N, N, N, N,
4273
4274 N, N, N, N, N, N, N, N,
4275
4276 N, N, N, N, N, N, N, N,
4277
4278 N, N, N, N, N, N, N, N,
4279
4280 N, N, N, N, N, N, N, N,
4281
4282 N, N, N, N, N, N, N, N,
4283
4284 N, N, N, N, N, N, N, N,
4285} };
4286
4287static const struct instr_dual instr_dual_0f_c3 = {
4288 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
4289};
4290
4291static const struct mode_dual mode_dual_63 = {
4292 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
4293};
4294
4295static const struct opcode opcode_table[256] = {
4296
4297 F6ALU(Lock, em_add),
4298 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4299 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4300
4301 F6ALU(Lock | PageTable, em_or),
4302 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4303 N,
4304
4305 F6ALU(Lock, em_adc),
4306 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4307 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4308
4309 F6ALU(Lock, em_sbb),
4310 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4311 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4312
4313 F6ALU(Lock | PageTable, em_and), N, N,
4314
4315 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4316
4317 F6ALU(Lock, em_xor), N, N,
4318
4319 F6ALU(NoWrite, em_cmp), N, N,
4320
4321 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4322
4323 X8(I(SrcReg | Stack, em_push)),
4324
4325 X8(I(DstReg | Stack, em_pop)),
4326
4327 I(ImplicitOps | Stack | No64, em_pusha),
4328 I(ImplicitOps | Stack | No64, em_popa),
4329 N, MD(ModRM, &mode_dual_63),
4330 N, N, N, N,
4331
4332 I(SrcImm | Mov | Stack, em_push),
4333 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4334 I(SrcImmByte | Mov | Stack, em_push),
4335 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4336 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in),
4337 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out),
4338
4339 X16(D(SrcImmByte | NearBranch)),
4340
4341 G(ByteOp | DstMem | SrcImm, group1),
4342 G(DstMem | SrcImm, group1),
4343 G(ByteOp | DstMem | SrcImm | No64, group1),
4344 G(DstMem | SrcImmByte, group1),
4345 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4346 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4347
4348 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4349 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4350 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4351 D(ModRM | SrcMem | NoAccess | DstReg),
4352 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4353 G(0, group1A),
4354
4355 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4356
4357 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4358 I(SrcImmFAddr | No64, em_call_far), N,
4359 II(ImplicitOps | Stack, em_pushf, pushf),
4360 II(ImplicitOps | Stack, em_popf, popf),
4361 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4362
4363 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4364 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4365 I2bv(SrcSI | DstDI | Mov | String, em_mov),
4366 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp_r),
4367
4368 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4369 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4370 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4371 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4372
4373 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4374
4375 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4376
4377 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4378 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4379 I(ImplicitOps | NearBranch, em_ret),
4380 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4381 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4382 G(ByteOp, group11), G(0, group11),
4383
4384 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4385 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4386 I(ImplicitOps, em_ret_far),
4387 D(ImplicitOps), DI(SrcImmByte, intn),
4388 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4389
4390 G(Src2One | ByteOp, group2), G(Src2One, group2),
4391 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4392 I(DstAcc | SrcImmUByte | No64, em_aam),
4393 I(DstAcc | SrcImmUByte | No64, em_aad),
4394 F(DstAcc | ByteOp | No64, em_salc),
4395 I(DstAcc | SrcXLat | ByteOp, em_mov),
4396
4397 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4398
4399 X3(I(SrcImmByte | NearBranch, em_loop)),
4400 I(SrcImmByte | NearBranch, em_jcxz),
4401 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4402 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4403
4404 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4405 I(SrcImmFAddr | No64, em_jmp_far),
4406 D(SrcImmByte | ImplicitOps | NearBranch),
4407 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4408 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4409
4410 N, DI(ImplicitOps, icebp), N, N,
4411 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4412 G(ByteOp, group3), G(0, group3),
4413
4414 D(ImplicitOps), D(ImplicitOps),
4415 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4416 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4417};
4418
4419static const struct opcode twobyte_table[256] = {
4420
4421 G(0, group6), GD(0, &group7), N, N,
4422 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4423 II(ImplicitOps | Priv, em_clts, clts), N,
4424 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4425 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4426
4427 N, N, N, N, N, N, N, N,
4428 D(ImplicitOps | ModRM | SrcMem | NoAccess),
4429 N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4430
4431 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4432 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4433 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4434 check_cr_write),
4435 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4436 check_dr_write),
4437 N, N, N, N,
4438 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4439 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4440 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4441 N, N, N, N,
4442
4443 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4444 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4445 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4446 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4447 I(ImplicitOps | EmulateOnUD, em_sysenter),
4448 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4449 N, N,
4450 N, N, N, N, N, N, N, N,
4451
4452 X16(D(DstReg | SrcMem | ModRM)),
4453
4454 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4455
4456 N, N, N, N,
4457 N, N, N, N,
4458 N, N, N, N,
4459 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4460
4461 N, N, N, N,
4462 N, N, N, N,
4463 N, N, N, N,
4464 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4465
4466 X16(D(SrcImm | NearBranch)),
4467
4468 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4469
4470 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4471 II(ImplicitOps, em_cpuid, cpuid),
4472 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4473 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4474 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4475
4476 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4477 II(EmulateOnUD | ImplicitOps, em_rsm, rsm),
4478 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4479 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4480 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4481 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4482
4483 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4484 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4485 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4486 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4487 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4488 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4489
4490 N, N,
4491 G(BitOp, group8),
4492 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4493 I(DstReg | SrcMem | ModRM, em_bsf_c),
4494 I(DstReg | SrcMem | ModRM, em_bsr_c),
4495 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4496
4497 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4498 N, ID(0, &instr_dual_0f_c3),
4499 N, N, N, GD(0, &group9),
4500
4501 X8(I(DstReg, em_bswap)),
4502
4503 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4504
4505 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4506 N, N, N, N, N, N, N, N,
4507
4508 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4509};
4510
4511static const struct instr_dual instr_dual_0f_38_f0 = {
4512 I(DstReg | SrcMem | Mov, em_movbe), N
4513};
4514
4515static const struct instr_dual instr_dual_0f_38_f1 = {
4516 I(DstMem | SrcReg | Mov, em_movbe), N
4517};
4518
4519static const struct gprefix three_byte_0f_38_f0 = {
4520 ID(0, &instr_dual_0f_38_f0), N, N, N
4521};
4522
4523static const struct gprefix three_byte_0f_38_f1 = {
4524 ID(0, &instr_dual_0f_38_f1), N, N, N
4525};
4526
4527
4528
4529
4530
4531static const struct opcode opcode_map_0f_38[256] = {
4532
4533 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4534
4535 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4536
4537 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4538 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4539
4540 N, N, X4(N), X8(N)
4541};
4542
4543#undef D
4544#undef N
4545#undef G
4546#undef GD
4547#undef I
4548#undef GP
4549#undef EXT
4550#undef MD
4551#undef ID
4552
4553#undef D2bv
4554#undef D2bvIP
4555#undef I2bv
4556#undef I2bvIP
4557#undef I6ALU
4558
4559static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4560{
4561 unsigned size;
4562
4563 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4564 if (size == 8)
4565 size = 4;
4566 return size;
4567}
4568
4569static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4570 unsigned size, bool sign_extension)
4571{
4572 int rc = X86EMUL_CONTINUE;
4573
4574 op->type = OP_IMM;
4575 op->bytes = size;
4576 op->addr.mem.ea = ctxt->_eip;
4577
4578 switch (op->bytes) {
4579 case 1:
4580 op->val = insn_fetch(s8, ctxt);
4581 break;
4582 case 2:
4583 op->val = insn_fetch(s16, ctxt);
4584 break;
4585 case 4:
4586 op->val = insn_fetch(s32, ctxt);
4587 break;
4588 case 8:
4589 op->val = insn_fetch(s64, ctxt);
4590 break;
4591 }
4592 if (!sign_extension) {
4593 switch (op->bytes) {
4594 case 1:
4595 op->val &= 0xff;
4596 break;
4597 case 2:
4598 op->val &= 0xffff;
4599 break;
4600 case 4:
4601 op->val &= 0xffffffff;
4602 break;
4603 }
4604 }
4605done:
4606 return rc;
4607}
4608
4609static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4610 unsigned d)
4611{
4612 int rc = X86EMUL_CONTINUE;
4613
4614 switch (d) {
4615 case OpReg:
4616 decode_register_operand(ctxt, op);
4617 break;
4618 case OpImmUByte:
4619 rc = decode_imm(ctxt, op, 1, false);
4620 break;
4621 case OpMem:
4622 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4623 mem_common:
4624 *op = ctxt->memop;
4625 ctxt->memopp = op;
4626 if (ctxt->d & BitOp)
4627 fetch_bit_operand(ctxt);
4628 op->orig_val = op->val;
4629 break;
4630 case OpMem64:
4631 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4632 goto mem_common;
4633 case OpAcc:
4634 op->type = OP_REG;
4635 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4636 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4637 fetch_register_operand(op);
4638 op->orig_val = op->val;
4639 break;
4640 case OpAccLo:
4641 op->type = OP_REG;
4642 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4643 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4644 fetch_register_operand(op);
4645 op->orig_val = op->val;
4646 break;
4647 case OpAccHi:
4648 if (ctxt->d & ByteOp) {
4649 op->type = OP_NONE;
4650 break;
4651 }
4652 op->type = OP_REG;
4653 op->bytes = ctxt->op_bytes;
4654 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4655 fetch_register_operand(op);
4656 op->orig_val = op->val;
4657 break;
4658 case OpDI:
4659 op->type = OP_MEM;
4660 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4661 op->addr.mem.ea =
4662 register_address(ctxt, VCPU_REGS_RDI);
4663 op->addr.mem.seg = VCPU_SREG_ES;
4664 op->val = 0;
4665 op->count = 1;
4666 break;
4667 case OpDX:
4668 op->type = OP_REG;
4669 op->bytes = 2;
4670 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4671 fetch_register_operand(op);
4672 break;
4673 case OpCL:
4674 op->type = OP_IMM;
4675 op->bytes = 1;
4676 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4677 break;
4678 case OpImmByte:
4679 rc = decode_imm(ctxt, op, 1, true);
4680 break;
4681 case OpOne:
4682 op->type = OP_IMM;
4683 op->bytes = 1;
4684 op->val = 1;
4685 break;
4686 case OpImm:
4687 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4688 break;
4689 case OpImm64:
4690 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4691 break;
4692 case OpMem8:
4693 ctxt->memop.bytes = 1;
4694 if (ctxt->memop.type == OP_REG) {
4695 ctxt->memop.addr.reg = decode_register(ctxt,
4696 ctxt->modrm_rm, true);
4697 fetch_register_operand(&ctxt->memop);
4698 }
4699 goto mem_common;
4700 case OpMem16:
4701 ctxt->memop.bytes = 2;
4702 goto mem_common;
4703 case OpMem32:
4704 ctxt->memop.bytes = 4;
4705 goto mem_common;
4706 case OpImmU16:
4707 rc = decode_imm(ctxt, op, 2, false);
4708 break;
4709 case OpImmU:
4710 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4711 break;
4712 case OpSI:
4713 op->type = OP_MEM;
4714 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4715 op->addr.mem.ea =
4716 register_address(ctxt, VCPU_REGS_RSI);
4717 op->addr.mem.seg = ctxt->seg_override;
4718 op->val = 0;
4719 op->count = 1;
4720 break;
4721 case OpXLat:
4722 op->type = OP_MEM;
4723 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4724 op->addr.mem.ea =
4725 address_mask(ctxt,
4726 reg_read(ctxt, VCPU_REGS_RBX) +
4727 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4728 op->addr.mem.seg = ctxt->seg_override;
4729 op->val = 0;
4730 break;
4731 case OpImmFAddr:
4732 op->type = OP_IMM;
4733 op->addr.mem.ea = ctxt->_eip;
4734 op->bytes = ctxt->op_bytes + 2;
4735 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4736 break;
4737 case OpMemFAddr:
4738 ctxt->memop.bytes = ctxt->op_bytes + 2;
4739 goto mem_common;
4740 case OpES:
4741 op->type = OP_IMM;
4742 op->val = VCPU_SREG_ES;
4743 break;
4744 case OpCS:
4745 op->type = OP_IMM;
4746 op->val = VCPU_SREG_CS;
4747 break;
4748 case OpSS:
4749 op->type = OP_IMM;
4750 op->val = VCPU_SREG_SS;
4751 break;
4752 case OpDS:
4753 op->type = OP_IMM;
4754 op->val = VCPU_SREG_DS;
4755 break;
4756 case OpFS:
4757 op->type = OP_IMM;
4758 op->val = VCPU_SREG_FS;
4759 break;
4760 case OpGS:
4761 op->type = OP_IMM;
4762 op->val = VCPU_SREG_GS;
4763 break;
4764 case OpImplicit:
4765
4766 default:
4767 op->type = OP_NONE;
4768 break;
4769 }
4770
4771done:
4772 return rc;
4773}
4774
4775int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4776{
4777 int rc = X86EMUL_CONTINUE;
4778 int mode = ctxt->mode;
4779 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4780 bool op_prefix = false;
4781 bool has_seg_override = false;
4782 struct opcode opcode;
4783
4784 ctxt->memop.type = OP_NONE;
4785 ctxt->memopp = NULL;
4786 ctxt->_eip = ctxt->eip;
4787 ctxt->fetch.ptr = ctxt->fetch.data;
4788 ctxt->fetch.end = ctxt->fetch.data + insn_len;
4789 ctxt->opcode_len = 1;
4790 if (insn_len > 0)
4791 memcpy(ctxt->fetch.data, insn, insn_len);
4792 else {
4793 rc = __do_insn_fetch_bytes(ctxt, 1);
4794 if (rc != X86EMUL_CONTINUE)
4795 return rc;
4796 }
4797
4798 switch (mode) {
4799 case X86EMUL_MODE_REAL:
4800 case X86EMUL_MODE_VM86:
4801 case X86EMUL_MODE_PROT16:
4802 def_op_bytes = def_ad_bytes = 2;
4803 break;
4804 case X86EMUL_MODE_PROT32:
4805 def_op_bytes = def_ad_bytes = 4;
4806 break;
4807#ifdef CONFIG_X86_64
4808 case X86EMUL_MODE_PROT64:
4809 def_op_bytes = 4;
4810 def_ad_bytes = 8;
4811 break;
4812#endif
4813 default:
4814 return EMULATION_FAILED;
4815 }
4816
4817 ctxt->op_bytes = def_op_bytes;
4818 ctxt->ad_bytes = def_ad_bytes;
4819
4820
4821 for (;;) {
4822 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4823 case 0x66:
4824 op_prefix = true;
4825
4826 ctxt->op_bytes = def_op_bytes ^ 6;
4827 break;
4828 case 0x67:
4829 if (mode == X86EMUL_MODE_PROT64)
4830
4831 ctxt->ad_bytes = def_ad_bytes ^ 12;
4832 else
4833
4834 ctxt->ad_bytes = def_ad_bytes ^ 6;
4835 break;
4836 case 0x26:
4837 case 0x2e:
4838 case 0x36:
4839 case 0x3e:
4840 has_seg_override = true;
4841 ctxt->seg_override = (ctxt->b >> 3) & 3;
4842 break;
4843 case 0x64:
4844 case 0x65:
4845 has_seg_override = true;
4846 ctxt->seg_override = ctxt->b & 7;
4847 break;
4848 case 0x40 ... 0x4f:
4849 if (mode != X86EMUL_MODE_PROT64)
4850 goto done_prefixes;
4851 ctxt->rex_prefix = ctxt->b;
4852 continue;
4853 case 0xf0:
4854 ctxt->lock_prefix = 1;
4855 break;
4856 case 0xf2:
4857 case 0xf3:
4858 ctxt->rep_prefix = ctxt->b;
4859 break;
4860 default:
4861 goto done_prefixes;
4862 }
4863
4864
4865
4866 ctxt->rex_prefix = 0;
4867 }
4868
4869done_prefixes:
4870
4871
4872 if (ctxt->rex_prefix & 8)
4873 ctxt->op_bytes = 8;
4874
4875
4876 opcode = opcode_table[ctxt->b];
4877
4878 if (ctxt->b == 0x0f) {
4879 ctxt->opcode_len = 2;
4880 ctxt->b = insn_fetch(u8, ctxt);
4881 opcode = twobyte_table[ctxt->b];
4882
4883
4884 if (ctxt->b == 0x38) {
4885 ctxt->opcode_len = 3;
4886 ctxt->b = insn_fetch(u8, ctxt);
4887 opcode = opcode_map_0f_38[ctxt->b];
4888 }
4889 }
4890 ctxt->d = opcode.flags;
4891
4892 if (ctxt->d & ModRM)
4893 ctxt->modrm = insn_fetch(u8, ctxt);
4894
4895
4896 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
4897 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
4898 ctxt->d = NotImpl;
4899 }
4900
4901 while (ctxt->d & GroupMask) {
4902 switch (ctxt->d & GroupMask) {
4903 case Group:
4904 goffset = (ctxt->modrm >> 3) & 7;
4905 opcode = opcode.u.group[goffset];
4906 break;
4907 case GroupDual:
4908 goffset = (ctxt->modrm >> 3) & 7;
4909 if ((ctxt->modrm >> 6) == 3)
4910 opcode = opcode.u.gdual->mod3[goffset];
4911 else
4912 opcode = opcode.u.gdual->mod012[goffset];
4913 break;
4914 case RMExt:
4915 goffset = ctxt->modrm & 7;
4916 opcode = opcode.u.group[goffset];
4917 break;
4918 case Prefix:
4919 if (ctxt->rep_prefix && op_prefix)
4920 return EMULATION_FAILED;
4921 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4922 switch (simd_prefix) {
4923 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4924 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4925 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4926 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4927 }
4928 break;
4929 case Escape:
4930 if (ctxt->modrm > 0xbf)
4931 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4932 else
4933 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4934 break;
4935 case InstrDual:
4936 if ((ctxt->modrm >> 6) == 3)
4937 opcode = opcode.u.idual->mod3;
4938 else
4939 opcode = opcode.u.idual->mod012;
4940 break;
4941 case ModeDual:
4942 if (ctxt->mode == X86EMUL_MODE_PROT64)
4943 opcode = opcode.u.mdual->mode64;
4944 else
4945 opcode = opcode.u.mdual->mode32;
4946 break;
4947 default:
4948 return EMULATION_FAILED;
4949 }
4950
4951 ctxt->d &= ~(u64)GroupMask;
4952 ctxt->d |= opcode.flags;
4953 }
4954
4955
4956 if (ctxt->d == 0)
4957 return EMULATION_FAILED;
4958
4959 ctxt->execute = opcode.u.execute;
4960
4961 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
4962 return EMULATION_FAILED;
4963
4964 if (unlikely(ctxt->d &
4965 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
4966 No16))) {
4967
4968
4969
4970
4971 ctxt->check_perm = opcode.check_perm;
4972 ctxt->intercept = opcode.intercept;
4973
4974 if (ctxt->d & NotImpl)
4975 return EMULATION_FAILED;
4976
4977 if (mode == X86EMUL_MODE_PROT64) {
4978 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
4979 ctxt->op_bytes = 8;
4980 else if (ctxt->d & NearBranch)
4981 ctxt->op_bytes = 8;
4982 }
4983
4984 if (ctxt->d & Op3264) {
4985 if (mode == X86EMUL_MODE_PROT64)
4986 ctxt->op_bytes = 8;
4987 else
4988 ctxt->op_bytes = 4;
4989 }
4990
4991 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
4992 ctxt->op_bytes = 4;
4993
4994 if (ctxt->d & Sse)
4995 ctxt->op_bytes = 16;
4996 else if (ctxt->d & Mmx)
4997 ctxt->op_bytes = 8;
4998 }
4999
5000
5001 if (ctxt->d & ModRM) {
5002 rc = decode_modrm(ctxt, &ctxt->memop);
5003 if (!has_seg_override) {
5004 has_seg_override = true;
5005 ctxt->seg_override = ctxt->modrm_seg;
5006 }
5007 } else if (ctxt->d & MemAbs)
5008 rc = decode_abs(ctxt, &ctxt->memop);
5009 if (rc != X86EMUL_CONTINUE)
5010 goto done;
5011
5012 if (!has_seg_override)
5013 ctxt->seg_override = VCPU_SREG_DS;
5014
5015 ctxt->memop.addr.mem.seg = ctxt->seg_override;
5016
5017
5018
5019
5020
5021 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
5022 if (rc != X86EMUL_CONTINUE)
5023 goto done;
5024
5025
5026
5027
5028
5029 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
5030 if (rc != X86EMUL_CONTINUE)
5031 goto done;
5032
5033
5034 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
5035
5036 if (ctxt->rip_relative)
5037 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
5038 ctxt->memopp->addr.mem.ea + ctxt->_eip);
5039
5040done:
5041 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5042}
5043
5044bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
5045{
5046 return ctxt->d & PageTable;
5047}
5048
5049static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
5050{
5051
5052
5053
5054
5055
5056
5057
5058 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
5059 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
5060 && (((ctxt->rep_prefix == REPE_PREFIX) &&
5061 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
5062 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
5063 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
5064 return true;
5065
5066 return false;
5067}
5068
5069static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
5070{
5071 bool fault = false;
5072
5073 ctxt->ops->get_fpu(ctxt);
5074 asm volatile("1: fwait \n\t"
5075 "2: \n\t"
5076 ".pushsection .fixup,\"ax\" \n\t"
5077 "3: \n\t"
5078 "movb $1, %[fault] \n\t"
5079 "jmp 2b \n\t"
5080 ".popsection \n\t"
5081 _ASM_EXTABLE(1b, 3b)
5082 : [fault]"+qm"(fault));
5083 ctxt->ops->put_fpu(ctxt);
5084
5085 if (unlikely(fault))
5086 return emulate_exception(ctxt, MF_VECTOR, 0, false);
5087
5088 return X86EMUL_CONTINUE;
5089}
5090
5091static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
5092 struct operand *op)
5093{
5094 if (op->type == OP_MM)
5095 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
5096}
5097
5098static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
5099{
5100 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5101 if (!(ctxt->d & ByteOp))
5102 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5103 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
5104 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5105 [fastop]"+S"(fop)
5106 : "c"(ctxt->src2.val));
5107 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5108 if (!fop)
5109 return emulate_de(ctxt);
5110 return X86EMUL_CONTINUE;
5111}
5112
5113void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5114{
5115 memset(&ctxt->rip_relative, 0,
5116 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
5117
5118 ctxt->io_read.pos = 0;
5119 ctxt->io_read.end = 0;
5120 ctxt->mem_read.end = 0;
5121}
5122
5123int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
5124{
5125 const struct x86_emulate_ops *ops = ctxt->ops;
5126 int rc = X86EMUL_CONTINUE;
5127 int saved_dst_type = ctxt->dst.type;
5128
5129 ctxt->mem_read.pos = 0;
5130
5131
5132 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
5133 rc = emulate_ud(ctxt);
5134 goto done;
5135 }
5136
5137 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
5138 rc = emulate_ud(ctxt);
5139 goto done;
5140 }
5141
5142 if (unlikely(ctxt->d &
5143 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
5144 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
5145 (ctxt->d & Undefined)) {
5146 rc = emulate_ud(ctxt);
5147 goto done;
5148 }
5149
5150 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
5151 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
5152 rc = emulate_ud(ctxt);
5153 goto done;
5154 }
5155
5156 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
5157 rc = emulate_nm(ctxt);
5158 goto done;
5159 }
5160
5161 if (ctxt->d & Mmx) {
5162 rc = flush_pending_x87_faults(ctxt);
5163 if (rc != X86EMUL_CONTINUE)
5164 goto done;
5165
5166
5167
5168
5169 fetch_possible_mmx_operand(ctxt, &ctxt->src);
5170 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
5171 if (!(ctxt->d & Mov))
5172 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
5173 }
5174
5175 if (unlikely(ctxt->emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
5176 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5177 X86_ICPT_PRE_EXCEPT);
5178 if (rc != X86EMUL_CONTINUE)
5179 goto done;
5180 }
5181
5182
5183 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
5184 rc = emulate_ud(ctxt);
5185 goto done;
5186 }
5187
5188
5189 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
5190 if (ctxt->d & PrivUD)
5191 rc = emulate_ud(ctxt);
5192 else
5193 rc = emulate_gp(ctxt, 0);
5194 goto done;
5195 }
5196
5197
5198 if (ctxt->d & CheckPerm) {
5199 rc = ctxt->check_perm(ctxt);
5200 if (rc != X86EMUL_CONTINUE)
5201 goto done;
5202 }
5203
5204 if (unlikely(ctxt->emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5205 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5206 X86_ICPT_POST_EXCEPT);
5207 if (rc != X86EMUL_CONTINUE)
5208 goto done;
5209 }
5210
5211 if (ctxt->rep_prefix && (ctxt->d & String)) {
5212
5213 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
5214 string_registers_quirk(ctxt);
5215 ctxt->eip = ctxt->_eip;
5216 ctxt->eflags &= ~X86_EFLAGS_RF;
5217 goto done;
5218 }
5219 }
5220 }
5221
5222 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
5223 rc = segmented_read(ctxt, ctxt->src.addr.mem,
5224 ctxt->src.valptr, ctxt->src.bytes);
5225 if (rc != X86EMUL_CONTINUE)
5226 goto done;
5227 ctxt->src.orig_val64 = ctxt->src.val64;
5228 }
5229
5230 if (ctxt->src2.type == OP_MEM) {
5231 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
5232 &ctxt->src2.val, ctxt->src2.bytes);
5233 if (rc != X86EMUL_CONTINUE)
5234 goto done;
5235 }
5236
5237 if ((ctxt->d & DstMask) == ImplicitOps)
5238 goto special_insn;
5239
5240
5241 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
5242
5243 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
5244 &ctxt->dst.val, ctxt->dst.bytes);
5245 if (rc != X86EMUL_CONTINUE) {
5246 if (!(ctxt->d & NoWrite) &&
5247 rc == X86EMUL_PROPAGATE_FAULT &&
5248 ctxt->exception.vector == PF_VECTOR)
5249 ctxt->exception.error_code |= PFERR_WRITE_MASK;
5250 goto done;
5251 }
5252 }
5253
5254 ctxt->dst.orig_val64 = ctxt->dst.val64;
5255
5256special_insn:
5257
5258 if (unlikely(ctxt->emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5259 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5260 X86_ICPT_POST_MEMACCESS);
5261 if (rc != X86EMUL_CONTINUE)
5262 goto done;
5263 }
5264
5265 if (ctxt->rep_prefix && (ctxt->d & String))
5266 ctxt->eflags |= X86_EFLAGS_RF;
5267 else
5268 ctxt->eflags &= ~X86_EFLAGS_RF;
5269
5270 if (ctxt->execute) {
5271 if (ctxt->d & Fastop) {
5272 void (*fop)(struct fastop *) = (void *)ctxt->execute;
5273 rc = fastop(ctxt, fop);
5274 if (rc != X86EMUL_CONTINUE)
5275 goto done;
5276 goto writeback;
5277 }
5278 rc = ctxt->execute(ctxt);
5279 if (rc != X86EMUL_CONTINUE)
5280 goto done;
5281 goto writeback;
5282 }
5283
5284 if (ctxt->opcode_len == 2)
5285 goto twobyte_insn;
5286 else if (ctxt->opcode_len == 3)
5287 goto threebyte_insn;
5288
5289 switch (ctxt->b) {
5290 case 0x70 ... 0x7f:
5291 if (test_cc(ctxt->b, ctxt->eflags))
5292 rc = jmp_rel(ctxt, ctxt->src.val);
5293 break;
5294 case 0x8d:
5295 ctxt->dst.val = ctxt->src.addr.mem.ea;
5296 break;
5297 case 0x90 ... 0x97:
5298 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5299 ctxt->dst.type = OP_NONE;
5300 else
5301 rc = em_xchg(ctxt);
5302 break;
5303 case 0x98:
5304 switch (ctxt->op_bytes) {
5305 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5306 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5307 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5308 }
5309 break;
5310 case 0xcc:
5311 rc = emulate_int(ctxt, 3);
5312 break;
5313 case 0xcd:
5314 rc = emulate_int(ctxt, ctxt->src.val);
5315 break;
5316 case 0xce:
5317 if (ctxt->eflags & X86_EFLAGS_OF)
5318 rc = emulate_int(ctxt, 4);
5319 break;
5320 case 0xe9:
5321 case 0xeb:
5322 rc = jmp_rel(ctxt, ctxt->src.val);
5323 ctxt->dst.type = OP_NONE;
5324 break;
5325 case 0xf4:
5326 ctxt->ops->halt(ctxt);
5327 break;
5328 case 0xf5:
5329
5330 ctxt->eflags ^= X86_EFLAGS_CF;
5331 break;
5332 case 0xf8:
5333 ctxt->eflags &= ~X86_EFLAGS_CF;
5334 break;
5335 case 0xf9:
5336 ctxt->eflags |= X86_EFLAGS_CF;
5337 break;
5338 case 0xfc:
5339 ctxt->eflags &= ~X86_EFLAGS_DF;
5340 break;
5341 case 0xfd:
5342 ctxt->eflags |= X86_EFLAGS_DF;
5343 break;
5344 default:
5345 goto cannot_emulate;
5346 }
5347
5348 if (rc != X86EMUL_CONTINUE)
5349 goto done;
5350
5351writeback:
5352 if (ctxt->d & SrcWrite) {
5353 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5354 rc = writeback(ctxt, &ctxt->src);
5355 if (rc != X86EMUL_CONTINUE)
5356 goto done;
5357 }
5358 if (!(ctxt->d & NoWrite)) {
5359 rc = writeback(ctxt, &ctxt->dst);
5360 if (rc != X86EMUL_CONTINUE)
5361 goto done;
5362 }
5363
5364
5365
5366
5367
5368 ctxt->dst.type = saved_dst_type;
5369
5370 if ((ctxt->d & SrcMask) == SrcSI)
5371 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5372
5373 if ((ctxt->d & DstMask) == DstDI)
5374 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5375
5376 if (ctxt->rep_prefix && (ctxt->d & String)) {
5377 unsigned int count;
5378 struct read_cache *r = &ctxt->io_read;
5379 if ((ctxt->d & SrcMask) == SrcSI)
5380 count = ctxt->src.count;
5381 else
5382 count = ctxt->dst.count;
5383 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5384
5385 if (!string_insn_completed(ctxt)) {
5386
5387
5388
5389
5390 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5391 (r->end == 0 || r->end != r->pos)) {
5392
5393
5394
5395
5396
5397 ctxt->mem_read.end = 0;
5398 writeback_registers(ctxt);
5399 return EMULATION_RESTART;
5400 }
5401 goto done;
5402 }
5403 ctxt->eflags &= ~X86_EFLAGS_RF;
5404 }
5405
5406 ctxt->eip = ctxt->_eip;
5407
5408done:
5409 if (rc == X86EMUL_PROPAGATE_FAULT) {
5410 WARN_ON(ctxt->exception.vector > 0x1f);
5411 ctxt->have_exception = true;
5412 }
5413 if (rc == X86EMUL_INTERCEPTED)
5414 return EMULATION_INTERCEPTED;
5415
5416 if (rc == X86EMUL_CONTINUE)
5417 writeback_registers(ctxt);
5418
5419 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5420
5421twobyte_insn:
5422 switch (ctxt->b) {
5423 case 0x09:
5424 (ctxt->ops->wbinvd)(ctxt);
5425 break;
5426 case 0x08:
5427 case 0x0d:
5428 case 0x18:
5429 case 0x1f:
5430 break;
5431 case 0x20:
5432 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5433 break;
5434 case 0x21:
5435 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5436 break;
5437 case 0x40 ... 0x4f:
5438 if (test_cc(ctxt->b, ctxt->eflags))
5439 ctxt->dst.val = ctxt->src.val;
5440 else if (ctxt->op_bytes != 4)
5441 ctxt->dst.type = OP_NONE;
5442 break;
5443 case 0x80 ... 0x8f:
5444 if (test_cc(ctxt->b, ctxt->eflags))
5445 rc = jmp_rel(ctxt, ctxt->src.val);
5446 break;
5447 case 0x90 ... 0x9f:
5448 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5449 break;
5450 case 0xb6 ... 0xb7:
5451 ctxt->dst.bytes = ctxt->op_bytes;
5452 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5453 : (u16) ctxt->src.val;
5454 break;
5455 case 0xbe ... 0xbf:
5456 ctxt->dst.bytes = ctxt->op_bytes;
5457 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5458 (s16) ctxt->src.val;
5459 break;
5460 default:
5461 goto cannot_emulate;
5462 }
5463
5464threebyte_insn:
5465
5466 if (rc != X86EMUL_CONTINUE)
5467 goto done;
5468
5469 goto writeback;
5470
5471cannot_emulate:
5472 return EMULATION_FAILED;
5473}
5474
5475void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5476{
5477 invalidate_registers(ctxt);
5478}
5479
5480void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5481{
5482 writeback_registers(ctxt);
5483}
5484