1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#include <linux/kvm_host.h>
24#include "kvm_cache_regs.h"
25#include <linux/module.h>
26#include <asm/kvm_emulate.h>
27#include <linux/stringify.h>
28#include <asm/debugreg.h>
29
30#include "x86.h"
31#include "tss.h"
32
33
34
35
36#define OpNone 0ull
37#define OpImplicit 1ull
38#define OpReg 2ull
39#define OpMem 3ull
40#define OpAcc 4ull
41#define OpDI 5ull
42#define OpMem64 6ull
43#define OpImmUByte 7ull
44#define OpDX 8ull
45#define OpCL 9ull
46#define OpImmByte 10ull
47#define OpOne 11ull
48#define OpImm 12ull
49#define OpMem16 13ull
50#define OpMem32 14ull
51#define OpImmU 15ull
52#define OpSI 16ull
53#define OpImmFAddr 17ull
54#define OpMemFAddr 18ull
55#define OpImmU16 19ull
56#define OpES 20ull
57#define OpCS 21ull
58#define OpSS 22ull
59#define OpDS 23ull
60#define OpFS 24ull
61#define OpGS 25ull
62#define OpMem8 26ull
63#define OpImm64 27ull
64#define OpXLat 28ull
65#define OpAccLo 29ull
66#define OpAccHi 30ull
67
68#define OpBits 5
69#define OpMask ((1ull << OpBits) - 1)
70
71
72
73
74
75
76
77
78
79
80
81#define ByteOp (1<<0)
82
83#define DstShift 1
84#define ImplicitOps (OpImplicit << DstShift)
85#define DstReg (OpReg << DstShift)
86#define DstMem (OpMem << DstShift)
87#define DstAcc (OpAcc << DstShift)
88#define DstDI (OpDI << DstShift)
89#define DstMem64 (OpMem64 << DstShift)
90#define DstMem16 (OpMem16 << DstShift)
91#define DstImmUByte (OpImmUByte << DstShift)
92#define DstDX (OpDX << DstShift)
93#define DstAccLo (OpAccLo << DstShift)
94#define DstMask (OpMask << DstShift)
95
96#define SrcShift 6
97#define SrcNone (OpNone << SrcShift)
98#define SrcReg (OpReg << SrcShift)
99#define SrcMem (OpMem << SrcShift)
100#define SrcMem16 (OpMem16 << SrcShift)
101#define SrcMem32 (OpMem32 << SrcShift)
102#define SrcImm (OpImm << SrcShift)
103#define SrcImmByte (OpImmByte << SrcShift)
104#define SrcOne (OpOne << SrcShift)
105#define SrcImmUByte (OpImmUByte << SrcShift)
106#define SrcImmU (OpImmU << SrcShift)
107#define SrcSI (OpSI << SrcShift)
108#define SrcXLat (OpXLat << SrcShift)
109#define SrcImmFAddr (OpImmFAddr << SrcShift)
110#define SrcMemFAddr (OpMemFAddr << SrcShift)
111#define SrcAcc (OpAcc << SrcShift)
112#define SrcImmU16 (OpImmU16 << SrcShift)
113#define SrcImm64 (OpImm64 << SrcShift)
114#define SrcDX (OpDX << SrcShift)
115#define SrcMem8 (OpMem8 << SrcShift)
116#define SrcAccHi (OpAccHi << SrcShift)
117#define SrcMask (OpMask << SrcShift)
118#define BitOp (1<<11)
119#define MemAbs (1<<12)
120#define String (1<<13)
121#define Stack (1<<14)
122#define GroupMask (7<<15)
123#define Group (1<<15)
124#define GroupDual (2<<15)
125#define Prefix (3<<15)
126#define RMExt (4<<15)
127#define Escape (5<<15)
128#define InstrDual (6<<15)
129#define ModeDual (7<<15)
130#define Sse (1<<18)
131
132#define ModRM (1<<19)
133
134#define Mov (1<<20)
135
136#define Prot (1<<21)
137#define EmulateOnUD (1<<22)
138#define NoAccess (1<<23)
139#define Op3264 (1<<24)
140#define Undefined (1<<25)
141#define Lock (1<<26)
142#define Priv (1<<27)
143#define No64 (1<<28)
144#define PageTable (1 << 29)
145#define NotImpl (1 << 30)
146
147#define Src2Shift (31)
148#define Src2None (OpNone << Src2Shift)
149#define Src2Mem (OpMem << Src2Shift)
150#define Src2CL (OpCL << Src2Shift)
151#define Src2ImmByte (OpImmByte << Src2Shift)
152#define Src2One (OpOne << Src2Shift)
153#define Src2Imm (OpImm << Src2Shift)
154#define Src2ES (OpES << Src2Shift)
155#define Src2CS (OpCS << Src2Shift)
156#define Src2SS (OpSS << Src2Shift)
157#define Src2DS (OpDS << Src2Shift)
158#define Src2FS (OpFS << Src2Shift)
159#define Src2GS (OpGS << Src2Shift)
160#define Src2Mask (OpMask << Src2Shift)
161#define Mmx ((u64)1 << 40)
162#define Aligned ((u64)1 << 41)
163#define Unaligned ((u64)1 << 42)
164#define Avx ((u64)1 << 43)
165#define Fastop ((u64)1 << 44)
166#define NoWrite ((u64)1 << 45)
167#define SrcWrite ((u64)1 << 46)
168#define NoMod ((u64)1 << 47)
169#define Intercept ((u64)1 << 48)
170#define CheckPerm ((u64)1 << 49)
171#define PrivUD ((u64)1 << 51)
172#define NearBranch ((u64)1 << 52)
173#define No16 ((u64)1 << 53)
174#define IncSP ((u64)1 << 54)
175
176#define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
177
178#define X2(x...) x, x
179#define X3(x...) X2(x), x
180#define X4(x...) X2(x), X2(x)
181#define X5(x...) X4(x), x
182#define X6(x...) X4(x), X2(x)
183#define X7(x...) X4(x), X3(x)
184#define X8(x...) X4(x), X4(x)
185#define X16(x...) X8(x), X8(x)
186
187#define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
188#define FASTOP_SIZE 8
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207struct fastop;
208
209struct opcode {
210 u64 flags : 56;
211 u64 intercept : 8;
212 union {
213 int (*execute)(struct x86_emulate_ctxt *ctxt);
214 const struct opcode *group;
215 const struct group_dual *gdual;
216 const struct gprefix *gprefix;
217 const struct escape *esc;
218 const struct instr_dual *idual;
219 const struct mode_dual *mdual;
220 void (*fastop)(struct fastop *fake);
221 } u;
222 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
223};
224
225struct group_dual {
226 struct opcode mod012[8];
227 struct opcode mod3[8];
228};
229
230struct gprefix {
231 struct opcode pfx_no;
232 struct opcode pfx_66;
233 struct opcode pfx_f2;
234 struct opcode pfx_f3;
235};
236
237struct escape {
238 struct opcode op[8];
239 struct opcode high[64];
240};
241
242struct instr_dual {
243 struct opcode mod012;
244 struct opcode mod3;
245};
246
247struct mode_dual {
248 struct opcode mode32;
249 struct opcode mode64;
250};
251
252#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
253
254enum x86_transfer_type {
255 X86_TRANSFER_NONE,
256 X86_TRANSFER_CALL_JMP,
257 X86_TRANSFER_RET,
258 X86_TRANSFER_TASK_SWITCH,
259};
260
261static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
262{
263 if (!(ctxt->regs_valid & (1 << nr))) {
264 ctxt->regs_valid |= 1 << nr;
265 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
266 }
267 return ctxt->_regs[nr];
268}
269
270static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
271{
272 ctxt->regs_valid |= 1 << nr;
273 ctxt->regs_dirty |= 1 << nr;
274 return &ctxt->_regs[nr];
275}
276
277static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
278{
279 reg_read(ctxt, nr);
280 return reg_write(ctxt, nr);
281}
282
283static void writeback_registers(struct x86_emulate_ctxt *ctxt)
284{
285 unsigned reg;
286
287 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
288 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
289}
290
291static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
292{
293 ctxt->regs_dirty = 0;
294 ctxt->regs_valid = 0;
295}
296
297
298
299
300
301#define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
302 X86_EFLAGS_PF|X86_EFLAGS_CF)
303
304#ifdef CONFIG_X86_64
305#define ON64(x) x
306#else
307#define ON64(x)
308#endif
309
310static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
311
312#define FOP_FUNC(name) \
313 ".align " __stringify(FASTOP_SIZE) " \n\t" \
314 ".type " name ", @function \n\t" \
315 name ":\n\t"
316
317#define FOP_RET "ret \n\t"
318
319#define FOP_START(op) \
320 extern void em_##op(struct fastop *fake); \
321 asm(".pushsection .text, \"ax\" \n\t" \
322 ".global em_" #op " \n\t" \
323 FOP_FUNC("em_" #op)
324
325#define FOP_END \
326 ".popsection")
327
328#define FOPNOP() \
329 FOP_FUNC(__stringify(__UNIQUE_ID(nop))) \
330 FOP_RET
331
332#define FOP1E(op, dst) \
333 FOP_FUNC(#op "_" #dst) \
334 "10: " #op " %" #dst " \n\t" FOP_RET
335
336#define FOP1EEX(op, dst) \
337 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
338
339#define FASTOP1(op) \
340 FOP_START(op) \
341 FOP1E(op##b, al) \
342 FOP1E(op##w, ax) \
343 FOP1E(op##l, eax) \
344 ON64(FOP1E(op##q, rax)) \
345 FOP_END
346
347
348#define FASTOP1SRC2(op, name) \
349 FOP_START(name) \
350 FOP1E(op, cl) \
351 FOP1E(op, cx) \
352 FOP1E(op, ecx) \
353 ON64(FOP1E(op, rcx)) \
354 FOP_END
355
356
357#define FASTOP1SRC2EX(op, name) \
358 FOP_START(name) \
359 FOP1EEX(op, cl) \
360 FOP1EEX(op, cx) \
361 FOP1EEX(op, ecx) \
362 ON64(FOP1EEX(op, rcx)) \
363 FOP_END
364
365#define FOP2E(op, dst, src) \
366 FOP_FUNC(#op "_" #dst "_" #src) \
367 #op " %" #src ", %" #dst " \n\t" FOP_RET
368
369#define FASTOP2(op) \
370 FOP_START(op) \
371 FOP2E(op##b, al, dl) \
372 FOP2E(op##w, ax, dx) \
373 FOP2E(op##l, eax, edx) \
374 ON64(FOP2E(op##q, rax, rdx)) \
375 FOP_END
376
377
378#define FASTOP2W(op) \
379 FOP_START(op) \
380 FOPNOP() \
381 FOP2E(op##w, ax, dx) \
382 FOP2E(op##l, eax, edx) \
383 ON64(FOP2E(op##q, rax, rdx)) \
384 FOP_END
385
386
387#define FASTOP2CL(op) \
388 FOP_START(op) \
389 FOP2E(op##b, al, cl) \
390 FOP2E(op##w, ax, cl) \
391 FOP2E(op##l, eax, cl) \
392 ON64(FOP2E(op##q, rax, cl)) \
393 FOP_END
394
395
396#define FASTOP2R(op, name) \
397 FOP_START(name) \
398 FOP2E(op##b, dl, al) \
399 FOP2E(op##w, dx, ax) \
400 FOP2E(op##l, edx, eax) \
401 ON64(FOP2E(op##q, rdx, rax)) \
402 FOP_END
403
404#define FOP3E(op, dst, src, src2) \
405 FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
406 #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
407
408
409#define FASTOP3WCL(op) \
410 FOP_START(op) \
411 FOPNOP() \
412 FOP3E(op##w, ax, dx, cl) \
413 FOP3E(op##l, eax, edx, cl) \
414 ON64(FOP3E(op##q, rax, rdx, cl)) \
415 FOP_END
416
417
418#define FOP_SETCC(op) \
419 ".align 4 \n\t" \
420 ".type " #op ", @function \n\t" \
421 #op ": \n\t" \
422 #op " %al \n\t" \
423 FOP_RET
424
425asm(".global kvm_fastop_exception \n"
426 "kvm_fastop_exception: xor %esi, %esi; ret");
427
428FOP_START(setcc)
429FOP_SETCC(seto)
430FOP_SETCC(setno)
431FOP_SETCC(setc)
432FOP_SETCC(setnc)
433FOP_SETCC(setz)
434FOP_SETCC(setnz)
435FOP_SETCC(setbe)
436FOP_SETCC(setnbe)
437FOP_SETCC(sets)
438FOP_SETCC(setns)
439FOP_SETCC(setp)
440FOP_SETCC(setnp)
441FOP_SETCC(setl)
442FOP_SETCC(setnl)
443FOP_SETCC(setle)
444FOP_SETCC(setnle)
445FOP_END;
446
447FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
448FOP_END;
449
450static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
451 enum x86_intercept intercept,
452 enum x86_intercept_stage stage)
453{
454 struct x86_instruction_info info = {
455 .intercept = intercept,
456 .rep_prefix = ctxt->rep_prefix,
457 .modrm_mod = ctxt->modrm_mod,
458 .modrm_reg = ctxt->modrm_reg,
459 .modrm_rm = ctxt->modrm_rm,
460 .src_val = ctxt->src.val64,
461 .dst_val = ctxt->dst.val64,
462 .src_bytes = ctxt->src.bytes,
463 .dst_bytes = ctxt->dst.bytes,
464 .ad_bytes = ctxt->ad_bytes,
465 .next_rip = ctxt->eip,
466 };
467
468 return ctxt->ops->intercept(ctxt, &info, stage);
469}
470
471static void assign_masked(ulong *dest, ulong src, ulong mask)
472{
473 *dest = (*dest & ~mask) | (src & mask);
474}
475
476static void assign_register(unsigned long *reg, u64 val, int bytes)
477{
478
479 switch (bytes) {
480 case 1:
481 *(u8 *)reg = (u8)val;
482 break;
483 case 2:
484 *(u16 *)reg = (u16)val;
485 break;
486 case 4:
487 *reg = (u32)val;
488 break;
489 case 8:
490 *reg = val;
491 break;
492 }
493}
494
495static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
496{
497 return (1UL << (ctxt->ad_bytes << 3)) - 1;
498}
499
500static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
501{
502 u16 sel;
503 struct desc_struct ss;
504
505 if (ctxt->mode == X86EMUL_MODE_PROT64)
506 return ~0UL;
507 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
508 return ~0U >> ((ss.d ^ 1) * 16);
509}
510
511static int stack_size(struct x86_emulate_ctxt *ctxt)
512{
513 return (__fls(stack_mask(ctxt)) + 1) >> 3;
514}
515
516
517static inline unsigned long
518address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
519{
520 if (ctxt->ad_bytes == sizeof(unsigned long))
521 return reg;
522 else
523 return reg & ad_mask(ctxt);
524}
525
526static inline unsigned long
527register_address(struct x86_emulate_ctxt *ctxt, int reg)
528{
529 return address_mask(ctxt, reg_read(ctxt, reg));
530}
531
532static void masked_increment(ulong *reg, ulong mask, int inc)
533{
534 assign_masked(reg, *reg + inc, mask);
535}
536
537static inline void
538register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
539{
540 ulong *preg = reg_rmw(ctxt, reg);
541
542 assign_register(preg, *preg + inc, ctxt->ad_bytes);
543}
544
545static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
546{
547 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
548}
549
550static u32 desc_limit_scaled(struct desc_struct *desc)
551{
552 u32 limit = get_desc_limit(desc);
553
554 return desc->g ? (limit << 12) | 0xfff : limit;
555}
556
557static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
558{
559 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
560 return 0;
561
562 return ctxt->ops->get_cached_segment_base(ctxt, seg);
563}
564
565static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
566 u32 error, bool valid)
567{
568 WARN_ON(vec > 0x1f);
569 ctxt->exception.vector = vec;
570 ctxt->exception.error_code = error;
571 ctxt->exception.error_code_valid = valid;
572 return X86EMUL_PROPAGATE_FAULT;
573}
574
575static int emulate_db(struct x86_emulate_ctxt *ctxt)
576{
577 return emulate_exception(ctxt, DB_VECTOR, 0, false);
578}
579
580static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
581{
582 return emulate_exception(ctxt, GP_VECTOR, err, true);
583}
584
585static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
586{
587 return emulate_exception(ctxt, SS_VECTOR, err, true);
588}
589
590static int emulate_ud(struct x86_emulate_ctxt *ctxt)
591{
592 return emulate_exception(ctxt, UD_VECTOR, 0, false);
593}
594
595static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
596{
597 return emulate_exception(ctxt, TS_VECTOR, err, true);
598}
599
600static int emulate_de(struct x86_emulate_ctxt *ctxt)
601{
602 return emulate_exception(ctxt, DE_VECTOR, 0, false);
603}
604
605static int emulate_nm(struct x86_emulate_ctxt *ctxt)
606{
607 return emulate_exception(ctxt, NM_VECTOR, 0, false);
608}
609
610static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
611{
612 u16 selector;
613 struct desc_struct desc;
614
615 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
616 return selector;
617}
618
619static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
620 unsigned seg)
621{
622 u16 dummy;
623 u32 base3;
624 struct desc_struct desc;
625
626 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
627 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
628}
629
630
631
632
633
634
635
636
637
638static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
639{
640 if (likely(size < 16))
641 return false;
642
643 if (ctxt->d & Aligned)
644 return true;
645 else if (ctxt->d & Unaligned)
646 return false;
647 else if (ctxt->d & Avx)
648 return false;
649 else
650 return true;
651}
652
653static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
654 struct segmented_address addr,
655 unsigned *max_size, unsigned size,
656 bool write, bool fetch,
657 enum x86emul_mode mode, ulong *linear)
658{
659 struct desc_struct desc;
660 bool usable;
661 ulong la;
662 u32 lim;
663 u16 sel;
664
665 la = seg_base(ctxt, addr.seg) + addr.ea;
666 *max_size = 0;
667 switch (mode) {
668 case X86EMUL_MODE_PROT64:
669 *linear = la;
670 if (is_noncanonical_address(la))
671 goto bad;
672
673 *max_size = min_t(u64, ~0u, (1ull << 48) - la);
674 if (size > *max_size)
675 goto bad;
676 break;
677 default:
678 *linear = la = (u32)la;
679 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
680 addr.seg);
681 if (!usable)
682 goto bad;
683
684 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
685 || !(desc.type & 2)) && write)
686 goto bad;
687
688 if (!fetch && (desc.type & 8) && !(desc.type & 2))
689 goto bad;
690 lim = desc_limit_scaled(&desc);
691 if (!(desc.type & 8) && (desc.type & 4)) {
692
693 if (addr.ea <= lim)
694 goto bad;
695 lim = desc.d ? 0xffffffff : 0xffff;
696 }
697 if (addr.ea > lim)
698 goto bad;
699 if (lim == 0xffffffff)
700 *max_size = ~0u;
701 else {
702 *max_size = (u64)lim + 1 - addr.ea;
703 if (size > *max_size)
704 goto bad;
705 }
706 break;
707 }
708 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
709 return emulate_gp(ctxt, 0);
710 return X86EMUL_CONTINUE;
711bad:
712 if (addr.seg == VCPU_SREG_SS)
713 return emulate_ss(ctxt, 0);
714 else
715 return emulate_gp(ctxt, 0);
716}
717
718static int linearize(struct x86_emulate_ctxt *ctxt,
719 struct segmented_address addr,
720 unsigned size, bool write,
721 ulong *linear)
722{
723 unsigned max_size;
724 return __linearize(ctxt, addr, &max_size, size, write, false,
725 ctxt->mode, linear);
726}
727
728static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
729 enum x86emul_mode mode)
730{
731 ulong linear;
732 int rc;
733 unsigned max_size;
734 struct segmented_address addr = { .seg = VCPU_SREG_CS,
735 .ea = dst };
736
737 if (ctxt->op_bytes != sizeof(unsigned long))
738 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
739 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
740 if (rc == X86EMUL_CONTINUE)
741 ctxt->_eip = addr.ea;
742 return rc;
743}
744
745static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
746{
747 return assign_eip(ctxt, dst, ctxt->mode);
748}
749
750static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
751 const struct desc_struct *cs_desc)
752{
753 enum x86emul_mode mode = ctxt->mode;
754 int rc;
755
756#ifdef CONFIG_X86_64
757 if (ctxt->mode >= X86EMUL_MODE_PROT16) {
758 if (cs_desc->l) {
759 u64 efer = 0;
760
761 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
762 if (efer & EFER_LMA)
763 mode = X86EMUL_MODE_PROT64;
764 } else
765 mode = X86EMUL_MODE_PROT32;
766 }
767#endif
768 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
769 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
770 rc = assign_eip(ctxt, dst, mode);
771 if (rc == X86EMUL_CONTINUE)
772 ctxt->mode = mode;
773 return rc;
774}
775
776static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
777{
778 return assign_eip_near(ctxt, ctxt->_eip + rel);
779}
780
781static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
782 struct segmented_address addr,
783 void *data,
784 unsigned size)
785{
786 int rc;
787 ulong linear;
788
789 rc = linearize(ctxt, addr, size, false, &linear);
790 if (rc != X86EMUL_CONTINUE)
791 return rc;
792 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
793}
794
795
796
797
798
799static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
800{
801 int rc;
802 unsigned size, max_size;
803 unsigned long linear;
804 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
805 struct segmented_address addr = { .seg = VCPU_SREG_CS,
806 .ea = ctxt->eip + cur_size };
807
808
809
810
811
812
813
814
815
816
817
818 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
819 &linear);
820 if (unlikely(rc != X86EMUL_CONTINUE))
821 return rc;
822
823 size = min_t(unsigned, 15UL ^ cur_size, max_size);
824 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
825
826
827
828
829
830
831
832 if (unlikely(size < op_size))
833 return emulate_gp(ctxt, 0);
834
835 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
836 size, &ctxt->exception);
837 if (unlikely(rc != X86EMUL_CONTINUE))
838 return rc;
839 ctxt->fetch.end += size;
840 return X86EMUL_CONTINUE;
841}
842
843static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
844 unsigned size)
845{
846 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
847
848 if (unlikely(done_size < size))
849 return __do_insn_fetch_bytes(ctxt, size - done_size);
850 else
851 return X86EMUL_CONTINUE;
852}
853
854
855#define insn_fetch(_type, _ctxt) \
856({ _type _x; \
857 \
858 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
859 if (rc != X86EMUL_CONTINUE) \
860 goto done; \
861 ctxt->_eip += sizeof(_type); \
862 _x = *(_type __aligned(1) *) ctxt->fetch.ptr; \
863 ctxt->fetch.ptr += sizeof(_type); \
864 _x; \
865})
866
867#define insn_fetch_arr(_arr, _size, _ctxt) \
868({ \
869 rc = do_insn_fetch_bytes(_ctxt, _size); \
870 if (rc != X86EMUL_CONTINUE) \
871 goto done; \
872 ctxt->_eip += (_size); \
873 memcpy(_arr, ctxt->fetch.ptr, _size); \
874 ctxt->fetch.ptr += (_size); \
875})
876
877
878
879
880
881
882static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
883 int byteop)
884{
885 void *p;
886 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
887
888 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
889 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
890 else
891 p = reg_rmw(ctxt, modrm_reg);
892 return p;
893}
894
895static int read_descriptor(struct x86_emulate_ctxt *ctxt,
896 struct segmented_address addr,
897 u16 *size, unsigned long *address, int op_bytes)
898{
899 int rc;
900
901 if (op_bytes == 2)
902 op_bytes = 3;
903 *address = 0;
904 rc = segmented_read_std(ctxt, addr, size, 2);
905 if (rc != X86EMUL_CONTINUE)
906 return rc;
907 addr.ea += 2;
908 rc = segmented_read_std(ctxt, addr, address, op_bytes);
909 return rc;
910}
911
912FASTOP2(add);
913FASTOP2(or);
914FASTOP2(adc);
915FASTOP2(sbb);
916FASTOP2(and);
917FASTOP2(sub);
918FASTOP2(xor);
919FASTOP2(cmp);
920FASTOP2(test);
921
922FASTOP1SRC2(mul, mul_ex);
923FASTOP1SRC2(imul, imul_ex);
924FASTOP1SRC2EX(div, div_ex);
925FASTOP1SRC2EX(idiv, idiv_ex);
926
927FASTOP3WCL(shld);
928FASTOP3WCL(shrd);
929
930FASTOP2W(imul);
931
932FASTOP1(not);
933FASTOP1(neg);
934FASTOP1(inc);
935FASTOP1(dec);
936
937FASTOP2CL(rol);
938FASTOP2CL(ror);
939FASTOP2CL(rcl);
940FASTOP2CL(rcr);
941FASTOP2CL(shl);
942FASTOP2CL(shr);
943FASTOP2CL(sar);
944
945FASTOP2W(bsf);
946FASTOP2W(bsr);
947FASTOP2W(bt);
948FASTOP2W(bts);
949FASTOP2W(btr);
950FASTOP2W(btc);
951
952FASTOP2(xadd);
953
954FASTOP2R(cmp, cmp_r);
955
956static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
957{
958
959 if (ctxt->src.val == 0)
960 ctxt->dst.type = OP_NONE;
961 return fastop(ctxt, em_bsf);
962}
963
964static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
965{
966
967 if (ctxt->src.val == 0)
968 ctxt->dst.type = OP_NONE;
969 return fastop(ctxt, em_bsr);
970}
971
972static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
973{
974 u8 rc;
975 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
976
977 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
978 asm("push %[flags]; popf; call *%[fastop]"
979 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
980 return rc;
981}
982
983static void fetch_register_operand(struct operand *op)
984{
985 switch (op->bytes) {
986 case 1:
987 op->val = *(u8 *)op->addr.reg;
988 break;
989 case 2:
990 op->val = *(u16 *)op->addr.reg;
991 break;
992 case 4:
993 op->val = *(u32 *)op->addr.reg;
994 break;
995 case 8:
996 op->val = *(u64 *)op->addr.reg;
997 break;
998 }
999}
1000
1001static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
1002{
1003 ctxt->ops->get_fpu(ctxt);
1004 switch (reg) {
1005 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
1006 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
1007 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
1008 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
1009 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
1010 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
1011 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
1012 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1013#ifdef CONFIG_X86_64
1014 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1015 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1016 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1017 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1018 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1019 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1020 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1021 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1022#endif
1023 default: BUG();
1024 }
1025 ctxt->ops->put_fpu(ctxt);
1026}
1027
1028static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1029 int reg)
1030{
1031 ctxt->ops->get_fpu(ctxt);
1032 switch (reg) {
1033 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1034 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1035 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1036 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1037 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1038 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1039 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1040 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1041#ifdef CONFIG_X86_64
1042 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1043 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1044 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1045 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1046 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1047 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1048 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1049 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1050#endif
1051 default: BUG();
1052 }
1053 ctxt->ops->put_fpu(ctxt);
1054}
1055
1056static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1057{
1058 ctxt->ops->get_fpu(ctxt);
1059 switch (reg) {
1060 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1061 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1062 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1063 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1064 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1065 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1066 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1067 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1068 default: BUG();
1069 }
1070 ctxt->ops->put_fpu(ctxt);
1071}
1072
1073static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1074{
1075 ctxt->ops->get_fpu(ctxt);
1076 switch (reg) {
1077 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1078 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1079 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1080 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1081 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1082 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1083 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1084 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1085 default: BUG();
1086 }
1087 ctxt->ops->put_fpu(ctxt);
1088}
1089
1090static int em_fninit(struct x86_emulate_ctxt *ctxt)
1091{
1092 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1093 return emulate_nm(ctxt);
1094
1095 ctxt->ops->get_fpu(ctxt);
1096 asm volatile("fninit");
1097 ctxt->ops->put_fpu(ctxt);
1098 return X86EMUL_CONTINUE;
1099}
1100
1101static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1102{
1103 u16 fcw;
1104
1105 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1106 return emulate_nm(ctxt);
1107
1108 ctxt->ops->get_fpu(ctxt);
1109 asm volatile("fnstcw %0": "+m"(fcw));
1110 ctxt->ops->put_fpu(ctxt);
1111
1112 ctxt->dst.val = fcw;
1113
1114 return X86EMUL_CONTINUE;
1115}
1116
1117static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1118{
1119 u16 fsw;
1120
1121 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1122 return emulate_nm(ctxt);
1123
1124 ctxt->ops->get_fpu(ctxt);
1125 asm volatile("fnstsw %0": "+m"(fsw));
1126 ctxt->ops->put_fpu(ctxt);
1127
1128 ctxt->dst.val = fsw;
1129
1130 return X86EMUL_CONTINUE;
1131}
1132
1133static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1134 struct operand *op)
1135{
1136 unsigned reg = ctxt->modrm_reg;
1137
1138 if (!(ctxt->d & ModRM))
1139 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1140
1141 if (ctxt->d & Sse) {
1142 op->type = OP_XMM;
1143 op->bytes = 16;
1144 op->addr.xmm = reg;
1145 read_sse_reg(ctxt, &op->vec_val, reg);
1146 return;
1147 }
1148 if (ctxt->d & Mmx) {
1149 reg &= 7;
1150 op->type = OP_MM;
1151 op->bytes = 8;
1152 op->addr.mm = reg;
1153 return;
1154 }
1155
1156 op->type = OP_REG;
1157 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1158 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1159
1160 fetch_register_operand(op);
1161 op->orig_val = op->val;
1162}
1163
1164static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1165{
1166 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1167 ctxt->modrm_seg = VCPU_SREG_SS;
1168}
1169
1170static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1171 struct operand *op)
1172{
1173 u8 sib;
1174 int index_reg, base_reg, scale;
1175 int rc = X86EMUL_CONTINUE;
1176 ulong modrm_ea = 0;
1177
1178 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8);
1179 index_reg = (ctxt->rex_prefix << 2) & 8;
1180 base_reg = (ctxt->rex_prefix << 3) & 8;
1181
1182 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1183 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1184 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1185 ctxt->modrm_seg = VCPU_SREG_DS;
1186
1187 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1188 op->type = OP_REG;
1189 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1190 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1191 ctxt->d & ByteOp);
1192 if (ctxt->d & Sse) {
1193 op->type = OP_XMM;
1194 op->bytes = 16;
1195 op->addr.xmm = ctxt->modrm_rm;
1196 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1197 return rc;
1198 }
1199 if (ctxt->d & Mmx) {
1200 op->type = OP_MM;
1201 op->bytes = 8;
1202 op->addr.mm = ctxt->modrm_rm & 7;
1203 return rc;
1204 }
1205 fetch_register_operand(op);
1206 return rc;
1207 }
1208
1209 op->type = OP_MEM;
1210
1211 if (ctxt->ad_bytes == 2) {
1212 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1213 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1214 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1215 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1216
1217
1218 switch (ctxt->modrm_mod) {
1219 case 0:
1220 if (ctxt->modrm_rm == 6)
1221 modrm_ea += insn_fetch(u16, ctxt);
1222 break;
1223 case 1:
1224 modrm_ea += insn_fetch(s8, ctxt);
1225 break;
1226 case 2:
1227 modrm_ea += insn_fetch(u16, ctxt);
1228 break;
1229 }
1230 switch (ctxt->modrm_rm) {
1231 case 0:
1232 modrm_ea += bx + si;
1233 break;
1234 case 1:
1235 modrm_ea += bx + di;
1236 break;
1237 case 2:
1238 modrm_ea += bp + si;
1239 break;
1240 case 3:
1241 modrm_ea += bp + di;
1242 break;
1243 case 4:
1244 modrm_ea += si;
1245 break;
1246 case 5:
1247 modrm_ea += di;
1248 break;
1249 case 6:
1250 if (ctxt->modrm_mod != 0)
1251 modrm_ea += bp;
1252 break;
1253 case 7:
1254 modrm_ea += bx;
1255 break;
1256 }
1257 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1258 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1259 ctxt->modrm_seg = VCPU_SREG_SS;
1260 modrm_ea = (u16)modrm_ea;
1261 } else {
1262
1263 if ((ctxt->modrm_rm & 7) == 4) {
1264 sib = insn_fetch(u8, ctxt);
1265 index_reg |= (sib >> 3) & 7;
1266 base_reg |= sib & 7;
1267 scale = sib >> 6;
1268
1269 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1270 modrm_ea += insn_fetch(s32, ctxt);
1271 else {
1272 modrm_ea += reg_read(ctxt, base_reg);
1273 adjust_modrm_seg(ctxt, base_reg);
1274
1275 if ((ctxt->d & IncSP) &&
1276 base_reg == VCPU_REGS_RSP)
1277 modrm_ea += ctxt->op_bytes;
1278 }
1279 if (index_reg != 4)
1280 modrm_ea += reg_read(ctxt, index_reg) << scale;
1281 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1282 modrm_ea += insn_fetch(s32, ctxt);
1283 if (ctxt->mode == X86EMUL_MODE_PROT64)
1284 ctxt->rip_relative = 1;
1285 } else {
1286 base_reg = ctxt->modrm_rm;
1287 modrm_ea += reg_read(ctxt, base_reg);
1288 adjust_modrm_seg(ctxt, base_reg);
1289 }
1290 switch (ctxt->modrm_mod) {
1291 case 1:
1292 modrm_ea += insn_fetch(s8, ctxt);
1293 break;
1294 case 2:
1295 modrm_ea += insn_fetch(s32, ctxt);
1296 break;
1297 }
1298 }
1299 op->addr.mem.ea = modrm_ea;
1300 if (ctxt->ad_bytes != 8)
1301 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1302
1303done:
1304 return rc;
1305}
1306
1307static int decode_abs(struct x86_emulate_ctxt *ctxt,
1308 struct operand *op)
1309{
1310 int rc = X86EMUL_CONTINUE;
1311
1312 op->type = OP_MEM;
1313 switch (ctxt->ad_bytes) {
1314 case 2:
1315 op->addr.mem.ea = insn_fetch(u16, ctxt);
1316 break;
1317 case 4:
1318 op->addr.mem.ea = insn_fetch(u32, ctxt);
1319 break;
1320 case 8:
1321 op->addr.mem.ea = insn_fetch(u64, ctxt);
1322 break;
1323 }
1324done:
1325 return rc;
1326}
1327
1328static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1329{
1330 long sv = 0, mask;
1331
1332 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1333 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1334
1335 if (ctxt->src.bytes == 2)
1336 sv = (s16)ctxt->src.val & (s16)mask;
1337 else if (ctxt->src.bytes == 4)
1338 sv = (s32)ctxt->src.val & (s32)mask;
1339 else
1340 sv = (s64)ctxt->src.val & (s64)mask;
1341
1342 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1343 ctxt->dst.addr.mem.ea + (sv >> 3));
1344 }
1345
1346
1347 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1348}
1349
1350static int read_emulated(struct x86_emulate_ctxt *ctxt,
1351 unsigned long addr, void *dest, unsigned size)
1352{
1353 int rc;
1354 struct read_cache *mc = &ctxt->mem_read;
1355
1356 if (mc->pos < mc->end)
1357 goto read_cached;
1358
1359 WARN_ON((mc->end + size) >= sizeof(mc->data));
1360
1361 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1362 &ctxt->exception);
1363 if (rc != X86EMUL_CONTINUE)
1364 return rc;
1365
1366 mc->end += size;
1367
1368read_cached:
1369 memcpy(dest, mc->data + mc->pos, size);
1370 mc->pos += size;
1371 return X86EMUL_CONTINUE;
1372}
1373
1374static int segmented_read(struct x86_emulate_ctxt *ctxt,
1375 struct segmented_address addr,
1376 void *data,
1377 unsigned size)
1378{
1379 int rc;
1380 ulong linear;
1381
1382 rc = linearize(ctxt, addr, size, false, &linear);
1383 if (rc != X86EMUL_CONTINUE)
1384 return rc;
1385 return read_emulated(ctxt, linear, data, size);
1386}
1387
1388static int segmented_write(struct x86_emulate_ctxt *ctxt,
1389 struct segmented_address addr,
1390 const void *data,
1391 unsigned size)
1392{
1393 int rc;
1394 ulong linear;
1395
1396 rc = linearize(ctxt, addr, size, true, &linear);
1397 if (rc != X86EMUL_CONTINUE)
1398 return rc;
1399 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1400 &ctxt->exception);
1401}
1402
1403static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1404 struct segmented_address addr,
1405 const void *orig_data, const void *data,
1406 unsigned size)
1407{
1408 int rc;
1409 ulong linear;
1410
1411 rc = linearize(ctxt, addr, size, true, &linear);
1412 if (rc != X86EMUL_CONTINUE)
1413 return rc;
1414 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1415 size, &ctxt->exception);
1416}
1417
1418static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1419 unsigned int size, unsigned short port,
1420 void *dest)
1421{
1422 struct read_cache *rc = &ctxt->io_read;
1423
1424 if (rc->pos == rc->end) {
1425 unsigned int in_page, n;
1426 unsigned int count = ctxt->rep_prefix ?
1427 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1428 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1429 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1430 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1431 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1432 if (n == 0)
1433 n = 1;
1434 rc->pos = rc->end = 0;
1435 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1436 return 0;
1437 rc->end = n * size;
1438 }
1439
1440 if (ctxt->rep_prefix && (ctxt->d & String) &&
1441 !(ctxt->eflags & X86_EFLAGS_DF)) {
1442 ctxt->dst.data = rc->data + rc->pos;
1443 ctxt->dst.type = OP_MEM_STR;
1444 ctxt->dst.count = (rc->end - rc->pos) / size;
1445 rc->pos = rc->end;
1446 } else {
1447 memcpy(dest, rc->data + rc->pos, size);
1448 rc->pos += size;
1449 }
1450 return 1;
1451}
1452
1453static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1454 u16 index, struct desc_struct *desc)
1455{
1456 struct desc_ptr dt;
1457 ulong addr;
1458
1459 ctxt->ops->get_idt(ctxt, &dt);
1460
1461 if (dt.size < index * 8 + 7)
1462 return emulate_gp(ctxt, index << 3 | 0x2);
1463
1464 addr = dt.address + index * 8;
1465 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1466 &ctxt->exception);
1467}
1468
1469static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1470 u16 selector, struct desc_ptr *dt)
1471{
1472 const struct x86_emulate_ops *ops = ctxt->ops;
1473 u32 base3 = 0;
1474
1475 if (selector & 1 << 2) {
1476 struct desc_struct desc;
1477 u16 sel;
1478
1479 memset (dt, 0, sizeof *dt);
1480 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1481 VCPU_SREG_LDTR))
1482 return;
1483
1484 dt->size = desc_limit_scaled(&desc);
1485 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1486 } else
1487 ops->get_gdt(ctxt, dt);
1488}
1489
1490static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1491 u16 selector, ulong *desc_addr_p)
1492{
1493 struct desc_ptr dt;
1494 u16 index = selector >> 3;
1495 ulong addr;
1496
1497 get_descriptor_table_ptr(ctxt, selector, &dt);
1498
1499 if (dt.size < index * 8 + 7)
1500 return emulate_gp(ctxt, selector & 0xfffc);
1501
1502 addr = dt.address + index * 8;
1503
1504#ifdef CONFIG_X86_64
1505 if (addr >> 32 != 0) {
1506 u64 efer = 0;
1507
1508 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1509 if (!(efer & EFER_LMA))
1510 addr &= (u32)-1;
1511 }
1512#endif
1513
1514 *desc_addr_p = addr;
1515 return X86EMUL_CONTINUE;
1516}
1517
1518
1519static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1520 u16 selector, struct desc_struct *desc,
1521 ulong *desc_addr_p)
1522{
1523 int rc;
1524
1525 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1526 if (rc != X86EMUL_CONTINUE)
1527 return rc;
1528
1529 return ctxt->ops->read_std(ctxt, *desc_addr_p, desc, sizeof(*desc),
1530 &ctxt->exception);
1531}
1532
1533
1534static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1535 u16 selector, struct desc_struct *desc)
1536{
1537 int rc;
1538 ulong addr;
1539
1540 rc = get_descriptor_ptr(ctxt, selector, &addr);
1541 if (rc != X86EMUL_CONTINUE)
1542 return rc;
1543
1544 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1545 &ctxt->exception);
1546}
1547
1548
1549static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1550 u16 selector, int seg, u8 cpl,
1551 enum x86_transfer_type transfer,
1552 struct desc_struct *desc)
1553{
1554 struct desc_struct seg_desc, old_desc;
1555 u8 dpl, rpl;
1556 unsigned err_vec = GP_VECTOR;
1557 u32 err_code = 0;
1558 bool null_selector = !(selector & ~0x3);
1559 ulong desc_addr;
1560 int ret;
1561 u16 dummy;
1562 u32 base3 = 0;
1563
1564 memset(&seg_desc, 0, sizeof seg_desc);
1565
1566 if (ctxt->mode == X86EMUL_MODE_REAL) {
1567
1568
1569 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1570 set_desc_base(&seg_desc, selector << 4);
1571 goto load;
1572 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1573
1574 set_desc_base(&seg_desc, selector << 4);
1575 set_desc_limit(&seg_desc, 0xffff);
1576 seg_desc.type = 3;
1577 seg_desc.p = 1;
1578 seg_desc.s = 1;
1579 seg_desc.dpl = 3;
1580 goto load;
1581 }
1582
1583 rpl = selector & 3;
1584
1585
1586 if ((seg == VCPU_SREG_CS
1587 || (seg == VCPU_SREG_SS
1588 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1589 || seg == VCPU_SREG_TR)
1590 && null_selector)
1591 goto exception;
1592
1593
1594 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1595 goto exception;
1596
1597 if (null_selector)
1598 goto load;
1599
1600 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1601 if (ret != X86EMUL_CONTINUE)
1602 return ret;
1603
1604 err_code = selector & 0xfffc;
1605 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1606 GP_VECTOR;
1607
1608
1609 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1610 if (transfer == X86_TRANSFER_CALL_JMP)
1611 return X86EMUL_UNHANDLEABLE;
1612 goto exception;
1613 }
1614
1615 if (!seg_desc.p) {
1616 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1617 goto exception;
1618 }
1619
1620 dpl = seg_desc.dpl;
1621
1622 switch (seg) {
1623 case VCPU_SREG_SS:
1624
1625
1626
1627
1628 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1629 goto exception;
1630 break;
1631 case VCPU_SREG_CS:
1632 if (!(seg_desc.type & 8))
1633 goto exception;
1634
1635 if (seg_desc.type & 4) {
1636
1637 if (dpl > cpl)
1638 goto exception;
1639 } else {
1640
1641 if (rpl > cpl || dpl != cpl)
1642 goto exception;
1643 }
1644
1645 if (seg_desc.d && seg_desc.l) {
1646 u64 efer = 0;
1647
1648 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1649 if (efer & EFER_LMA)
1650 goto exception;
1651 }
1652
1653
1654 selector = (selector & 0xfffc) | cpl;
1655 break;
1656 case VCPU_SREG_TR:
1657 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1658 goto exception;
1659 old_desc = seg_desc;
1660 seg_desc.type |= 2;
1661 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1662 sizeof(seg_desc), &ctxt->exception);
1663 if (ret != X86EMUL_CONTINUE)
1664 return ret;
1665 break;
1666 case VCPU_SREG_LDTR:
1667 if (seg_desc.s || seg_desc.type != 2)
1668 goto exception;
1669 break;
1670 default:
1671
1672
1673
1674
1675
1676 if ((seg_desc.type & 0xa) == 0x8 ||
1677 (((seg_desc.type & 0xc) != 0xc) &&
1678 (rpl > dpl && cpl > dpl)))
1679 goto exception;
1680 break;
1681 }
1682
1683 if (seg_desc.s) {
1684
1685 if (!(seg_desc.type & 1)) {
1686 seg_desc.type |= 1;
1687 ret = write_segment_descriptor(ctxt, selector,
1688 &seg_desc);
1689 if (ret != X86EMUL_CONTINUE)
1690 return ret;
1691 }
1692 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1693 ret = ctxt->ops->read_std(ctxt, desc_addr+8, &base3,
1694 sizeof(base3), &ctxt->exception);
1695 if (ret != X86EMUL_CONTINUE)
1696 return ret;
1697 if (is_noncanonical_address(get_desc_base(&seg_desc) |
1698 ((u64)base3 << 32)))
1699 return emulate_gp(ctxt, 0);
1700 }
1701load:
1702 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1703 if (desc)
1704 *desc = seg_desc;
1705 return X86EMUL_CONTINUE;
1706exception:
1707 return emulate_exception(ctxt, err_vec, err_code, true);
1708}
1709
1710static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1711 u16 selector, int seg)
1712{
1713 u8 cpl = ctxt->ops->cpl(ctxt);
1714 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1715 X86_TRANSFER_NONE, NULL);
1716}
1717
1718static void write_register_operand(struct operand *op)
1719{
1720 return assign_register(op->addr.reg, op->val, op->bytes);
1721}
1722
1723static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1724{
1725 switch (op->type) {
1726 case OP_REG:
1727 write_register_operand(op);
1728 break;
1729 case OP_MEM:
1730 if (ctxt->lock_prefix)
1731 return segmented_cmpxchg(ctxt,
1732 op->addr.mem,
1733 &op->orig_val,
1734 &op->val,
1735 op->bytes);
1736 else
1737 return segmented_write(ctxt,
1738 op->addr.mem,
1739 &op->val,
1740 op->bytes);
1741 break;
1742 case OP_MEM_STR:
1743 return segmented_write(ctxt,
1744 op->addr.mem,
1745 op->data,
1746 op->bytes * op->count);
1747 break;
1748 case OP_XMM:
1749 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1750 break;
1751 case OP_MM:
1752 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1753 break;
1754 case OP_NONE:
1755
1756 break;
1757 default:
1758 break;
1759 }
1760 return X86EMUL_CONTINUE;
1761}
1762
1763static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1764{
1765 struct segmented_address addr;
1766
1767 rsp_increment(ctxt, -bytes);
1768 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1769 addr.seg = VCPU_SREG_SS;
1770
1771 return segmented_write(ctxt, addr, data, bytes);
1772}
1773
1774static int em_push(struct x86_emulate_ctxt *ctxt)
1775{
1776
1777 ctxt->dst.type = OP_NONE;
1778 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1779}
1780
1781static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1782 void *dest, int len)
1783{
1784 int rc;
1785 struct segmented_address addr;
1786
1787 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1788 addr.seg = VCPU_SREG_SS;
1789 rc = segmented_read(ctxt, addr, dest, len);
1790 if (rc != X86EMUL_CONTINUE)
1791 return rc;
1792
1793 rsp_increment(ctxt, len);
1794 return rc;
1795}
1796
1797static int em_pop(struct x86_emulate_ctxt *ctxt)
1798{
1799 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1800}
1801
1802static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1803 void *dest, int len)
1804{
1805 int rc;
1806 unsigned long val, change_mask;
1807 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1808 int cpl = ctxt->ops->cpl(ctxt);
1809
1810 rc = emulate_pop(ctxt, &val, len);
1811 if (rc != X86EMUL_CONTINUE)
1812 return rc;
1813
1814 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1815 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1816 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1817 X86_EFLAGS_AC | X86_EFLAGS_ID;
1818
1819 switch(ctxt->mode) {
1820 case X86EMUL_MODE_PROT64:
1821 case X86EMUL_MODE_PROT32:
1822 case X86EMUL_MODE_PROT16:
1823 if (cpl == 0)
1824 change_mask |= X86_EFLAGS_IOPL;
1825 if (cpl <= iopl)
1826 change_mask |= X86_EFLAGS_IF;
1827 break;
1828 case X86EMUL_MODE_VM86:
1829 if (iopl < 3)
1830 return emulate_gp(ctxt, 0);
1831 change_mask |= X86_EFLAGS_IF;
1832 break;
1833 default:
1834 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1835 break;
1836 }
1837
1838 *(unsigned long *)dest =
1839 (ctxt->eflags & ~change_mask) | (val & change_mask);
1840
1841 return rc;
1842}
1843
1844static int em_popf(struct x86_emulate_ctxt *ctxt)
1845{
1846 ctxt->dst.type = OP_REG;
1847 ctxt->dst.addr.reg = &ctxt->eflags;
1848 ctxt->dst.bytes = ctxt->op_bytes;
1849 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1850}
1851
1852static int em_enter(struct x86_emulate_ctxt *ctxt)
1853{
1854 int rc;
1855 unsigned frame_size = ctxt->src.val;
1856 unsigned nesting_level = ctxt->src2.val & 31;
1857 ulong rbp;
1858
1859 if (nesting_level)
1860 return X86EMUL_UNHANDLEABLE;
1861
1862 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1863 rc = push(ctxt, &rbp, stack_size(ctxt));
1864 if (rc != X86EMUL_CONTINUE)
1865 return rc;
1866 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1867 stack_mask(ctxt));
1868 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1869 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1870 stack_mask(ctxt));
1871 return X86EMUL_CONTINUE;
1872}
1873
1874static int em_leave(struct x86_emulate_ctxt *ctxt)
1875{
1876 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1877 stack_mask(ctxt));
1878 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1879}
1880
1881static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1882{
1883 int seg = ctxt->src2.val;
1884
1885 ctxt->src.val = get_segment_selector(ctxt, seg);
1886 if (ctxt->op_bytes == 4) {
1887 rsp_increment(ctxt, -2);
1888 ctxt->op_bytes = 2;
1889 }
1890
1891 return em_push(ctxt);
1892}
1893
1894static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1895{
1896 int seg = ctxt->src2.val;
1897 unsigned long selector;
1898 int rc;
1899
1900 rc = emulate_pop(ctxt, &selector, 2);
1901 if (rc != X86EMUL_CONTINUE)
1902 return rc;
1903
1904 if (ctxt->modrm_reg == VCPU_SREG_SS)
1905 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1906 if (ctxt->op_bytes > 2)
1907 rsp_increment(ctxt, ctxt->op_bytes - 2);
1908
1909 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1910 return rc;
1911}
1912
1913static int em_pusha(struct x86_emulate_ctxt *ctxt)
1914{
1915 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1916 int rc = X86EMUL_CONTINUE;
1917 int reg = VCPU_REGS_RAX;
1918
1919 while (reg <= VCPU_REGS_RDI) {
1920 (reg == VCPU_REGS_RSP) ?
1921 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1922
1923 rc = em_push(ctxt);
1924 if (rc != X86EMUL_CONTINUE)
1925 return rc;
1926
1927 ++reg;
1928 }
1929
1930 return rc;
1931}
1932
1933static int em_pushf(struct x86_emulate_ctxt *ctxt)
1934{
1935 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
1936 return em_push(ctxt);
1937}
1938
1939static int em_popa(struct x86_emulate_ctxt *ctxt)
1940{
1941 int rc = X86EMUL_CONTINUE;
1942 int reg = VCPU_REGS_RDI;
1943 u32 val;
1944
1945 while (reg >= VCPU_REGS_RAX) {
1946 if (reg == VCPU_REGS_RSP) {
1947 rsp_increment(ctxt, ctxt->op_bytes);
1948 --reg;
1949 }
1950
1951 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
1952 if (rc != X86EMUL_CONTINUE)
1953 break;
1954 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
1955 --reg;
1956 }
1957 return rc;
1958}
1959
1960static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1961{
1962 const struct x86_emulate_ops *ops = ctxt->ops;
1963 int rc;
1964 struct desc_ptr dt;
1965 gva_t cs_addr;
1966 gva_t eip_addr;
1967 u16 cs, eip;
1968
1969
1970 ctxt->src.val = ctxt->eflags;
1971 rc = em_push(ctxt);
1972 if (rc != X86EMUL_CONTINUE)
1973 return rc;
1974
1975 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
1976
1977 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1978 rc = em_push(ctxt);
1979 if (rc != X86EMUL_CONTINUE)
1980 return rc;
1981
1982 ctxt->src.val = ctxt->_eip;
1983 rc = em_push(ctxt);
1984 if (rc != X86EMUL_CONTINUE)
1985 return rc;
1986
1987 ops->get_idt(ctxt, &dt);
1988
1989 eip_addr = dt.address + (irq << 2);
1990 cs_addr = dt.address + (irq << 2) + 2;
1991
1992 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1993 if (rc != X86EMUL_CONTINUE)
1994 return rc;
1995
1996 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1997 if (rc != X86EMUL_CONTINUE)
1998 return rc;
1999
2000 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2001 if (rc != X86EMUL_CONTINUE)
2002 return rc;
2003
2004 ctxt->_eip = eip;
2005
2006 return rc;
2007}
2008
2009int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2010{
2011 int rc;
2012
2013 invalidate_registers(ctxt);
2014 rc = __emulate_int_real(ctxt, irq);
2015 if (rc == X86EMUL_CONTINUE)
2016 writeback_registers(ctxt);
2017 return rc;
2018}
2019
2020static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2021{
2022 switch(ctxt->mode) {
2023 case X86EMUL_MODE_REAL:
2024 return __emulate_int_real(ctxt, irq);
2025 case X86EMUL_MODE_VM86:
2026 case X86EMUL_MODE_PROT16:
2027 case X86EMUL_MODE_PROT32:
2028 case X86EMUL_MODE_PROT64:
2029 default:
2030
2031 return X86EMUL_UNHANDLEABLE;
2032 }
2033}
2034
2035static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2036{
2037 int rc = X86EMUL_CONTINUE;
2038 unsigned long temp_eip = 0;
2039 unsigned long temp_eflags = 0;
2040 unsigned long cs = 0;
2041 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2042 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2043 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2044 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2045 X86_EFLAGS_AC | X86_EFLAGS_ID |
2046 X86_EFLAGS_FIXED;
2047 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2048 X86_EFLAGS_VIP;
2049
2050
2051
2052 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2053
2054 if (rc != X86EMUL_CONTINUE)
2055 return rc;
2056
2057 if (temp_eip & ~0xffff)
2058 return emulate_gp(ctxt, 0);
2059
2060 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2061
2062 if (rc != X86EMUL_CONTINUE)
2063 return rc;
2064
2065 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2066
2067 if (rc != X86EMUL_CONTINUE)
2068 return rc;
2069
2070 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2071
2072 if (rc != X86EMUL_CONTINUE)
2073 return rc;
2074
2075 ctxt->_eip = temp_eip;
2076
2077 if (ctxt->op_bytes == 4)
2078 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2079 else if (ctxt->op_bytes == 2) {
2080 ctxt->eflags &= ~0xffff;
2081 ctxt->eflags |= temp_eflags;
2082 }
2083
2084 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK;
2085 ctxt->eflags |= X86_EFLAGS_FIXED;
2086 ctxt->ops->set_nmi_mask(ctxt, false);
2087
2088 return rc;
2089}
2090
2091static int em_iret(struct x86_emulate_ctxt *ctxt)
2092{
2093 switch(ctxt->mode) {
2094 case X86EMUL_MODE_REAL:
2095 return emulate_iret_real(ctxt);
2096 case X86EMUL_MODE_VM86:
2097 case X86EMUL_MODE_PROT16:
2098 case X86EMUL_MODE_PROT32:
2099 case X86EMUL_MODE_PROT64:
2100 default:
2101
2102 return X86EMUL_UNHANDLEABLE;
2103 }
2104}
2105
2106static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2107{
2108 int rc;
2109 unsigned short sel, old_sel;
2110 struct desc_struct old_desc, new_desc;
2111 const struct x86_emulate_ops *ops = ctxt->ops;
2112 u8 cpl = ctxt->ops->cpl(ctxt);
2113
2114
2115 if (ctxt->mode == X86EMUL_MODE_PROT64)
2116 ops->get_segment(ctxt, &old_sel, &old_desc, NULL,
2117 VCPU_SREG_CS);
2118
2119 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2120
2121 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2122 X86_TRANSFER_CALL_JMP,
2123 &new_desc);
2124 if (rc != X86EMUL_CONTINUE)
2125 return rc;
2126
2127 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2128 if (rc != X86EMUL_CONTINUE) {
2129 WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64);
2130
2131 ops->set_segment(ctxt, old_sel, &old_desc, 0, VCPU_SREG_CS);
2132 return rc;
2133 }
2134 return rc;
2135}
2136
2137static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2138{
2139 return assign_eip_near(ctxt, ctxt->src.val);
2140}
2141
2142static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2143{
2144 int rc;
2145 long int old_eip;
2146
2147 old_eip = ctxt->_eip;
2148 rc = assign_eip_near(ctxt, ctxt->src.val);
2149 if (rc != X86EMUL_CONTINUE)
2150 return rc;
2151 ctxt->src.val = old_eip;
2152 rc = em_push(ctxt);
2153 return rc;
2154}
2155
2156static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2157{
2158 u64 old = ctxt->dst.orig_val64;
2159
2160 if (ctxt->dst.bytes == 16)
2161 return X86EMUL_UNHANDLEABLE;
2162
2163 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2164 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2165 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2166 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2167 ctxt->eflags &= ~X86_EFLAGS_ZF;
2168 } else {
2169 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2170 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2171
2172 ctxt->eflags |= X86_EFLAGS_ZF;
2173 }
2174 return X86EMUL_CONTINUE;
2175}
2176
2177static int em_ret(struct x86_emulate_ctxt *ctxt)
2178{
2179 int rc;
2180 unsigned long eip;
2181
2182 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2183 if (rc != X86EMUL_CONTINUE)
2184 return rc;
2185
2186 return assign_eip_near(ctxt, eip);
2187}
2188
2189static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2190{
2191 int rc;
2192 unsigned long eip, cs;
2193 u16 old_cs;
2194 int cpl = ctxt->ops->cpl(ctxt);
2195 struct desc_struct old_desc, new_desc;
2196 const struct x86_emulate_ops *ops = ctxt->ops;
2197
2198 if (ctxt->mode == X86EMUL_MODE_PROT64)
2199 ops->get_segment(ctxt, &old_cs, &old_desc, NULL,
2200 VCPU_SREG_CS);
2201
2202 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2203 if (rc != X86EMUL_CONTINUE)
2204 return rc;
2205 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2206 if (rc != X86EMUL_CONTINUE)
2207 return rc;
2208
2209 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2210 return X86EMUL_UNHANDLEABLE;
2211 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2212 X86_TRANSFER_RET,
2213 &new_desc);
2214 if (rc != X86EMUL_CONTINUE)
2215 return rc;
2216 rc = assign_eip_far(ctxt, eip, &new_desc);
2217 if (rc != X86EMUL_CONTINUE) {
2218 WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64);
2219 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
2220 }
2221 return rc;
2222}
2223
2224static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2225{
2226 int rc;
2227
2228 rc = em_ret_far(ctxt);
2229 if (rc != X86EMUL_CONTINUE)
2230 return rc;
2231 rsp_increment(ctxt, ctxt->src.val);
2232 return X86EMUL_CONTINUE;
2233}
2234
2235static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2236{
2237
2238 ctxt->dst.orig_val = ctxt->dst.val;
2239 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2240 ctxt->src.orig_val = ctxt->src.val;
2241 ctxt->src.val = ctxt->dst.orig_val;
2242 fastop(ctxt, em_cmp);
2243
2244 if (ctxt->eflags & X86_EFLAGS_ZF) {
2245
2246 ctxt->src.type = OP_NONE;
2247 ctxt->dst.val = ctxt->src.orig_val;
2248 } else {
2249
2250 ctxt->src.type = OP_REG;
2251 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2252 ctxt->src.val = ctxt->dst.orig_val;
2253
2254 ctxt->dst.val = ctxt->dst.orig_val;
2255 }
2256 return X86EMUL_CONTINUE;
2257}
2258
2259static int em_lseg(struct x86_emulate_ctxt *ctxt)
2260{
2261 int seg = ctxt->src2.val;
2262 unsigned short sel;
2263 int rc;
2264
2265 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2266
2267 rc = load_segment_descriptor(ctxt, sel, seg);
2268 if (rc != X86EMUL_CONTINUE)
2269 return rc;
2270
2271 ctxt->dst.val = ctxt->src.val;
2272 return rc;
2273}
2274
2275static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
2276{
2277 u32 eax, ebx, ecx, edx;
2278
2279 eax = 0x80000001;
2280 ecx = 0;
2281 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2282 return edx & bit(X86_FEATURE_LM);
2283}
2284
2285#define GET_SMSTATE(type, smbase, offset) \
2286 ({ \
2287 type __val; \
2288 int r = ctxt->ops->read_phys(ctxt, smbase + offset, &__val, \
2289 sizeof(__val)); \
2290 if (r != X86EMUL_CONTINUE) \
2291 return X86EMUL_UNHANDLEABLE; \
2292 __val; \
2293 })
2294
2295static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
2296{
2297 desc->g = (flags >> 23) & 1;
2298 desc->d = (flags >> 22) & 1;
2299 desc->l = (flags >> 21) & 1;
2300 desc->avl = (flags >> 20) & 1;
2301 desc->p = (flags >> 15) & 1;
2302 desc->dpl = (flags >> 13) & 3;
2303 desc->s = (flags >> 12) & 1;
2304 desc->type = (flags >> 8) & 15;
2305}
2306
2307static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2308{
2309 struct desc_struct desc;
2310 int offset;
2311 u16 selector;
2312
2313 selector = GET_SMSTATE(u32, smbase, 0x7fa8 + n * 4);
2314
2315 if (n < 3)
2316 offset = 0x7f84 + n * 12;
2317 else
2318 offset = 0x7f2c + (n - 3) * 12;
2319
2320 set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2321 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2322 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, offset));
2323 ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
2324 return X86EMUL_CONTINUE;
2325}
2326
2327static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2328{
2329 struct desc_struct desc;
2330 int offset;
2331 u16 selector;
2332 u32 base3;
2333
2334 offset = 0x7e00 + n * 16;
2335
2336 selector = GET_SMSTATE(u16, smbase, offset);
2337 rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smbase, offset + 2) << 8);
2338 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2339 set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2340 base3 = GET_SMSTATE(u32, smbase, offset + 12);
2341
2342 ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
2343 return X86EMUL_CONTINUE;
2344}
2345
2346static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,
2347 u64 cr0, u64 cr4)
2348{
2349 int bad;
2350
2351
2352
2353
2354
2355
2356 bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2357 if (bad)
2358 return X86EMUL_UNHANDLEABLE;
2359
2360 bad = ctxt->ops->set_cr(ctxt, 0, cr0);
2361 if (bad)
2362 return X86EMUL_UNHANDLEABLE;
2363
2364 if (cr4 & X86_CR4_PCIDE) {
2365 bad = ctxt->ops->set_cr(ctxt, 4, cr4);
2366 if (bad)
2367 return X86EMUL_UNHANDLEABLE;
2368 }
2369
2370 return X86EMUL_CONTINUE;
2371}
2372
2373static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt, u64 smbase)
2374{
2375 struct desc_struct desc;
2376 struct desc_ptr dt;
2377 u16 selector;
2378 u32 val, cr0, cr4;
2379 int i;
2380
2381 cr0 = GET_SMSTATE(u32, smbase, 0x7ffc);
2382 ctxt->ops->set_cr(ctxt, 3, GET_SMSTATE(u32, smbase, 0x7ff8));
2383 ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7ff4) | X86_EFLAGS_FIXED;
2384 ctxt->_eip = GET_SMSTATE(u32, smbase, 0x7ff0);
2385
2386 for (i = 0; i < 8; i++)
2387 *reg_write(ctxt, i) = GET_SMSTATE(u32, smbase, 0x7fd0 + i * 4);
2388
2389 val = GET_SMSTATE(u32, smbase, 0x7fcc);
2390 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2391 val = GET_SMSTATE(u32, smbase, 0x7fc8);
2392 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2393
2394 selector = GET_SMSTATE(u32, smbase, 0x7fc4);
2395 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f64));
2396 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f60));
2397 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f5c));
2398 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);
2399
2400 selector = GET_SMSTATE(u32, smbase, 0x7fc0);
2401 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f80));
2402 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f7c));
2403 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f78));
2404 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);
2405
2406 dt.address = GET_SMSTATE(u32, smbase, 0x7f74);
2407 dt.size = GET_SMSTATE(u32, smbase, 0x7f70);
2408 ctxt->ops->set_gdt(ctxt, &dt);
2409
2410 dt.address = GET_SMSTATE(u32, smbase, 0x7f58);
2411 dt.size = GET_SMSTATE(u32, smbase, 0x7f54);
2412 ctxt->ops->set_idt(ctxt, &dt);
2413
2414 for (i = 0; i < 6; i++) {
2415 int r = rsm_load_seg_32(ctxt, smbase, i);
2416 if (r != X86EMUL_CONTINUE)
2417 return r;
2418 }
2419
2420 cr4 = GET_SMSTATE(u32, smbase, 0x7f14);
2421
2422 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7ef8));
2423
2424 return rsm_enter_protected_mode(ctxt, cr0, cr4);
2425}
2426
2427static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt, u64 smbase)
2428{
2429 struct desc_struct desc;
2430 struct desc_ptr dt;
2431 u64 val, cr0, cr4;
2432 u32 base3;
2433 u16 selector;
2434 int i, r;
2435
2436 for (i = 0; i < 16; i++)
2437 *reg_write(ctxt, i) = GET_SMSTATE(u64, smbase, 0x7ff8 - i * 8);
2438
2439 ctxt->_eip = GET_SMSTATE(u64, smbase, 0x7f78);
2440 ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7f70) | X86_EFLAGS_FIXED;
2441
2442 val = GET_SMSTATE(u32, smbase, 0x7f68);
2443 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2444 val = GET_SMSTATE(u32, smbase, 0x7f60);
2445 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2446
2447 cr0 = GET_SMSTATE(u64, smbase, 0x7f58);
2448 ctxt->ops->set_cr(ctxt, 3, GET_SMSTATE(u64, smbase, 0x7f50));
2449 cr4 = GET_SMSTATE(u64, smbase, 0x7f48);
2450 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7f00));
2451 val = GET_SMSTATE(u64, smbase, 0x7ed0);
2452 ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA);
2453
2454 selector = GET_SMSTATE(u32, smbase, 0x7e90);
2455 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e92) << 8);
2456 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e94));
2457 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e98));
2458 base3 = GET_SMSTATE(u32, smbase, 0x7e9c);
2459 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);
2460
2461 dt.size = GET_SMSTATE(u32, smbase, 0x7e84);
2462 dt.address = GET_SMSTATE(u64, smbase, 0x7e88);
2463 ctxt->ops->set_idt(ctxt, &dt);
2464
2465 selector = GET_SMSTATE(u32, smbase, 0x7e70);
2466 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e72) << 8);
2467 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e74));
2468 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e78));
2469 base3 = GET_SMSTATE(u32, smbase, 0x7e7c);
2470 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);
2471
2472 dt.size = GET_SMSTATE(u32, smbase, 0x7e64);
2473 dt.address = GET_SMSTATE(u64, smbase, 0x7e68);
2474 ctxt->ops->set_gdt(ctxt, &dt);
2475
2476 r = rsm_enter_protected_mode(ctxt, cr0, cr4);
2477 if (r != X86EMUL_CONTINUE)
2478 return r;
2479
2480 for (i = 0; i < 6; i++) {
2481 r = rsm_load_seg_64(ctxt, smbase, i);
2482 if (r != X86EMUL_CONTINUE)
2483 return r;
2484 }
2485
2486 return X86EMUL_CONTINUE;
2487}
2488
2489static int em_rsm(struct x86_emulate_ctxt *ctxt)
2490{
2491 unsigned long cr0, cr4, efer;
2492 u64 smbase;
2493 int ret;
2494
2495 if ((ctxt->emul_flags & X86EMUL_SMM_MASK) == 0)
2496 return emulate_ud(ctxt);
2497
2498
2499
2500
2501
2502
2503 cr4 = ctxt->ops->get_cr(ctxt, 4);
2504 if (emulator_has_longmode(ctxt)) {
2505 struct desc_struct cs_desc;
2506
2507
2508 if (cr4 & X86_CR4_PCIDE) {
2509 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2510 cr4 &= ~X86_CR4_PCIDE;
2511 }
2512
2513
2514 memset(&cs_desc, 0, sizeof(cs_desc));
2515 cs_desc.type = 0xb;
2516 cs_desc.s = cs_desc.g = cs_desc.p = 1;
2517 ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);
2518 }
2519
2520
2521 cr0 = ctxt->ops->get_cr(ctxt, 0);
2522 if (cr0 & X86_CR0_PE)
2523 ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
2524
2525
2526 if (cr4 & X86_CR4_PAE)
2527 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);
2528
2529
2530 efer = 0;
2531 ctxt->ops->set_msr(ctxt, MSR_EFER, efer);
2532
2533 smbase = ctxt->ops->get_smbase(ctxt);
2534 if (emulator_has_longmode(ctxt))
2535 ret = rsm_load_state_64(ctxt, smbase + 0x8000);
2536 else
2537 ret = rsm_load_state_32(ctxt, smbase + 0x8000);
2538
2539 if (ret != X86EMUL_CONTINUE) {
2540
2541 return X86EMUL_UNHANDLEABLE;
2542 }
2543
2544 if ((ctxt->emul_flags & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
2545 ctxt->ops->set_nmi_mask(ctxt, false);
2546
2547 ctxt->emul_flags &= ~X86EMUL_SMM_INSIDE_NMI_MASK;
2548 ctxt->emul_flags &= ~X86EMUL_SMM_MASK;
2549 return X86EMUL_CONTINUE;
2550}
2551
2552static void
2553setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2554 struct desc_struct *cs, struct desc_struct *ss)
2555{
2556 cs->l = 0;
2557 set_desc_base(cs, 0);
2558 cs->g = 1;
2559 set_desc_limit(cs, 0xfffff);
2560 cs->type = 0x0b;
2561 cs->s = 1;
2562 cs->dpl = 0;
2563 cs->p = 1;
2564 cs->d = 1;
2565 cs->avl = 0;
2566
2567 set_desc_base(ss, 0);
2568 set_desc_limit(ss, 0xfffff);
2569 ss->g = 1;
2570 ss->s = 1;
2571 ss->type = 0x03;
2572 ss->d = 1;
2573 ss->dpl = 0;
2574 ss->p = 1;
2575 ss->l = 0;
2576 ss->avl = 0;
2577}
2578
2579static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2580{
2581 u32 eax, ebx, ecx, edx;
2582
2583 eax = ecx = 0;
2584 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2585 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2586 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2587 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2588}
2589
2590static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2591{
2592 const struct x86_emulate_ops *ops = ctxt->ops;
2593 u32 eax, ebx, ecx, edx;
2594
2595
2596
2597
2598
2599 if (ctxt->mode == X86EMUL_MODE_PROT64)
2600 return true;
2601
2602 eax = 0x00000000;
2603 ecx = 0x00000000;
2604 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2605
2606
2607
2608
2609
2610
2611
2612
2613 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2614 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2615 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2616 return false;
2617
2618
2619 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2620 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2621 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2622 return true;
2623
2624
2625 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2626 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2627 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2628 return true;
2629
2630
2631 return false;
2632}
2633
2634static int em_syscall(struct x86_emulate_ctxt *ctxt)
2635{
2636 const struct x86_emulate_ops *ops = ctxt->ops;
2637 struct desc_struct cs, ss;
2638 u64 msr_data;
2639 u16 cs_sel, ss_sel;
2640 u64 efer = 0;
2641
2642
2643 if (ctxt->mode == X86EMUL_MODE_REAL ||
2644 ctxt->mode == X86EMUL_MODE_VM86)
2645 return emulate_ud(ctxt);
2646
2647 if (!(em_syscall_is_enabled(ctxt)))
2648 return emulate_ud(ctxt);
2649
2650 ops->get_msr(ctxt, MSR_EFER, &efer);
2651 setup_syscalls_segments(ctxt, &cs, &ss);
2652
2653 if (!(efer & EFER_SCE))
2654 return emulate_ud(ctxt);
2655
2656 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2657 msr_data >>= 32;
2658 cs_sel = (u16)(msr_data & 0xfffc);
2659 ss_sel = (u16)(msr_data + 8);
2660
2661 if (efer & EFER_LMA) {
2662 cs.d = 0;
2663 cs.l = 1;
2664 }
2665 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2666 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2667
2668 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2669 if (efer & EFER_LMA) {
2670#ifdef CONFIG_X86_64
2671 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2672
2673 ops->get_msr(ctxt,
2674 ctxt->mode == X86EMUL_MODE_PROT64 ?
2675 MSR_LSTAR : MSR_CSTAR, &msr_data);
2676 ctxt->_eip = msr_data;
2677
2678 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2679 ctxt->eflags &= ~msr_data;
2680 ctxt->eflags |= X86_EFLAGS_FIXED;
2681#endif
2682 } else {
2683
2684 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2685 ctxt->_eip = (u32)msr_data;
2686
2687 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2688 }
2689
2690 return X86EMUL_CONTINUE;
2691}
2692
2693static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2694{
2695 const struct x86_emulate_ops *ops = ctxt->ops;
2696 struct desc_struct cs, ss;
2697 u64 msr_data;
2698 u16 cs_sel, ss_sel;
2699 u64 efer = 0;
2700
2701 ops->get_msr(ctxt, MSR_EFER, &efer);
2702
2703 if (ctxt->mode == X86EMUL_MODE_REAL)
2704 return emulate_gp(ctxt, 0);
2705
2706
2707
2708
2709
2710 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2711 && !vendor_intel(ctxt))
2712 return emulate_ud(ctxt);
2713
2714
2715 if (ctxt->mode == X86EMUL_MODE_PROT64)
2716 return X86EMUL_UNHANDLEABLE;
2717
2718 setup_syscalls_segments(ctxt, &cs, &ss);
2719
2720 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2721 if ((msr_data & 0xfffc) == 0x0)
2722 return emulate_gp(ctxt, 0);
2723
2724 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2725 cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2726 ss_sel = cs_sel + 8;
2727 if (efer & EFER_LMA) {
2728 cs.d = 0;
2729 cs.l = 1;
2730 }
2731
2732 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2733 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2734
2735 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2736 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2737
2738 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2739 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2740 (u32)msr_data;
2741
2742 return X86EMUL_CONTINUE;
2743}
2744
2745static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2746{
2747 const struct x86_emulate_ops *ops = ctxt->ops;
2748 struct desc_struct cs, ss;
2749 u64 msr_data, rcx, rdx;
2750 int usermode;
2751 u16 cs_sel = 0, ss_sel = 0;
2752
2753
2754 if (ctxt->mode == X86EMUL_MODE_REAL ||
2755 ctxt->mode == X86EMUL_MODE_VM86)
2756 return emulate_gp(ctxt, 0);
2757
2758 setup_syscalls_segments(ctxt, &cs, &ss);
2759
2760 if ((ctxt->rex_prefix & 0x8) != 0x0)
2761 usermode = X86EMUL_MODE_PROT64;
2762 else
2763 usermode = X86EMUL_MODE_PROT32;
2764
2765 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2766 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2767
2768 cs.dpl = 3;
2769 ss.dpl = 3;
2770 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2771 switch (usermode) {
2772 case X86EMUL_MODE_PROT32:
2773 cs_sel = (u16)(msr_data + 16);
2774 if ((msr_data & 0xfffc) == 0x0)
2775 return emulate_gp(ctxt, 0);
2776 ss_sel = (u16)(msr_data + 24);
2777 rcx = (u32)rcx;
2778 rdx = (u32)rdx;
2779 break;
2780 case X86EMUL_MODE_PROT64:
2781 cs_sel = (u16)(msr_data + 32);
2782 if (msr_data == 0x0)
2783 return emulate_gp(ctxt, 0);
2784 ss_sel = cs_sel + 8;
2785 cs.d = 0;
2786 cs.l = 1;
2787 if (is_noncanonical_address(rcx) ||
2788 is_noncanonical_address(rdx))
2789 return emulate_gp(ctxt, 0);
2790 break;
2791 }
2792 cs_sel |= SEGMENT_RPL_MASK;
2793 ss_sel |= SEGMENT_RPL_MASK;
2794
2795 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2796 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2797
2798 ctxt->_eip = rdx;
2799 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2800
2801 return X86EMUL_CONTINUE;
2802}
2803
2804static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2805{
2806 int iopl;
2807 if (ctxt->mode == X86EMUL_MODE_REAL)
2808 return false;
2809 if (ctxt->mode == X86EMUL_MODE_VM86)
2810 return true;
2811 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2812 return ctxt->ops->cpl(ctxt) > iopl;
2813}
2814
2815static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2816 u16 port, u16 len)
2817{
2818 const struct x86_emulate_ops *ops = ctxt->ops;
2819 struct desc_struct tr_seg;
2820 u32 base3;
2821 int r;
2822 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2823 unsigned mask = (1 << len) - 1;
2824 unsigned long base;
2825
2826 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2827 if (!tr_seg.p)
2828 return false;
2829 if (desc_limit_scaled(&tr_seg) < 103)
2830 return false;
2831 base = get_desc_base(&tr_seg);
2832#ifdef CONFIG_X86_64
2833 base |= ((u64)base3) << 32;
2834#endif
2835 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2836 if (r != X86EMUL_CONTINUE)
2837 return false;
2838 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2839 return false;
2840 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2841 if (r != X86EMUL_CONTINUE)
2842 return false;
2843 if ((perm >> bit_idx) & mask)
2844 return false;
2845 return true;
2846}
2847
2848static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2849 u16 port, u16 len)
2850{
2851 if (ctxt->perm_ok)
2852 return true;
2853
2854 if (emulator_bad_iopl(ctxt))
2855 if (!emulator_io_port_access_allowed(ctxt, port, len))
2856 return false;
2857
2858 ctxt->perm_ok = true;
2859
2860 return true;
2861}
2862
2863static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
2864{
2865
2866
2867
2868
2869#ifdef CONFIG_X86_64
2870 if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
2871 return;
2872
2873 *reg_write(ctxt, VCPU_REGS_RCX) = 0;
2874
2875 switch (ctxt->b) {
2876 case 0xa4:
2877 case 0xa5:
2878 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
2879
2880 case 0xaa:
2881 case 0xab:
2882 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
2883 }
2884#endif
2885}
2886
2887static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2888 struct tss_segment_16 *tss)
2889{
2890 tss->ip = ctxt->_eip;
2891 tss->flag = ctxt->eflags;
2892 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2893 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2894 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2895 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2896 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2897 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2898 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2899 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2900
2901 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2902 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2903 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2904 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2905 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2906}
2907
2908static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2909 struct tss_segment_16 *tss)
2910{
2911 int ret;
2912 u8 cpl;
2913
2914 ctxt->_eip = tss->ip;
2915 ctxt->eflags = tss->flag | 2;
2916 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2917 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2918 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2919 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2920 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2921 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2922 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2923 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2924
2925
2926
2927
2928
2929 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2930 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2931 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2932 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2933 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2934
2935 cpl = tss->cs & 3;
2936
2937
2938
2939
2940
2941 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
2942 X86_TRANSFER_TASK_SWITCH, NULL);
2943 if (ret != X86EMUL_CONTINUE)
2944 return ret;
2945 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
2946 X86_TRANSFER_TASK_SWITCH, NULL);
2947 if (ret != X86EMUL_CONTINUE)
2948 return ret;
2949 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
2950 X86_TRANSFER_TASK_SWITCH, NULL);
2951 if (ret != X86EMUL_CONTINUE)
2952 return ret;
2953 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
2954 X86_TRANSFER_TASK_SWITCH, NULL);
2955 if (ret != X86EMUL_CONTINUE)
2956 return ret;
2957 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
2958 X86_TRANSFER_TASK_SWITCH, NULL);
2959 if (ret != X86EMUL_CONTINUE)
2960 return ret;
2961
2962 return X86EMUL_CONTINUE;
2963}
2964
2965static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2966 u16 tss_selector, u16 old_tss_sel,
2967 ulong old_tss_base, struct desc_struct *new_desc)
2968{
2969 const struct x86_emulate_ops *ops = ctxt->ops;
2970 struct tss_segment_16 tss_seg;
2971 int ret;
2972 u32 new_tss_base = get_desc_base(new_desc);
2973
2974 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2975 &ctxt->exception);
2976 if (ret != X86EMUL_CONTINUE)
2977 return ret;
2978
2979 save_state_to_tss16(ctxt, &tss_seg);
2980
2981 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2982 &ctxt->exception);
2983 if (ret != X86EMUL_CONTINUE)
2984 return ret;
2985
2986 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2987 &ctxt->exception);
2988 if (ret != X86EMUL_CONTINUE)
2989 return ret;
2990
2991 if (old_tss_sel != 0xffff) {
2992 tss_seg.prev_task_link = old_tss_sel;
2993
2994 ret = ops->write_std(ctxt, new_tss_base,
2995 &tss_seg.prev_task_link,
2996 sizeof tss_seg.prev_task_link,
2997 &ctxt->exception);
2998 if (ret != X86EMUL_CONTINUE)
2999 return ret;
3000 }
3001
3002 return load_state_from_tss16(ctxt, &tss_seg);
3003}
3004
3005static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
3006 struct tss_segment_32 *tss)
3007{
3008
3009 tss->eip = ctxt->_eip;
3010 tss->eflags = ctxt->eflags;
3011 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
3012 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
3013 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
3014 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
3015 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
3016 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
3017 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
3018 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
3019
3020 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3021 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3022 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3023 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3024 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
3025 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
3026}
3027
3028static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
3029 struct tss_segment_32 *tss)
3030{
3031 int ret;
3032 u8 cpl;
3033
3034 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
3035 return emulate_gp(ctxt, 0);
3036 ctxt->_eip = tss->eip;
3037 ctxt->eflags = tss->eflags | 2;
3038
3039
3040 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
3041 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
3042 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
3043 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
3044 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
3045 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
3046 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
3047 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
3048
3049
3050
3051
3052
3053
3054 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
3055 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3056 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3057 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3058 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3059 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
3060 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
3061
3062
3063
3064
3065
3066
3067 if (ctxt->eflags & X86_EFLAGS_VM) {
3068 ctxt->mode = X86EMUL_MODE_VM86;
3069 cpl = 3;
3070 } else {
3071 ctxt->mode = X86EMUL_MODE_PROT32;
3072 cpl = tss->cs & 3;
3073 }
3074
3075
3076
3077
3078
3079 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
3080 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
3081 if (ret != X86EMUL_CONTINUE)
3082 return ret;
3083 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3084 X86_TRANSFER_TASK_SWITCH, NULL);
3085 if (ret != X86EMUL_CONTINUE)
3086 return ret;
3087 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3088 X86_TRANSFER_TASK_SWITCH, NULL);
3089 if (ret != X86EMUL_CONTINUE)
3090 return ret;
3091 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3092 X86_TRANSFER_TASK_SWITCH, NULL);
3093 if (ret != X86EMUL_CONTINUE)
3094 return ret;
3095 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3096 X86_TRANSFER_TASK_SWITCH, NULL);
3097 if (ret != X86EMUL_CONTINUE)
3098 return ret;
3099 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
3100 X86_TRANSFER_TASK_SWITCH, NULL);
3101 if (ret != X86EMUL_CONTINUE)
3102 return ret;
3103 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
3104 X86_TRANSFER_TASK_SWITCH, NULL);
3105
3106 return ret;
3107}
3108
3109static int task_switch_32(struct x86_emulate_ctxt *ctxt,
3110 u16 tss_selector, u16 old_tss_sel,
3111 ulong old_tss_base, struct desc_struct *new_desc)
3112{
3113 const struct x86_emulate_ops *ops = ctxt->ops;
3114 struct tss_segment_32 tss_seg;
3115 int ret;
3116 u32 new_tss_base = get_desc_base(new_desc);
3117 u32 eip_offset = offsetof(struct tss_segment_32, eip);
3118 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3119
3120 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
3121 &ctxt->exception);
3122 if (ret != X86EMUL_CONTINUE)
3123 return ret;
3124
3125 save_state_to_tss32(ctxt, &tss_seg);
3126
3127
3128 ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
3129 ldt_sel_offset - eip_offset, &ctxt->exception);
3130 if (ret != X86EMUL_CONTINUE)
3131 return ret;
3132
3133 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
3134 &ctxt->exception);
3135 if (ret != X86EMUL_CONTINUE)
3136 return ret;
3137
3138 if (old_tss_sel != 0xffff) {
3139 tss_seg.prev_task_link = old_tss_sel;
3140
3141 ret = ops->write_std(ctxt, new_tss_base,
3142 &tss_seg.prev_task_link,
3143 sizeof tss_seg.prev_task_link,
3144 &ctxt->exception);
3145 if (ret != X86EMUL_CONTINUE)
3146 return ret;
3147 }
3148
3149 return load_state_from_tss32(ctxt, &tss_seg);
3150}
3151
3152static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
3153 u16 tss_selector, int idt_index, int reason,
3154 bool has_error_code, u32 error_code)
3155{
3156 const struct x86_emulate_ops *ops = ctxt->ops;
3157 struct desc_struct curr_tss_desc, next_tss_desc;
3158 int ret;
3159 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
3160 ulong old_tss_base =
3161 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
3162 u32 desc_limit;
3163 ulong desc_addr, dr7;
3164
3165
3166
3167 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
3168 if (ret != X86EMUL_CONTINUE)
3169 return ret;
3170 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
3171 if (ret != X86EMUL_CONTINUE)
3172 return ret;
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184 if (reason == TASK_SWITCH_GATE) {
3185 if (idt_index != -1) {
3186
3187 struct desc_struct task_gate_desc;
3188 int dpl;
3189
3190 ret = read_interrupt_descriptor(ctxt, idt_index,
3191 &task_gate_desc);
3192 if (ret != X86EMUL_CONTINUE)
3193 return ret;
3194
3195 dpl = task_gate_desc.dpl;
3196 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
3197 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
3198 }
3199 }
3200
3201 desc_limit = desc_limit_scaled(&next_tss_desc);
3202 if (!next_tss_desc.p ||
3203 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
3204 desc_limit < 0x2b)) {
3205 return emulate_ts(ctxt, tss_selector & 0xfffc);
3206 }
3207
3208 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
3209 curr_tss_desc.type &= ~(1 << 1);
3210 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
3211 }
3212
3213 if (reason == TASK_SWITCH_IRET)
3214 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
3215
3216
3217
3218 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
3219 old_tss_sel = 0xffff;
3220
3221 if (next_tss_desc.type & 8)
3222 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
3223 old_tss_base, &next_tss_desc);
3224 else
3225 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
3226 old_tss_base, &next_tss_desc);
3227 if (ret != X86EMUL_CONTINUE)
3228 return ret;
3229
3230 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
3231 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
3232
3233 if (reason != TASK_SWITCH_IRET) {
3234 next_tss_desc.type |= (1 << 1);
3235 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
3236 }
3237
3238 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
3239 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
3240
3241 if (has_error_code) {
3242 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
3243 ctxt->lock_prefix = 0;
3244 ctxt->src.val = (unsigned long) error_code;
3245 ret = em_push(ctxt);
3246 }
3247
3248 ops->get_dr(ctxt, 7, &dr7);
3249 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));
3250
3251 return ret;
3252}
3253
3254int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
3255 u16 tss_selector, int idt_index, int reason,
3256 bool has_error_code, u32 error_code)
3257{
3258 int rc;
3259
3260 invalidate_registers(ctxt);
3261 ctxt->_eip = ctxt->eip;
3262 ctxt->dst.type = OP_NONE;
3263
3264 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
3265 has_error_code, error_code);
3266
3267 if (rc == X86EMUL_CONTINUE) {
3268 ctxt->eip = ctxt->_eip;
3269 writeback_registers(ctxt);
3270 }
3271
3272 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3273}
3274
3275static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
3276 struct operand *op)
3277{
3278 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
3279
3280 register_address_increment(ctxt, reg, df * op->bytes);
3281 op->addr.mem.ea = register_address(ctxt, reg);
3282}
3283
3284static int em_das(struct x86_emulate_ctxt *ctxt)
3285{
3286 u8 al, old_al;
3287 bool af, cf, old_cf;
3288
3289 cf = ctxt->eflags & X86_EFLAGS_CF;
3290 al = ctxt->dst.val;
3291
3292 old_al = al;
3293 old_cf = cf;
3294 cf = false;
3295 af = ctxt->eflags & X86_EFLAGS_AF;
3296 if ((al & 0x0f) > 9 || af) {
3297 al -= 6;
3298 cf = old_cf | (al >= 250);
3299 af = true;
3300 } else {
3301 af = false;
3302 }
3303 if (old_al > 0x99 || old_cf) {
3304 al -= 0x60;
3305 cf = true;
3306 }
3307
3308 ctxt->dst.val = al;
3309
3310 ctxt->src.type = OP_IMM;
3311 ctxt->src.val = 0;
3312 ctxt->src.bytes = 1;
3313 fastop(ctxt, em_or);
3314 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3315 if (cf)
3316 ctxt->eflags |= X86_EFLAGS_CF;
3317 if (af)
3318 ctxt->eflags |= X86_EFLAGS_AF;
3319 return X86EMUL_CONTINUE;
3320}
3321
3322static int em_aam(struct x86_emulate_ctxt *ctxt)
3323{
3324 u8 al, ah;
3325
3326 if (ctxt->src.val == 0)
3327 return emulate_de(ctxt);
3328
3329 al = ctxt->dst.val & 0xff;
3330 ah = al / ctxt->src.val;
3331 al %= ctxt->src.val;
3332
3333 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3334
3335
3336 ctxt->src.type = OP_IMM;
3337 ctxt->src.val = 0;
3338 ctxt->src.bytes = 1;
3339 fastop(ctxt, em_or);
3340
3341 return X86EMUL_CONTINUE;
3342}
3343
3344static int em_aad(struct x86_emulate_ctxt *ctxt)
3345{
3346 u8 al = ctxt->dst.val & 0xff;
3347 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3348
3349 al = (al + (ah * ctxt->src.val)) & 0xff;
3350
3351 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3352
3353
3354 ctxt->src.type = OP_IMM;
3355 ctxt->src.val = 0;
3356 ctxt->src.bytes = 1;
3357 fastop(ctxt, em_or);
3358
3359 return X86EMUL_CONTINUE;
3360}
3361
3362static int em_call(struct x86_emulate_ctxt *ctxt)
3363{
3364 int rc;
3365 long rel = ctxt->src.val;
3366
3367 ctxt->src.val = (unsigned long)ctxt->_eip;
3368 rc = jmp_rel(ctxt, rel);
3369 if (rc != X86EMUL_CONTINUE)
3370 return rc;
3371 return em_push(ctxt);
3372}
3373
3374static int em_call_far(struct x86_emulate_ctxt *ctxt)
3375{
3376 u16 sel, old_cs;
3377 ulong old_eip;
3378 int rc;
3379 struct desc_struct old_desc, new_desc;
3380 const struct x86_emulate_ops *ops = ctxt->ops;
3381 int cpl = ctxt->ops->cpl(ctxt);
3382 enum x86emul_mode prev_mode = ctxt->mode;
3383
3384 old_eip = ctxt->_eip;
3385 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3386
3387 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3388 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3389 X86_TRANSFER_CALL_JMP, &new_desc);
3390 if (rc != X86EMUL_CONTINUE)
3391 return rc;
3392
3393 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3394 if (rc != X86EMUL_CONTINUE)
3395 goto fail;
3396
3397 ctxt->src.val = old_cs;
3398 rc = em_push(ctxt);
3399 if (rc != X86EMUL_CONTINUE)
3400 goto fail;
3401
3402 ctxt->src.val = old_eip;
3403 rc = em_push(ctxt);
3404
3405
3406 if (rc != X86EMUL_CONTINUE) {
3407 pr_warn_once("faulting far call emulation tainted memory\n");
3408 goto fail;
3409 }
3410 return rc;
3411fail:
3412 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3413 ctxt->mode = prev_mode;
3414 return rc;
3415
3416}
3417
3418static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3419{
3420 int rc;
3421 unsigned long eip;
3422
3423 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3424 if (rc != X86EMUL_CONTINUE)
3425 return rc;
3426 rc = assign_eip_near(ctxt, eip);
3427 if (rc != X86EMUL_CONTINUE)
3428 return rc;
3429 rsp_increment(ctxt, ctxt->src.val);
3430 return X86EMUL_CONTINUE;
3431}
3432
3433static int em_xchg(struct x86_emulate_ctxt *ctxt)
3434{
3435
3436 ctxt->src.val = ctxt->dst.val;
3437 write_register_operand(&ctxt->src);
3438
3439
3440 ctxt->dst.val = ctxt->src.orig_val;
3441 ctxt->lock_prefix = 1;
3442 return X86EMUL_CONTINUE;
3443}
3444
3445static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3446{
3447 ctxt->dst.val = ctxt->src2.val;
3448 return fastop(ctxt, em_imul);
3449}
3450
3451static int em_cwd(struct x86_emulate_ctxt *ctxt)
3452{
3453 ctxt->dst.type = OP_REG;
3454 ctxt->dst.bytes = ctxt->src.bytes;
3455 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3456 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3457
3458 return X86EMUL_CONTINUE;
3459}
3460
3461static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3462{
3463 u64 tsc = 0;
3464
3465 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3466 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3467 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3468 return X86EMUL_CONTINUE;
3469}
3470
3471static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3472{
3473 u64 pmc;
3474
3475 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3476 return emulate_gp(ctxt, 0);
3477 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3478 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3479 return X86EMUL_CONTINUE;
3480}
3481
3482static int em_mov(struct x86_emulate_ctxt *ctxt)
3483{
3484 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3485 return X86EMUL_CONTINUE;
3486}
3487
3488#define FFL(x) bit(X86_FEATURE_##x)
3489
3490static int em_movbe(struct x86_emulate_ctxt *ctxt)
3491{
3492 u32 ebx, ecx, edx, eax = 1;
3493 u16 tmp;
3494
3495
3496
3497
3498 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3499 if (!(ecx & FFL(MOVBE)))
3500 return emulate_ud(ctxt);
3501
3502 switch (ctxt->op_bytes) {
3503 case 2:
3504
3505
3506
3507
3508
3509
3510
3511
3512 tmp = (u16)ctxt->src.val;
3513 ctxt->dst.val &= ~0xffffUL;
3514 ctxt->dst.val |= (unsigned long)swab16(tmp);
3515 break;
3516 case 4:
3517 ctxt->dst.val = swab32((u32)ctxt->src.val);
3518 break;
3519 case 8:
3520 ctxt->dst.val = swab64(ctxt->src.val);
3521 break;
3522 default:
3523 BUG();
3524 }
3525 return X86EMUL_CONTINUE;
3526}
3527
3528static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3529{
3530 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3531 return emulate_gp(ctxt, 0);
3532
3533
3534 ctxt->dst.type = OP_NONE;
3535 return X86EMUL_CONTINUE;
3536}
3537
3538static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3539{
3540 unsigned long val;
3541
3542 if (ctxt->mode == X86EMUL_MODE_PROT64)
3543 val = ctxt->src.val & ~0ULL;
3544 else
3545 val = ctxt->src.val & ~0U;
3546
3547
3548 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3549 return emulate_gp(ctxt, 0);
3550
3551
3552 ctxt->dst.type = OP_NONE;
3553 return X86EMUL_CONTINUE;
3554}
3555
3556static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3557{
3558 u64 msr_data;
3559
3560 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3561 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3562 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3563 return emulate_gp(ctxt, 0);
3564
3565 return X86EMUL_CONTINUE;
3566}
3567
3568static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3569{
3570 u64 msr_data;
3571
3572 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3573 return emulate_gp(ctxt, 0);
3574
3575 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3576 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3577 return X86EMUL_CONTINUE;
3578}
3579
3580static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3581{
3582 if (ctxt->modrm_reg > VCPU_SREG_GS)
3583 return emulate_ud(ctxt);
3584
3585 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3586 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3587 ctxt->dst.bytes = 2;
3588 return X86EMUL_CONTINUE;
3589}
3590
3591static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3592{
3593 u16 sel = ctxt->src.val;
3594
3595 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3596 return emulate_ud(ctxt);
3597
3598 if (ctxt->modrm_reg == VCPU_SREG_SS)
3599 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3600
3601
3602 ctxt->dst.type = OP_NONE;
3603 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3604}
3605
3606static int em_lldt(struct x86_emulate_ctxt *ctxt)
3607{
3608 u16 sel = ctxt->src.val;
3609
3610
3611 ctxt->dst.type = OP_NONE;
3612 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3613}
3614
3615static int em_ltr(struct x86_emulate_ctxt *ctxt)
3616{
3617 u16 sel = ctxt->src.val;
3618
3619
3620 ctxt->dst.type = OP_NONE;
3621 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3622}
3623
3624static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3625{
3626 int rc;
3627 ulong linear;
3628
3629 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3630 if (rc == X86EMUL_CONTINUE)
3631 ctxt->ops->invlpg(ctxt, linear);
3632
3633 ctxt->dst.type = OP_NONE;
3634 return X86EMUL_CONTINUE;
3635}
3636
3637static int em_clts(struct x86_emulate_ctxt *ctxt)
3638{
3639 ulong cr0;
3640
3641 cr0 = ctxt->ops->get_cr(ctxt, 0);
3642 cr0 &= ~X86_CR0_TS;
3643 ctxt->ops->set_cr(ctxt, 0, cr0);
3644 return X86EMUL_CONTINUE;
3645}
3646
3647static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3648{
3649 int rc = ctxt->ops->fix_hypercall(ctxt);
3650
3651 if (rc != X86EMUL_CONTINUE)
3652 return rc;
3653
3654
3655 ctxt->_eip = ctxt->eip;
3656
3657 ctxt->dst.type = OP_NONE;
3658 return X86EMUL_CONTINUE;
3659}
3660
3661static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3662 void (*get)(struct x86_emulate_ctxt *ctxt,
3663 struct desc_ptr *ptr))
3664{
3665 struct desc_ptr desc_ptr;
3666
3667 if (ctxt->mode == X86EMUL_MODE_PROT64)
3668 ctxt->op_bytes = 8;
3669 get(ctxt, &desc_ptr);
3670 if (ctxt->op_bytes == 2) {
3671 ctxt->op_bytes = 4;
3672 desc_ptr.address &= 0x00ffffff;
3673 }
3674
3675 ctxt->dst.type = OP_NONE;
3676 return segmented_write(ctxt, ctxt->dst.addr.mem,
3677 &desc_ptr, 2 + ctxt->op_bytes);
3678}
3679
3680static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3681{
3682 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3683}
3684
3685static int em_sidt(struct x86_emulate_ctxt *ctxt)
3686{
3687 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3688}
3689
3690static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3691{
3692 struct desc_ptr desc_ptr;
3693 int rc;
3694
3695 if (ctxt->mode == X86EMUL_MODE_PROT64)
3696 ctxt->op_bytes = 8;
3697 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3698 &desc_ptr.size, &desc_ptr.address,
3699 ctxt->op_bytes);
3700 if (rc != X86EMUL_CONTINUE)
3701 return rc;
3702 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3703 is_noncanonical_address(desc_ptr.address))
3704 return emulate_gp(ctxt, 0);
3705 if (lgdt)
3706 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3707 else
3708 ctxt->ops->set_idt(ctxt, &desc_ptr);
3709
3710 ctxt->dst.type = OP_NONE;
3711 return X86EMUL_CONTINUE;
3712}
3713
3714static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3715{
3716 return em_lgdt_lidt(ctxt, true);
3717}
3718
3719static int em_lidt(struct x86_emulate_ctxt *ctxt)
3720{
3721 return em_lgdt_lidt(ctxt, false);
3722}
3723
3724static int em_smsw(struct x86_emulate_ctxt *ctxt)
3725{
3726 if (ctxt->dst.type == OP_MEM)
3727 ctxt->dst.bytes = 2;
3728 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3729 return X86EMUL_CONTINUE;
3730}
3731
3732static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3733{
3734 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3735 | (ctxt->src.val & 0x0f));
3736 ctxt->dst.type = OP_NONE;
3737 return X86EMUL_CONTINUE;
3738}
3739
3740static int em_loop(struct x86_emulate_ctxt *ctxt)
3741{
3742 int rc = X86EMUL_CONTINUE;
3743
3744 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3745 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3746 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3747 rc = jmp_rel(ctxt, ctxt->src.val);
3748
3749 return rc;
3750}
3751
3752static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3753{
3754 int rc = X86EMUL_CONTINUE;
3755
3756 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3757 rc = jmp_rel(ctxt, ctxt->src.val);
3758
3759 return rc;
3760}
3761
3762static int em_in(struct x86_emulate_ctxt *ctxt)
3763{
3764 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3765 &ctxt->dst.val))
3766 return X86EMUL_IO_NEEDED;
3767
3768 return X86EMUL_CONTINUE;
3769}
3770
3771static int em_out(struct x86_emulate_ctxt *ctxt)
3772{
3773 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3774 &ctxt->src.val, 1);
3775
3776 ctxt->dst.type = OP_NONE;
3777 return X86EMUL_CONTINUE;
3778}
3779
3780static int em_cli(struct x86_emulate_ctxt *ctxt)
3781{
3782 if (emulator_bad_iopl(ctxt))
3783 return emulate_gp(ctxt, 0);
3784
3785 ctxt->eflags &= ~X86_EFLAGS_IF;
3786 return X86EMUL_CONTINUE;
3787}
3788
3789static int em_sti(struct x86_emulate_ctxt *ctxt)
3790{
3791 if (emulator_bad_iopl(ctxt))
3792 return emulate_gp(ctxt, 0);
3793
3794 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3795 ctxt->eflags |= X86_EFLAGS_IF;
3796 return X86EMUL_CONTINUE;
3797}
3798
3799static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3800{
3801 u32 eax, ebx, ecx, edx;
3802
3803 eax = reg_read(ctxt, VCPU_REGS_RAX);
3804 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3805 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3806 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3807 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3808 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3809 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3810 return X86EMUL_CONTINUE;
3811}
3812
3813static int em_sahf(struct x86_emulate_ctxt *ctxt)
3814{
3815 u32 flags;
3816
3817 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
3818 X86_EFLAGS_SF;
3819 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3820
3821 ctxt->eflags &= ~0xffUL;
3822 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3823 return X86EMUL_CONTINUE;
3824}
3825
3826static int em_lahf(struct x86_emulate_ctxt *ctxt)
3827{
3828 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3829 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3830 return X86EMUL_CONTINUE;
3831}
3832
3833static int em_bswap(struct x86_emulate_ctxt *ctxt)
3834{
3835 switch (ctxt->op_bytes) {
3836#ifdef CONFIG_X86_64
3837 case 8:
3838 asm("bswap %0" : "+r"(ctxt->dst.val));
3839 break;
3840#endif
3841 default:
3842 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3843 break;
3844 }
3845 return X86EMUL_CONTINUE;
3846}
3847
3848static int em_clflush(struct x86_emulate_ctxt *ctxt)
3849{
3850
3851 return X86EMUL_CONTINUE;
3852}
3853
3854static int em_movsxd(struct x86_emulate_ctxt *ctxt)
3855{
3856 ctxt->dst.val = (s32) ctxt->src.val;
3857 return X86EMUL_CONTINUE;
3858}
3859
3860static bool valid_cr(int nr)
3861{
3862 switch (nr) {
3863 case 0:
3864 case 2 ... 4:
3865 case 8:
3866 return true;
3867 default:
3868 return false;
3869 }
3870}
3871
3872static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3873{
3874 if (!valid_cr(ctxt->modrm_reg))
3875 return emulate_ud(ctxt);
3876
3877 return X86EMUL_CONTINUE;
3878}
3879
3880static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3881{
3882 u64 new_val = ctxt->src.val64;
3883 int cr = ctxt->modrm_reg;
3884 u64 efer = 0;
3885
3886 static u64 cr_reserved_bits[] = {
3887 0xffffffff00000000ULL,
3888 0, 0, 0,
3889 CR4_RESERVED_BITS,
3890 0, 0, 0,
3891 CR8_RESERVED_BITS,
3892 };
3893
3894 if (!valid_cr(cr))
3895 return emulate_ud(ctxt);
3896
3897 if (new_val & cr_reserved_bits[cr])
3898 return emulate_gp(ctxt, 0);
3899
3900 switch (cr) {
3901 case 0: {
3902 u64 cr4;
3903 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3904 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3905 return emulate_gp(ctxt, 0);
3906
3907 cr4 = ctxt->ops->get_cr(ctxt, 4);
3908 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3909
3910 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3911 !(cr4 & X86_CR4_PAE))
3912 return emulate_gp(ctxt, 0);
3913
3914 break;
3915 }
3916 case 3: {
3917 u64 rsvd = 0;
3918
3919 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3920 if (efer & EFER_LMA)
3921 rsvd = CR3_L_MODE_RESERVED_BITS & ~CR3_PCID_INVD;
3922
3923 if (new_val & rsvd)
3924 return emulate_gp(ctxt, 0);
3925
3926 break;
3927 }
3928 case 4: {
3929 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3930
3931 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3932 return emulate_gp(ctxt, 0);
3933
3934 break;
3935 }
3936 }
3937
3938 return X86EMUL_CONTINUE;
3939}
3940
3941static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3942{
3943 unsigned long dr7;
3944
3945 ctxt->ops->get_dr(ctxt, 7, &dr7);
3946
3947
3948 return dr7 & (1 << 13);
3949}
3950
3951static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3952{
3953 int dr = ctxt->modrm_reg;
3954 u64 cr4;
3955
3956 if (dr > 7)
3957 return emulate_ud(ctxt);
3958
3959 cr4 = ctxt->ops->get_cr(ctxt, 4);
3960 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3961 return emulate_ud(ctxt);
3962
3963 if (check_dr7_gd(ctxt)) {
3964 ulong dr6;
3965
3966 ctxt->ops->get_dr(ctxt, 6, &dr6);
3967 dr6 &= ~15;
3968 dr6 |= DR6_BD | DR6_RTM;
3969 ctxt->ops->set_dr(ctxt, 6, dr6);
3970 return emulate_db(ctxt);
3971 }
3972
3973 return X86EMUL_CONTINUE;
3974}
3975
3976static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3977{
3978 u64 new_val = ctxt->src.val64;
3979 int dr = ctxt->modrm_reg;
3980
3981 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3982 return emulate_gp(ctxt, 0);
3983
3984 return check_dr_read(ctxt);
3985}
3986
3987static int check_svme(struct x86_emulate_ctxt *ctxt)
3988{
3989 u64 efer;
3990
3991 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3992
3993 if (!(efer & EFER_SVME))
3994 return emulate_ud(ctxt);
3995
3996 return X86EMUL_CONTINUE;
3997}
3998
3999static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
4000{
4001 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
4002
4003
4004 if (rax & 0xffff000000000000ULL)
4005 return emulate_gp(ctxt, 0);
4006
4007 return check_svme(ctxt);
4008}
4009
4010static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
4011{
4012 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4013
4014 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
4015 return emulate_ud(ctxt);
4016
4017 return X86EMUL_CONTINUE;
4018}
4019
4020static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
4021{
4022 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4023 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
4024
4025 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
4026 ctxt->ops->check_pmc(ctxt, rcx))
4027 return emulate_gp(ctxt, 0);
4028
4029 return X86EMUL_CONTINUE;
4030}
4031
4032static int check_perm_in(struct x86_emulate_ctxt *ctxt)
4033{
4034 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
4035 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
4036 return emulate_gp(ctxt, 0);
4037
4038 return X86EMUL_CONTINUE;
4039}
4040
4041static int check_perm_out(struct x86_emulate_ctxt *ctxt)
4042{
4043 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
4044 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
4045 return emulate_gp(ctxt, 0);
4046
4047 return X86EMUL_CONTINUE;
4048}
4049
4050#define D(_y) { .flags = (_y) }
4051#define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4052#define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4053 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4054#define N D(NotImpl)
4055#define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4056#define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4057#define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4058#define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4059#define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4060#define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4061#define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4062#define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4063#define II(_f, _e, _i) \
4064 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4065#define IIP(_f, _e, _i, _p) \
4066 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4067 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4068#define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4069
4070#define D2bv(_f) D((_f) | ByteOp), D(_f)
4071#define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4072#define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4073#define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4074#define I2bvIP(_f, _e, _i, _p) \
4075 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4076
4077#define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4078 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4079 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4080
4081static const struct opcode group7_rm0[] = {
4082 N,
4083 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
4084 N, N, N, N, N, N,
4085};
4086
4087static const struct opcode group7_rm1[] = {
4088 DI(SrcNone | Priv, monitor),
4089 DI(SrcNone | Priv, mwait),
4090 N, N, N, N, N, N,
4091};
4092
4093static const struct opcode group7_rm3[] = {
4094 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
4095 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
4096 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
4097 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
4098 DIP(SrcNone | Prot | Priv, stgi, check_svme),
4099 DIP(SrcNone | Prot | Priv, clgi, check_svme),
4100 DIP(SrcNone | Prot | Priv, skinit, check_svme),
4101 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
4102};
4103
4104static const struct opcode group7_rm7[] = {
4105 N,
4106 DIP(SrcNone, rdtscp, check_rdtsc),
4107 N, N, N, N, N, N,
4108};
4109
4110static const struct opcode group1[] = {
4111 F(Lock, em_add),
4112 F(Lock | PageTable, em_or),
4113 F(Lock, em_adc),
4114 F(Lock, em_sbb),
4115 F(Lock | PageTable, em_and),
4116 F(Lock, em_sub),
4117 F(Lock, em_xor),
4118 F(NoWrite, em_cmp),
4119};
4120
4121static const struct opcode group1A[] = {
4122 I(DstMem | SrcNone | Mov | Stack | IncSP, em_pop), N, N, N, N, N, N, N,
4123};
4124
4125static const struct opcode group2[] = {
4126 F(DstMem | ModRM, em_rol),
4127 F(DstMem | ModRM, em_ror),
4128 F(DstMem | ModRM, em_rcl),
4129 F(DstMem | ModRM, em_rcr),
4130 F(DstMem | ModRM, em_shl),
4131 F(DstMem | ModRM, em_shr),
4132 F(DstMem | ModRM, em_shl),
4133 F(DstMem | ModRM, em_sar),
4134};
4135
4136static const struct opcode group3[] = {
4137 F(DstMem | SrcImm | NoWrite, em_test),
4138 F(DstMem | SrcImm | NoWrite, em_test),
4139 F(DstMem | SrcNone | Lock, em_not),
4140 F(DstMem | SrcNone | Lock, em_neg),
4141 F(DstXacc | Src2Mem, em_mul_ex),
4142 F(DstXacc | Src2Mem, em_imul_ex),
4143 F(DstXacc | Src2Mem, em_div_ex),
4144 F(DstXacc | Src2Mem, em_idiv_ex),
4145};
4146
4147static const struct opcode group4[] = {
4148 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
4149 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
4150 N, N, N, N, N, N,
4151};
4152
4153static const struct opcode group5[] = {
4154 F(DstMem | SrcNone | Lock, em_inc),
4155 F(DstMem | SrcNone | Lock, em_dec),
4156 I(SrcMem | NearBranch, em_call_near_abs),
4157 I(SrcMemFAddr | ImplicitOps, em_call_far),
4158 I(SrcMem | NearBranch, em_jmp_abs),
4159 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
4160 I(SrcMem | Stack, em_push), D(Undefined),
4161};
4162
4163static const struct opcode group6[] = {
4164 DI(Prot | DstMem, sldt),
4165 DI(Prot | DstMem, str),
4166 II(Prot | Priv | SrcMem16, em_lldt, lldt),
4167 II(Prot | Priv | SrcMem16, em_ltr, ltr),
4168 N, N, N, N,
4169};
4170
4171static const struct group_dual group7 = { {
4172 II(Mov | DstMem, em_sgdt, sgdt),
4173 II(Mov | DstMem, em_sidt, sidt),
4174 II(SrcMem | Priv, em_lgdt, lgdt),
4175 II(SrcMem | Priv, em_lidt, lidt),
4176 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4177 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4178 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
4179}, {
4180 EXT(0, group7_rm0),
4181 EXT(0, group7_rm1),
4182 N, EXT(0, group7_rm3),
4183 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4184 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4185 EXT(0, group7_rm7),
4186} };
4187
4188static const struct opcode group8[] = {
4189 N, N, N, N,
4190 F(DstMem | SrcImmByte | NoWrite, em_bt),
4191 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
4192 F(DstMem | SrcImmByte | Lock, em_btr),
4193 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
4194};
4195
4196static const struct group_dual group9 = { {
4197 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
4198}, {
4199 N, N, N, N, N, N, N, N,
4200} };
4201
4202static const struct opcode group11[] = {
4203 I(DstMem | SrcImm | Mov | PageTable, em_mov),
4204 X7(D(Undefined)),
4205};
4206
4207static const struct gprefix pfx_0f_ae_7 = {
4208 I(SrcMem | ByteOp, em_clflush), N, N, N,
4209};
4210
4211static const struct group_dual group15 = { {
4212 N, N, N, N, N, N, N, GP(0, &pfx_0f_ae_7),
4213}, {
4214 N, N, N, N, N, N, N, N,
4215} };
4216
4217static const struct gprefix pfx_0f_6f_0f_7f = {
4218 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
4219};
4220
4221static const struct instr_dual instr_dual_0f_2b = {
4222 I(0, em_mov), N
4223};
4224
4225static const struct gprefix pfx_0f_2b = {
4226 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
4227};
4228
4229static const struct gprefix pfx_0f_28_0f_29 = {
4230 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4231};
4232
4233static const struct gprefix pfx_0f_e7 = {
4234 N, I(Sse, em_mov), N, N,
4235};
4236
4237static const struct escape escape_d9 = { {
4238 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
4239}, {
4240
4241 N, N, N, N, N, N, N, N,
4242
4243 N, N, N, N, N, N, N, N,
4244
4245 N, N, N, N, N, N, N, N,
4246
4247 N, N, N, N, N, N, N, N,
4248
4249 N, N, N, N, N, N, N, N,
4250
4251 N, N, N, N, N, N, N, N,
4252
4253 N, N, N, N, N, N, N, N,
4254
4255 N, N, N, N, N, N, N, N,
4256} };
4257
4258static const struct escape escape_db = { {
4259 N, N, N, N, N, N, N, N,
4260}, {
4261
4262 N, N, N, N, N, N, N, N,
4263
4264 N, N, N, N, N, N, N, N,
4265
4266 N, N, N, N, N, N, N, N,
4267
4268 N, N, N, N, N, N, N, N,
4269
4270 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
4271
4272 N, N, N, N, N, N, N, N,
4273
4274 N, N, N, N, N, N, N, N,
4275
4276 N, N, N, N, N, N, N, N,
4277} };
4278
4279static const struct escape escape_dd = { {
4280 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4281}, {
4282
4283 N, N, N, N, N, N, N, N,
4284
4285 N, N, N, N, N, N, N, N,
4286
4287 N, N, N, N, N, N, N, N,
4288
4289 N, N, N, N, N, N, N, N,
4290
4291 N, N, N, N, N, N, N, N,
4292
4293 N, N, N, N, N, N, N, N,
4294
4295 N, N, N, N, N, N, N, N,
4296
4297 N, N, N, N, N, N, N, N,
4298} };
4299
4300static const struct instr_dual instr_dual_0f_c3 = {
4301 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
4302};
4303
4304static const struct mode_dual mode_dual_63 = {
4305 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
4306};
4307
4308static const struct opcode opcode_table[256] = {
4309
4310 F6ALU(Lock, em_add),
4311 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4312 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4313
4314 F6ALU(Lock | PageTable, em_or),
4315 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4316 N,
4317
4318 F6ALU(Lock, em_adc),
4319 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4320 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4321
4322 F6ALU(Lock, em_sbb),
4323 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4324 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4325
4326 F6ALU(Lock | PageTable, em_and), N, N,
4327
4328 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4329
4330 F6ALU(Lock, em_xor), N, N,
4331
4332 F6ALU(NoWrite, em_cmp), N, N,
4333
4334 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4335
4336 X8(I(SrcReg | Stack, em_push)),
4337
4338 X8(I(DstReg | Stack, em_pop)),
4339
4340 I(ImplicitOps | Stack | No64, em_pusha),
4341 I(ImplicitOps | Stack | No64, em_popa),
4342 N, MD(ModRM, &mode_dual_63),
4343 N, N, N, N,
4344
4345 I(SrcImm | Mov | Stack, em_push),
4346 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4347 I(SrcImmByte | Mov | Stack, em_push),
4348 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4349 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in),
4350 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out),
4351
4352 X16(D(SrcImmByte | NearBranch)),
4353
4354 G(ByteOp | DstMem | SrcImm, group1),
4355 G(DstMem | SrcImm, group1),
4356 G(ByteOp | DstMem | SrcImm | No64, group1),
4357 G(DstMem | SrcImmByte, group1),
4358 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4359 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4360
4361 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4362 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4363 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4364 D(ModRM | SrcMem | NoAccess | DstReg),
4365 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4366 G(0, group1A),
4367
4368 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4369
4370 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4371 I(SrcImmFAddr | No64, em_call_far), N,
4372 II(ImplicitOps | Stack, em_pushf, pushf),
4373 II(ImplicitOps | Stack, em_popf, popf),
4374 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4375
4376 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4377 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4378 I2bv(SrcSI | DstDI | Mov | String, em_mov),
4379 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp_r),
4380
4381 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4382 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4383 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4384 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4385
4386 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4387
4388 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4389
4390 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4391 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4392 I(ImplicitOps | NearBranch, em_ret),
4393 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4394 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4395 G(ByteOp, group11), G(0, group11),
4396
4397 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4398 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4399 I(ImplicitOps, em_ret_far),
4400 D(ImplicitOps), DI(SrcImmByte, intn),
4401 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4402
4403 G(Src2One | ByteOp, group2), G(Src2One, group2),
4404 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4405 I(DstAcc | SrcImmUByte | No64, em_aam),
4406 I(DstAcc | SrcImmUByte | No64, em_aad),
4407 F(DstAcc | ByteOp | No64, em_salc),
4408 I(DstAcc | SrcXLat | ByteOp, em_mov),
4409
4410 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4411
4412 X3(I(SrcImmByte | NearBranch, em_loop)),
4413 I(SrcImmByte | NearBranch, em_jcxz),
4414 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4415 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4416
4417 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4418 I(SrcImmFAddr | No64, em_jmp_far),
4419 D(SrcImmByte | ImplicitOps | NearBranch),
4420 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4421 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4422
4423 N, DI(ImplicitOps, icebp), N, N,
4424 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4425 G(ByteOp, group3), G(0, group3),
4426
4427 D(ImplicitOps), D(ImplicitOps),
4428 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4429 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4430};
4431
4432static const struct opcode twobyte_table[256] = {
4433
4434 G(0, group6), GD(0, &group7), N, N,
4435 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4436 II(ImplicitOps | Priv, em_clts, clts), N,
4437 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4438 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4439
4440 N, N, N, N, N, N, N, N,
4441 D(ImplicitOps | ModRM | SrcMem | NoAccess),
4442 N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4443
4444 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4445 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4446 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4447 check_cr_write),
4448 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4449 check_dr_write),
4450 N, N, N, N,
4451 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4452 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4453 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4454 N, N, N, N,
4455
4456 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4457 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4458 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4459 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4460 I(ImplicitOps | EmulateOnUD, em_sysenter),
4461 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4462 N, N,
4463 N, N, N, N, N, N, N, N,
4464
4465 X16(D(DstReg | SrcMem | ModRM)),
4466
4467 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4468
4469 N, N, N, N,
4470 N, N, N, N,
4471 N, N, N, N,
4472 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4473
4474 N, N, N, N,
4475 N, N, N, N,
4476 N, N, N, N,
4477 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4478
4479 X16(D(SrcImm | NearBranch)),
4480
4481 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4482
4483 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4484 II(ImplicitOps, em_cpuid, cpuid),
4485 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4486 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4487 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4488
4489 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4490 II(EmulateOnUD | ImplicitOps, em_rsm, rsm),
4491 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4492 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4493 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4494 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4495
4496 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4497 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4498 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4499 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4500 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4501 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4502
4503 N, N,
4504 G(BitOp, group8),
4505 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4506 I(DstReg | SrcMem | ModRM, em_bsf_c),
4507 I(DstReg | SrcMem | ModRM, em_bsr_c),
4508 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4509
4510 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4511 N, ID(0, &instr_dual_0f_c3),
4512 N, N, N, GD(0, &group9),
4513
4514 X8(I(DstReg, em_bswap)),
4515
4516 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4517
4518 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4519 N, N, N, N, N, N, N, N,
4520
4521 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4522};
4523
4524static const struct instr_dual instr_dual_0f_38_f0 = {
4525 I(DstReg | SrcMem | Mov, em_movbe), N
4526};
4527
4528static const struct instr_dual instr_dual_0f_38_f1 = {
4529 I(DstMem | SrcReg | Mov, em_movbe), N
4530};
4531
4532static const struct gprefix three_byte_0f_38_f0 = {
4533 ID(0, &instr_dual_0f_38_f0), N, N, N
4534};
4535
4536static const struct gprefix three_byte_0f_38_f1 = {
4537 ID(0, &instr_dual_0f_38_f1), N, N, N
4538};
4539
4540
4541
4542
4543
4544static const struct opcode opcode_map_0f_38[256] = {
4545
4546 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4547
4548 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4549
4550 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4551 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4552
4553 N, N, X4(N), X8(N)
4554};
4555
4556#undef D
4557#undef N
4558#undef G
4559#undef GD
4560#undef I
4561#undef GP
4562#undef EXT
4563#undef MD
4564#undef ID
4565
4566#undef D2bv
4567#undef D2bvIP
4568#undef I2bv
4569#undef I2bvIP
4570#undef I6ALU
4571
4572static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4573{
4574 unsigned size;
4575
4576 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4577 if (size == 8)
4578 size = 4;
4579 return size;
4580}
4581
4582static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4583 unsigned size, bool sign_extension)
4584{
4585 int rc = X86EMUL_CONTINUE;
4586
4587 op->type = OP_IMM;
4588 op->bytes = size;
4589 op->addr.mem.ea = ctxt->_eip;
4590
4591 switch (op->bytes) {
4592 case 1:
4593 op->val = insn_fetch(s8, ctxt);
4594 break;
4595 case 2:
4596 op->val = insn_fetch(s16, ctxt);
4597 break;
4598 case 4:
4599 op->val = insn_fetch(s32, ctxt);
4600 break;
4601 case 8:
4602 op->val = insn_fetch(s64, ctxt);
4603 break;
4604 }
4605 if (!sign_extension) {
4606 switch (op->bytes) {
4607 case 1:
4608 op->val &= 0xff;
4609 break;
4610 case 2:
4611 op->val &= 0xffff;
4612 break;
4613 case 4:
4614 op->val &= 0xffffffff;
4615 break;
4616 }
4617 }
4618done:
4619 return rc;
4620}
4621
4622static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4623 unsigned d)
4624{
4625 int rc = X86EMUL_CONTINUE;
4626
4627 switch (d) {
4628 case OpReg:
4629 decode_register_operand(ctxt, op);
4630 break;
4631 case OpImmUByte:
4632 rc = decode_imm(ctxt, op, 1, false);
4633 break;
4634 case OpMem:
4635 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4636 mem_common:
4637 *op = ctxt->memop;
4638 ctxt->memopp = op;
4639 if (ctxt->d & BitOp)
4640 fetch_bit_operand(ctxt);
4641 op->orig_val = op->val;
4642 break;
4643 case OpMem64:
4644 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4645 goto mem_common;
4646 case OpAcc:
4647 op->type = OP_REG;
4648 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4649 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4650 fetch_register_operand(op);
4651 op->orig_val = op->val;
4652 break;
4653 case OpAccLo:
4654 op->type = OP_REG;
4655 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4656 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4657 fetch_register_operand(op);
4658 op->orig_val = op->val;
4659 break;
4660 case OpAccHi:
4661 if (ctxt->d & ByteOp) {
4662 op->type = OP_NONE;
4663 break;
4664 }
4665 op->type = OP_REG;
4666 op->bytes = ctxt->op_bytes;
4667 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4668 fetch_register_operand(op);
4669 op->orig_val = op->val;
4670 break;
4671 case OpDI:
4672 op->type = OP_MEM;
4673 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4674 op->addr.mem.ea =
4675 register_address(ctxt, VCPU_REGS_RDI);
4676 op->addr.mem.seg = VCPU_SREG_ES;
4677 op->val = 0;
4678 op->count = 1;
4679 break;
4680 case OpDX:
4681 op->type = OP_REG;
4682 op->bytes = 2;
4683 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4684 fetch_register_operand(op);
4685 break;
4686 case OpCL:
4687 op->type = OP_IMM;
4688 op->bytes = 1;
4689 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4690 break;
4691 case OpImmByte:
4692 rc = decode_imm(ctxt, op, 1, true);
4693 break;
4694 case OpOne:
4695 op->type = OP_IMM;
4696 op->bytes = 1;
4697 op->val = 1;
4698 break;
4699 case OpImm:
4700 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4701 break;
4702 case OpImm64:
4703 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4704 break;
4705 case OpMem8:
4706 ctxt->memop.bytes = 1;
4707 if (ctxt->memop.type == OP_REG) {
4708 ctxt->memop.addr.reg = decode_register(ctxt,
4709 ctxt->modrm_rm, true);
4710 fetch_register_operand(&ctxt->memop);
4711 }
4712 goto mem_common;
4713 case OpMem16:
4714 ctxt->memop.bytes = 2;
4715 goto mem_common;
4716 case OpMem32:
4717 ctxt->memop.bytes = 4;
4718 goto mem_common;
4719 case OpImmU16:
4720 rc = decode_imm(ctxt, op, 2, false);
4721 break;
4722 case OpImmU:
4723 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4724 break;
4725 case OpSI:
4726 op->type = OP_MEM;
4727 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4728 op->addr.mem.ea =
4729 register_address(ctxt, VCPU_REGS_RSI);
4730 op->addr.mem.seg = ctxt->seg_override;
4731 op->val = 0;
4732 op->count = 1;
4733 break;
4734 case OpXLat:
4735 op->type = OP_MEM;
4736 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4737 op->addr.mem.ea =
4738 address_mask(ctxt,
4739 reg_read(ctxt, VCPU_REGS_RBX) +
4740 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4741 op->addr.mem.seg = ctxt->seg_override;
4742 op->val = 0;
4743 break;
4744 case OpImmFAddr:
4745 op->type = OP_IMM;
4746 op->addr.mem.ea = ctxt->_eip;
4747 op->bytes = ctxt->op_bytes + 2;
4748 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4749 break;
4750 case OpMemFAddr:
4751 ctxt->memop.bytes = ctxt->op_bytes + 2;
4752 goto mem_common;
4753 case OpES:
4754 op->type = OP_IMM;
4755 op->val = VCPU_SREG_ES;
4756 break;
4757 case OpCS:
4758 op->type = OP_IMM;
4759 op->val = VCPU_SREG_CS;
4760 break;
4761 case OpSS:
4762 op->type = OP_IMM;
4763 op->val = VCPU_SREG_SS;
4764 break;
4765 case OpDS:
4766 op->type = OP_IMM;
4767 op->val = VCPU_SREG_DS;
4768 break;
4769 case OpFS:
4770 op->type = OP_IMM;
4771 op->val = VCPU_SREG_FS;
4772 break;
4773 case OpGS:
4774 op->type = OP_IMM;
4775 op->val = VCPU_SREG_GS;
4776 break;
4777 case OpImplicit:
4778
4779 default:
4780 op->type = OP_NONE;
4781 break;
4782 }
4783
4784done:
4785 return rc;
4786}
4787
4788int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4789{
4790 int rc = X86EMUL_CONTINUE;
4791 int mode = ctxt->mode;
4792 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4793 bool op_prefix = false;
4794 bool has_seg_override = false;
4795 struct opcode opcode;
4796
4797 ctxt->memop.type = OP_NONE;
4798 ctxt->memopp = NULL;
4799 ctxt->_eip = ctxt->eip;
4800 ctxt->fetch.ptr = ctxt->fetch.data;
4801 ctxt->fetch.end = ctxt->fetch.data + insn_len;
4802 ctxt->opcode_len = 1;
4803 if (insn_len > 0)
4804 memcpy(ctxt->fetch.data, insn, insn_len);
4805 else {
4806 rc = __do_insn_fetch_bytes(ctxt, 1);
4807 if (rc != X86EMUL_CONTINUE)
4808 return rc;
4809 }
4810
4811 switch (mode) {
4812 case X86EMUL_MODE_REAL:
4813 case X86EMUL_MODE_VM86:
4814 case X86EMUL_MODE_PROT16:
4815 def_op_bytes = def_ad_bytes = 2;
4816 break;
4817 case X86EMUL_MODE_PROT32:
4818 def_op_bytes = def_ad_bytes = 4;
4819 break;
4820#ifdef CONFIG_X86_64
4821 case X86EMUL_MODE_PROT64:
4822 def_op_bytes = 4;
4823 def_ad_bytes = 8;
4824 break;
4825#endif
4826 default:
4827 return EMULATION_FAILED;
4828 }
4829
4830 ctxt->op_bytes = def_op_bytes;
4831 ctxt->ad_bytes = def_ad_bytes;
4832
4833
4834 for (;;) {
4835 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4836 case 0x66:
4837 op_prefix = true;
4838
4839 ctxt->op_bytes = def_op_bytes ^ 6;
4840 break;
4841 case 0x67:
4842 if (mode == X86EMUL_MODE_PROT64)
4843
4844 ctxt->ad_bytes = def_ad_bytes ^ 12;
4845 else
4846
4847 ctxt->ad_bytes = def_ad_bytes ^ 6;
4848 break;
4849 case 0x26:
4850 case 0x2e:
4851 case 0x36:
4852 case 0x3e:
4853 has_seg_override = true;
4854 ctxt->seg_override = (ctxt->b >> 3) & 3;
4855 break;
4856 case 0x64:
4857 case 0x65:
4858 has_seg_override = true;
4859 ctxt->seg_override = ctxt->b & 7;
4860 break;
4861 case 0x40 ... 0x4f:
4862 if (mode != X86EMUL_MODE_PROT64)
4863 goto done_prefixes;
4864 ctxt->rex_prefix = ctxt->b;
4865 continue;
4866 case 0xf0:
4867 ctxt->lock_prefix = 1;
4868 break;
4869 case 0xf2:
4870 case 0xf3:
4871 ctxt->rep_prefix = ctxt->b;
4872 break;
4873 default:
4874 goto done_prefixes;
4875 }
4876
4877
4878
4879 ctxt->rex_prefix = 0;
4880 }
4881
4882done_prefixes:
4883
4884
4885 if (ctxt->rex_prefix & 8)
4886 ctxt->op_bytes = 8;
4887
4888
4889 opcode = opcode_table[ctxt->b];
4890
4891 if (ctxt->b == 0x0f) {
4892 ctxt->opcode_len = 2;
4893 ctxt->b = insn_fetch(u8, ctxt);
4894 opcode = twobyte_table[ctxt->b];
4895
4896
4897 if (ctxt->b == 0x38) {
4898 ctxt->opcode_len = 3;
4899 ctxt->b = insn_fetch(u8, ctxt);
4900 opcode = opcode_map_0f_38[ctxt->b];
4901 }
4902 }
4903 ctxt->d = opcode.flags;
4904
4905 if (ctxt->d & ModRM)
4906 ctxt->modrm = insn_fetch(u8, ctxt);
4907
4908
4909 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
4910 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
4911 ctxt->d = NotImpl;
4912 }
4913
4914 while (ctxt->d & GroupMask) {
4915 switch (ctxt->d & GroupMask) {
4916 case Group:
4917 goffset = (ctxt->modrm >> 3) & 7;
4918 opcode = opcode.u.group[goffset];
4919 break;
4920 case GroupDual:
4921 goffset = (ctxt->modrm >> 3) & 7;
4922 if ((ctxt->modrm >> 6) == 3)
4923 opcode = opcode.u.gdual->mod3[goffset];
4924 else
4925 opcode = opcode.u.gdual->mod012[goffset];
4926 break;
4927 case RMExt:
4928 goffset = ctxt->modrm & 7;
4929 opcode = opcode.u.group[goffset];
4930 break;
4931 case Prefix:
4932 if (ctxt->rep_prefix && op_prefix)
4933 return EMULATION_FAILED;
4934 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4935 switch (simd_prefix) {
4936 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4937 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4938 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4939 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4940 }
4941 break;
4942 case Escape:
4943 if (ctxt->modrm > 0xbf)
4944 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4945 else
4946 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4947 break;
4948 case InstrDual:
4949 if ((ctxt->modrm >> 6) == 3)
4950 opcode = opcode.u.idual->mod3;
4951 else
4952 opcode = opcode.u.idual->mod012;
4953 break;
4954 case ModeDual:
4955 if (ctxt->mode == X86EMUL_MODE_PROT64)
4956 opcode = opcode.u.mdual->mode64;
4957 else
4958 opcode = opcode.u.mdual->mode32;
4959 break;
4960 default:
4961 return EMULATION_FAILED;
4962 }
4963
4964 ctxt->d &= ~(u64)GroupMask;
4965 ctxt->d |= opcode.flags;
4966 }
4967
4968
4969 if (ctxt->d == 0)
4970 return EMULATION_FAILED;
4971
4972 ctxt->execute = opcode.u.execute;
4973
4974 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
4975 return EMULATION_FAILED;
4976
4977 if (unlikely(ctxt->d &
4978 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
4979 No16))) {
4980
4981
4982
4983
4984 ctxt->check_perm = opcode.check_perm;
4985 ctxt->intercept = opcode.intercept;
4986
4987 if (ctxt->d & NotImpl)
4988 return EMULATION_FAILED;
4989
4990 if (mode == X86EMUL_MODE_PROT64) {
4991 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
4992 ctxt->op_bytes = 8;
4993 else if (ctxt->d & NearBranch)
4994 ctxt->op_bytes = 8;
4995 }
4996
4997 if (ctxt->d & Op3264) {
4998 if (mode == X86EMUL_MODE_PROT64)
4999 ctxt->op_bytes = 8;
5000 else
5001 ctxt->op_bytes = 4;
5002 }
5003
5004 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
5005 ctxt->op_bytes = 4;
5006
5007 if (ctxt->d & Sse)
5008 ctxt->op_bytes = 16;
5009 else if (ctxt->d & Mmx)
5010 ctxt->op_bytes = 8;
5011 }
5012
5013
5014 if (ctxt->d & ModRM) {
5015 rc = decode_modrm(ctxt, &ctxt->memop);
5016 if (!has_seg_override) {
5017 has_seg_override = true;
5018 ctxt->seg_override = ctxt->modrm_seg;
5019 }
5020 } else if (ctxt->d & MemAbs)
5021 rc = decode_abs(ctxt, &ctxt->memop);
5022 if (rc != X86EMUL_CONTINUE)
5023 goto done;
5024
5025 if (!has_seg_override)
5026 ctxt->seg_override = VCPU_SREG_DS;
5027
5028 ctxt->memop.addr.mem.seg = ctxt->seg_override;
5029
5030
5031
5032
5033
5034 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
5035 if (rc != X86EMUL_CONTINUE)
5036 goto done;
5037
5038
5039
5040
5041
5042 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
5043 if (rc != X86EMUL_CONTINUE)
5044 goto done;
5045
5046
5047 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
5048
5049 if (ctxt->rip_relative)
5050 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
5051 ctxt->memopp->addr.mem.ea + ctxt->_eip);
5052
5053done:
5054 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5055}
5056
5057bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
5058{
5059 return ctxt->d & PageTable;
5060}
5061
5062static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
5063{
5064
5065
5066
5067
5068
5069
5070
5071 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
5072 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
5073 && (((ctxt->rep_prefix == REPE_PREFIX) &&
5074 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
5075 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
5076 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
5077 return true;
5078
5079 return false;
5080}
5081
5082static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
5083{
5084 bool fault = false;
5085
5086 ctxt->ops->get_fpu(ctxt);
5087 asm volatile("1: fwait \n\t"
5088 "2: \n\t"
5089 ".pushsection .fixup,\"ax\" \n\t"
5090 "3: \n\t"
5091 "movb $1, %[fault] \n\t"
5092 "jmp 2b \n\t"
5093 ".popsection \n\t"
5094 _ASM_EXTABLE(1b, 3b)
5095 : [fault]"+qm"(fault));
5096 ctxt->ops->put_fpu(ctxt);
5097
5098 if (unlikely(fault))
5099 return emulate_exception(ctxt, MF_VECTOR, 0, false);
5100
5101 return X86EMUL_CONTINUE;
5102}
5103
5104static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
5105 struct operand *op)
5106{
5107 if (op->type == OP_MM)
5108 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
5109}
5110
5111static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
5112{
5113 register void *__sp asm(_ASM_SP);
5114 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5115
5116 if (!(ctxt->d & ByteOp))
5117 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5118
5119 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
5120 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5121 [fastop]"+S"(fop), "+r"(__sp)
5122 : "c"(ctxt->src2.val));
5123
5124 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5125 if (!fop)
5126 return emulate_de(ctxt);
5127 return X86EMUL_CONTINUE;
5128}
5129
5130void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5131{
5132 memset(&ctxt->rip_relative, 0,
5133 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
5134
5135 ctxt->io_read.pos = 0;
5136 ctxt->io_read.end = 0;
5137 ctxt->mem_read.end = 0;
5138}
5139
5140int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
5141{
5142 const struct x86_emulate_ops *ops = ctxt->ops;
5143 int rc = X86EMUL_CONTINUE;
5144 int saved_dst_type = ctxt->dst.type;
5145
5146 ctxt->mem_read.pos = 0;
5147
5148
5149 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
5150 rc = emulate_ud(ctxt);
5151 goto done;
5152 }
5153
5154 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
5155 rc = emulate_ud(ctxt);
5156 goto done;
5157 }
5158
5159 if (unlikely(ctxt->d &
5160 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
5161 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
5162 (ctxt->d & Undefined)) {
5163 rc = emulate_ud(ctxt);
5164 goto done;
5165 }
5166
5167 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
5168 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
5169 rc = emulate_ud(ctxt);
5170 goto done;
5171 }
5172
5173 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
5174 rc = emulate_nm(ctxt);
5175 goto done;
5176 }
5177
5178 if (ctxt->d & Mmx) {
5179 rc = flush_pending_x87_faults(ctxt);
5180 if (rc != X86EMUL_CONTINUE)
5181 goto done;
5182
5183
5184
5185
5186 fetch_possible_mmx_operand(ctxt, &ctxt->src);
5187 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
5188 if (!(ctxt->d & Mov))
5189 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
5190 }
5191
5192 if (unlikely(ctxt->emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
5193 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5194 X86_ICPT_PRE_EXCEPT);
5195 if (rc != X86EMUL_CONTINUE)
5196 goto done;
5197 }
5198
5199
5200 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
5201 rc = emulate_ud(ctxt);
5202 goto done;
5203 }
5204
5205
5206 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
5207 if (ctxt->d & PrivUD)
5208 rc = emulate_ud(ctxt);
5209 else
5210 rc = emulate_gp(ctxt, 0);
5211 goto done;
5212 }
5213
5214
5215 if (ctxt->d & CheckPerm) {
5216 rc = ctxt->check_perm(ctxt);
5217 if (rc != X86EMUL_CONTINUE)
5218 goto done;
5219 }
5220
5221 if (unlikely(ctxt->emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5222 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5223 X86_ICPT_POST_EXCEPT);
5224 if (rc != X86EMUL_CONTINUE)
5225 goto done;
5226 }
5227
5228 if (ctxt->rep_prefix && (ctxt->d & String)) {
5229
5230 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
5231 string_registers_quirk(ctxt);
5232 ctxt->eip = ctxt->_eip;
5233 ctxt->eflags &= ~X86_EFLAGS_RF;
5234 goto done;
5235 }
5236 }
5237 }
5238
5239 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
5240 rc = segmented_read(ctxt, ctxt->src.addr.mem,
5241 ctxt->src.valptr, ctxt->src.bytes);
5242 if (rc != X86EMUL_CONTINUE)
5243 goto done;
5244 ctxt->src.orig_val64 = ctxt->src.val64;
5245 }
5246
5247 if (ctxt->src2.type == OP_MEM) {
5248 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
5249 &ctxt->src2.val, ctxt->src2.bytes);
5250 if (rc != X86EMUL_CONTINUE)
5251 goto done;
5252 }
5253
5254 if ((ctxt->d & DstMask) == ImplicitOps)
5255 goto special_insn;
5256
5257
5258 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
5259
5260 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
5261 &ctxt->dst.val, ctxt->dst.bytes);
5262 if (rc != X86EMUL_CONTINUE) {
5263 if (!(ctxt->d & NoWrite) &&
5264 rc == X86EMUL_PROPAGATE_FAULT &&
5265 ctxt->exception.vector == PF_VECTOR)
5266 ctxt->exception.error_code |= PFERR_WRITE_MASK;
5267 goto done;
5268 }
5269 }
5270
5271 ctxt->dst.orig_val64 = ctxt->dst.val64;
5272
5273special_insn:
5274
5275 if (unlikely(ctxt->emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5276 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5277 X86_ICPT_POST_MEMACCESS);
5278 if (rc != X86EMUL_CONTINUE)
5279 goto done;
5280 }
5281
5282 if (ctxt->rep_prefix && (ctxt->d & String))
5283 ctxt->eflags |= X86_EFLAGS_RF;
5284 else
5285 ctxt->eflags &= ~X86_EFLAGS_RF;
5286
5287 if (ctxt->execute) {
5288 if (ctxt->d & Fastop) {
5289 void (*fop)(struct fastop *) = (void *)ctxt->execute;
5290 rc = fastop(ctxt, fop);
5291 if (rc != X86EMUL_CONTINUE)
5292 goto done;
5293 goto writeback;
5294 }
5295 rc = ctxt->execute(ctxt);
5296 if (rc != X86EMUL_CONTINUE)
5297 goto done;
5298 goto writeback;
5299 }
5300
5301 if (ctxt->opcode_len == 2)
5302 goto twobyte_insn;
5303 else if (ctxt->opcode_len == 3)
5304 goto threebyte_insn;
5305
5306 switch (ctxt->b) {
5307 case 0x70 ... 0x7f:
5308 if (test_cc(ctxt->b, ctxt->eflags))
5309 rc = jmp_rel(ctxt, ctxt->src.val);
5310 break;
5311 case 0x8d:
5312 ctxt->dst.val = ctxt->src.addr.mem.ea;
5313 break;
5314 case 0x90 ... 0x97:
5315 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5316 ctxt->dst.type = OP_NONE;
5317 else
5318 rc = em_xchg(ctxt);
5319 break;
5320 case 0x98:
5321 switch (ctxt->op_bytes) {
5322 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5323 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5324 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5325 }
5326 break;
5327 case 0xcc:
5328 rc = emulate_int(ctxt, 3);
5329 break;
5330 case 0xcd:
5331 rc = emulate_int(ctxt, ctxt->src.val);
5332 break;
5333 case 0xce:
5334 if (ctxt->eflags & X86_EFLAGS_OF)
5335 rc = emulate_int(ctxt, 4);
5336 break;
5337 case 0xe9:
5338 case 0xeb:
5339 rc = jmp_rel(ctxt, ctxt->src.val);
5340 ctxt->dst.type = OP_NONE;
5341 break;
5342 case 0xf4:
5343 ctxt->ops->halt(ctxt);
5344 break;
5345 case 0xf5:
5346
5347 ctxt->eflags ^= X86_EFLAGS_CF;
5348 break;
5349 case 0xf8:
5350 ctxt->eflags &= ~X86_EFLAGS_CF;
5351 break;
5352 case 0xf9:
5353 ctxt->eflags |= X86_EFLAGS_CF;
5354 break;
5355 case 0xfc:
5356 ctxt->eflags &= ~X86_EFLAGS_DF;
5357 break;
5358 case 0xfd:
5359 ctxt->eflags |= X86_EFLAGS_DF;
5360 break;
5361 default:
5362 goto cannot_emulate;
5363 }
5364
5365 if (rc != X86EMUL_CONTINUE)
5366 goto done;
5367
5368writeback:
5369 if (ctxt->d & SrcWrite) {
5370 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5371 rc = writeback(ctxt, &ctxt->src);
5372 if (rc != X86EMUL_CONTINUE)
5373 goto done;
5374 }
5375 if (!(ctxt->d & NoWrite)) {
5376 rc = writeback(ctxt, &ctxt->dst);
5377 if (rc != X86EMUL_CONTINUE)
5378 goto done;
5379 }
5380
5381
5382
5383
5384
5385 ctxt->dst.type = saved_dst_type;
5386
5387 if ((ctxt->d & SrcMask) == SrcSI)
5388 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5389
5390 if ((ctxt->d & DstMask) == DstDI)
5391 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5392
5393 if (ctxt->rep_prefix && (ctxt->d & String)) {
5394 unsigned int count;
5395 struct read_cache *r = &ctxt->io_read;
5396 if ((ctxt->d & SrcMask) == SrcSI)
5397 count = ctxt->src.count;
5398 else
5399 count = ctxt->dst.count;
5400 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5401
5402 if (!string_insn_completed(ctxt)) {
5403
5404
5405
5406
5407 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5408 (r->end == 0 || r->end != r->pos)) {
5409
5410
5411
5412
5413
5414 ctxt->mem_read.end = 0;
5415 writeback_registers(ctxt);
5416 return EMULATION_RESTART;
5417 }
5418 goto done;
5419 }
5420 ctxt->eflags &= ~X86_EFLAGS_RF;
5421 }
5422
5423 ctxt->eip = ctxt->_eip;
5424
5425done:
5426 if (rc == X86EMUL_PROPAGATE_FAULT) {
5427 WARN_ON(ctxt->exception.vector > 0x1f);
5428 ctxt->have_exception = true;
5429 }
5430 if (rc == X86EMUL_INTERCEPTED)
5431 return EMULATION_INTERCEPTED;
5432
5433 if (rc == X86EMUL_CONTINUE)
5434 writeback_registers(ctxt);
5435
5436 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5437
5438twobyte_insn:
5439 switch (ctxt->b) {
5440 case 0x09:
5441 (ctxt->ops->wbinvd)(ctxt);
5442 break;
5443 case 0x08:
5444 case 0x0d:
5445 case 0x18:
5446 case 0x1f:
5447 break;
5448 case 0x20:
5449 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5450 break;
5451 case 0x21:
5452 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5453 break;
5454 case 0x40 ... 0x4f:
5455 if (test_cc(ctxt->b, ctxt->eflags))
5456 ctxt->dst.val = ctxt->src.val;
5457 else if (ctxt->op_bytes != 4)
5458 ctxt->dst.type = OP_NONE;
5459 break;
5460 case 0x80 ... 0x8f:
5461 if (test_cc(ctxt->b, ctxt->eflags))
5462 rc = jmp_rel(ctxt, ctxt->src.val);
5463 break;
5464 case 0x90 ... 0x9f:
5465 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5466 break;
5467 case 0xb6 ... 0xb7:
5468 ctxt->dst.bytes = ctxt->op_bytes;
5469 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5470 : (u16) ctxt->src.val;
5471 break;
5472 case 0xbe ... 0xbf:
5473 ctxt->dst.bytes = ctxt->op_bytes;
5474 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5475 (s16) ctxt->src.val;
5476 break;
5477 default:
5478 goto cannot_emulate;
5479 }
5480
5481threebyte_insn:
5482
5483 if (rc != X86EMUL_CONTINUE)
5484 goto done;
5485
5486 goto writeback;
5487
5488cannot_emulate:
5489 return EMULATION_FAILED;
5490}
5491
5492void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5493{
5494 invalidate_registers(ctxt);
5495}
5496
5497void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5498{
5499 writeback_registers(ctxt);
5500}
5501