1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#include <linux/kvm_host.h>
24#include "kvm_cache_regs.h"
25#include <linux/module.h>
26#include <asm/kvm_emulate.h>
27#include <linux/stringify.h>
28
29#include "x86.h"
30#include "tss.h"
31
32
33
34
35#define OpNone 0ull
36#define OpImplicit 1ull
37#define OpReg 2ull
38#define OpMem 3ull
39#define OpAcc 4ull
40#define OpDI 5ull
41#define OpMem64 6ull
42#define OpImmUByte 7ull
43#define OpDX 8ull
44#define OpCL 9ull
45#define OpImmByte 10ull
46#define OpOne 11ull
47#define OpImm 12ull
48#define OpMem16 13ull
49#define OpMem32 14ull
50#define OpImmU 15ull
51#define OpSI 16ull
52#define OpImmFAddr 17ull
53#define OpMemFAddr 18ull
54#define OpImmU16 19ull
55#define OpES 20ull
56#define OpCS 21ull
57#define OpSS 22ull
58#define OpDS 23ull
59#define OpFS 24ull
60#define OpGS 25ull
61#define OpMem8 26ull
62#define OpImm64 27ull
63#define OpXLat 28ull
64#define OpAccLo 29ull
65#define OpAccHi 30ull
66
67#define OpBits 5
68#define OpMask ((1ull << OpBits) - 1)
69
70
71
72
73
74
75
76
77
78
79
80#define ByteOp (1<<0)
81
82#define DstShift 1
83#define ImplicitOps (OpImplicit << DstShift)
84#define DstReg (OpReg << DstShift)
85#define DstMem (OpMem << DstShift)
86#define DstAcc (OpAcc << DstShift)
87#define DstDI (OpDI << DstShift)
88#define DstMem64 (OpMem64 << DstShift)
89#define DstImmUByte (OpImmUByte << DstShift)
90#define DstDX (OpDX << DstShift)
91#define DstAccLo (OpAccLo << DstShift)
92#define DstMask (OpMask << DstShift)
93
94#define SrcShift 6
95#define SrcNone (OpNone << SrcShift)
96#define SrcReg (OpReg << SrcShift)
97#define SrcMem (OpMem << SrcShift)
98#define SrcMem16 (OpMem16 << SrcShift)
99#define SrcMem32 (OpMem32 << SrcShift)
100#define SrcImm (OpImm << SrcShift)
101#define SrcImmByte (OpImmByte << SrcShift)
102#define SrcOne (OpOne << SrcShift)
103#define SrcImmUByte (OpImmUByte << SrcShift)
104#define SrcImmU (OpImmU << SrcShift)
105#define SrcSI (OpSI << SrcShift)
106#define SrcXLat (OpXLat << SrcShift)
107#define SrcImmFAddr (OpImmFAddr << SrcShift)
108#define SrcMemFAddr (OpMemFAddr << SrcShift)
109#define SrcAcc (OpAcc << SrcShift)
110#define SrcImmU16 (OpImmU16 << SrcShift)
111#define SrcImm64 (OpImm64 << SrcShift)
112#define SrcDX (OpDX << SrcShift)
113#define SrcMem8 (OpMem8 << SrcShift)
114#define SrcAccHi (OpAccHi << SrcShift)
115#define SrcMask (OpMask << SrcShift)
116#define BitOp (1<<11)
117#define MemAbs (1<<12)
118#define String (1<<13)
119#define Stack (1<<14)
120#define GroupMask (7<<15)
121#define Group (1<<15)
122#define GroupDual (2<<15)
123#define Prefix (3<<15)
124#define RMExt (4<<15)
125#define Escape (5<<15)
126#define Sse (1<<18)
127
128#define ModRM (1<<19)
129
130#define Mov (1<<20)
131
132#define Prot (1<<21)
133#define VendorSpecific (1<<22)
134#define NoAccess (1<<23)
135#define Op3264 (1<<24)
136#define Undefined (1<<25)
137#define Lock (1<<26)
138#define Priv (1<<27)
139#define No64 (1<<28)
140#define PageTable (1 << 29)
141#define NotImpl (1 << 30)
142
143#define Src2Shift (31)
144#define Src2None (OpNone << Src2Shift)
145#define Src2Mem (OpMem << Src2Shift)
146#define Src2CL (OpCL << Src2Shift)
147#define Src2ImmByte (OpImmByte << Src2Shift)
148#define Src2One (OpOne << Src2Shift)
149#define Src2Imm (OpImm << Src2Shift)
150#define Src2ES (OpES << Src2Shift)
151#define Src2CS (OpCS << Src2Shift)
152#define Src2SS (OpSS << Src2Shift)
153#define Src2DS (OpDS << Src2Shift)
154#define Src2FS (OpFS << Src2Shift)
155#define Src2GS (OpGS << Src2Shift)
156#define Src2Mask (OpMask << Src2Shift)
157#define Mmx ((u64)1 << 40)
158#define Aligned ((u64)1 << 41)
159#define Unaligned ((u64)1 << 42)
160#define Avx ((u64)1 << 43)
161#define Fastop ((u64)1 << 44)
162#define NoWrite ((u64)1 << 45)
163#define SrcWrite ((u64)1 << 46)
164
165#define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
166
167#define X2(x...) x, x
168#define X3(x...) X2(x), x
169#define X4(x...) X2(x), X2(x)
170#define X5(x...) X4(x), x
171#define X6(x...) X4(x), X2(x)
172#define X7(x...) X4(x), X3(x)
173#define X8(x...) X4(x), X4(x)
174#define X16(x...) X8(x), X8(x)
175
176#define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
177#define FASTOP_SIZE 8
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196struct fastop;
197
198struct opcode {
199 u64 flags : 56;
200 u64 intercept : 8;
201 union {
202 int (*execute)(struct x86_emulate_ctxt *ctxt);
203 const struct opcode *group;
204 const struct group_dual *gdual;
205 const struct gprefix *gprefix;
206 const struct escape *esc;
207 void (*fastop)(struct fastop *fake);
208 } u;
209 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
210};
211
212struct group_dual {
213 struct opcode mod012[8];
214 struct opcode mod3[8];
215};
216
217struct gprefix {
218 struct opcode pfx_no;
219 struct opcode pfx_66;
220 struct opcode pfx_f2;
221 struct opcode pfx_f3;
222};
223
224struct escape {
225 struct opcode op[8];
226 struct opcode high[64];
227};
228
229
230#define EFLG_ID (1<<21)
231#define EFLG_VIP (1<<20)
232#define EFLG_VIF (1<<19)
233#define EFLG_AC (1<<18)
234#define EFLG_VM (1<<17)
235#define EFLG_RF (1<<16)
236#define EFLG_IOPL (3<<12)
237#define EFLG_NT (1<<14)
238#define EFLG_OF (1<<11)
239#define EFLG_DF (1<<10)
240#define EFLG_IF (1<<9)
241#define EFLG_TF (1<<8)
242#define EFLG_SF (1<<7)
243#define EFLG_ZF (1<<6)
244#define EFLG_AF (1<<4)
245#define EFLG_PF (1<<2)
246#define EFLG_CF (1<<0)
247
248#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
249#define EFLG_RESERVED_ONE_MASK 2
250
251static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
252{
253 if (!(ctxt->regs_valid & (1 << nr))) {
254 ctxt->regs_valid |= 1 << nr;
255 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
256 }
257 return ctxt->_regs[nr];
258}
259
260static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
261{
262 ctxt->regs_valid |= 1 << nr;
263 ctxt->regs_dirty |= 1 << nr;
264 return &ctxt->_regs[nr];
265}
266
267static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
268{
269 reg_read(ctxt, nr);
270 return reg_write(ctxt, nr);
271}
272
273static void writeback_registers(struct x86_emulate_ctxt *ctxt)
274{
275 unsigned reg;
276
277 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
278 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
279}
280
281static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
282{
283 ctxt->regs_dirty = 0;
284 ctxt->regs_valid = 0;
285}
286
287
288
289
290
291#define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
292
293#ifdef CONFIG_X86_64
294#define ON64(x) x
295#else
296#define ON64(x)
297#endif
298
299static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
300
301#define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
302#define FOP_RET "ret \n\t"
303
304#define FOP_START(op) \
305 extern void em_##op(struct fastop *fake); \
306 asm(".pushsection .text, \"ax\" \n\t" \
307 ".global em_" #op " \n\t" \
308 FOP_ALIGN \
309 "em_" #op ": \n\t"
310
311#define FOP_END \
312 ".popsection")
313
314#define FOPNOP() FOP_ALIGN FOP_RET
315
316#define FOP1E(op, dst) \
317 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
318
319#define FOP1EEX(op, dst) \
320 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
321
322#define FASTOP1(op) \
323 FOP_START(op) \
324 FOP1E(op##b, al) \
325 FOP1E(op##w, ax) \
326 FOP1E(op##l, eax) \
327 ON64(FOP1E(op##q, rax)) \
328 FOP_END
329
330
331#define FASTOP1SRC2(op, name) \
332 FOP_START(name) \
333 FOP1E(op, cl) \
334 FOP1E(op, cx) \
335 FOP1E(op, ecx) \
336 ON64(FOP1E(op, rcx)) \
337 FOP_END
338
339
340#define FASTOP1SRC2EX(op, name) \
341 FOP_START(name) \
342 FOP1EEX(op, cl) \
343 FOP1EEX(op, cx) \
344 FOP1EEX(op, ecx) \
345 ON64(FOP1EEX(op, rcx)) \
346 FOP_END
347
348#define FOP2E(op, dst, src) \
349 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
350
351#define FASTOP2(op) \
352 FOP_START(op) \
353 FOP2E(op##b, al, dl) \
354 FOP2E(op##w, ax, dx) \
355 FOP2E(op##l, eax, edx) \
356 ON64(FOP2E(op##q, rax, rdx)) \
357 FOP_END
358
359
360#define FASTOP2W(op) \
361 FOP_START(op) \
362 FOPNOP() \
363 FOP2E(op##w, ax, dx) \
364 FOP2E(op##l, eax, edx) \
365 ON64(FOP2E(op##q, rax, rdx)) \
366 FOP_END
367
368
369#define FASTOP2CL(op) \
370 FOP_START(op) \
371 FOP2E(op##b, al, cl) \
372 FOP2E(op##w, ax, cl) \
373 FOP2E(op##l, eax, cl) \
374 ON64(FOP2E(op##q, rax, cl)) \
375 FOP_END
376
377#define FOP3E(op, dst, src, src2) \
378 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
379
380
381#define FASTOP3WCL(op) \
382 FOP_START(op) \
383 FOPNOP() \
384 FOP3E(op##w, ax, dx, cl) \
385 FOP3E(op##l, eax, edx, cl) \
386 ON64(FOP3E(op##q, rax, rdx, cl)) \
387 FOP_END
388
389
390#define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
391
392asm(".global kvm_fastop_exception \n"
393 "kvm_fastop_exception: xor %esi, %esi; ret");
394
395FOP_START(setcc)
396FOP_SETCC(seto)
397FOP_SETCC(setno)
398FOP_SETCC(setc)
399FOP_SETCC(setnc)
400FOP_SETCC(setz)
401FOP_SETCC(setnz)
402FOP_SETCC(setbe)
403FOP_SETCC(setnbe)
404FOP_SETCC(sets)
405FOP_SETCC(setns)
406FOP_SETCC(setp)
407FOP_SETCC(setnp)
408FOP_SETCC(setl)
409FOP_SETCC(setnl)
410FOP_SETCC(setle)
411FOP_SETCC(setnle)
412FOP_END;
413
414FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
415FOP_END;
416
417static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
418 enum x86_intercept intercept,
419 enum x86_intercept_stage stage)
420{
421 struct x86_instruction_info info = {
422 .intercept = intercept,
423 .rep_prefix = ctxt->rep_prefix,
424 .modrm_mod = ctxt->modrm_mod,
425 .modrm_reg = ctxt->modrm_reg,
426 .modrm_rm = ctxt->modrm_rm,
427 .src_val = ctxt->src.val64,
428 .src_bytes = ctxt->src.bytes,
429 .dst_bytes = ctxt->dst.bytes,
430 .ad_bytes = ctxt->ad_bytes,
431 .next_rip = ctxt->eip,
432 };
433
434 return ctxt->ops->intercept(ctxt, &info, stage);
435}
436
437static void assign_masked(ulong *dest, ulong src, ulong mask)
438{
439 *dest = (*dest & ~mask) | (src & mask);
440}
441
442static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
443{
444 return (1UL << (ctxt->ad_bytes << 3)) - 1;
445}
446
447static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
448{
449 u16 sel;
450 struct desc_struct ss;
451
452 if (ctxt->mode == X86EMUL_MODE_PROT64)
453 return ~0UL;
454 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
455 return ~0U >> ((ss.d ^ 1) * 16);
456}
457
458static int stack_size(struct x86_emulate_ctxt *ctxt)
459{
460 return (__fls(stack_mask(ctxt)) + 1) >> 3;
461}
462
463
464static inline unsigned long
465address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
466{
467 if (ctxt->ad_bytes == sizeof(unsigned long))
468 return reg;
469 else
470 return reg & ad_mask(ctxt);
471}
472
473static inline unsigned long
474register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
475{
476 return address_mask(ctxt, reg);
477}
478
479static void masked_increment(ulong *reg, ulong mask, int inc)
480{
481 assign_masked(reg, *reg + inc, mask);
482}
483
484static inline void
485register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
486{
487 ulong mask;
488
489 if (ctxt->ad_bytes == sizeof(unsigned long))
490 mask = ~0UL;
491 else
492 mask = ad_mask(ctxt);
493 masked_increment(reg, mask, inc);
494}
495
496static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
497{
498 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
499}
500
501static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
502{
503 register_address_increment(ctxt, &ctxt->_eip, rel);
504}
505
506static u32 desc_limit_scaled(struct desc_struct *desc)
507{
508 u32 limit = get_desc_limit(desc);
509
510 return desc->g ? (limit << 12) | 0xfff : limit;
511}
512
513static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
514{
515 ctxt->has_seg_override = true;
516 ctxt->seg_override = seg;
517}
518
519static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
520{
521 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
522 return 0;
523
524 return ctxt->ops->get_cached_segment_base(ctxt, seg);
525}
526
527static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
528{
529 if (!ctxt->has_seg_override)
530 return 0;
531
532 return ctxt->seg_override;
533}
534
535static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
536 u32 error, bool valid)
537{
538 ctxt->exception.vector = vec;
539 ctxt->exception.error_code = error;
540 ctxt->exception.error_code_valid = valid;
541 return X86EMUL_PROPAGATE_FAULT;
542}
543
544static int emulate_db(struct x86_emulate_ctxt *ctxt)
545{
546 return emulate_exception(ctxt, DB_VECTOR, 0, false);
547}
548
549static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
550{
551 return emulate_exception(ctxt, GP_VECTOR, err, true);
552}
553
554static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
555{
556 return emulate_exception(ctxt, SS_VECTOR, err, true);
557}
558
559static int emulate_ud(struct x86_emulate_ctxt *ctxt)
560{
561 return emulate_exception(ctxt, UD_VECTOR, 0, false);
562}
563
564static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
565{
566 return emulate_exception(ctxt, TS_VECTOR, err, true);
567}
568
569static int emulate_de(struct x86_emulate_ctxt *ctxt)
570{
571 return emulate_exception(ctxt, DE_VECTOR, 0, false);
572}
573
574static int emulate_nm(struct x86_emulate_ctxt *ctxt)
575{
576 return emulate_exception(ctxt, NM_VECTOR, 0, false);
577}
578
579static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
580{
581 u16 selector;
582 struct desc_struct desc;
583
584 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
585 return selector;
586}
587
588static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
589 unsigned seg)
590{
591 u16 dummy;
592 u32 base3;
593 struct desc_struct desc;
594
595 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
596 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
597}
598
599
600
601
602
603
604
605
606
607static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
608{
609 if (likely(size < 16))
610 return false;
611
612 if (ctxt->d & Aligned)
613 return true;
614 else if (ctxt->d & Unaligned)
615 return false;
616 else if (ctxt->d & Avx)
617 return false;
618 else
619 return true;
620}
621
622static int __linearize(struct x86_emulate_ctxt *ctxt,
623 struct segmented_address addr,
624 unsigned size, bool write, bool fetch,
625 ulong *linear)
626{
627 struct desc_struct desc;
628 bool usable;
629 ulong la;
630 u32 lim;
631 u16 sel;
632 unsigned cpl;
633
634 la = seg_base(ctxt, addr.seg) + addr.ea;
635 switch (ctxt->mode) {
636 case X86EMUL_MODE_PROT64:
637 if (((signed long)la << 16) >> 16 != la)
638 return emulate_gp(ctxt, 0);
639 break;
640 default:
641 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
642 addr.seg);
643 if (!usable)
644 goto bad;
645
646 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
647 || !(desc.type & 2)) && write)
648 goto bad;
649
650 if (!fetch && (desc.type & 8) && !(desc.type & 2))
651 goto bad;
652 lim = desc_limit_scaled(&desc);
653 if ((desc.type & 8) || !(desc.type & 4)) {
654
655 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
656 goto bad;
657 } else {
658
659 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
660 goto bad;
661 lim = desc.d ? 0xffffffff : 0xffff;
662 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
663 goto bad;
664 }
665 cpl = ctxt->ops->cpl(ctxt);
666 if (!(desc.type & 8)) {
667
668 if (cpl > desc.dpl)
669 goto bad;
670 } else if ((desc.type & 8) && !(desc.type & 4)) {
671
672 if (cpl != desc.dpl)
673 goto bad;
674 } else if ((desc.type & 8) && (desc.type & 4)) {
675
676 if (cpl < desc.dpl)
677 goto bad;
678 }
679 break;
680 }
681 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
682 la &= (u32)-1;
683 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
684 return emulate_gp(ctxt, 0);
685 *linear = la;
686 return X86EMUL_CONTINUE;
687bad:
688 if (addr.seg == VCPU_SREG_SS)
689 return emulate_ss(ctxt, sel);
690 else
691 return emulate_gp(ctxt, sel);
692}
693
694static int linearize(struct x86_emulate_ctxt *ctxt,
695 struct segmented_address addr,
696 unsigned size, bool write,
697 ulong *linear)
698{
699 return __linearize(ctxt, addr, size, write, false, linear);
700}
701
702
703static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
704 struct segmented_address addr,
705 void *data,
706 unsigned size)
707{
708 int rc;
709 ulong linear;
710
711 rc = linearize(ctxt, addr, size, false, &linear);
712 if (rc != X86EMUL_CONTINUE)
713 return rc;
714 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
715}
716
717
718
719
720
721
722
723
724static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
725{
726 struct fetch_cache *fc = &ctxt->fetch;
727 int rc;
728 int size, cur_size;
729
730 if (ctxt->_eip == fc->end) {
731 unsigned long linear;
732 struct segmented_address addr = { .seg = VCPU_SREG_CS,
733 .ea = ctxt->_eip };
734 cur_size = fc->end - fc->start;
735 size = min(15UL - cur_size,
736 PAGE_SIZE - offset_in_page(ctxt->_eip));
737 rc = __linearize(ctxt, addr, size, false, true, &linear);
738 if (unlikely(rc != X86EMUL_CONTINUE))
739 return rc;
740 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
741 size, &ctxt->exception);
742 if (unlikely(rc != X86EMUL_CONTINUE))
743 return rc;
744 fc->end += size;
745 }
746 *dest = fc->data[ctxt->_eip - fc->start];
747 ctxt->_eip++;
748 return X86EMUL_CONTINUE;
749}
750
751static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
752 void *dest, unsigned size)
753{
754 int rc;
755
756
757 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
758 return X86EMUL_UNHANDLEABLE;
759 while (size--) {
760 rc = do_insn_fetch_byte(ctxt, dest++);
761 if (rc != X86EMUL_CONTINUE)
762 return rc;
763 }
764 return X86EMUL_CONTINUE;
765}
766
767
768#define insn_fetch(_type, _ctxt) \
769({ unsigned long _x; \
770 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
771 if (rc != X86EMUL_CONTINUE) \
772 goto done; \
773 (_type)_x; \
774})
775
776#define insn_fetch_arr(_arr, _size, _ctxt) \
777({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
778 if (rc != X86EMUL_CONTINUE) \
779 goto done; \
780})
781
782
783
784
785
786
787static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
788 int highbyte_regs)
789{
790 void *p;
791
792 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
793 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
794 else
795 p = reg_rmw(ctxt, modrm_reg);
796 return p;
797}
798
799static int read_descriptor(struct x86_emulate_ctxt *ctxt,
800 struct segmented_address addr,
801 u16 *size, unsigned long *address, int op_bytes)
802{
803 int rc;
804
805 if (op_bytes == 2)
806 op_bytes = 3;
807 *address = 0;
808 rc = segmented_read_std(ctxt, addr, size, 2);
809 if (rc != X86EMUL_CONTINUE)
810 return rc;
811 addr.ea += 2;
812 rc = segmented_read_std(ctxt, addr, address, op_bytes);
813 return rc;
814}
815
816FASTOP2(add);
817FASTOP2(or);
818FASTOP2(adc);
819FASTOP2(sbb);
820FASTOP2(and);
821FASTOP2(sub);
822FASTOP2(xor);
823FASTOP2(cmp);
824FASTOP2(test);
825
826FASTOP1SRC2(mul, mul_ex);
827FASTOP1SRC2(imul, imul_ex);
828FASTOP1SRC2EX(div, div_ex);
829FASTOP1SRC2EX(idiv, idiv_ex);
830
831FASTOP3WCL(shld);
832FASTOP3WCL(shrd);
833
834FASTOP2W(imul);
835
836FASTOP1(not);
837FASTOP1(neg);
838FASTOP1(inc);
839FASTOP1(dec);
840
841FASTOP2CL(rol);
842FASTOP2CL(ror);
843FASTOP2CL(rcl);
844FASTOP2CL(rcr);
845FASTOP2CL(shl);
846FASTOP2CL(shr);
847FASTOP2CL(sar);
848
849FASTOP2W(bsf);
850FASTOP2W(bsr);
851FASTOP2W(bt);
852FASTOP2W(bts);
853FASTOP2W(btr);
854FASTOP2W(btc);
855
856FASTOP2(xadd);
857
858static u8 test_cc(unsigned int condition, unsigned long flags)
859{
860 u8 rc;
861 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
862
863 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
864 asm("push %[flags]; popf; call *%[fastop]"
865 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
866 return rc;
867}
868
869static void fetch_register_operand(struct operand *op)
870{
871 switch (op->bytes) {
872 case 1:
873 op->val = *(u8 *)op->addr.reg;
874 break;
875 case 2:
876 op->val = *(u16 *)op->addr.reg;
877 break;
878 case 4:
879 op->val = *(u32 *)op->addr.reg;
880 break;
881 case 8:
882 op->val = *(u64 *)op->addr.reg;
883 break;
884 }
885}
886
887static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
888{
889 ctxt->ops->get_fpu(ctxt);
890 switch (reg) {
891 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
892 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
893 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
894 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
895 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
896 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
897 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
898 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
899#ifdef CONFIG_X86_64
900 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
901 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
902 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
903 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
904 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
905 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
906 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
907 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
908#endif
909 default: BUG();
910 }
911 ctxt->ops->put_fpu(ctxt);
912}
913
914static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
915 int reg)
916{
917 ctxt->ops->get_fpu(ctxt);
918 switch (reg) {
919 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
920 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
921 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
922 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
923 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
924 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
925 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
926 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
927#ifdef CONFIG_X86_64
928 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
929 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
930 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
931 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
932 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
933 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
934 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
935 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
936#endif
937 default: BUG();
938 }
939 ctxt->ops->put_fpu(ctxt);
940}
941
942static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
943{
944 ctxt->ops->get_fpu(ctxt);
945 switch (reg) {
946 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
947 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
948 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
949 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
950 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
951 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
952 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
953 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
954 default: BUG();
955 }
956 ctxt->ops->put_fpu(ctxt);
957}
958
959static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
960{
961 ctxt->ops->get_fpu(ctxt);
962 switch (reg) {
963 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
964 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
965 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
966 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
967 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
968 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
969 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
970 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
971 default: BUG();
972 }
973 ctxt->ops->put_fpu(ctxt);
974}
975
976static int em_fninit(struct x86_emulate_ctxt *ctxt)
977{
978 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
979 return emulate_nm(ctxt);
980
981 ctxt->ops->get_fpu(ctxt);
982 asm volatile("fninit");
983 ctxt->ops->put_fpu(ctxt);
984 return X86EMUL_CONTINUE;
985}
986
987static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
988{
989 u16 fcw;
990
991 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
992 return emulate_nm(ctxt);
993
994 ctxt->ops->get_fpu(ctxt);
995 asm volatile("fnstcw %0": "+m"(fcw));
996 ctxt->ops->put_fpu(ctxt);
997
998
999 ctxt->dst.bytes = 2;
1000 ctxt->dst.val = fcw;
1001
1002 return X86EMUL_CONTINUE;
1003}
1004
1005static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1006{
1007 u16 fsw;
1008
1009 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1010 return emulate_nm(ctxt);
1011
1012 ctxt->ops->get_fpu(ctxt);
1013 asm volatile("fnstsw %0": "+m"(fsw));
1014 ctxt->ops->put_fpu(ctxt);
1015
1016
1017 ctxt->dst.bytes = 2;
1018 ctxt->dst.val = fsw;
1019
1020 return X86EMUL_CONTINUE;
1021}
1022
1023static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1024 struct operand *op)
1025{
1026 unsigned reg = ctxt->modrm_reg;
1027 int highbyte_regs = ctxt->rex_prefix == 0;
1028
1029 if (!(ctxt->d & ModRM))
1030 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1031
1032 if (ctxt->d & Sse) {
1033 op->type = OP_XMM;
1034 op->bytes = 16;
1035 op->addr.xmm = reg;
1036 read_sse_reg(ctxt, &op->vec_val, reg);
1037 return;
1038 }
1039 if (ctxt->d & Mmx) {
1040 reg &= 7;
1041 op->type = OP_MM;
1042 op->bytes = 8;
1043 op->addr.mm = reg;
1044 return;
1045 }
1046
1047 op->type = OP_REG;
1048 if (ctxt->d & ByteOp) {
1049 op->addr.reg = decode_register(ctxt, reg, highbyte_regs);
1050 op->bytes = 1;
1051 } else {
1052 op->addr.reg = decode_register(ctxt, reg, 0);
1053 op->bytes = ctxt->op_bytes;
1054 }
1055 fetch_register_operand(op);
1056 op->orig_val = op->val;
1057}
1058
1059static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1060{
1061 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1062 ctxt->modrm_seg = VCPU_SREG_SS;
1063}
1064
1065static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1066 struct operand *op)
1067{
1068 u8 sib;
1069 int index_reg = 0, base_reg = 0, scale;
1070 int rc = X86EMUL_CONTINUE;
1071 ulong modrm_ea = 0;
1072
1073 if (ctxt->rex_prefix) {
1074 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1;
1075 index_reg = (ctxt->rex_prefix & 2) << 2;
1076 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3;
1077 }
1078
1079 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1080 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1081 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1082 ctxt->modrm_seg = VCPU_SREG_DS;
1083
1084 if (ctxt->modrm_mod == 3) {
1085 int highbyte_regs = ctxt->rex_prefix == 0;
1086
1087 op->type = OP_REG;
1088 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1089 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1090 highbyte_regs && (ctxt->d & ByteOp));
1091 if (ctxt->d & Sse) {
1092 op->type = OP_XMM;
1093 op->bytes = 16;
1094 op->addr.xmm = ctxt->modrm_rm;
1095 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1096 return rc;
1097 }
1098 if (ctxt->d & Mmx) {
1099 op->type = OP_MM;
1100 op->bytes = 8;
1101 op->addr.xmm = ctxt->modrm_rm & 7;
1102 return rc;
1103 }
1104 fetch_register_operand(op);
1105 return rc;
1106 }
1107
1108 op->type = OP_MEM;
1109
1110 if (ctxt->ad_bytes == 2) {
1111 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1112 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1113 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1114 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1115
1116
1117 switch (ctxt->modrm_mod) {
1118 case 0:
1119 if (ctxt->modrm_rm == 6)
1120 modrm_ea += insn_fetch(u16, ctxt);
1121 break;
1122 case 1:
1123 modrm_ea += insn_fetch(s8, ctxt);
1124 break;
1125 case 2:
1126 modrm_ea += insn_fetch(u16, ctxt);
1127 break;
1128 }
1129 switch (ctxt->modrm_rm) {
1130 case 0:
1131 modrm_ea += bx + si;
1132 break;
1133 case 1:
1134 modrm_ea += bx + di;
1135 break;
1136 case 2:
1137 modrm_ea += bp + si;
1138 break;
1139 case 3:
1140 modrm_ea += bp + di;
1141 break;
1142 case 4:
1143 modrm_ea += si;
1144 break;
1145 case 5:
1146 modrm_ea += di;
1147 break;
1148 case 6:
1149 if (ctxt->modrm_mod != 0)
1150 modrm_ea += bp;
1151 break;
1152 case 7:
1153 modrm_ea += bx;
1154 break;
1155 }
1156 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1157 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1158 ctxt->modrm_seg = VCPU_SREG_SS;
1159 modrm_ea = (u16)modrm_ea;
1160 } else {
1161
1162 if ((ctxt->modrm_rm & 7) == 4) {
1163 sib = insn_fetch(u8, ctxt);
1164 index_reg |= (sib >> 3) & 7;
1165 base_reg |= sib & 7;
1166 scale = sib >> 6;
1167
1168 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1169 modrm_ea += insn_fetch(s32, ctxt);
1170 else {
1171 modrm_ea += reg_read(ctxt, base_reg);
1172 adjust_modrm_seg(ctxt, base_reg);
1173 }
1174 if (index_reg != 4)
1175 modrm_ea += reg_read(ctxt, index_reg) << scale;
1176 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1177 if (ctxt->mode == X86EMUL_MODE_PROT64)
1178 ctxt->rip_relative = 1;
1179 } else {
1180 base_reg = ctxt->modrm_rm;
1181 modrm_ea += reg_read(ctxt, base_reg);
1182 adjust_modrm_seg(ctxt, base_reg);
1183 }
1184 switch (ctxt->modrm_mod) {
1185 case 0:
1186 if (ctxt->modrm_rm == 5)
1187 modrm_ea += insn_fetch(s32, ctxt);
1188 break;
1189 case 1:
1190 modrm_ea += insn_fetch(s8, ctxt);
1191 break;
1192 case 2:
1193 modrm_ea += insn_fetch(s32, ctxt);
1194 break;
1195 }
1196 }
1197 op->addr.mem.ea = modrm_ea;
1198done:
1199 return rc;
1200}
1201
1202static int decode_abs(struct x86_emulate_ctxt *ctxt,
1203 struct operand *op)
1204{
1205 int rc = X86EMUL_CONTINUE;
1206
1207 op->type = OP_MEM;
1208 switch (ctxt->ad_bytes) {
1209 case 2:
1210 op->addr.mem.ea = insn_fetch(u16, ctxt);
1211 break;
1212 case 4:
1213 op->addr.mem.ea = insn_fetch(u32, ctxt);
1214 break;
1215 case 8:
1216 op->addr.mem.ea = insn_fetch(u64, ctxt);
1217 break;
1218 }
1219done:
1220 return rc;
1221}
1222
1223static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1224{
1225 long sv = 0, mask;
1226
1227 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1228 mask = ~(ctxt->dst.bytes * 8 - 1);
1229
1230 if (ctxt->src.bytes == 2)
1231 sv = (s16)ctxt->src.val & (s16)mask;
1232 else if (ctxt->src.bytes == 4)
1233 sv = (s32)ctxt->src.val & (s32)mask;
1234
1235 ctxt->dst.addr.mem.ea += (sv >> 3);
1236 }
1237
1238
1239 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1240}
1241
1242static int read_emulated(struct x86_emulate_ctxt *ctxt,
1243 unsigned long addr, void *dest, unsigned size)
1244{
1245 int rc;
1246 struct read_cache *mc = &ctxt->mem_read;
1247
1248 if (mc->pos < mc->end)
1249 goto read_cached;
1250
1251 WARN_ON((mc->end + size) >= sizeof(mc->data));
1252
1253 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1254 &ctxt->exception);
1255 if (rc != X86EMUL_CONTINUE)
1256 return rc;
1257
1258 mc->end += size;
1259
1260read_cached:
1261 memcpy(dest, mc->data + mc->pos, size);
1262 mc->pos += size;
1263 return X86EMUL_CONTINUE;
1264}
1265
1266static int segmented_read(struct x86_emulate_ctxt *ctxt,
1267 struct segmented_address addr,
1268 void *data,
1269 unsigned size)
1270{
1271 int rc;
1272 ulong linear;
1273
1274 rc = linearize(ctxt, addr, size, false, &linear);
1275 if (rc != X86EMUL_CONTINUE)
1276 return rc;
1277 return read_emulated(ctxt, linear, data, size);
1278}
1279
1280static int segmented_write(struct x86_emulate_ctxt *ctxt,
1281 struct segmented_address addr,
1282 const void *data,
1283 unsigned size)
1284{
1285 int rc;
1286 ulong linear;
1287
1288 rc = linearize(ctxt, addr, size, true, &linear);
1289 if (rc != X86EMUL_CONTINUE)
1290 return rc;
1291 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1292 &ctxt->exception);
1293}
1294
1295static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1296 struct segmented_address addr,
1297 const void *orig_data, const void *data,
1298 unsigned size)
1299{
1300 int rc;
1301 ulong linear;
1302
1303 rc = linearize(ctxt, addr, size, true, &linear);
1304 if (rc != X86EMUL_CONTINUE)
1305 return rc;
1306 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1307 size, &ctxt->exception);
1308}
1309
1310static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1311 unsigned int size, unsigned short port,
1312 void *dest)
1313{
1314 struct read_cache *rc = &ctxt->io_read;
1315
1316 if (rc->pos == rc->end) {
1317 unsigned int in_page, n;
1318 unsigned int count = ctxt->rep_prefix ?
1319 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1320 in_page = (ctxt->eflags & EFLG_DF) ?
1321 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1322 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1323 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1324 count);
1325 if (n == 0)
1326 n = 1;
1327 rc->pos = rc->end = 0;
1328 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1329 return 0;
1330 rc->end = n * size;
1331 }
1332
1333 if (ctxt->rep_prefix && !(ctxt->eflags & EFLG_DF)) {
1334 ctxt->dst.data = rc->data + rc->pos;
1335 ctxt->dst.type = OP_MEM_STR;
1336 ctxt->dst.count = (rc->end - rc->pos) / size;
1337 rc->pos = rc->end;
1338 } else {
1339 memcpy(dest, rc->data + rc->pos, size);
1340 rc->pos += size;
1341 }
1342 return 1;
1343}
1344
1345static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1346 u16 index, struct desc_struct *desc)
1347{
1348 struct desc_ptr dt;
1349 ulong addr;
1350
1351 ctxt->ops->get_idt(ctxt, &dt);
1352
1353 if (dt.size < index * 8 + 7)
1354 return emulate_gp(ctxt, index << 3 | 0x2);
1355
1356 addr = dt.address + index * 8;
1357 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1358 &ctxt->exception);
1359}
1360
1361static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1362 u16 selector, struct desc_ptr *dt)
1363{
1364 const struct x86_emulate_ops *ops = ctxt->ops;
1365
1366 if (selector & 1 << 2) {
1367 struct desc_struct desc;
1368 u16 sel;
1369
1370 memset (dt, 0, sizeof *dt);
1371 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1372 return;
1373
1374 dt->size = desc_limit_scaled(&desc);
1375 dt->address = get_desc_base(&desc);
1376 } else
1377 ops->get_gdt(ctxt, dt);
1378}
1379
1380
1381static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1382 u16 selector, struct desc_struct *desc,
1383 ulong *desc_addr_p)
1384{
1385 struct desc_ptr dt;
1386 u16 index = selector >> 3;
1387 ulong addr;
1388
1389 get_descriptor_table_ptr(ctxt, selector, &dt);
1390
1391 if (dt.size < index * 8 + 7)
1392 return emulate_gp(ctxt, selector & 0xfffc);
1393
1394 *desc_addr_p = addr = dt.address + index * 8;
1395 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1396 &ctxt->exception);
1397}
1398
1399
1400static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1401 u16 selector, struct desc_struct *desc)
1402{
1403 struct desc_ptr dt;
1404 u16 index = selector >> 3;
1405 ulong addr;
1406
1407 get_descriptor_table_ptr(ctxt, selector, &dt);
1408
1409 if (dt.size < index * 8 + 7)
1410 return emulate_gp(ctxt, selector & 0xfffc);
1411
1412 addr = dt.address + index * 8;
1413 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1414 &ctxt->exception);
1415}
1416
1417
1418static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1419 u16 selector, int seg)
1420{
1421 struct desc_struct seg_desc, old_desc;
1422 u8 dpl, rpl, cpl;
1423 unsigned err_vec = GP_VECTOR;
1424 u32 err_code = 0;
1425 bool null_selector = !(selector & ~0x3);
1426 ulong desc_addr;
1427 int ret;
1428 u16 dummy;
1429
1430 memset(&seg_desc, 0, sizeof seg_desc);
1431
1432 if (ctxt->mode == X86EMUL_MODE_REAL) {
1433
1434
1435 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1436 set_desc_base(&seg_desc, selector << 4);
1437 goto load;
1438 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1439
1440 set_desc_base(&seg_desc, selector << 4);
1441 set_desc_limit(&seg_desc, 0xffff);
1442 seg_desc.type = 3;
1443 seg_desc.p = 1;
1444 seg_desc.s = 1;
1445 seg_desc.dpl = 3;
1446 goto load;
1447 }
1448
1449 rpl = selector & 3;
1450 cpl = ctxt->ops->cpl(ctxt);
1451
1452
1453 if ((seg == VCPU_SREG_CS
1454 || (seg == VCPU_SREG_SS
1455 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1456 || seg == VCPU_SREG_TR)
1457 && null_selector)
1458 goto exception;
1459
1460
1461 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1462 goto exception;
1463
1464 if (null_selector)
1465 goto load;
1466
1467 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1468 if (ret != X86EMUL_CONTINUE)
1469 return ret;
1470
1471 err_code = selector & 0xfffc;
1472 err_vec = GP_VECTOR;
1473
1474
1475 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1476 goto exception;
1477
1478 if (!seg_desc.p) {
1479 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1480 goto exception;
1481 }
1482
1483 dpl = seg_desc.dpl;
1484
1485 switch (seg) {
1486 case VCPU_SREG_SS:
1487
1488
1489
1490
1491 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1492 goto exception;
1493 break;
1494 case VCPU_SREG_CS:
1495 if (!(seg_desc.type & 8))
1496 goto exception;
1497
1498 if (seg_desc.type & 4) {
1499
1500 if (dpl > cpl)
1501 goto exception;
1502 } else {
1503
1504 if (rpl > cpl || dpl != cpl)
1505 goto exception;
1506 }
1507
1508 selector = (selector & 0xfffc) | cpl;
1509 break;
1510 case VCPU_SREG_TR:
1511 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1512 goto exception;
1513 old_desc = seg_desc;
1514 seg_desc.type |= 2;
1515 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1516 sizeof(seg_desc), &ctxt->exception);
1517 if (ret != X86EMUL_CONTINUE)
1518 return ret;
1519 break;
1520 case VCPU_SREG_LDTR:
1521 if (seg_desc.s || seg_desc.type != 2)
1522 goto exception;
1523 break;
1524 default:
1525
1526
1527
1528
1529
1530 if ((seg_desc.type & 0xa) == 0x8 ||
1531 (((seg_desc.type & 0xc) != 0xc) &&
1532 (rpl > dpl && cpl > dpl)))
1533 goto exception;
1534 break;
1535 }
1536
1537 if (seg_desc.s) {
1538
1539 seg_desc.type |= 1;
1540 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1541 if (ret != X86EMUL_CONTINUE)
1542 return ret;
1543 }
1544load:
1545 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1546 return X86EMUL_CONTINUE;
1547exception:
1548 emulate_exception(ctxt, err_vec, err_code, true);
1549 return X86EMUL_PROPAGATE_FAULT;
1550}
1551
1552static void write_register_operand(struct operand *op)
1553{
1554
1555 switch (op->bytes) {
1556 case 1:
1557 *(u8 *)op->addr.reg = (u8)op->val;
1558 break;
1559 case 2:
1560 *(u16 *)op->addr.reg = (u16)op->val;
1561 break;
1562 case 4:
1563 *op->addr.reg = (u32)op->val;
1564 break;
1565 case 8:
1566 *op->addr.reg = op->val;
1567 break;
1568 }
1569}
1570
1571static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1572{
1573 int rc;
1574
1575 switch (op->type) {
1576 case OP_REG:
1577 write_register_operand(op);
1578 break;
1579 case OP_MEM:
1580 if (ctxt->lock_prefix)
1581 rc = segmented_cmpxchg(ctxt,
1582 op->addr.mem,
1583 &op->orig_val,
1584 &op->val,
1585 op->bytes);
1586 else
1587 rc = segmented_write(ctxt,
1588 op->addr.mem,
1589 &op->val,
1590 op->bytes);
1591 if (rc != X86EMUL_CONTINUE)
1592 return rc;
1593 break;
1594 case OP_MEM_STR:
1595 rc = segmented_write(ctxt,
1596 op->addr.mem,
1597 op->data,
1598 op->bytes * op->count);
1599 if (rc != X86EMUL_CONTINUE)
1600 return rc;
1601 break;
1602 case OP_XMM:
1603 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1604 break;
1605 case OP_MM:
1606 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1607 break;
1608 case OP_NONE:
1609
1610 break;
1611 default:
1612 break;
1613 }
1614 return X86EMUL_CONTINUE;
1615}
1616
1617static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1618{
1619 struct segmented_address addr;
1620
1621 rsp_increment(ctxt, -bytes);
1622 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1623 addr.seg = VCPU_SREG_SS;
1624
1625 return segmented_write(ctxt, addr, data, bytes);
1626}
1627
1628static int em_push(struct x86_emulate_ctxt *ctxt)
1629{
1630
1631 ctxt->dst.type = OP_NONE;
1632 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1633}
1634
1635static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1636 void *dest, int len)
1637{
1638 int rc;
1639 struct segmented_address addr;
1640
1641 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1642 addr.seg = VCPU_SREG_SS;
1643 rc = segmented_read(ctxt, addr, dest, len);
1644 if (rc != X86EMUL_CONTINUE)
1645 return rc;
1646
1647 rsp_increment(ctxt, len);
1648 return rc;
1649}
1650
1651static int em_pop(struct x86_emulate_ctxt *ctxt)
1652{
1653 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1654}
1655
1656static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1657 void *dest, int len)
1658{
1659 int rc;
1660 unsigned long val, change_mask;
1661 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1662 int cpl = ctxt->ops->cpl(ctxt);
1663
1664 rc = emulate_pop(ctxt, &val, len);
1665 if (rc != X86EMUL_CONTINUE)
1666 return rc;
1667
1668 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1669 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1670
1671 switch(ctxt->mode) {
1672 case X86EMUL_MODE_PROT64:
1673 case X86EMUL_MODE_PROT32:
1674 case X86EMUL_MODE_PROT16:
1675 if (cpl == 0)
1676 change_mask |= EFLG_IOPL;
1677 if (cpl <= iopl)
1678 change_mask |= EFLG_IF;
1679 break;
1680 case X86EMUL_MODE_VM86:
1681 if (iopl < 3)
1682 return emulate_gp(ctxt, 0);
1683 change_mask |= EFLG_IF;
1684 break;
1685 default:
1686 change_mask |= (EFLG_IOPL | EFLG_IF);
1687 break;
1688 }
1689
1690 *(unsigned long *)dest =
1691 (ctxt->eflags & ~change_mask) | (val & change_mask);
1692
1693 return rc;
1694}
1695
1696static int em_popf(struct x86_emulate_ctxt *ctxt)
1697{
1698 ctxt->dst.type = OP_REG;
1699 ctxt->dst.addr.reg = &ctxt->eflags;
1700 ctxt->dst.bytes = ctxt->op_bytes;
1701 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1702}
1703
1704static int em_enter(struct x86_emulate_ctxt *ctxt)
1705{
1706 int rc;
1707 unsigned frame_size = ctxt->src.val;
1708 unsigned nesting_level = ctxt->src2.val & 31;
1709 ulong rbp;
1710
1711 if (nesting_level)
1712 return X86EMUL_UNHANDLEABLE;
1713
1714 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1715 rc = push(ctxt, &rbp, stack_size(ctxt));
1716 if (rc != X86EMUL_CONTINUE)
1717 return rc;
1718 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1719 stack_mask(ctxt));
1720 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1721 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1722 stack_mask(ctxt));
1723 return X86EMUL_CONTINUE;
1724}
1725
1726static int em_leave(struct x86_emulate_ctxt *ctxt)
1727{
1728 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1729 stack_mask(ctxt));
1730 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1731}
1732
1733static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1734{
1735 int seg = ctxt->src2.val;
1736
1737 ctxt->src.val = get_segment_selector(ctxt, seg);
1738
1739 return em_push(ctxt);
1740}
1741
1742static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1743{
1744 int seg = ctxt->src2.val;
1745 unsigned long selector;
1746 int rc;
1747
1748 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1749 if (rc != X86EMUL_CONTINUE)
1750 return rc;
1751
1752 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1753 return rc;
1754}
1755
1756static int em_pusha(struct x86_emulate_ctxt *ctxt)
1757{
1758 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1759 int rc = X86EMUL_CONTINUE;
1760 int reg = VCPU_REGS_RAX;
1761
1762 while (reg <= VCPU_REGS_RDI) {
1763 (reg == VCPU_REGS_RSP) ?
1764 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1765
1766 rc = em_push(ctxt);
1767 if (rc != X86EMUL_CONTINUE)
1768 return rc;
1769
1770 ++reg;
1771 }
1772
1773 return rc;
1774}
1775
1776static int em_pushf(struct x86_emulate_ctxt *ctxt)
1777{
1778 ctxt->src.val = (unsigned long)ctxt->eflags;
1779 return em_push(ctxt);
1780}
1781
1782static int em_popa(struct x86_emulate_ctxt *ctxt)
1783{
1784 int rc = X86EMUL_CONTINUE;
1785 int reg = VCPU_REGS_RDI;
1786
1787 while (reg >= VCPU_REGS_RAX) {
1788 if (reg == VCPU_REGS_RSP) {
1789 rsp_increment(ctxt, ctxt->op_bytes);
1790 --reg;
1791 }
1792
1793 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1794 if (rc != X86EMUL_CONTINUE)
1795 break;
1796 --reg;
1797 }
1798 return rc;
1799}
1800
1801static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1802{
1803 const struct x86_emulate_ops *ops = ctxt->ops;
1804 int rc;
1805 struct desc_ptr dt;
1806 gva_t cs_addr;
1807 gva_t eip_addr;
1808 u16 cs, eip;
1809
1810
1811 ctxt->src.val = ctxt->eflags;
1812 rc = em_push(ctxt);
1813 if (rc != X86EMUL_CONTINUE)
1814 return rc;
1815
1816 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1817
1818 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1819 rc = em_push(ctxt);
1820 if (rc != X86EMUL_CONTINUE)
1821 return rc;
1822
1823 ctxt->src.val = ctxt->_eip;
1824 rc = em_push(ctxt);
1825 if (rc != X86EMUL_CONTINUE)
1826 return rc;
1827
1828 ops->get_idt(ctxt, &dt);
1829
1830 eip_addr = dt.address + (irq << 2);
1831 cs_addr = dt.address + (irq << 2) + 2;
1832
1833 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1834 if (rc != X86EMUL_CONTINUE)
1835 return rc;
1836
1837 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1838 if (rc != X86EMUL_CONTINUE)
1839 return rc;
1840
1841 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1842 if (rc != X86EMUL_CONTINUE)
1843 return rc;
1844
1845 ctxt->_eip = eip;
1846
1847 return rc;
1848}
1849
1850int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1851{
1852 int rc;
1853
1854 invalidate_registers(ctxt);
1855 rc = __emulate_int_real(ctxt, irq);
1856 if (rc == X86EMUL_CONTINUE)
1857 writeback_registers(ctxt);
1858 return rc;
1859}
1860
1861static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1862{
1863 switch(ctxt->mode) {
1864 case X86EMUL_MODE_REAL:
1865 return __emulate_int_real(ctxt, irq);
1866 case X86EMUL_MODE_VM86:
1867 case X86EMUL_MODE_PROT16:
1868 case X86EMUL_MODE_PROT32:
1869 case X86EMUL_MODE_PROT64:
1870 default:
1871
1872 return X86EMUL_UNHANDLEABLE;
1873 }
1874}
1875
1876static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1877{
1878 int rc = X86EMUL_CONTINUE;
1879 unsigned long temp_eip = 0;
1880 unsigned long temp_eflags = 0;
1881 unsigned long cs = 0;
1882 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1883 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1884 EFLG_AC | EFLG_ID | (1 << 1);
1885 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1886
1887
1888
1889 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1890
1891 if (rc != X86EMUL_CONTINUE)
1892 return rc;
1893
1894 if (temp_eip & ~0xffff)
1895 return emulate_gp(ctxt, 0);
1896
1897 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1898
1899 if (rc != X86EMUL_CONTINUE)
1900 return rc;
1901
1902 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1903
1904 if (rc != X86EMUL_CONTINUE)
1905 return rc;
1906
1907 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1908
1909 if (rc != X86EMUL_CONTINUE)
1910 return rc;
1911
1912 ctxt->_eip = temp_eip;
1913
1914
1915 if (ctxt->op_bytes == 4)
1916 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1917 else if (ctxt->op_bytes == 2) {
1918 ctxt->eflags &= ~0xffff;
1919 ctxt->eflags |= temp_eflags;
1920 }
1921
1922 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK;
1923 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1924
1925 return rc;
1926}
1927
1928static int em_iret(struct x86_emulate_ctxt *ctxt)
1929{
1930 switch(ctxt->mode) {
1931 case X86EMUL_MODE_REAL:
1932 return emulate_iret_real(ctxt);
1933 case X86EMUL_MODE_VM86:
1934 case X86EMUL_MODE_PROT16:
1935 case X86EMUL_MODE_PROT32:
1936 case X86EMUL_MODE_PROT64:
1937 default:
1938
1939 return X86EMUL_UNHANDLEABLE;
1940 }
1941}
1942
1943static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1944{
1945 int rc;
1946 unsigned short sel;
1947
1948 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1949
1950 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1951 if (rc != X86EMUL_CONTINUE)
1952 return rc;
1953
1954 ctxt->_eip = 0;
1955 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
1956 return X86EMUL_CONTINUE;
1957}
1958
1959static int em_grp45(struct x86_emulate_ctxt *ctxt)
1960{
1961 int rc = X86EMUL_CONTINUE;
1962
1963 switch (ctxt->modrm_reg) {
1964 case 2: {
1965 long int old_eip;
1966 old_eip = ctxt->_eip;
1967 ctxt->_eip = ctxt->src.val;
1968 ctxt->src.val = old_eip;
1969 rc = em_push(ctxt);
1970 break;
1971 }
1972 case 4:
1973 ctxt->_eip = ctxt->src.val;
1974 break;
1975 case 5:
1976 rc = em_jmp_far(ctxt);
1977 break;
1978 case 6:
1979 rc = em_push(ctxt);
1980 break;
1981 }
1982 return rc;
1983}
1984
1985static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
1986{
1987 u64 old = ctxt->dst.orig_val64;
1988
1989 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
1990 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
1991 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
1992 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
1993 ctxt->eflags &= ~EFLG_ZF;
1994 } else {
1995 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
1996 (u32) reg_read(ctxt, VCPU_REGS_RBX);
1997
1998 ctxt->eflags |= EFLG_ZF;
1999 }
2000 return X86EMUL_CONTINUE;
2001}
2002
2003static int em_ret(struct x86_emulate_ctxt *ctxt)
2004{
2005 ctxt->dst.type = OP_REG;
2006 ctxt->dst.addr.reg = &ctxt->_eip;
2007 ctxt->dst.bytes = ctxt->op_bytes;
2008 return em_pop(ctxt);
2009}
2010
2011static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2012{
2013 int rc;
2014 unsigned long cs;
2015
2016 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2017 if (rc != X86EMUL_CONTINUE)
2018 return rc;
2019 if (ctxt->op_bytes == 4)
2020 ctxt->_eip = (u32)ctxt->_eip;
2021 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2022 if (rc != X86EMUL_CONTINUE)
2023 return rc;
2024 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2025 return rc;
2026}
2027
2028static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2029{
2030
2031 ctxt->src.orig_val = ctxt->src.val;
2032 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
2033 fastop(ctxt, em_cmp);
2034
2035 if (ctxt->eflags & EFLG_ZF) {
2036
2037 ctxt->dst.val = ctxt->src.orig_val;
2038 } else {
2039
2040 ctxt->dst.type = OP_REG;
2041 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2042 }
2043 return X86EMUL_CONTINUE;
2044}
2045
2046static int em_lseg(struct x86_emulate_ctxt *ctxt)
2047{
2048 int seg = ctxt->src2.val;
2049 unsigned short sel;
2050 int rc;
2051
2052 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2053
2054 rc = load_segment_descriptor(ctxt, sel, seg);
2055 if (rc != X86EMUL_CONTINUE)
2056 return rc;
2057
2058 ctxt->dst.val = ctxt->src.val;
2059 return rc;
2060}
2061
2062static void
2063setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2064 struct desc_struct *cs, struct desc_struct *ss)
2065{
2066 cs->l = 0;
2067 set_desc_base(cs, 0);
2068 cs->g = 1;
2069 set_desc_limit(cs, 0xfffff);
2070 cs->type = 0x0b;
2071 cs->s = 1;
2072 cs->dpl = 0;
2073 cs->p = 1;
2074 cs->d = 1;
2075 cs->avl = 0;
2076
2077 set_desc_base(ss, 0);
2078 set_desc_limit(ss, 0xfffff);
2079 ss->g = 1;
2080 ss->s = 1;
2081 ss->type = 0x03;
2082 ss->d = 1;
2083 ss->dpl = 0;
2084 ss->p = 1;
2085 ss->l = 0;
2086 ss->avl = 0;
2087}
2088
2089static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2090{
2091 u32 eax, ebx, ecx, edx;
2092
2093 eax = ecx = 0;
2094 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2095 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2096 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2097 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2098}
2099
2100static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2101{
2102 const struct x86_emulate_ops *ops = ctxt->ops;
2103 u32 eax, ebx, ecx, edx;
2104
2105
2106
2107
2108
2109 if (ctxt->mode == X86EMUL_MODE_PROT64)
2110 return true;
2111
2112 eax = 0x00000000;
2113 ecx = 0x00000000;
2114 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2115
2116
2117
2118
2119
2120
2121
2122
2123 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2124 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2125 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2126 return false;
2127
2128
2129 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2130 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2131 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2132 return true;
2133
2134
2135 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2136 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2137 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2138 return true;
2139
2140
2141 return false;
2142}
2143
2144static int em_syscall(struct x86_emulate_ctxt *ctxt)
2145{
2146 const struct x86_emulate_ops *ops = ctxt->ops;
2147 struct desc_struct cs, ss;
2148 u64 msr_data;
2149 u16 cs_sel, ss_sel;
2150 u64 efer = 0;
2151
2152
2153 if (ctxt->mode == X86EMUL_MODE_REAL ||
2154 ctxt->mode == X86EMUL_MODE_VM86)
2155 return emulate_ud(ctxt);
2156
2157 if (!(em_syscall_is_enabled(ctxt)))
2158 return emulate_ud(ctxt);
2159
2160 ops->get_msr(ctxt, MSR_EFER, &efer);
2161 setup_syscalls_segments(ctxt, &cs, &ss);
2162
2163 if (!(efer & EFER_SCE))
2164 return emulate_ud(ctxt);
2165
2166 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2167 msr_data >>= 32;
2168 cs_sel = (u16)(msr_data & 0xfffc);
2169 ss_sel = (u16)(msr_data + 8);
2170
2171 if (efer & EFER_LMA) {
2172 cs.d = 0;
2173 cs.l = 1;
2174 }
2175 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2176 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2177
2178 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2179 if (efer & EFER_LMA) {
2180#ifdef CONFIG_X86_64
2181 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2182
2183 ops->get_msr(ctxt,
2184 ctxt->mode == X86EMUL_MODE_PROT64 ?
2185 MSR_LSTAR : MSR_CSTAR, &msr_data);
2186 ctxt->_eip = msr_data;
2187
2188 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2189 ctxt->eflags &= ~(msr_data | EFLG_RF);
2190#endif
2191 } else {
2192
2193 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2194 ctxt->_eip = (u32)msr_data;
2195
2196 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2197 }
2198
2199 return X86EMUL_CONTINUE;
2200}
2201
2202static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2203{
2204 const struct x86_emulate_ops *ops = ctxt->ops;
2205 struct desc_struct cs, ss;
2206 u64 msr_data;
2207 u16 cs_sel, ss_sel;
2208 u64 efer = 0;
2209
2210 ops->get_msr(ctxt, MSR_EFER, &efer);
2211
2212 if (ctxt->mode == X86EMUL_MODE_REAL)
2213 return emulate_gp(ctxt, 0);
2214
2215
2216
2217
2218
2219 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2220 && !vendor_intel(ctxt))
2221 return emulate_ud(ctxt);
2222
2223
2224
2225
2226 if (ctxt->mode == X86EMUL_MODE_PROT64)
2227 return emulate_ud(ctxt);
2228
2229 setup_syscalls_segments(ctxt, &cs, &ss);
2230
2231 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2232 switch (ctxt->mode) {
2233 case X86EMUL_MODE_PROT32:
2234 if ((msr_data & 0xfffc) == 0x0)
2235 return emulate_gp(ctxt, 0);
2236 break;
2237 case X86EMUL_MODE_PROT64:
2238 if (msr_data == 0x0)
2239 return emulate_gp(ctxt, 0);
2240 break;
2241 default:
2242 break;
2243 }
2244
2245 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2246 cs_sel = (u16)msr_data;
2247 cs_sel &= ~SELECTOR_RPL_MASK;
2248 ss_sel = cs_sel + 8;
2249 ss_sel &= ~SELECTOR_RPL_MASK;
2250 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2251 cs.d = 0;
2252 cs.l = 1;
2253 }
2254
2255 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2256 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2257
2258 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2259 ctxt->_eip = msr_data;
2260
2261 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2262 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2263
2264 return X86EMUL_CONTINUE;
2265}
2266
2267static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2268{
2269 const struct x86_emulate_ops *ops = ctxt->ops;
2270 struct desc_struct cs, ss;
2271 u64 msr_data;
2272 int usermode;
2273 u16 cs_sel = 0, ss_sel = 0;
2274
2275
2276 if (ctxt->mode == X86EMUL_MODE_REAL ||
2277 ctxt->mode == X86EMUL_MODE_VM86)
2278 return emulate_gp(ctxt, 0);
2279
2280 setup_syscalls_segments(ctxt, &cs, &ss);
2281
2282 if ((ctxt->rex_prefix & 0x8) != 0x0)
2283 usermode = X86EMUL_MODE_PROT64;
2284 else
2285 usermode = X86EMUL_MODE_PROT32;
2286
2287 cs.dpl = 3;
2288 ss.dpl = 3;
2289 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2290 switch (usermode) {
2291 case X86EMUL_MODE_PROT32:
2292 cs_sel = (u16)(msr_data + 16);
2293 if ((msr_data & 0xfffc) == 0x0)
2294 return emulate_gp(ctxt, 0);
2295 ss_sel = (u16)(msr_data + 24);
2296 break;
2297 case X86EMUL_MODE_PROT64:
2298 cs_sel = (u16)(msr_data + 32);
2299 if (msr_data == 0x0)
2300 return emulate_gp(ctxt, 0);
2301 ss_sel = cs_sel + 8;
2302 cs.d = 0;
2303 cs.l = 1;
2304 break;
2305 }
2306 cs_sel |= SELECTOR_RPL_MASK;
2307 ss_sel |= SELECTOR_RPL_MASK;
2308
2309 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2310 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2311
2312 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2313 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2314
2315 return X86EMUL_CONTINUE;
2316}
2317
2318static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2319{
2320 int iopl;
2321 if (ctxt->mode == X86EMUL_MODE_REAL)
2322 return false;
2323 if (ctxt->mode == X86EMUL_MODE_VM86)
2324 return true;
2325 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2326 return ctxt->ops->cpl(ctxt) > iopl;
2327}
2328
2329static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2330 u16 port, u16 len)
2331{
2332 const struct x86_emulate_ops *ops = ctxt->ops;
2333 struct desc_struct tr_seg;
2334 u32 base3;
2335 int r;
2336 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2337 unsigned mask = (1 << len) - 1;
2338 unsigned long base;
2339
2340 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2341 if (!tr_seg.p)
2342 return false;
2343 if (desc_limit_scaled(&tr_seg) < 103)
2344 return false;
2345 base = get_desc_base(&tr_seg);
2346#ifdef CONFIG_X86_64
2347 base |= ((u64)base3) << 32;
2348#endif
2349 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2350 if (r != X86EMUL_CONTINUE)
2351 return false;
2352 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2353 return false;
2354 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2355 if (r != X86EMUL_CONTINUE)
2356 return false;
2357 if ((perm >> bit_idx) & mask)
2358 return false;
2359 return true;
2360}
2361
2362static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2363 u16 port, u16 len)
2364{
2365 if (ctxt->perm_ok)
2366 return true;
2367
2368 if (emulator_bad_iopl(ctxt))
2369 if (!emulator_io_port_access_allowed(ctxt, port, len))
2370 return false;
2371
2372 ctxt->perm_ok = true;
2373
2374 return true;
2375}
2376
2377static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2378 struct tss_segment_16 *tss)
2379{
2380 tss->ip = ctxt->_eip;
2381 tss->flag = ctxt->eflags;
2382 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2383 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2384 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2385 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2386 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2387 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2388 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2389 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2390
2391 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2392 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2393 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2394 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2395 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2396}
2397
2398static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2399 struct tss_segment_16 *tss)
2400{
2401 int ret;
2402
2403 ctxt->_eip = tss->ip;
2404 ctxt->eflags = tss->flag | 2;
2405 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2406 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2407 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2408 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2409 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2410 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2411 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2412 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2413
2414
2415
2416
2417
2418 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2419 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2420 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2421 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2422 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2423
2424
2425
2426
2427
2428 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2429 if (ret != X86EMUL_CONTINUE)
2430 return ret;
2431 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2432 if (ret != X86EMUL_CONTINUE)
2433 return ret;
2434 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2435 if (ret != X86EMUL_CONTINUE)
2436 return ret;
2437 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2438 if (ret != X86EMUL_CONTINUE)
2439 return ret;
2440 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2441 if (ret != X86EMUL_CONTINUE)
2442 return ret;
2443
2444 return X86EMUL_CONTINUE;
2445}
2446
2447static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2448 u16 tss_selector, u16 old_tss_sel,
2449 ulong old_tss_base, struct desc_struct *new_desc)
2450{
2451 const struct x86_emulate_ops *ops = ctxt->ops;
2452 struct tss_segment_16 tss_seg;
2453 int ret;
2454 u32 new_tss_base = get_desc_base(new_desc);
2455
2456 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2457 &ctxt->exception);
2458 if (ret != X86EMUL_CONTINUE)
2459
2460 return ret;
2461
2462 save_state_to_tss16(ctxt, &tss_seg);
2463
2464 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2465 &ctxt->exception);
2466 if (ret != X86EMUL_CONTINUE)
2467
2468 return ret;
2469
2470 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2471 &ctxt->exception);
2472 if (ret != X86EMUL_CONTINUE)
2473
2474 return ret;
2475
2476 if (old_tss_sel != 0xffff) {
2477 tss_seg.prev_task_link = old_tss_sel;
2478
2479 ret = ops->write_std(ctxt, new_tss_base,
2480 &tss_seg.prev_task_link,
2481 sizeof tss_seg.prev_task_link,
2482 &ctxt->exception);
2483 if (ret != X86EMUL_CONTINUE)
2484
2485 return ret;
2486 }
2487
2488 return load_state_from_tss16(ctxt, &tss_seg);
2489}
2490
2491static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2492 struct tss_segment_32 *tss)
2493{
2494 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2495 tss->eip = ctxt->_eip;
2496 tss->eflags = ctxt->eflags;
2497 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2498 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2499 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2500 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2501 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2502 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2503 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2504 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2505
2506 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2507 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2508 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2509 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2510 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2511 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2512 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2513}
2514
2515static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2516 struct tss_segment_32 *tss)
2517{
2518 int ret;
2519
2520 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2521 return emulate_gp(ctxt, 0);
2522 ctxt->_eip = tss->eip;
2523 ctxt->eflags = tss->eflags | 2;
2524
2525
2526 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2527 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2528 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2529 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2530 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2531 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2532 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2533 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2534
2535
2536
2537
2538
2539 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2540 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2541 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2542 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2543 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2544 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2545 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558 if (ctxt->eflags & X86_EFLAGS_VM)
2559 ctxt->mode = X86EMUL_MODE_VM86;
2560 else
2561 ctxt->mode = X86EMUL_MODE_PROT32;
2562
2563 ctxt->ops->set_rflags(ctxt, ctxt->eflags);
2564
2565
2566
2567
2568
2569 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2570 if (ret != X86EMUL_CONTINUE)
2571 return ret;
2572 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2573 if (ret != X86EMUL_CONTINUE)
2574 return ret;
2575 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2576 if (ret != X86EMUL_CONTINUE)
2577 return ret;
2578 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2579 if (ret != X86EMUL_CONTINUE)
2580 return ret;
2581 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2582 if (ret != X86EMUL_CONTINUE)
2583 return ret;
2584 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2585 if (ret != X86EMUL_CONTINUE)
2586 return ret;
2587 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2588 if (ret != X86EMUL_CONTINUE)
2589 return ret;
2590
2591 return X86EMUL_CONTINUE;
2592}
2593
2594static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2595 u16 tss_selector, u16 old_tss_sel,
2596 ulong old_tss_base, struct desc_struct *new_desc)
2597{
2598 const struct x86_emulate_ops *ops = ctxt->ops;
2599 struct tss_segment_32 tss_seg;
2600 int ret;
2601 u32 new_tss_base = get_desc_base(new_desc);
2602
2603 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2604 &ctxt->exception);
2605 if (ret != X86EMUL_CONTINUE)
2606
2607 return ret;
2608
2609 save_state_to_tss32(ctxt, &tss_seg);
2610
2611 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2612 &ctxt->exception);
2613 if (ret != X86EMUL_CONTINUE)
2614
2615 return ret;
2616
2617 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2618 &ctxt->exception);
2619 if (ret != X86EMUL_CONTINUE)
2620
2621 return ret;
2622
2623 if (old_tss_sel != 0xffff) {
2624 tss_seg.prev_task_link = old_tss_sel;
2625
2626 ret = ops->write_std(ctxt, new_tss_base,
2627 &tss_seg.prev_task_link,
2628 sizeof tss_seg.prev_task_link,
2629 &ctxt->exception);
2630 if (ret != X86EMUL_CONTINUE)
2631
2632 return ret;
2633 }
2634
2635 return load_state_from_tss32(ctxt, &tss_seg);
2636}
2637
2638static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2639 u16 tss_selector, int idt_index, int reason,
2640 bool has_error_code, u32 error_code)
2641{
2642 const struct x86_emulate_ops *ops = ctxt->ops;
2643 struct desc_struct curr_tss_desc, next_tss_desc;
2644 int ret;
2645 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2646 ulong old_tss_base =
2647 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2648 u32 desc_limit;
2649 ulong desc_addr;
2650
2651
2652
2653 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2654 if (ret != X86EMUL_CONTINUE)
2655 return ret;
2656 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2657 if (ret != X86EMUL_CONTINUE)
2658 return ret;
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669 if (reason == TASK_SWITCH_GATE) {
2670 if (idt_index != -1) {
2671
2672 struct desc_struct task_gate_desc;
2673 int dpl;
2674
2675 ret = read_interrupt_descriptor(ctxt, idt_index,
2676 &task_gate_desc);
2677 if (ret != X86EMUL_CONTINUE)
2678 return ret;
2679
2680 dpl = task_gate_desc.dpl;
2681 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2682 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2683 }
2684 } else if (reason != TASK_SWITCH_IRET) {
2685 int dpl = next_tss_desc.dpl;
2686 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2687 return emulate_gp(ctxt, tss_selector);
2688 }
2689
2690
2691 desc_limit = desc_limit_scaled(&next_tss_desc);
2692 if (!next_tss_desc.p ||
2693 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2694 desc_limit < 0x2b)) {
2695 emulate_ts(ctxt, tss_selector & 0xfffc);
2696 return X86EMUL_PROPAGATE_FAULT;
2697 }
2698
2699 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2700 curr_tss_desc.type &= ~(1 << 1);
2701 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2702 }
2703
2704 if (reason == TASK_SWITCH_IRET)
2705 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2706
2707
2708
2709 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2710 old_tss_sel = 0xffff;
2711
2712 if (next_tss_desc.type & 8)
2713 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2714 old_tss_base, &next_tss_desc);
2715 else
2716 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2717 old_tss_base, &next_tss_desc);
2718 if (ret != X86EMUL_CONTINUE)
2719 return ret;
2720
2721 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2722 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2723
2724 if (reason != TASK_SWITCH_IRET) {
2725 next_tss_desc.type |= (1 << 1);
2726 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2727 }
2728
2729 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2730 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2731
2732 if (has_error_code) {
2733 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2734 ctxt->lock_prefix = 0;
2735 ctxt->src.val = (unsigned long) error_code;
2736 ret = em_push(ctxt);
2737 }
2738
2739 return ret;
2740}
2741
2742int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2743 u16 tss_selector, int idt_index, int reason,
2744 bool has_error_code, u32 error_code)
2745{
2746 int rc;
2747
2748 invalidate_registers(ctxt);
2749 ctxt->_eip = ctxt->eip;
2750 ctxt->dst.type = OP_NONE;
2751
2752 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2753 has_error_code, error_code);
2754
2755 if (rc == X86EMUL_CONTINUE) {
2756 ctxt->eip = ctxt->_eip;
2757 writeback_registers(ctxt);
2758 }
2759
2760 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2761}
2762
2763static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2764 struct operand *op)
2765{
2766 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2767
2768 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2769 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2770}
2771
2772static int em_das(struct x86_emulate_ctxt *ctxt)
2773{
2774 u8 al, old_al;
2775 bool af, cf, old_cf;
2776
2777 cf = ctxt->eflags & X86_EFLAGS_CF;
2778 al = ctxt->dst.val;
2779
2780 old_al = al;
2781 old_cf = cf;
2782 cf = false;
2783 af = ctxt->eflags & X86_EFLAGS_AF;
2784 if ((al & 0x0f) > 9 || af) {
2785 al -= 6;
2786 cf = old_cf | (al >= 250);
2787 af = true;
2788 } else {
2789 af = false;
2790 }
2791 if (old_al > 0x99 || old_cf) {
2792 al -= 0x60;
2793 cf = true;
2794 }
2795
2796 ctxt->dst.val = al;
2797
2798 ctxt->src.type = OP_IMM;
2799 ctxt->src.val = 0;
2800 ctxt->src.bytes = 1;
2801 fastop(ctxt, em_or);
2802 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2803 if (cf)
2804 ctxt->eflags |= X86_EFLAGS_CF;
2805 if (af)
2806 ctxt->eflags |= X86_EFLAGS_AF;
2807 return X86EMUL_CONTINUE;
2808}
2809
2810static int em_aam(struct x86_emulate_ctxt *ctxt)
2811{
2812 u8 al, ah;
2813
2814 if (ctxt->src.val == 0)
2815 return emulate_de(ctxt);
2816
2817 al = ctxt->dst.val & 0xff;
2818 ah = al / ctxt->src.val;
2819 al %= ctxt->src.val;
2820
2821 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
2822
2823
2824 ctxt->src.type = OP_IMM;
2825 ctxt->src.val = 0;
2826 ctxt->src.bytes = 1;
2827 fastop(ctxt, em_or);
2828
2829 return X86EMUL_CONTINUE;
2830}
2831
2832static int em_aad(struct x86_emulate_ctxt *ctxt)
2833{
2834 u8 al = ctxt->dst.val & 0xff;
2835 u8 ah = (ctxt->dst.val >> 8) & 0xff;
2836
2837 al = (al + (ah * ctxt->src.val)) & 0xff;
2838
2839 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
2840
2841
2842 ctxt->src.type = OP_IMM;
2843 ctxt->src.val = 0;
2844 ctxt->src.bytes = 1;
2845 fastop(ctxt, em_or);
2846
2847 return X86EMUL_CONTINUE;
2848}
2849
2850static int em_call(struct x86_emulate_ctxt *ctxt)
2851{
2852 long rel = ctxt->src.val;
2853
2854 ctxt->src.val = (unsigned long)ctxt->_eip;
2855 jmp_rel(ctxt, rel);
2856 return em_push(ctxt);
2857}
2858
2859static int em_call_far(struct x86_emulate_ctxt *ctxt)
2860{
2861 u16 sel, old_cs;
2862 ulong old_eip;
2863 int rc;
2864
2865 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2866 old_eip = ctxt->_eip;
2867
2868 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2869 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2870 return X86EMUL_CONTINUE;
2871
2872 ctxt->_eip = 0;
2873 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2874
2875 ctxt->src.val = old_cs;
2876 rc = em_push(ctxt);
2877 if (rc != X86EMUL_CONTINUE)
2878 return rc;
2879
2880 ctxt->src.val = old_eip;
2881 return em_push(ctxt);
2882}
2883
2884static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2885{
2886 int rc;
2887
2888 ctxt->dst.type = OP_REG;
2889 ctxt->dst.addr.reg = &ctxt->_eip;
2890 ctxt->dst.bytes = ctxt->op_bytes;
2891 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
2892 if (rc != X86EMUL_CONTINUE)
2893 return rc;
2894 rsp_increment(ctxt, ctxt->src.val);
2895 return X86EMUL_CONTINUE;
2896}
2897
2898static int em_xchg(struct x86_emulate_ctxt *ctxt)
2899{
2900
2901 ctxt->src.val = ctxt->dst.val;
2902 write_register_operand(&ctxt->src);
2903
2904
2905 ctxt->dst.val = ctxt->src.orig_val;
2906 ctxt->lock_prefix = 1;
2907 return X86EMUL_CONTINUE;
2908}
2909
2910static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2911{
2912 ctxt->dst.val = ctxt->src2.val;
2913 return fastop(ctxt, em_imul);
2914}
2915
2916static int em_cwd(struct x86_emulate_ctxt *ctxt)
2917{
2918 ctxt->dst.type = OP_REG;
2919 ctxt->dst.bytes = ctxt->src.bytes;
2920 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
2921 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2922
2923 return X86EMUL_CONTINUE;
2924}
2925
2926static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2927{
2928 u64 tsc = 0;
2929
2930 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2931 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
2932 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
2933 return X86EMUL_CONTINUE;
2934}
2935
2936static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
2937{
2938 u64 pmc;
2939
2940 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
2941 return emulate_gp(ctxt, 0);
2942 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
2943 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
2944 return X86EMUL_CONTINUE;
2945}
2946
2947static int em_mov(struct x86_emulate_ctxt *ctxt)
2948{
2949 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
2950 return X86EMUL_CONTINUE;
2951}
2952
2953static int em_cr_write(struct x86_emulate_ctxt *ctxt)
2954{
2955 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
2956 return emulate_gp(ctxt, 0);
2957
2958
2959 ctxt->dst.type = OP_NONE;
2960 return X86EMUL_CONTINUE;
2961}
2962
2963static int em_dr_write(struct x86_emulate_ctxt *ctxt)
2964{
2965 unsigned long val;
2966
2967 if (ctxt->mode == X86EMUL_MODE_PROT64)
2968 val = ctxt->src.val & ~0ULL;
2969 else
2970 val = ctxt->src.val & ~0U;
2971
2972
2973 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
2974 return emulate_gp(ctxt, 0);
2975
2976
2977 ctxt->dst.type = OP_NONE;
2978 return X86EMUL_CONTINUE;
2979}
2980
2981static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
2982{
2983 u64 msr_data;
2984
2985 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
2986 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
2987 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
2988 return emulate_gp(ctxt, 0);
2989
2990 return X86EMUL_CONTINUE;
2991}
2992
2993static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
2994{
2995 u64 msr_data;
2996
2997 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
2998 return emulate_gp(ctxt, 0);
2999
3000 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3001 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3002 return X86EMUL_CONTINUE;
3003}
3004
3005static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3006{
3007 if (ctxt->modrm_reg > VCPU_SREG_GS)
3008 return emulate_ud(ctxt);
3009
3010 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3011 return X86EMUL_CONTINUE;
3012}
3013
3014static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3015{
3016 u16 sel = ctxt->src.val;
3017
3018 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3019 return emulate_ud(ctxt);
3020
3021 if (ctxt->modrm_reg == VCPU_SREG_SS)
3022 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3023
3024
3025 ctxt->dst.type = OP_NONE;
3026 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3027}
3028
3029static int em_lldt(struct x86_emulate_ctxt *ctxt)
3030{
3031 u16 sel = ctxt->src.val;
3032
3033
3034 ctxt->dst.type = OP_NONE;
3035 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3036}
3037
3038static int em_ltr(struct x86_emulate_ctxt *ctxt)
3039{
3040 u16 sel = ctxt->src.val;
3041
3042
3043 ctxt->dst.type = OP_NONE;
3044 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3045}
3046
3047static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3048{
3049 int rc;
3050 ulong linear;
3051
3052 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3053 if (rc == X86EMUL_CONTINUE)
3054 ctxt->ops->invlpg(ctxt, linear);
3055
3056 ctxt->dst.type = OP_NONE;
3057 return X86EMUL_CONTINUE;
3058}
3059
3060static int em_clts(struct x86_emulate_ctxt *ctxt)
3061{
3062 ulong cr0;
3063
3064 cr0 = ctxt->ops->get_cr(ctxt, 0);
3065 cr0 &= ~X86_CR0_TS;
3066 ctxt->ops->set_cr(ctxt, 0, cr0);
3067 return X86EMUL_CONTINUE;
3068}
3069
3070static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3071{
3072 int rc;
3073
3074 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3075 return X86EMUL_UNHANDLEABLE;
3076
3077 rc = ctxt->ops->fix_hypercall(ctxt);
3078 if (rc != X86EMUL_CONTINUE)
3079 return rc;
3080
3081
3082 ctxt->_eip = ctxt->eip;
3083
3084 ctxt->dst.type = OP_NONE;
3085 return X86EMUL_CONTINUE;
3086}
3087
3088static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3089 void (*get)(struct x86_emulate_ctxt *ctxt,
3090 struct desc_ptr *ptr))
3091{
3092 struct desc_ptr desc_ptr;
3093
3094 if (ctxt->mode == X86EMUL_MODE_PROT64)
3095 ctxt->op_bytes = 8;
3096 get(ctxt, &desc_ptr);
3097 if (ctxt->op_bytes == 2) {
3098 ctxt->op_bytes = 4;
3099 desc_ptr.address &= 0x00ffffff;
3100 }
3101
3102 ctxt->dst.type = OP_NONE;
3103 return segmented_write(ctxt, ctxt->dst.addr.mem,
3104 &desc_ptr, 2 + ctxt->op_bytes);
3105}
3106
3107static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3108{
3109 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3110}
3111
3112static int em_sidt(struct x86_emulate_ctxt *ctxt)
3113{
3114 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3115}
3116
3117static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3118{
3119 struct desc_ptr desc_ptr;
3120 int rc;
3121
3122 if (ctxt->mode == X86EMUL_MODE_PROT64)
3123 ctxt->op_bytes = 8;
3124 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3125 &desc_ptr.size, &desc_ptr.address,
3126 ctxt->op_bytes);
3127 if (rc != X86EMUL_CONTINUE)
3128 return rc;
3129 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3130
3131 ctxt->dst.type = OP_NONE;
3132 return X86EMUL_CONTINUE;
3133}
3134
3135static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3136{
3137 int rc;
3138
3139 rc = ctxt->ops->fix_hypercall(ctxt);
3140
3141
3142 ctxt->dst.type = OP_NONE;
3143 return rc;
3144}
3145
3146static int em_lidt(struct x86_emulate_ctxt *ctxt)
3147{
3148 struct desc_ptr desc_ptr;
3149 int rc;
3150
3151 if (ctxt->mode == X86EMUL_MODE_PROT64)
3152 ctxt->op_bytes = 8;
3153 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3154 &desc_ptr.size, &desc_ptr.address,
3155 ctxt->op_bytes);
3156 if (rc != X86EMUL_CONTINUE)
3157 return rc;
3158 ctxt->ops->set_idt(ctxt, &desc_ptr);
3159
3160 ctxt->dst.type = OP_NONE;
3161 return X86EMUL_CONTINUE;
3162}
3163
3164static int em_smsw(struct x86_emulate_ctxt *ctxt)
3165{
3166 ctxt->dst.bytes = 2;
3167 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3168 return X86EMUL_CONTINUE;
3169}
3170
3171static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3172{
3173 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3174 | (ctxt->src.val & 0x0f));
3175 ctxt->dst.type = OP_NONE;
3176 return X86EMUL_CONTINUE;
3177}
3178
3179static int em_loop(struct x86_emulate_ctxt *ctxt)
3180{
3181 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3182 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3183 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3184 jmp_rel(ctxt, ctxt->src.val);
3185
3186 return X86EMUL_CONTINUE;
3187}
3188
3189static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3190{
3191 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3192 jmp_rel(ctxt, ctxt->src.val);
3193
3194 return X86EMUL_CONTINUE;
3195}
3196
3197static int em_in(struct x86_emulate_ctxt *ctxt)
3198{
3199 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3200 &ctxt->dst.val))
3201 return X86EMUL_IO_NEEDED;
3202
3203 return X86EMUL_CONTINUE;
3204}
3205
3206static int em_out(struct x86_emulate_ctxt *ctxt)
3207{
3208 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3209 &ctxt->src.val, 1);
3210
3211 ctxt->dst.type = OP_NONE;
3212 return X86EMUL_CONTINUE;
3213}
3214
3215static int em_cli(struct x86_emulate_ctxt *ctxt)
3216{
3217 if (emulator_bad_iopl(ctxt))
3218 return emulate_gp(ctxt, 0);
3219
3220 ctxt->eflags &= ~X86_EFLAGS_IF;
3221 return X86EMUL_CONTINUE;
3222}
3223
3224static int em_sti(struct x86_emulate_ctxt *ctxt)
3225{
3226 if (emulator_bad_iopl(ctxt))
3227 return emulate_gp(ctxt, 0);
3228
3229 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3230 ctxt->eflags |= X86_EFLAGS_IF;
3231 return X86EMUL_CONTINUE;
3232}
3233
3234static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3235{
3236 u32 eax, ebx, ecx, edx;
3237
3238 eax = reg_read(ctxt, VCPU_REGS_RAX);
3239 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3240 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3241 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3242 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3243 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3244 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3245 return X86EMUL_CONTINUE;
3246}
3247
3248static int em_lahf(struct x86_emulate_ctxt *ctxt)
3249{
3250 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3251 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3252 return X86EMUL_CONTINUE;
3253}
3254
3255static int em_bswap(struct x86_emulate_ctxt *ctxt)
3256{
3257 switch (ctxt->op_bytes) {
3258#ifdef CONFIG_X86_64
3259 case 8:
3260 asm("bswap %0" : "+r"(ctxt->dst.val));
3261 break;
3262#endif
3263 default:
3264 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3265 break;
3266 }
3267 return X86EMUL_CONTINUE;
3268}
3269
3270static bool valid_cr(int nr)
3271{
3272 switch (nr) {
3273 case 0:
3274 case 2 ... 4:
3275 case 8:
3276 return true;
3277 default:
3278 return false;
3279 }
3280}
3281
3282static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3283{
3284 if (!valid_cr(ctxt->modrm_reg))
3285 return emulate_ud(ctxt);
3286
3287 return X86EMUL_CONTINUE;
3288}
3289
3290static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3291{
3292 u64 new_val = ctxt->src.val64;
3293 int cr = ctxt->modrm_reg;
3294 u64 efer = 0;
3295
3296 static u64 cr_reserved_bits[] = {
3297 0xffffffff00000000ULL,
3298 0, 0, 0,
3299 CR4_RESERVED_BITS,
3300 0, 0, 0,
3301 CR8_RESERVED_BITS,
3302 };
3303
3304 if (!valid_cr(cr))
3305 return emulate_ud(ctxt);
3306
3307 if (new_val & cr_reserved_bits[cr])
3308 return emulate_gp(ctxt, 0);
3309
3310 switch (cr) {
3311 case 0: {
3312 u64 cr4;
3313 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3314 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3315 return emulate_gp(ctxt, 0);
3316
3317 cr4 = ctxt->ops->get_cr(ctxt, 4);
3318 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3319
3320 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3321 !(cr4 & X86_CR4_PAE))
3322 return emulate_gp(ctxt, 0);
3323
3324 break;
3325 }
3326 case 3: {
3327 u64 rsvd = 0;
3328
3329 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3330 if (efer & EFER_LMA)
3331 rsvd = CR3_L_MODE_RESERVED_BITS;
3332 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3333 rsvd = CR3_PAE_RESERVED_BITS;
3334 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3335 rsvd = CR3_NONPAE_RESERVED_BITS;
3336
3337 if (new_val & rsvd)
3338 return emulate_gp(ctxt, 0);
3339
3340 break;
3341 }
3342 case 4: {
3343 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3344
3345 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3346 return emulate_gp(ctxt, 0);
3347
3348 break;
3349 }
3350 }
3351
3352 return X86EMUL_CONTINUE;
3353}
3354
3355static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3356{
3357 unsigned long dr7;
3358
3359 ctxt->ops->get_dr(ctxt, 7, &dr7);
3360
3361
3362 return dr7 & (1 << 13);
3363}
3364
3365static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3366{
3367 int dr = ctxt->modrm_reg;
3368 u64 cr4;
3369
3370 if (dr > 7)
3371 return emulate_ud(ctxt);
3372
3373 cr4 = ctxt->ops->get_cr(ctxt, 4);
3374 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3375 return emulate_ud(ctxt);
3376
3377 if (check_dr7_gd(ctxt))
3378 return emulate_db(ctxt);
3379
3380 return X86EMUL_CONTINUE;
3381}
3382
3383static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3384{
3385 u64 new_val = ctxt->src.val64;
3386 int dr = ctxt->modrm_reg;
3387
3388 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3389 return emulate_gp(ctxt, 0);
3390
3391 return check_dr_read(ctxt);
3392}
3393
3394static int check_svme(struct x86_emulate_ctxt *ctxt)
3395{
3396 u64 efer;
3397
3398 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3399
3400 if (!(efer & EFER_SVME))
3401 return emulate_ud(ctxt);
3402
3403 return X86EMUL_CONTINUE;
3404}
3405
3406static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3407{
3408 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3409
3410
3411 if (rax & 0xffff000000000000ULL)
3412 return emulate_gp(ctxt, 0);
3413
3414 return check_svme(ctxt);
3415}
3416
3417static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3418{
3419 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3420
3421 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3422 return emulate_ud(ctxt);
3423
3424 return X86EMUL_CONTINUE;
3425}
3426
3427static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3428{
3429 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3430 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3431
3432 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3433 (rcx > 3))
3434 return emulate_gp(ctxt, 0);
3435
3436 return X86EMUL_CONTINUE;
3437}
3438
3439static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3440{
3441 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3442 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3443 return emulate_gp(ctxt, 0);
3444
3445 return X86EMUL_CONTINUE;
3446}
3447
3448static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3449{
3450 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3451 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3452 return emulate_gp(ctxt, 0);
3453
3454 return X86EMUL_CONTINUE;
3455}
3456
3457#define D(_y) { .flags = (_y) }
3458#define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3459#define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3460 .check_perm = (_p) }
3461#define N D(NotImpl)
3462#define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3463#define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3464#define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3465#define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3466#define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3467#define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3468#define II(_f, _e, _i) \
3469 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3470#define IIP(_f, _e, _i, _p) \
3471 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3472 .check_perm = (_p) }
3473#define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3474
3475#define D2bv(_f) D((_f) | ByteOp), D(_f)
3476#define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3477#define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3478#define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3479#define I2bvIP(_f, _e, _i, _p) \
3480 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3481
3482#define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3483 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3484 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3485
3486static const struct opcode group7_rm1[] = {
3487 DI(SrcNone | Priv, monitor),
3488 DI(SrcNone | Priv, mwait),
3489 N, N, N, N, N, N,
3490};
3491
3492static const struct opcode group7_rm3[] = {
3493 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3494 II(SrcNone | Prot | VendorSpecific, em_vmmcall, vmmcall),
3495 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3496 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3497 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3498 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3499 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3500 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3501};
3502
3503static const struct opcode group7_rm7[] = {
3504 N,
3505 DIP(SrcNone, rdtscp, check_rdtsc),
3506 N, N, N, N, N, N,
3507};
3508
3509static const struct opcode group1[] = {
3510 F(Lock, em_add),
3511 F(Lock | PageTable, em_or),
3512 F(Lock, em_adc),
3513 F(Lock, em_sbb),
3514 F(Lock | PageTable, em_and),
3515 F(Lock, em_sub),
3516 F(Lock, em_xor),
3517 F(NoWrite, em_cmp),
3518};
3519
3520static const struct opcode group1A[] = {
3521 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3522};
3523
3524static const struct opcode group2[] = {
3525 F(DstMem | ModRM, em_rol),
3526 F(DstMem | ModRM, em_ror),
3527 F(DstMem | ModRM, em_rcl),
3528 F(DstMem | ModRM, em_rcr),
3529 F(DstMem | ModRM, em_shl),
3530 F(DstMem | ModRM, em_shr),
3531 F(DstMem | ModRM, em_shl),
3532 F(DstMem | ModRM, em_sar),
3533};
3534
3535static const struct opcode group3[] = {
3536 F(DstMem | SrcImm | NoWrite, em_test),
3537 F(DstMem | SrcImm | NoWrite, em_test),
3538 F(DstMem | SrcNone | Lock, em_not),
3539 F(DstMem | SrcNone | Lock, em_neg),
3540 F(DstXacc | Src2Mem, em_mul_ex),
3541 F(DstXacc | Src2Mem, em_imul_ex),
3542 F(DstXacc | Src2Mem, em_div_ex),
3543 F(DstXacc | Src2Mem, em_idiv_ex),
3544};
3545
3546static const struct opcode group4[] = {
3547 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3548 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3549 N, N, N, N, N, N,
3550};
3551
3552static const struct opcode group5[] = {
3553 F(DstMem | SrcNone | Lock, em_inc),
3554 F(DstMem | SrcNone | Lock, em_dec),
3555 I(SrcMem | Stack, em_grp45),
3556 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3557 I(SrcMem | Stack, em_grp45),
3558 I(SrcMemFAddr | ImplicitOps, em_grp45),
3559 I(SrcMem | Stack, em_grp45), D(Undefined),
3560};
3561
3562static const struct opcode group6[] = {
3563 DI(Prot, sldt),
3564 DI(Prot, str),
3565 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3566 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3567 N, N, N, N,
3568};
3569
3570static const struct group_dual group7 = { {
3571 II(Mov | DstMem | Priv, em_sgdt, sgdt),
3572 II(Mov | DstMem | Priv, em_sidt, sidt),
3573 II(SrcMem | Priv, em_lgdt, lgdt),
3574 II(SrcMem | Priv, em_lidt, lidt),
3575 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3576 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3577 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3578}, {
3579 I(SrcNone | Priv | VendorSpecific, em_vmcall),
3580 EXT(0, group7_rm1),
3581 N, EXT(0, group7_rm3),
3582 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3583 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3584 EXT(0, group7_rm7),
3585} };
3586
3587static const struct opcode group8[] = {
3588 N, N, N, N,
3589 F(DstMem | SrcImmByte | NoWrite, em_bt),
3590 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3591 F(DstMem | SrcImmByte | Lock, em_btr),
3592 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3593};
3594
3595static const struct group_dual group9 = { {
3596 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3597}, {
3598 N, N, N, N, N, N, N, N,
3599} };
3600
3601static const struct opcode group11[] = {
3602 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3603 X7(D(Undefined)),
3604};
3605
3606static const struct gprefix pfx_0f_6f_0f_7f = {
3607 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3608};
3609
3610static const struct gprefix pfx_vmovntpx = {
3611 I(0, em_mov), N, N, N,
3612};
3613
3614static const struct escape escape_d9 = { {
3615 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3616}, {
3617
3618 N, N, N, N, N, N, N, N,
3619
3620 N, N, N, N, N, N, N, N,
3621
3622 N, N, N, N, N, N, N, N,
3623
3624 N, N, N, N, N, N, N, N,
3625
3626 N, N, N, N, N, N, N, N,
3627
3628 N, N, N, N, N, N, N, N,
3629
3630 N, N, N, N, N, N, N, N,
3631
3632 N, N, N, N, N, N, N, N,
3633} };
3634
3635static const struct escape escape_db = { {
3636 N, N, N, N, N, N, N, N,
3637}, {
3638
3639 N, N, N, N, N, N, N, N,
3640
3641 N, N, N, N, N, N, N, N,
3642
3643 N, N, N, N, N, N, N, N,
3644
3645 N, N, N, N, N, N, N, N,
3646
3647 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3648
3649 N, N, N, N, N, N, N, N,
3650
3651 N, N, N, N, N, N, N, N,
3652
3653 N, N, N, N, N, N, N, N,
3654} };
3655
3656static const struct escape escape_dd = { {
3657 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3658}, {
3659
3660 N, N, N, N, N, N, N, N,
3661
3662 N, N, N, N, N, N, N, N,
3663
3664 N, N, N, N, N, N, N, N,
3665
3666 N, N, N, N, N, N, N, N,
3667
3668 N, N, N, N, N, N, N, N,
3669
3670 N, N, N, N, N, N, N, N,
3671
3672 N, N, N, N, N, N, N, N,
3673
3674 N, N, N, N, N, N, N, N,
3675} };
3676
3677static const struct opcode opcode_table[256] = {
3678
3679 F6ALU(Lock, em_add),
3680 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3681 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3682
3683 F6ALU(Lock | PageTable, em_or),
3684 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3685 N,
3686
3687 F6ALU(Lock, em_adc),
3688 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3689 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3690
3691 F6ALU(Lock, em_sbb),
3692 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3693 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3694
3695 F6ALU(Lock | PageTable, em_and), N, N,
3696
3697 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3698
3699 F6ALU(Lock, em_xor), N, N,
3700
3701 F6ALU(NoWrite, em_cmp), N, N,
3702
3703 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
3704
3705 X8(I(SrcReg | Stack, em_push)),
3706
3707 X8(I(DstReg | Stack, em_pop)),
3708
3709 I(ImplicitOps | Stack | No64, em_pusha),
3710 I(ImplicitOps | Stack | No64, em_popa),
3711 N, D(DstReg | SrcMem32 | ModRM | Mov) ,
3712 N, N, N, N,
3713
3714 I(SrcImm | Mov | Stack, em_push),
3715 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3716 I(SrcImmByte | Mov | Stack, em_push),
3717 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3718 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in),
3719 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out),
3720
3721 X16(D(SrcImmByte)),
3722
3723 G(ByteOp | DstMem | SrcImm, group1),
3724 G(DstMem | SrcImm, group1),
3725 G(ByteOp | DstMem | SrcImm | No64, group1),
3726 G(DstMem | SrcImmByte, group1),
3727 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3728 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3729
3730 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3731 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3732 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3733 D(ModRM | SrcMem | NoAccess | DstReg),
3734 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3735 G(0, group1A),
3736
3737 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3738
3739 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3740 I(SrcImmFAddr | No64, em_call_far), N,
3741 II(ImplicitOps | Stack, em_pushf, pushf),
3742 II(ImplicitOps | Stack, em_popf, popf), N, I(ImplicitOps, em_lahf),
3743
3744 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3745 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3746 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3747 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3748
3749 F2bv(DstAcc | SrcImm | NoWrite, em_test),
3750 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3751 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3752 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3753
3754 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3755
3756 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3757
3758 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
3759 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3760 I(ImplicitOps | Stack, em_ret),
3761 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3762 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3763 G(ByteOp, group11), G(0, group11),
3764
3765 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3766 N, I(ImplicitOps | Stack, em_ret_far),
3767 D(ImplicitOps), DI(SrcImmByte, intn),
3768 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3769
3770 G(Src2One | ByteOp, group2), G(Src2One, group2),
3771 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
3772 I(DstAcc | SrcImmUByte | No64, em_aam),
3773 I(DstAcc | SrcImmUByte | No64, em_aad),
3774 F(DstAcc | ByteOp | No64, em_salc),
3775 I(DstAcc | SrcXLat | ByteOp, em_mov),
3776
3777 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3778
3779 X3(I(SrcImmByte, em_loop)),
3780 I(SrcImmByte, em_jcxz),
3781 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3782 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3783
3784 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3785 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3786 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3787 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3788
3789 N, DI(ImplicitOps, icebp), N, N,
3790 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3791 G(ByteOp, group3), G(0, group3),
3792
3793 D(ImplicitOps), D(ImplicitOps),
3794 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3795 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3796};
3797
3798static const struct opcode twobyte_table[256] = {
3799
3800 G(0, group6), GD(0, &group7), N, N,
3801 N, I(ImplicitOps | VendorSpecific, em_syscall),
3802 II(ImplicitOps | Priv, em_clts, clts), N,
3803 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3804 N, D(ImplicitOps | ModRM), N, N,
3805
3806 N, N, N, N, N, N, N, N,
3807 D(ImplicitOps | ModRM), N, N, N, N, N, N, D(ImplicitOps | ModRM),
3808
3809 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3810 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3811 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
3812 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
3813 N, N, N, N,
3814 N, N, N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
3815 N, N, N, N,
3816
3817 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3818 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3819 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3820 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3821 I(ImplicitOps | VendorSpecific, em_sysenter),
3822 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3823 N, N,
3824 N, N, N, N, N, N, N, N,
3825
3826 X16(D(DstReg | SrcMem | ModRM | Mov)),
3827
3828 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3829
3830 N, N, N, N,
3831 N, N, N, N,
3832 N, N, N, N,
3833 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3834
3835 N, N, N, N,
3836 N, N, N, N,
3837 N, N, N, N,
3838 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3839
3840 X16(D(SrcImm)),
3841
3842 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3843
3844 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3845 II(ImplicitOps, em_cpuid, cpuid),
3846 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
3847 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
3848 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
3849
3850 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
3851 DI(ImplicitOps, rsm),
3852 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
3853 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
3854 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
3855 D(ModRM), F(DstReg | SrcMem | ModRM, em_imul),
3856
3857 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
3858 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
3859 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
3860 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
3861 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
3862 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3863
3864 N, N,
3865 G(BitOp, group8),
3866 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
3867 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
3868 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
3869
3870 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
3871 N, D(DstMem | SrcReg | ModRM | Mov),
3872 N, N, N, GD(0, &group9),
3873
3874 X8(I(DstReg, em_bswap)),
3875
3876 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3877
3878 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3879
3880 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
3881};
3882
3883#undef D
3884#undef N
3885#undef G
3886#undef GD
3887#undef I
3888#undef GP
3889#undef EXT
3890
3891#undef D2bv
3892#undef D2bvIP
3893#undef I2bv
3894#undef I2bvIP
3895#undef I6ALU
3896
3897static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
3898{
3899 unsigned size;
3900
3901 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3902 if (size == 8)
3903 size = 4;
3904 return size;
3905}
3906
3907static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
3908 unsigned size, bool sign_extension)
3909{
3910 int rc = X86EMUL_CONTINUE;
3911
3912 op->type = OP_IMM;
3913 op->bytes = size;
3914 op->addr.mem.ea = ctxt->_eip;
3915
3916 switch (op->bytes) {
3917 case 1:
3918 op->val = insn_fetch(s8, ctxt);
3919 break;
3920 case 2:
3921 op->val = insn_fetch(s16, ctxt);
3922 break;
3923 case 4:
3924 op->val = insn_fetch(s32, ctxt);
3925 break;
3926 case 8:
3927 op->val = insn_fetch(s64, ctxt);
3928 break;
3929 }
3930 if (!sign_extension) {
3931 switch (op->bytes) {
3932 case 1:
3933 op->val &= 0xff;
3934 break;
3935 case 2:
3936 op->val &= 0xffff;
3937 break;
3938 case 4:
3939 op->val &= 0xffffffff;
3940 break;
3941 }
3942 }
3943done:
3944 return rc;
3945}
3946
3947static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
3948 unsigned d)
3949{
3950 int rc = X86EMUL_CONTINUE;
3951
3952 switch (d) {
3953 case OpReg:
3954 decode_register_operand(ctxt, op);
3955 break;
3956 case OpImmUByte:
3957 rc = decode_imm(ctxt, op, 1, false);
3958 break;
3959 case OpMem:
3960 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3961 mem_common:
3962 *op = ctxt->memop;
3963 ctxt->memopp = op;
3964 if ((ctxt->d & BitOp) && op == &ctxt->dst)
3965 fetch_bit_operand(ctxt);
3966 op->orig_val = op->val;
3967 break;
3968 case OpMem64:
3969 ctxt->memop.bytes = 8;
3970 goto mem_common;
3971 case OpAcc:
3972 op->type = OP_REG;
3973 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3974 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
3975 fetch_register_operand(op);
3976 op->orig_val = op->val;
3977 break;
3978 case OpAccLo:
3979 op->type = OP_REG;
3980 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
3981 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
3982 fetch_register_operand(op);
3983 op->orig_val = op->val;
3984 break;
3985 case OpAccHi:
3986 if (ctxt->d & ByteOp) {
3987 op->type = OP_NONE;
3988 break;
3989 }
3990 op->type = OP_REG;
3991 op->bytes = ctxt->op_bytes;
3992 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3993 fetch_register_operand(op);
3994 op->orig_val = op->val;
3995 break;
3996 case OpDI:
3997 op->type = OP_MEM;
3998 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3999 op->addr.mem.ea =
4000 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4001 op->addr.mem.seg = VCPU_SREG_ES;
4002 op->val = 0;
4003 op->count = 1;
4004 break;
4005 case OpDX:
4006 op->type = OP_REG;
4007 op->bytes = 2;
4008 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4009 fetch_register_operand(op);
4010 break;
4011 case OpCL:
4012 op->bytes = 1;
4013 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4014 break;
4015 case OpImmByte:
4016 rc = decode_imm(ctxt, op, 1, true);
4017 break;
4018 case OpOne:
4019 op->bytes = 1;
4020 op->val = 1;
4021 break;
4022 case OpImm:
4023 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4024 break;
4025 case OpImm64:
4026 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4027 break;
4028 case OpMem8:
4029 ctxt->memop.bytes = 1;
4030 if (ctxt->memop.type == OP_REG) {
4031 ctxt->memop.addr.reg = decode_register(ctxt, ctxt->modrm_rm, 1);
4032 fetch_register_operand(&ctxt->memop);
4033 }
4034 goto mem_common;
4035 case OpMem16:
4036 ctxt->memop.bytes = 2;
4037 goto mem_common;
4038 case OpMem32:
4039 ctxt->memop.bytes = 4;
4040 goto mem_common;
4041 case OpImmU16:
4042 rc = decode_imm(ctxt, op, 2, false);
4043 break;
4044 case OpImmU:
4045 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4046 break;
4047 case OpSI:
4048 op->type = OP_MEM;
4049 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4050 op->addr.mem.ea =
4051 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4052 op->addr.mem.seg = seg_override(ctxt);
4053 op->val = 0;
4054 op->count = 1;
4055 break;
4056 case OpXLat:
4057 op->type = OP_MEM;
4058 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4059 op->addr.mem.ea =
4060 register_address(ctxt,
4061 reg_read(ctxt, VCPU_REGS_RBX) +
4062 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4063 op->addr.mem.seg = seg_override(ctxt);
4064 op->val = 0;
4065 break;
4066 case OpImmFAddr:
4067 op->type = OP_IMM;
4068 op->addr.mem.ea = ctxt->_eip;
4069 op->bytes = ctxt->op_bytes + 2;
4070 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4071 break;
4072 case OpMemFAddr:
4073 ctxt->memop.bytes = ctxt->op_bytes + 2;
4074 goto mem_common;
4075 case OpES:
4076 op->val = VCPU_SREG_ES;
4077 break;
4078 case OpCS:
4079 op->val = VCPU_SREG_CS;
4080 break;
4081 case OpSS:
4082 op->val = VCPU_SREG_SS;
4083 break;
4084 case OpDS:
4085 op->val = VCPU_SREG_DS;
4086 break;
4087 case OpFS:
4088 op->val = VCPU_SREG_FS;
4089 break;
4090 case OpGS:
4091 op->val = VCPU_SREG_GS;
4092 break;
4093 case OpImplicit:
4094
4095 default:
4096 op->type = OP_NONE;
4097 break;
4098 }
4099
4100done:
4101 return rc;
4102}
4103
4104int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4105{
4106 int rc = X86EMUL_CONTINUE;
4107 int mode = ctxt->mode;
4108 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4109 bool op_prefix = false;
4110 struct opcode opcode;
4111
4112 ctxt->memop.type = OP_NONE;
4113 ctxt->memopp = NULL;
4114 ctxt->_eip = ctxt->eip;
4115 ctxt->fetch.start = ctxt->_eip;
4116 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4117 if (insn_len > 0)
4118 memcpy(ctxt->fetch.data, insn, insn_len);
4119
4120 switch (mode) {
4121 case X86EMUL_MODE_REAL:
4122 case X86EMUL_MODE_VM86:
4123 case X86EMUL_MODE_PROT16:
4124 def_op_bytes = def_ad_bytes = 2;
4125 break;
4126 case X86EMUL_MODE_PROT32:
4127 def_op_bytes = def_ad_bytes = 4;
4128 break;
4129#ifdef CONFIG_X86_64
4130 case X86EMUL_MODE_PROT64:
4131 def_op_bytes = 4;
4132 def_ad_bytes = 8;
4133 break;
4134#endif
4135 default:
4136 return EMULATION_FAILED;
4137 }
4138
4139 ctxt->op_bytes = def_op_bytes;
4140 ctxt->ad_bytes = def_ad_bytes;
4141
4142
4143 for (;;) {
4144 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4145 case 0x66:
4146 op_prefix = true;
4147
4148 ctxt->op_bytes = def_op_bytes ^ 6;
4149 break;
4150 case 0x67:
4151 if (mode == X86EMUL_MODE_PROT64)
4152
4153 ctxt->ad_bytes = def_ad_bytes ^ 12;
4154 else
4155
4156 ctxt->ad_bytes = def_ad_bytes ^ 6;
4157 break;
4158 case 0x26:
4159 case 0x2e:
4160 case 0x36:
4161 case 0x3e:
4162 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4163 break;
4164 case 0x64:
4165 case 0x65:
4166 set_seg_override(ctxt, ctxt->b & 7);
4167 break;
4168 case 0x40 ... 0x4f:
4169 if (mode != X86EMUL_MODE_PROT64)
4170 goto done_prefixes;
4171 ctxt->rex_prefix = ctxt->b;
4172 continue;
4173 case 0xf0:
4174 ctxt->lock_prefix = 1;
4175 break;
4176 case 0xf2:
4177 case 0xf3:
4178 ctxt->rep_prefix = ctxt->b;
4179 break;
4180 default:
4181 goto done_prefixes;
4182 }
4183
4184
4185
4186 ctxt->rex_prefix = 0;
4187 }
4188
4189done_prefixes:
4190
4191
4192 if (ctxt->rex_prefix & 8)
4193 ctxt->op_bytes = 8;
4194
4195
4196 opcode = opcode_table[ctxt->b];
4197
4198 if (ctxt->b == 0x0f) {
4199 ctxt->twobyte = 1;
4200 ctxt->b = insn_fetch(u8, ctxt);
4201 opcode = twobyte_table[ctxt->b];
4202 }
4203 ctxt->d = opcode.flags;
4204
4205 if (ctxt->d & ModRM)
4206 ctxt->modrm = insn_fetch(u8, ctxt);
4207
4208 while (ctxt->d & GroupMask) {
4209 switch (ctxt->d & GroupMask) {
4210 case Group:
4211 goffset = (ctxt->modrm >> 3) & 7;
4212 opcode = opcode.u.group[goffset];
4213 break;
4214 case GroupDual:
4215 goffset = (ctxt->modrm >> 3) & 7;
4216 if ((ctxt->modrm >> 6) == 3)
4217 opcode = opcode.u.gdual->mod3[goffset];
4218 else
4219 opcode = opcode.u.gdual->mod012[goffset];
4220 break;
4221 case RMExt:
4222 goffset = ctxt->modrm & 7;
4223 opcode = opcode.u.group[goffset];
4224 break;
4225 case Prefix:
4226 if (ctxt->rep_prefix && op_prefix)
4227 return EMULATION_FAILED;
4228 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4229 switch (simd_prefix) {
4230 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4231 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4232 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4233 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4234 }
4235 break;
4236 case Escape:
4237 if (ctxt->modrm > 0xbf)
4238 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4239 else
4240 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4241 break;
4242 default:
4243 return EMULATION_FAILED;
4244 }
4245
4246 ctxt->d &= ~(u64)GroupMask;
4247 ctxt->d |= opcode.flags;
4248 }
4249
4250 ctxt->execute = opcode.u.execute;
4251 ctxt->check_perm = opcode.check_perm;
4252 ctxt->intercept = opcode.intercept;
4253
4254
4255 if (ctxt->d == 0 || (ctxt->d & NotImpl))
4256 return EMULATION_FAILED;
4257
4258 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
4259 return EMULATION_FAILED;
4260
4261 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4262 ctxt->op_bytes = 8;
4263
4264 if (ctxt->d & Op3264) {
4265 if (mode == X86EMUL_MODE_PROT64)
4266 ctxt->op_bytes = 8;
4267 else
4268 ctxt->op_bytes = 4;
4269 }
4270
4271 if (ctxt->d & Sse)
4272 ctxt->op_bytes = 16;
4273 else if (ctxt->d & Mmx)
4274 ctxt->op_bytes = 8;
4275
4276
4277 if (ctxt->d & ModRM) {
4278 rc = decode_modrm(ctxt, &ctxt->memop);
4279 if (!ctxt->has_seg_override)
4280 set_seg_override(ctxt, ctxt->modrm_seg);
4281 } else if (ctxt->d & MemAbs)
4282 rc = decode_abs(ctxt, &ctxt->memop);
4283 if (rc != X86EMUL_CONTINUE)
4284 goto done;
4285
4286 if (!ctxt->has_seg_override)
4287 set_seg_override(ctxt, VCPU_SREG_DS);
4288
4289 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4290
4291 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4292 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4293
4294
4295
4296
4297
4298 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4299 if (rc != X86EMUL_CONTINUE)
4300 goto done;
4301
4302
4303
4304
4305
4306 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4307 if (rc != X86EMUL_CONTINUE)
4308 goto done;
4309
4310
4311 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4312
4313done:
4314 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4315 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4316
4317 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4318}
4319
4320bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4321{
4322 return ctxt->d & PageTable;
4323}
4324
4325static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4326{
4327
4328
4329
4330
4331
4332
4333
4334 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4335 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4336 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4337 ((ctxt->eflags & EFLG_ZF) == 0))
4338 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4339 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4340 return true;
4341
4342 return false;
4343}
4344
4345static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4346{
4347 bool fault = false;
4348
4349 ctxt->ops->get_fpu(ctxt);
4350 asm volatile("1: fwait \n\t"
4351 "2: \n\t"
4352 ".pushsection .fixup,\"ax\" \n\t"
4353 "3: \n\t"
4354 "movb $1, %[fault] \n\t"
4355 "jmp 2b \n\t"
4356 ".popsection \n\t"
4357 _ASM_EXTABLE(1b, 3b)
4358 : [fault]"+qm"(fault));
4359 ctxt->ops->put_fpu(ctxt);
4360
4361 if (unlikely(fault))
4362 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4363
4364 return X86EMUL_CONTINUE;
4365}
4366
4367static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4368 struct operand *op)
4369{
4370 if (op->type == OP_MM)
4371 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4372}
4373
4374static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4375{
4376 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4377 if (!(ctxt->d & ByteOp))
4378 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4379 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4380 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
4381 [fastop]"+S"(fop)
4382 : "c"(ctxt->src2.val));
4383 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4384 if (!fop)
4385 return emulate_de(ctxt);
4386 return X86EMUL_CONTINUE;
4387}
4388
4389int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4390{
4391 const struct x86_emulate_ops *ops = ctxt->ops;
4392 int rc = X86EMUL_CONTINUE;
4393 int saved_dst_type = ctxt->dst.type;
4394
4395 ctxt->mem_read.pos = 0;
4396
4397 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4398 (ctxt->d & Undefined)) {
4399 rc = emulate_ud(ctxt);
4400 goto done;
4401 }
4402
4403
4404 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4405 rc = emulate_ud(ctxt);
4406 goto done;
4407 }
4408
4409 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4410 rc = emulate_ud(ctxt);
4411 goto done;
4412 }
4413
4414 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4415 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4416 rc = emulate_ud(ctxt);
4417 goto done;
4418 }
4419
4420 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4421 rc = emulate_nm(ctxt);
4422 goto done;
4423 }
4424
4425 if (ctxt->d & Mmx) {
4426 rc = flush_pending_x87_faults(ctxt);
4427 if (rc != X86EMUL_CONTINUE)
4428 goto done;
4429
4430
4431
4432
4433 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4434 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4435 if (!(ctxt->d & Mov))
4436 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4437 }
4438
4439 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4440 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4441 X86_ICPT_PRE_EXCEPT);
4442 if (rc != X86EMUL_CONTINUE)
4443 goto done;
4444 }
4445
4446
4447 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4448 rc = emulate_gp(ctxt, 0);
4449 goto done;
4450 }
4451
4452
4453 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4454 rc = emulate_ud(ctxt);
4455 goto done;
4456 }
4457
4458
4459 if (ctxt->check_perm) {
4460 rc = ctxt->check_perm(ctxt);
4461 if (rc != X86EMUL_CONTINUE)
4462 goto done;
4463 }
4464
4465 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4466 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4467 X86_ICPT_POST_EXCEPT);
4468 if (rc != X86EMUL_CONTINUE)
4469 goto done;
4470 }
4471
4472 if (ctxt->rep_prefix && (ctxt->d & String)) {
4473
4474 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4475 ctxt->eip = ctxt->_eip;
4476 goto done;
4477 }
4478 }
4479
4480 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4481 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4482 ctxt->src.valptr, ctxt->src.bytes);
4483 if (rc != X86EMUL_CONTINUE)
4484 goto done;
4485 ctxt->src.orig_val64 = ctxt->src.val64;
4486 }
4487
4488 if (ctxt->src2.type == OP_MEM) {
4489 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4490 &ctxt->src2.val, ctxt->src2.bytes);
4491 if (rc != X86EMUL_CONTINUE)
4492 goto done;
4493 }
4494
4495 if ((ctxt->d & DstMask) == ImplicitOps)
4496 goto special_insn;
4497
4498
4499 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4500
4501 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4502 &ctxt->dst.val, ctxt->dst.bytes);
4503 if (rc != X86EMUL_CONTINUE)
4504 goto done;
4505 }
4506 ctxt->dst.orig_val = ctxt->dst.val;
4507
4508special_insn:
4509
4510 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4511 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4512 X86_ICPT_POST_MEMACCESS);
4513 if (rc != X86EMUL_CONTINUE)
4514 goto done;
4515 }
4516
4517 if (ctxt->execute) {
4518 if (ctxt->d & Fastop) {
4519 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4520 rc = fastop(ctxt, fop);
4521 if (rc != X86EMUL_CONTINUE)
4522 goto done;
4523 goto writeback;
4524 }
4525 rc = ctxt->execute(ctxt);
4526 if (rc != X86EMUL_CONTINUE)
4527 goto done;
4528 goto writeback;
4529 }
4530
4531 if (ctxt->twobyte)
4532 goto twobyte_insn;
4533
4534 switch (ctxt->b) {
4535 case 0x63:
4536 if (ctxt->mode != X86EMUL_MODE_PROT64)
4537 goto cannot_emulate;
4538 ctxt->dst.val = (s32) ctxt->src.val;
4539 break;
4540 case 0x70 ... 0x7f:
4541 if (test_cc(ctxt->b, ctxt->eflags))
4542 jmp_rel(ctxt, ctxt->src.val);
4543 break;
4544 case 0x8d:
4545 ctxt->dst.val = ctxt->src.addr.mem.ea;
4546 break;
4547 case 0x90 ... 0x97:
4548 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4549 break;
4550 rc = em_xchg(ctxt);
4551 break;
4552 case 0x98:
4553 switch (ctxt->op_bytes) {
4554 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4555 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4556 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4557 }
4558 break;
4559 case 0xcc:
4560 rc = emulate_int(ctxt, 3);
4561 break;
4562 case 0xcd:
4563 rc = emulate_int(ctxt, ctxt->src.val);
4564 break;
4565 case 0xce:
4566 if (ctxt->eflags & EFLG_OF)
4567 rc = emulate_int(ctxt, 4);
4568 break;
4569 case 0xe9:
4570 case 0xeb:
4571 jmp_rel(ctxt, ctxt->src.val);
4572 ctxt->dst.type = OP_NONE;
4573 break;
4574 case 0xf4:
4575 ctxt->ops->halt(ctxt);
4576 break;
4577 case 0xf5:
4578
4579 ctxt->eflags ^= EFLG_CF;
4580 break;
4581 case 0xf8:
4582 ctxt->eflags &= ~EFLG_CF;
4583 break;
4584 case 0xf9:
4585 ctxt->eflags |= EFLG_CF;
4586 break;
4587 case 0xfc:
4588 ctxt->eflags &= ~EFLG_DF;
4589 break;
4590 case 0xfd:
4591 ctxt->eflags |= EFLG_DF;
4592 break;
4593 default:
4594 goto cannot_emulate;
4595 }
4596
4597 if (rc != X86EMUL_CONTINUE)
4598 goto done;
4599
4600writeback:
4601 if (!(ctxt->d & NoWrite)) {
4602 rc = writeback(ctxt, &ctxt->dst);
4603 if (rc != X86EMUL_CONTINUE)
4604 goto done;
4605 }
4606 if (ctxt->d & SrcWrite) {
4607 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
4608 rc = writeback(ctxt, &ctxt->src);
4609 if (rc != X86EMUL_CONTINUE)
4610 goto done;
4611 }
4612
4613
4614
4615
4616
4617 ctxt->dst.type = saved_dst_type;
4618
4619 if ((ctxt->d & SrcMask) == SrcSI)
4620 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4621
4622 if ((ctxt->d & DstMask) == DstDI)
4623 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4624
4625 if (ctxt->rep_prefix && (ctxt->d & String)) {
4626 unsigned int count;
4627 struct read_cache *r = &ctxt->io_read;
4628 if ((ctxt->d & SrcMask) == SrcSI)
4629 count = ctxt->src.count;
4630 else
4631 count = ctxt->dst.count;
4632 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4633 -count);
4634
4635 if (!string_insn_completed(ctxt)) {
4636
4637
4638
4639
4640 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4641 (r->end == 0 || r->end != r->pos)) {
4642
4643
4644
4645
4646
4647 ctxt->mem_read.end = 0;
4648 writeback_registers(ctxt);
4649 return EMULATION_RESTART;
4650 }
4651 goto done;
4652 }
4653 }
4654
4655 ctxt->eip = ctxt->_eip;
4656
4657done:
4658 if (rc == X86EMUL_PROPAGATE_FAULT)
4659 ctxt->have_exception = true;
4660 if (rc == X86EMUL_INTERCEPTED)
4661 return EMULATION_INTERCEPTED;
4662
4663 if (rc == X86EMUL_CONTINUE)
4664 writeback_registers(ctxt);
4665
4666 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4667
4668twobyte_insn:
4669 switch (ctxt->b) {
4670 case 0x09:
4671 (ctxt->ops->wbinvd)(ctxt);
4672 break;
4673 case 0x08:
4674 case 0x0d:
4675 case 0x18:
4676 case 0x1f:
4677 break;
4678 case 0x20:
4679 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4680 break;
4681 case 0x21:
4682 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4683 break;
4684 case 0x40 ... 0x4f:
4685 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4686 if (!test_cc(ctxt->b, ctxt->eflags))
4687 ctxt->dst.type = OP_NONE;
4688 break;
4689 case 0x80 ... 0x8f:
4690 if (test_cc(ctxt->b, ctxt->eflags))
4691 jmp_rel(ctxt, ctxt->src.val);
4692 break;
4693 case 0x90 ... 0x9f:
4694 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4695 break;
4696 case 0xae:
4697 break;
4698 case 0xb6 ... 0xb7:
4699 ctxt->dst.bytes = ctxt->op_bytes;
4700 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4701 : (u16) ctxt->src.val;
4702 break;
4703 case 0xbe ... 0xbf:
4704 ctxt->dst.bytes = ctxt->op_bytes;
4705 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4706 (s16) ctxt->src.val;
4707 break;
4708 case 0xc3:
4709 ctxt->dst.bytes = ctxt->op_bytes;
4710 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4711 (u64) ctxt->src.val;
4712 break;
4713 default:
4714 goto cannot_emulate;
4715 }
4716
4717 if (rc != X86EMUL_CONTINUE)
4718 goto done;
4719
4720 goto writeback;
4721
4722cannot_emulate:
4723 return EMULATION_FAILED;
4724}
4725
4726void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4727{
4728 invalidate_registers(ctxt);
4729}
4730
4731void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4732{
4733 writeback_registers(ctxt);
4734}
4735