1#define pr_fmt(fmt) "SVM: " fmt
2
3#include <linux/kvm_host.h>
4
5#include "irq.h"
6#include "mmu.h"
7#include "kvm_cache_regs.h"
8#include "x86.h"
9#include "cpuid.h"
10#include "pmu.h"
11
12#include <linux/module.h>
13#include <linux/mod_devicetable.h>
14#include <linux/kernel.h>
15#include <linux/vmalloc.h>
16#include <linux/highmem.h>
17#include <linux/amd-iommu.h>
18#include <linux/sched.h>
19#include <linux/trace_events.h>
20#include <linux/slab.h>
21#include <linux/hashtable.h>
22#include <linux/objtool.h>
23#include <linux/psp-sev.h>
24#include <linux/file.h>
25#include <linux/pagemap.h>
26#include <linux/swap.h>
27#include <linux/rwsem.h>
28
29#include <asm/apic.h>
30#include <asm/perf_event.h>
31#include <asm/tlbflush.h>
32#include <asm/desc.h>
33#include <asm/debugreg.h>
34#include <asm/kvm_para.h>
35#include <asm/irq_remapping.h>
36#include <asm/spec-ctrl.h>
37#include <asm/cpu_device_id.h>
38#include <asm/traps.h>
39
40#include <asm/virtext.h>
41#include "trace.h"
42
43#include "svm.h"
44#include "svm_ops.h"
45
46#include "kvm_onhyperv.h"
47#include "svm_onhyperv.h"
48
49MODULE_AUTHOR("Qumranet");
50MODULE_LICENSE("GPL");
51
52#ifdef MODULE
53static const struct x86_cpu_id svm_cpu_id[] = {
54 X86_MATCH_FEATURE(X86_FEATURE_SVM, NULL),
55 {}
56};
57MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);
58#endif
59
60#define SEG_TYPE_LDT 2
61#define SEG_TYPE_BUSY_TSS16 3
62
63#define SVM_FEATURE_LBRV (1 << 1)
64#define SVM_FEATURE_SVML (1 << 2)
65#define SVM_FEATURE_TSC_RATE (1 << 4)
66#define SVM_FEATURE_VMCB_CLEAN (1 << 5)
67#define SVM_FEATURE_FLUSH_ASID (1 << 6)
68#define SVM_FEATURE_DECODE_ASSIST (1 << 7)
69#define SVM_FEATURE_PAUSE_FILTER (1 << 10)
70
71#define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
72
73#define TSC_RATIO_RSVD 0xffffff0000000000ULL
74#define TSC_RATIO_MIN 0x0000000000000001ULL
75#define TSC_RATIO_MAX 0x000000ffffffffffULL
76
77static bool erratum_383_found __read_mostly;
78
79u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
80
81
82
83
84
85static uint64_t osvw_len = 4, osvw_status;
86
87static DEFINE_PER_CPU(u64, current_tsc_ratio);
88#define TSC_RATIO_DEFAULT 0x0100000000ULL
89
90static const struct svm_direct_access_msrs {
91 u32 index;
92 bool always;
93} direct_access_msrs[MAX_DIRECT_ACCESS_MSRS] = {
94 { .index = MSR_STAR, .always = true },
95 { .index = MSR_IA32_SYSENTER_CS, .always = true },
96 { .index = MSR_IA32_SYSENTER_EIP, .always = false },
97 { .index = MSR_IA32_SYSENTER_ESP, .always = false },
98#ifdef CONFIG_X86_64
99 { .index = MSR_GS_BASE, .always = true },
100 { .index = MSR_FS_BASE, .always = true },
101 { .index = MSR_KERNEL_GS_BASE, .always = true },
102 { .index = MSR_LSTAR, .always = true },
103 { .index = MSR_CSTAR, .always = true },
104 { .index = MSR_SYSCALL_MASK, .always = true },
105#endif
106 { .index = MSR_IA32_SPEC_CTRL, .always = false },
107 { .index = MSR_IA32_PRED_CMD, .always = false },
108 { .index = MSR_IA32_LASTBRANCHFROMIP, .always = false },
109 { .index = MSR_IA32_LASTBRANCHTOIP, .always = false },
110 { .index = MSR_IA32_LASTINTFROMIP, .always = false },
111 { .index = MSR_IA32_LASTINTTOIP, .always = false },
112 { .index = MSR_EFER, .always = false },
113 { .index = MSR_IA32_CR_PAT, .always = false },
114 { .index = MSR_AMD64_SEV_ES_GHCB, .always = true },
115 { .index = MSR_INVALID, .always = false },
116};
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148static unsigned short pause_filter_thresh = KVM_DEFAULT_PLE_GAP;
149module_param(pause_filter_thresh, ushort, 0444);
150
151static unsigned short pause_filter_count = KVM_SVM_DEFAULT_PLE_WINDOW;
152module_param(pause_filter_count, ushort, 0444);
153
154
155static unsigned short pause_filter_count_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
156module_param(pause_filter_count_grow, ushort, 0444);
157
158
159static unsigned short pause_filter_count_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
160module_param(pause_filter_count_shrink, ushort, 0444);
161
162
163static unsigned short pause_filter_count_max = KVM_SVM_DEFAULT_PLE_WINDOW_MAX;
164module_param(pause_filter_count_max, ushort, 0444);
165
166
167
168
169
170bool npt_enabled = true;
171module_param_named(npt, npt_enabled, bool, 0444);
172
173
174static int nested = true;
175module_param(nested, int, S_IRUGO);
176
177
178static int nrips = true;
179module_param(nrips, int, 0444);
180
181
182static int vls = true;
183module_param(vls, int, 0444);
184
185
186static int vgif = true;
187module_param(vgif, int, 0444);
188
189
190
191
192
193static bool avic;
194module_param(avic, bool, 0444);
195
196bool __read_mostly dump_invalid_vmcb;
197module_param(dump_invalid_vmcb, bool, 0644);
198
199
200bool intercept_smi = true;
201module_param(intercept_smi, bool, 0444);
202
203
204static bool svm_gp_erratum_intercept = true;
205
206static u8 rsm_ins_bytes[] = "\x0f\xaa";
207
208static unsigned long iopm_base;
209
210struct kvm_ldttss_desc {
211 u16 limit0;
212 u16 base0;
213 unsigned base1:8, type:5, dpl:2, p:1;
214 unsigned limit1:4, zero0:3, g:1, base2:8;
215 u32 base3;
216 u32 zero1;
217} __attribute__((packed));
218
219DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
220
221
222
223
224
225
226
227
228static int tsc_aux_uret_slot __read_mostly = -1;
229
230static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
231
232#define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
233#define MSRS_RANGE_SIZE 2048
234#define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
235
236u32 svm_msrpm_offset(u32 msr)
237{
238 u32 offset;
239 int i;
240
241 for (i = 0; i < NUM_MSR_MAPS; i++) {
242 if (msr < msrpm_ranges[i] ||
243 msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
244 continue;
245
246 offset = (msr - msrpm_ranges[i]) / 4;
247 offset += (i * MSRS_RANGE_SIZE);
248
249
250 return offset / 4;
251 }
252
253
254 return MSR_INVALID;
255}
256
257#define MAX_INST_SIZE 15
258
259static int get_max_npt_level(void)
260{
261#ifdef CONFIG_X86_64
262 return pgtable_l5_enabled() ? PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL;
263#else
264 return PT32E_ROOT_LEVEL;
265#endif
266}
267
268int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
269{
270 struct vcpu_svm *svm = to_svm(vcpu);
271 u64 old_efer = vcpu->arch.efer;
272 vcpu->arch.efer = efer;
273
274 if (!npt_enabled) {
275
276 efer |= EFER_NX;
277
278 if (!(efer & EFER_LMA))
279 efer &= ~EFER_LME;
280 }
281
282 if ((old_efer & EFER_SVME) != (efer & EFER_SVME)) {
283 if (!(efer & EFER_SVME)) {
284 svm_leave_nested(svm);
285 svm_set_gif(svm, true);
286
287 if (!enable_vmware_backdoor)
288 clr_exception_intercept(svm, GP_VECTOR);
289
290
291
292
293
294
295 if (!is_smm(vcpu))
296 svm_free_nested(svm);
297
298 } else {
299 int ret = svm_allocate_nested(svm);
300
301 if (ret) {
302 vcpu->arch.efer = old_efer;
303 return ret;
304 }
305
306 if (svm_gp_erratum_intercept)
307 set_exception_intercept(svm, GP_VECTOR);
308 }
309 }
310
311 svm->vmcb->save.efer = efer | EFER_SVME;
312 vmcb_mark_dirty(svm->vmcb, VMCB_CR);
313 return 0;
314}
315
316static int is_external_interrupt(u32 info)
317{
318 info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
319 return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
320}
321
322static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
323{
324 struct vcpu_svm *svm = to_svm(vcpu);
325 u32 ret = 0;
326
327 if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
328 ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
329 return ret;
330}
331
332static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
333{
334 struct vcpu_svm *svm = to_svm(vcpu);
335
336 if (mask == 0)
337 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
338 else
339 svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
340
341}
342
343static int skip_emulated_instruction(struct kvm_vcpu *vcpu)
344{
345 struct vcpu_svm *svm = to_svm(vcpu);
346
347
348
349
350
351 if (sev_es_guest(vcpu->kvm))
352 goto done;
353
354 if (nrips && svm->vmcb->control.next_rip != 0) {
355 WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
356 svm->next_rip = svm->vmcb->control.next_rip;
357 }
358
359 if (!svm->next_rip) {
360 if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP))
361 return 0;
362 } else {
363 kvm_rip_write(vcpu, svm->next_rip);
364 }
365
366done:
367 svm_set_interrupt_shadow(vcpu, 0);
368
369 return 1;
370}
371
372static void svm_queue_exception(struct kvm_vcpu *vcpu)
373{
374 struct vcpu_svm *svm = to_svm(vcpu);
375 unsigned nr = vcpu->arch.exception.nr;
376 bool has_error_code = vcpu->arch.exception.has_error_code;
377 u32 error_code = vcpu->arch.exception.error_code;
378
379 kvm_deliver_exception_payload(vcpu);
380
381 if (nr == BP_VECTOR && !nrips) {
382 unsigned long rip, old_rip = kvm_rip_read(vcpu);
383
384
385
386
387
388
389
390
391 (void)skip_emulated_instruction(vcpu);
392 rip = kvm_rip_read(vcpu);
393 svm->int3_rip = rip + svm->vmcb->save.cs.base;
394 svm->int3_injected = rip - old_rip;
395 }
396
397 svm->vmcb->control.event_inj = nr
398 | SVM_EVTINJ_VALID
399 | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
400 | SVM_EVTINJ_TYPE_EXEPT;
401 svm->vmcb->control.event_inj_err = error_code;
402}
403
404static void svm_init_erratum_383(void)
405{
406 u32 low, high;
407 int err;
408 u64 val;
409
410 if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
411 return;
412
413
414 val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
415 if (err)
416 return;
417
418 val |= (1ULL << 47);
419
420 low = lower_32_bits(val);
421 high = upper_32_bits(val);
422
423 native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);
424
425 erratum_383_found = true;
426}
427
428static void svm_init_osvw(struct kvm_vcpu *vcpu)
429{
430
431
432
433
434 vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
435 vcpu->arch.osvw.status = osvw_status & ~(6ULL);
436
437
438
439
440
441
442
443
444
445 if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
446 vcpu->arch.osvw.status |= 1;
447}
448
449static int has_svm(void)
450{
451 const char *msg;
452
453 if (!cpu_has_svm(&msg)) {
454 printk(KERN_INFO "has_svm: %s\n", msg);
455 return 0;
456 }
457
458 if (sev_active()) {
459 pr_info("KVM is unsupported when running as an SEV guest\n");
460 return 0;
461 }
462
463 return 1;
464}
465
466static void svm_hardware_disable(void)
467{
468
469 if (static_cpu_has(X86_FEATURE_TSCRATEMSR))
470 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
471
472 cpu_svm_disable();
473
474 amd_pmu_disable_virt();
475}
476
477static int svm_hardware_enable(void)
478{
479
480 struct svm_cpu_data *sd;
481 uint64_t efer;
482 struct desc_struct *gdt;
483 int me = raw_smp_processor_id();
484
485 rdmsrl(MSR_EFER, efer);
486 if (efer & EFER_SVME)
487 return -EBUSY;
488
489 if (!has_svm()) {
490 pr_err("%s: err EOPNOTSUPP on %d\n", __func__, me);
491 return -EINVAL;
492 }
493 sd = per_cpu(svm_data, me);
494 if (!sd) {
495 pr_err("%s: svm_data is NULL on %d\n", __func__, me);
496 return -EINVAL;
497 }
498
499 sd->asid_generation = 1;
500 sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
501 sd->next_asid = sd->max_asid + 1;
502 sd->min_asid = max_sev_asid + 1;
503
504 gdt = get_current_gdt_rw();
505 sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
506
507 wrmsrl(MSR_EFER, efer | EFER_SVME);
508
509 wrmsrl(MSR_VM_HSAVE_PA, __sme_page_pa(sd->save_area));
510
511 if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
512 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
513 __this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT);
514 }
515
516
517
518
519
520
521
522
523
524
525
526 if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
527 uint64_t len, status = 0;
528 int err;
529
530 len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err);
531 if (!err)
532 status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS,
533 &err);
534
535 if (err)
536 osvw_status = osvw_len = 0;
537 else {
538 if (len < osvw_len)
539 osvw_len = len;
540 osvw_status |= status;
541 osvw_status &= (1ULL << osvw_len) - 1;
542 }
543 } else
544 osvw_status = osvw_len = 0;
545
546 svm_init_erratum_383();
547
548 amd_pmu_enable_virt();
549
550 return 0;
551}
552
553static void svm_cpu_uninit(int cpu)
554{
555 struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
556
557 if (!sd)
558 return;
559
560 per_cpu(svm_data, cpu) = NULL;
561 kfree(sd->sev_vmcbs);
562 __free_page(sd->save_area);
563 kfree(sd);
564}
565
566static int svm_cpu_init(int cpu)
567{
568 struct svm_cpu_data *sd;
569 int ret = -ENOMEM;
570
571 sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
572 if (!sd)
573 return ret;
574 sd->cpu = cpu;
575 sd->save_area = alloc_page(GFP_KERNEL);
576 if (!sd->save_area)
577 goto free_cpu_data;
578
579 clear_page(page_address(sd->save_area));
580
581 ret = sev_cpu_init(sd);
582 if (ret)
583 goto free_save_area;
584
585 per_cpu(svm_data, cpu) = sd;
586
587 return 0;
588
589free_save_area:
590 __free_page(sd->save_area);
591free_cpu_data:
592 kfree(sd);
593 return ret;
594
595}
596
597static int direct_access_msr_slot(u32 msr)
598{
599 u32 i;
600
601 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
602 if (direct_access_msrs[i].index == msr)
603 return i;
604
605 return -ENOENT;
606}
607
608static void set_shadow_msr_intercept(struct kvm_vcpu *vcpu, u32 msr, int read,
609 int write)
610{
611 struct vcpu_svm *svm = to_svm(vcpu);
612 int slot = direct_access_msr_slot(msr);
613
614 if (slot == -ENOENT)
615 return;
616
617
618 if (read)
619 set_bit(slot, svm->shadow_msr_intercept.read);
620 else
621 clear_bit(slot, svm->shadow_msr_intercept.read);
622
623 if (write)
624 set_bit(slot, svm->shadow_msr_intercept.write);
625 else
626 clear_bit(slot, svm->shadow_msr_intercept.write);
627}
628
629static bool valid_msr_intercept(u32 index)
630{
631 return direct_access_msr_slot(index) != -ENOENT;
632}
633
634static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr)
635{
636 u8 bit_write;
637 unsigned long tmp;
638 u32 offset;
639 u32 *msrpm;
640
641 msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm:
642 to_svm(vcpu)->msrpm;
643
644 offset = svm_msrpm_offset(msr);
645 bit_write = 2 * (msr & 0x0f) + 1;
646 tmp = msrpm[offset];
647
648 BUG_ON(offset == MSR_INVALID);
649
650 return !!test_bit(bit_write, &tmp);
651}
652
653static void set_msr_interception_bitmap(struct kvm_vcpu *vcpu, u32 *msrpm,
654 u32 msr, int read, int write)
655{
656 u8 bit_read, bit_write;
657 unsigned long tmp;
658 u32 offset;
659
660
661
662
663
664 WARN_ON(!valid_msr_intercept(msr));
665
666
667 if (read && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_READ))
668 read = 0;
669
670 if (write && !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_WRITE))
671 write = 0;
672
673 offset = svm_msrpm_offset(msr);
674 bit_read = 2 * (msr & 0x0f);
675 bit_write = 2 * (msr & 0x0f) + 1;
676 tmp = msrpm[offset];
677
678 BUG_ON(offset == MSR_INVALID);
679
680 read ? clear_bit(bit_read, &tmp) : set_bit(bit_read, &tmp);
681 write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);
682
683 msrpm[offset] = tmp;
684
685 svm_hv_vmcb_dirty_nested_enlightenments(vcpu);
686
687}
688
689void set_msr_interception(struct kvm_vcpu *vcpu, u32 *msrpm, u32 msr,
690 int read, int write)
691{
692 set_shadow_msr_intercept(vcpu, msr, read, write);
693 set_msr_interception_bitmap(vcpu, msrpm, msr, read, write);
694}
695
696u32 *svm_vcpu_alloc_msrpm(void)
697{
698 unsigned int order = get_order(MSRPM_SIZE);
699 struct page *pages = alloc_pages(GFP_KERNEL_ACCOUNT, order);
700 u32 *msrpm;
701
702 if (!pages)
703 return NULL;
704
705 msrpm = page_address(pages);
706 memset(msrpm, 0xff, PAGE_SIZE * (1 << order));
707
708 return msrpm;
709}
710
711void svm_vcpu_init_msrpm(struct kvm_vcpu *vcpu, u32 *msrpm)
712{
713 int i;
714
715 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
716 if (!direct_access_msrs[i].always)
717 continue;
718 set_msr_interception(vcpu, msrpm, direct_access_msrs[i].index, 1, 1);
719 }
720}
721
722
723void svm_vcpu_free_msrpm(u32 *msrpm)
724{
725 __free_pages(virt_to_page(msrpm), get_order(MSRPM_SIZE));
726}
727
728static void svm_msr_filter_changed(struct kvm_vcpu *vcpu)
729{
730 struct vcpu_svm *svm = to_svm(vcpu);
731 u32 i;
732
733
734
735
736
737
738 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
739 u32 msr = direct_access_msrs[i].index;
740 u32 read = test_bit(i, svm->shadow_msr_intercept.read);
741 u32 write = test_bit(i, svm->shadow_msr_intercept.write);
742
743 set_msr_interception_bitmap(vcpu, svm->msrpm, msr, read, write);
744 }
745}
746
747static void add_msr_offset(u32 offset)
748{
749 int i;
750
751 for (i = 0; i < MSRPM_OFFSETS; ++i) {
752
753
754 if (msrpm_offsets[i] == offset)
755 return;
756
757
758 if (msrpm_offsets[i] != MSR_INVALID)
759 continue;
760
761
762 msrpm_offsets[i] = offset;
763
764 return;
765 }
766
767
768
769
770
771 BUG();
772}
773
774static void init_msrpm_offsets(void)
775{
776 int i;
777
778 memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));
779
780 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
781 u32 offset;
782
783 offset = svm_msrpm_offset(direct_access_msrs[i].index);
784 BUG_ON(offset == MSR_INVALID);
785
786 add_msr_offset(offset);
787 }
788}
789
790static void svm_enable_lbrv(struct kvm_vcpu *vcpu)
791{
792 struct vcpu_svm *svm = to_svm(vcpu);
793
794 svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK;
795 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
796 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
797 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
798 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
799}
800
801static void svm_disable_lbrv(struct kvm_vcpu *vcpu)
802{
803 struct vcpu_svm *svm = to_svm(vcpu);
804
805 svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK;
806 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
807 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
808 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
809 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
810}
811
812void disable_nmi_singlestep(struct vcpu_svm *svm)
813{
814 svm->nmi_singlestep = false;
815
816 if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) {
817
818 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
819 svm->vmcb->save.rflags &= ~X86_EFLAGS_TF;
820 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
821 svm->vmcb->save.rflags &= ~X86_EFLAGS_RF;
822 }
823}
824
825static void grow_ple_window(struct kvm_vcpu *vcpu)
826{
827 struct vcpu_svm *svm = to_svm(vcpu);
828 struct vmcb_control_area *control = &svm->vmcb->control;
829 int old = control->pause_filter_count;
830
831 control->pause_filter_count = __grow_ple_window(old,
832 pause_filter_count,
833 pause_filter_count_grow,
834 pause_filter_count_max);
835
836 if (control->pause_filter_count != old) {
837 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
838 trace_kvm_ple_window_update(vcpu->vcpu_id,
839 control->pause_filter_count, old);
840 }
841}
842
843static void shrink_ple_window(struct kvm_vcpu *vcpu)
844{
845 struct vcpu_svm *svm = to_svm(vcpu);
846 struct vmcb_control_area *control = &svm->vmcb->control;
847 int old = control->pause_filter_count;
848
849 control->pause_filter_count =
850 __shrink_ple_window(old,
851 pause_filter_count,
852 pause_filter_count_shrink,
853 pause_filter_count);
854 if (control->pause_filter_count != old) {
855 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
856 trace_kvm_ple_window_update(vcpu->vcpu_id,
857 control->pause_filter_count, old);
858 }
859}
860
861
862
863
864
865
866
867static __init void svm_adjust_mmio_mask(void)
868{
869 unsigned int enc_bit, mask_bit;
870 u64 msr, mask;
871
872
873 if (cpuid_eax(0x80000000) < 0x8000001f)
874 return;
875
876
877 rdmsrl(MSR_AMD64_SYSCFG, msr);
878 if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT))
879 return;
880
881 enc_bit = cpuid_ebx(0x8000001f) & 0x3f;
882 mask_bit = boot_cpu_data.x86_phys_bits;
883
884
885 if (enc_bit == mask_bit)
886 mask_bit++;
887
888
889
890
891
892
893
894
895
896
897 mask = (mask_bit < 52) ? rsvd_bits(mask_bit, 51) | PT_PRESENT_MASK : 0;
898
899 kvm_mmu_set_mmio_spte_mask(mask, mask, PT_WRITABLE_MASK | PT_USER_MASK);
900}
901
902static void svm_hardware_teardown(void)
903{
904 int cpu;
905
906 sev_hardware_teardown();
907
908 for_each_possible_cpu(cpu)
909 svm_cpu_uninit(cpu);
910
911 __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT),
912 get_order(IOPM_SIZE));
913 iopm_base = 0;
914}
915
916static __init void svm_set_cpu_caps(void)
917{
918 kvm_set_cpu_caps();
919
920 supported_xss = 0;
921
922
923 if (nested) {
924 kvm_cpu_cap_set(X86_FEATURE_SVM);
925
926 if (nrips)
927 kvm_cpu_cap_set(X86_FEATURE_NRIPS);
928
929 if (npt_enabled)
930 kvm_cpu_cap_set(X86_FEATURE_NPT);
931
932
933 kvm_cpu_cap_set(X86_FEATURE_SVME_ADDR_CHK);
934 }
935
936
937 if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) ||
938 boot_cpu_has(X86_FEATURE_AMD_SSBD))
939 kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD);
940
941
942 sev_set_cpu_caps();
943}
944
945static __init int svm_hardware_setup(void)
946{
947 int cpu;
948 struct page *iopm_pages;
949 void *iopm_va;
950 int r;
951 unsigned int order = get_order(IOPM_SIZE);
952
953
954
955
956
957 if (!boot_cpu_has(X86_FEATURE_NX)) {
958 pr_err_ratelimited("NX (Execute Disable) not supported\n");
959 return -EOPNOTSUPP;
960 }
961 kvm_enable_efer_bits(EFER_NX);
962
963 iopm_pages = alloc_pages(GFP_KERNEL, order);
964
965 if (!iopm_pages)
966 return -ENOMEM;
967
968 iopm_va = page_address(iopm_pages);
969 memset(iopm_va, 0xff, PAGE_SIZE * (1 << order));
970 iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
971
972 init_msrpm_offsets();
973
974 supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR);
975
976 if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
977 kvm_enable_efer_bits(EFER_FFXSR);
978
979 if (boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
980 kvm_has_tsc_control = true;
981 kvm_max_tsc_scaling_ratio = TSC_RATIO_MAX;
982 kvm_tsc_scaling_ratio_frac_bits = 32;
983 }
984
985 tsc_aux_uret_slot = kvm_add_user_return_msr(MSR_TSC_AUX);
986
987
988 if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
989 pause_filter_count = 0;
990 pause_filter_thresh = 0;
991 } else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) {
992 pause_filter_thresh = 0;
993 }
994
995 if (nested) {
996 printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
997 kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
998 }
999
1000
1001
1002
1003
1004
1005 if (!IS_ENABLED(CONFIG_X86_64) && !IS_ENABLED(CONFIG_X86_PAE))
1006 npt_enabled = false;
1007
1008 if (!boot_cpu_has(X86_FEATURE_NPT))
1009 npt_enabled = false;
1010
1011
1012 kvm_configure_mmu(npt_enabled, get_max_npt_level(),
1013 get_max_npt_level(), PG_LEVEL_1G);
1014 pr_info("kvm: Nested Paging %sabled\n", npt_enabled ? "en" : "dis");
1015
1016
1017 sev_hardware_setup();
1018
1019 svm_hv_hardware_setup();
1020
1021 svm_adjust_mmio_mask();
1022
1023 for_each_possible_cpu(cpu) {
1024 r = svm_cpu_init(cpu);
1025 if (r)
1026 goto err;
1027 }
1028
1029 if (nrips) {
1030 if (!boot_cpu_has(X86_FEATURE_NRIPS))
1031 nrips = false;
1032 }
1033
1034 enable_apicv = avic = avic && npt_enabled && boot_cpu_has(X86_FEATURE_AVIC);
1035
1036 if (enable_apicv) {
1037 pr_info("AVIC enabled\n");
1038
1039 amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier);
1040 }
1041
1042 if (vls) {
1043 if (!npt_enabled ||
1044 !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) ||
1045 !IS_ENABLED(CONFIG_X86_64)) {
1046 vls = false;
1047 } else {
1048 pr_info("Virtual VMLOAD VMSAVE supported\n");
1049 }
1050 }
1051
1052 if (boot_cpu_has(X86_FEATURE_SVME_ADDR_CHK))
1053 svm_gp_erratum_intercept = false;
1054
1055 if (vgif) {
1056 if (!boot_cpu_has(X86_FEATURE_VGIF))
1057 vgif = false;
1058 else
1059 pr_info("Virtual GIF supported\n");
1060 }
1061
1062 svm_set_cpu_caps();
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077 allow_smaller_maxphyaddr = !npt_enabled;
1078
1079 return 0;
1080
1081err:
1082 svm_hardware_teardown();
1083 return r;
1084}
1085
1086static void init_seg(struct vmcb_seg *seg)
1087{
1088 seg->selector = 0;
1089 seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
1090 SVM_SELECTOR_WRITE_MASK;
1091 seg->limit = 0xffff;
1092 seg->base = 0;
1093}
1094
1095static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
1096{
1097 seg->selector = 0;
1098 seg->attrib = SVM_SELECTOR_P_MASK | type;
1099 seg->limit = 0xffff;
1100 seg->base = 0;
1101}
1102
1103static u64 svm_get_l2_tsc_offset(struct kvm_vcpu *vcpu)
1104{
1105 struct vcpu_svm *svm = to_svm(vcpu);
1106
1107 return svm->nested.ctl.tsc_offset;
1108}
1109
1110static u64 svm_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu)
1111{
1112 return kvm_default_tsc_scaling_ratio;
1113}
1114
1115static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
1116{
1117 struct vcpu_svm *svm = to_svm(vcpu);
1118
1119 svm->vmcb01.ptr->control.tsc_offset = vcpu->arch.l1_tsc_offset;
1120 svm->vmcb->control.tsc_offset = offset;
1121 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1122}
1123
1124static void svm_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 multiplier)
1125{
1126 wrmsrl(MSR_AMD64_TSC_RATIO, multiplier);
1127}
1128
1129
1130static void svm_recalc_instruction_intercepts(struct kvm_vcpu *vcpu,
1131 struct vcpu_svm *svm)
1132{
1133
1134
1135
1136
1137 if (kvm_cpu_cap_has(X86_FEATURE_INVPCID)) {
1138 if (!npt_enabled ||
1139 !guest_cpuid_has(&svm->vcpu, X86_FEATURE_INVPCID))
1140 svm_set_intercept(svm, INTERCEPT_INVPCID);
1141 else
1142 svm_clr_intercept(svm, INTERCEPT_INVPCID);
1143 }
1144
1145 if (kvm_cpu_cap_has(X86_FEATURE_RDTSCP)) {
1146 if (guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))
1147 svm_clr_intercept(svm, INTERCEPT_RDTSCP);
1148 else
1149 svm_set_intercept(svm, INTERCEPT_RDTSCP);
1150 }
1151}
1152
1153static void init_vmcb(struct kvm_vcpu *vcpu)
1154{
1155 struct vcpu_svm *svm = to_svm(vcpu);
1156 struct vmcb_control_area *control = &svm->vmcb->control;
1157 struct vmcb_save_area *save = &svm->vmcb->save;
1158
1159 svm_set_intercept(svm, INTERCEPT_CR0_READ);
1160 svm_set_intercept(svm, INTERCEPT_CR3_READ);
1161 svm_set_intercept(svm, INTERCEPT_CR4_READ);
1162 svm_set_intercept(svm, INTERCEPT_CR0_WRITE);
1163 svm_set_intercept(svm, INTERCEPT_CR3_WRITE);
1164 svm_set_intercept(svm, INTERCEPT_CR4_WRITE);
1165 if (!kvm_vcpu_apicv_active(vcpu))
1166 svm_set_intercept(svm, INTERCEPT_CR8_WRITE);
1167
1168 set_dr_intercepts(svm);
1169
1170 set_exception_intercept(svm, PF_VECTOR);
1171 set_exception_intercept(svm, UD_VECTOR);
1172 set_exception_intercept(svm, MC_VECTOR);
1173 set_exception_intercept(svm, AC_VECTOR);
1174 set_exception_intercept(svm, DB_VECTOR);
1175
1176
1177
1178
1179
1180
1181 if (enable_vmware_backdoor)
1182 set_exception_intercept(svm, GP_VECTOR);
1183
1184 svm_set_intercept(svm, INTERCEPT_INTR);
1185 svm_set_intercept(svm, INTERCEPT_NMI);
1186
1187 if (intercept_smi)
1188 svm_set_intercept(svm, INTERCEPT_SMI);
1189
1190 svm_set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
1191 svm_set_intercept(svm, INTERCEPT_RDPMC);
1192 svm_set_intercept(svm, INTERCEPT_CPUID);
1193 svm_set_intercept(svm, INTERCEPT_INVD);
1194 svm_set_intercept(svm, INTERCEPT_INVLPG);
1195 svm_set_intercept(svm, INTERCEPT_INVLPGA);
1196 svm_set_intercept(svm, INTERCEPT_IOIO_PROT);
1197 svm_set_intercept(svm, INTERCEPT_MSR_PROT);
1198 svm_set_intercept(svm, INTERCEPT_TASK_SWITCH);
1199 svm_set_intercept(svm, INTERCEPT_SHUTDOWN);
1200 svm_set_intercept(svm, INTERCEPT_VMRUN);
1201 svm_set_intercept(svm, INTERCEPT_VMMCALL);
1202 svm_set_intercept(svm, INTERCEPT_VMLOAD);
1203 svm_set_intercept(svm, INTERCEPT_VMSAVE);
1204 svm_set_intercept(svm, INTERCEPT_STGI);
1205 svm_set_intercept(svm, INTERCEPT_CLGI);
1206 svm_set_intercept(svm, INTERCEPT_SKINIT);
1207 svm_set_intercept(svm, INTERCEPT_WBINVD);
1208 svm_set_intercept(svm, INTERCEPT_XSETBV);
1209 svm_set_intercept(svm, INTERCEPT_RDPRU);
1210 svm_set_intercept(svm, INTERCEPT_RSM);
1211
1212 if (!kvm_mwait_in_guest(vcpu->kvm)) {
1213 svm_set_intercept(svm, INTERCEPT_MONITOR);
1214 svm_set_intercept(svm, INTERCEPT_MWAIT);
1215 }
1216
1217 if (!kvm_hlt_in_guest(vcpu->kvm))
1218 svm_set_intercept(svm, INTERCEPT_HLT);
1219
1220 control->iopm_base_pa = __sme_set(iopm_base);
1221 control->msrpm_base_pa = __sme_set(__pa(svm->msrpm));
1222 control->int_ctl = V_INTR_MASKING_MASK;
1223
1224 init_seg(&save->es);
1225 init_seg(&save->ss);
1226 init_seg(&save->ds);
1227 init_seg(&save->fs);
1228 init_seg(&save->gs);
1229
1230 save->cs.selector = 0xf000;
1231 save->cs.base = 0xffff0000;
1232
1233 save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
1234 SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
1235 save->cs.limit = 0xffff;
1236
1237 save->gdtr.base = 0;
1238 save->gdtr.limit = 0xffff;
1239 save->idtr.base = 0;
1240 save->idtr.limit = 0xffff;
1241
1242 init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
1243 init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
1244
1245 if (npt_enabled) {
1246
1247 control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE;
1248 svm_clr_intercept(svm, INTERCEPT_INVLPG);
1249 clr_exception_intercept(svm, PF_VECTOR);
1250 svm_clr_intercept(svm, INTERCEPT_CR3_READ);
1251 svm_clr_intercept(svm, INTERCEPT_CR3_WRITE);
1252 save->g_pat = vcpu->arch.pat;
1253 save->cr3 = 0;
1254 }
1255 svm->current_vmcb->asid_generation = 0;
1256 svm->asid = 0;
1257
1258 svm->nested.vmcb12_gpa = INVALID_GPA;
1259 svm->nested.last_vmcb12_gpa = INVALID_GPA;
1260
1261 if (!kvm_pause_in_guest(vcpu->kvm)) {
1262 control->pause_filter_count = pause_filter_count;
1263 if (pause_filter_thresh)
1264 control->pause_filter_thresh = pause_filter_thresh;
1265 svm_set_intercept(svm, INTERCEPT_PAUSE);
1266 } else {
1267 svm_clr_intercept(svm, INTERCEPT_PAUSE);
1268 }
1269
1270 svm_recalc_instruction_intercepts(vcpu, svm);
1271
1272
1273
1274
1275
1276 if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
1277 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
1278
1279 if (kvm_vcpu_apicv_active(vcpu))
1280 avic_init_vmcb(svm);
1281
1282 if (vgif) {
1283 svm_clr_intercept(svm, INTERCEPT_STGI);
1284 svm_clr_intercept(svm, INTERCEPT_CLGI);
1285 svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK;
1286 }
1287
1288 if (sev_guest(vcpu->kvm)) {
1289 svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ENABLE;
1290 clr_exception_intercept(svm, UD_VECTOR);
1291
1292 if (sev_es_guest(vcpu->kvm)) {
1293
1294 sev_es_init_vmcb(svm);
1295 }
1296 }
1297
1298 svm_hv_init_vmcb(svm->vmcb);
1299
1300 vmcb_mark_all_dirty(svm->vmcb);
1301
1302 enable_gif(svm);
1303
1304}
1305
1306static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
1307{
1308 struct vcpu_svm *svm = to_svm(vcpu);
1309
1310 svm->spec_ctrl = 0;
1311 svm->virt_spec_ctrl = 0;
1312
1313 init_vmcb(vcpu);
1314}
1315
1316void svm_switch_vmcb(struct vcpu_svm *svm, struct kvm_vmcb_info *target_vmcb)
1317{
1318 svm->current_vmcb = target_vmcb;
1319 svm->vmcb = target_vmcb->ptr;
1320}
1321
1322static int svm_create_vcpu(struct kvm_vcpu *vcpu)
1323{
1324 struct vcpu_svm *svm;
1325 struct page *vmcb01_page;
1326 struct page *vmsa_page = NULL;
1327 int err;
1328
1329 BUILD_BUG_ON(offsetof(struct vcpu_svm, vcpu) != 0);
1330 svm = to_svm(vcpu);
1331
1332 err = -ENOMEM;
1333 vmcb01_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
1334 if (!vmcb01_page)
1335 goto out;
1336
1337 if (sev_es_guest(vcpu->kvm)) {
1338
1339
1340
1341
1342 vmsa_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
1343 if (!vmsa_page)
1344 goto error_free_vmcb_page;
1345
1346
1347
1348
1349
1350
1351
1352 kvm_free_guest_fpu(vcpu);
1353 }
1354
1355 err = avic_init_vcpu(svm);
1356 if (err)
1357 goto error_free_vmsa_page;
1358
1359
1360
1361
1362 if (irqchip_in_kernel(vcpu->kvm) && kvm_apicv_activated(vcpu->kvm))
1363 svm->avic_is_running = true;
1364
1365 svm->msrpm = svm_vcpu_alloc_msrpm();
1366 if (!svm->msrpm) {
1367 err = -ENOMEM;
1368 goto error_free_vmsa_page;
1369 }
1370
1371 svm->vmcb01.ptr = page_address(vmcb01_page);
1372 svm->vmcb01.pa = __sme_set(page_to_pfn(vmcb01_page) << PAGE_SHIFT);
1373
1374 if (vmsa_page)
1375 svm->vmsa = page_address(vmsa_page);
1376
1377 svm->guest_state_loaded = false;
1378
1379 svm_switch_vmcb(svm, &svm->vmcb01);
1380 init_vmcb(vcpu);
1381
1382 svm_vcpu_init_msrpm(vcpu, svm->msrpm);
1383
1384 svm_init_osvw(vcpu);
1385 vcpu->arch.microcode_version = 0x01000065;
1386
1387 if (sev_es_guest(vcpu->kvm))
1388
1389 sev_es_create_vcpu(svm);
1390
1391 return 0;
1392
1393error_free_vmsa_page:
1394 if (vmsa_page)
1395 __free_page(vmsa_page);
1396error_free_vmcb_page:
1397 __free_page(vmcb01_page);
1398out:
1399 return err;
1400}
1401
1402static void svm_clear_current_vmcb(struct vmcb *vmcb)
1403{
1404 int i;
1405
1406 for_each_online_cpu(i)
1407 cmpxchg(&per_cpu(svm_data, i)->current_vmcb, vmcb, NULL);
1408}
1409
1410static void svm_free_vcpu(struct kvm_vcpu *vcpu)
1411{
1412 struct vcpu_svm *svm = to_svm(vcpu);
1413
1414
1415
1416
1417
1418
1419 svm_clear_current_vmcb(svm->vmcb);
1420
1421 svm_free_nested(svm);
1422
1423 sev_free_vcpu(vcpu);
1424
1425 __free_page(pfn_to_page(__sme_clr(svm->vmcb01.pa) >> PAGE_SHIFT));
1426 __free_pages(virt_to_page(svm->msrpm), get_order(MSRPM_SIZE));
1427}
1428
1429static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
1430{
1431 struct vcpu_svm *svm = to_svm(vcpu);
1432 struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu);
1433
1434 if (sev_es_guest(vcpu->kvm))
1435 sev_es_unmap_ghcb(svm);
1436
1437 if (svm->guest_state_loaded)
1438 return;
1439
1440
1441
1442
1443
1444 if (sev_es_guest(vcpu->kvm)) {
1445 sev_es_prepare_guest_switch(svm, vcpu->cpu);
1446 } else {
1447 vmsave(__sme_page_pa(sd->save_area));
1448 }
1449
1450 if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
1451 u64 tsc_ratio = vcpu->arch.tsc_scaling_ratio;
1452 if (tsc_ratio != __this_cpu_read(current_tsc_ratio)) {
1453 __this_cpu_write(current_tsc_ratio, tsc_ratio);
1454 wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio);
1455 }
1456 }
1457
1458 if (likely(tsc_aux_uret_slot >= 0))
1459 kvm_set_user_return_msr(tsc_aux_uret_slot, svm->tsc_aux, -1ull);
1460
1461 svm->guest_state_loaded = true;
1462}
1463
1464static void svm_prepare_host_switch(struct kvm_vcpu *vcpu)
1465{
1466 to_svm(vcpu)->guest_state_loaded = false;
1467}
1468
1469static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1470{
1471 struct vcpu_svm *svm = to_svm(vcpu);
1472 struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
1473
1474 if (sd->current_vmcb != svm->vmcb) {
1475 sd->current_vmcb = svm->vmcb;
1476 indirect_branch_prediction_barrier();
1477 }
1478 if (kvm_vcpu_apicv_active(vcpu))
1479 avic_vcpu_load(vcpu, cpu);
1480}
1481
1482static void svm_vcpu_put(struct kvm_vcpu *vcpu)
1483{
1484 if (kvm_vcpu_apicv_active(vcpu))
1485 avic_vcpu_put(vcpu);
1486
1487 svm_prepare_host_switch(vcpu);
1488
1489 ++vcpu->stat.host_state_reload;
1490}
1491
1492static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
1493{
1494 struct vcpu_svm *svm = to_svm(vcpu);
1495 unsigned long rflags = svm->vmcb->save.rflags;
1496
1497 if (svm->nmi_singlestep) {
1498
1499 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
1500 rflags &= ~X86_EFLAGS_TF;
1501 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
1502 rflags &= ~X86_EFLAGS_RF;
1503 }
1504 return rflags;
1505}
1506
1507static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
1508{
1509 if (to_svm(vcpu)->nmi_singlestep)
1510 rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
1511
1512
1513
1514
1515
1516
1517 to_svm(vcpu)->vmcb->save.rflags = rflags;
1518}
1519
1520static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
1521{
1522 switch (reg) {
1523 case VCPU_EXREG_PDPTR:
1524 BUG_ON(!npt_enabled);
1525 load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
1526 break;
1527 default:
1528 KVM_BUG_ON(1, vcpu->kvm);
1529 }
1530}
1531
1532static void svm_set_vintr(struct vcpu_svm *svm)
1533{
1534 struct vmcb_control_area *control;
1535
1536
1537
1538
1539 WARN_ON(kvm_apicv_activated(svm->vcpu.kvm));
1540
1541 svm_set_intercept(svm, INTERCEPT_VINTR);
1542
1543
1544
1545
1546
1547 control = &svm->vmcb->control;
1548 control->int_vector = 0x0;
1549 control->int_ctl &= ~V_INTR_PRIO_MASK;
1550 control->int_ctl |= V_IRQ_MASK |
1551 (( 0xf) << V_INTR_PRIO_SHIFT);
1552 vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
1553}
1554
1555static void svm_clear_vintr(struct vcpu_svm *svm)
1556{
1557 svm_clr_intercept(svm, INTERCEPT_VINTR);
1558
1559
1560 svm->vmcb->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK;
1561 if (is_guest_mode(&svm->vcpu)) {
1562 svm->vmcb01.ptr->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK;
1563
1564 WARN_ON((svm->vmcb->control.int_ctl & V_TPR_MASK) !=
1565 (svm->nested.ctl.int_ctl & V_TPR_MASK));
1566
1567 svm->vmcb->control.int_ctl |= svm->nested.ctl.int_ctl &
1568 V_IRQ_INJECTION_BITS_MASK;
1569
1570 svm->vmcb->control.int_vector = svm->nested.ctl.int_vector;
1571 }
1572
1573 vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
1574}
1575
1576static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
1577{
1578 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1579 struct vmcb_save_area *save01 = &to_svm(vcpu)->vmcb01.ptr->save;
1580
1581 switch (seg) {
1582 case VCPU_SREG_CS: return &save->cs;
1583 case VCPU_SREG_DS: return &save->ds;
1584 case VCPU_SREG_ES: return &save->es;
1585 case VCPU_SREG_FS: return &save01->fs;
1586 case VCPU_SREG_GS: return &save01->gs;
1587 case VCPU_SREG_SS: return &save->ss;
1588 case VCPU_SREG_TR: return &save01->tr;
1589 case VCPU_SREG_LDTR: return &save01->ldtr;
1590 }
1591 BUG();
1592 return NULL;
1593}
1594
1595static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
1596{
1597 struct vmcb_seg *s = svm_seg(vcpu, seg);
1598
1599 return s->base;
1600}
1601
1602static void svm_get_segment(struct kvm_vcpu *vcpu,
1603 struct kvm_segment *var, int seg)
1604{
1605 struct vmcb_seg *s = svm_seg(vcpu, seg);
1606
1607 var->base = s->base;
1608 var->limit = s->limit;
1609 var->selector = s->selector;
1610 var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
1611 var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
1612 var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
1613 var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
1614 var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
1615 var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
1616 var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626 var->g = s->limit > 0xfffff;
1627
1628
1629
1630
1631
1632 var->unusable = !var->present;
1633
1634 switch (seg) {
1635 case VCPU_SREG_TR:
1636
1637
1638
1639
1640 var->type |= 0x2;
1641 break;
1642 case VCPU_SREG_DS:
1643 case VCPU_SREG_ES:
1644 case VCPU_SREG_FS:
1645 case VCPU_SREG_GS:
1646
1647
1648
1649
1650
1651
1652
1653 if (!var->unusable)
1654 var->type |= 0x1;
1655 break;
1656 case VCPU_SREG_SS:
1657
1658
1659
1660
1661
1662
1663 if (var->unusable)
1664 var->db = 0;
1665
1666 var->dpl = to_svm(vcpu)->vmcb->save.cpl;
1667 break;
1668 }
1669}
1670
1671static int svm_get_cpl(struct kvm_vcpu *vcpu)
1672{
1673 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1674
1675 return save->cpl;
1676}
1677
1678static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1679{
1680 struct vcpu_svm *svm = to_svm(vcpu);
1681
1682 dt->size = svm->vmcb->save.idtr.limit;
1683 dt->address = svm->vmcb->save.idtr.base;
1684}
1685
1686static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1687{
1688 struct vcpu_svm *svm = to_svm(vcpu);
1689
1690 svm->vmcb->save.idtr.limit = dt->size;
1691 svm->vmcb->save.idtr.base = dt->address ;
1692 vmcb_mark_dirty(svm->vmcb, VMCB_DT);
1693}
1694
1695static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1696{
1697 struct vcpu_svm *svm = to_svm(vcpu);
1698
1699 dt->size = svm->vmcb->save.gdtr.limit;
1700 dt->address = svm->vmcb->save.gdtr.base;
1701}
1702
1703static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1704{
1705 struct vcpu_svm *svm = to_svm(vcpu);
1706
1707 svm->vmcb->save.gdtr.limit = dt->size;
1708 svm->vmcb->save.gdtr.base = dt->address ;
1709 vmcb_mark_dirty(svm->vmcb, VMCB_DT);
1710}
1711
1712void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1713{
1714 struct vcpu_svm *svm = to_svm(vcpu);
1715 u64 hcr0 = cr0;
1716
1717#ifdef CONFIG_X86_64
1718 if (vcpu->arch.efer & EFER_LME && !vcpu->arch.guest_state_protected) {
1719 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
1720 vcpu->arch.efer |= EFER_LMA;
1721 svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
1722 }
1723
1724 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
1725 vcpu->arch.efer &= ~EFER_LMA;
1726 svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
1727 }
1728 }
1729#endif
1730 vcpu->arch.cr0 = cr0;
1731
1732 if (!npt_enabled)
1733 hcr0 |= X86_CR0_PG | X86_CR0_WP;
1734
1735
1736
1737
1738
1739
1740 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
1741 hcr0 &= ~(X86_CR0_CD | X86_CR0_NW);
1742
1743 svm->vmcb->save.cr0 = hcr0;
1744 vmcb_mark_dirty(svm->vmcb, VMCB_CR);
1745
1746
1747
1748
1749
1750 if (sev_es_guest(vcpu->kvm))
1751 return;
1752
1753 if (hcr0 == cr0) {
1754
1755 svm_clr_intercept(svm, INTERCEPT_CR0_READ);
1756 svm_clr_intercept(svm, INTERCEPT_CR0_WRITE);
1757 } else {
1758 svm_set_intercept(svm, INTERCEPT_CR0_READ);
1759 svm_set_intercept(svm, INTERCEPT_CR0_WRITE);
1760 }
1761}
1762
1763static bool svm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1764{
1765 return true;
1766}
1767
1768void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1769{
1770 unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
1771 unsigned long old_cr4 = vcpu->arch.cr4;
1772
1773 if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
1774 svm_flush_tlb(vcpu);
1775
1776 vcpu->arch.cr4 = cr4;
1777 if (!npt_enabled)
1778 cr4 |= X86_CR4_PAE;
1779 cr4 |= host_cr4_mce;
1780 to_svm(vcpu)->vmcb->save.cr4 = cr4;
1781 vmcb_mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
1782
1783 if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE))
1784 kvm_update_cpuid_runtime(vcpu);
1785}
1786
1787static void svm_set_segment(struct kvm_vcpu *vcpu,
1788 struct kvm_segment *var, int seg)
1789{
1790 struct vcpu_svm *svm = to_svm(vcpu);
1791 struct vmcb_seg *s = svm_seg(vcpu, seg);
1792
1793 s->base = var->base;
1794 s->limit = var->limit;
1795 s->selector = var->selector;
1796 s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
1797 s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
1798 s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
1799 s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT;
1800 s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
1801 s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
1802 s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
1803 s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
1804
1805
1806
1807
1808
1809
1810
1811 if (seg == VCPU_SREG_SS)
1812
1813 svm->vmcb->save.cpl = (var->dpl & 3);
1814
1815 vmcb_mark_dirty(svm->vmcb, VMCB_SEG);
1816}
1817
1818static void svm_update_exception_bitmap(struct kvm_vcpu *vcpu)
1819{
1820 struct vcpu_svm *svm = to_svm(vcpu);
1821
1822 clr_exception_intercept(svm, BP_VECTOR);
1823
1824 if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
1825 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
1826 set_exception_intercept(svm, BP_VECTOR);
1827 }
1828}
1829
1830static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
1831{
1832 if (sd->next_asid > sd->max_asid) {
1833 ++sd->asid_generation;
1834 sd->next_asid = sd->min_asid;
1835 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
1836 vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
1837 }
1838
1839 svm->current_vmcb->asid_generation = sd->asid_generation;
1840 svm->asid = sd->next_asid++;
1841}
1842
1843static void svm_set_dr6(struct vcpu_svm *svm, unsigned long value)
1844{
1845 struct vmcb *vmcb = svm->vmcb;
1846
1847 if (svm->vcpu.arch.guest_state_protected)
1848 return;
1849
1850 if (unlikely(value != vmcb->save.dr6)) {
1851 vmcb->save.dr6 = value;
1852 vmcb_mark_dirty(vmcb, VMCB_DR);
1853 }
1854}
1855
1856static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
1857{
1858 struct vcpu_svm *svm = to_svm(vcpu);
1859
1860 if (vcpu->arch.guest_state_protected)
1861 return;
1862
1863 get_debugreg(vcpu->arch.db[0], 0);
1864 get_debugreg(vcpu->arch.db[1], 1);
1865 get_debugreg(vcpu->arch.db[2], 2);
1866 get_debugreg(vcpu->arch.db[3], 3);
1867
1868
1869
1870
1871 vcpu->arch.dr6 = svm->vmcb->save.dr6;
1872 vcpu->arch.dr7 = svm->vmcb->save.dr7;
1873 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
1874 set_dr_intercepts(svm);
1875}
1876
1877static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
1878{
1879 struct vcpu_svm *svm = to_svm(vcpu);
1880
1881 if (vcpu->arch.guest_state_protected)
1882 return;
1883
1884 svm->vmcb->save.dr7 = value;
1885 vmcb_mark_dirty(svm->vmcb, VMCB_DR);
1886}
1887
1888static int pf_interception(struct kvm_vcpu *vcpu)
1889{
1890 struct vcpu_svm *svm = to_svm(vcpu);
1891
1892 u64 fault_address = svm->vmcb->control.exit_info_2;
1893 u64 error_code = svm->vmcb->control.exit_info_1;
1894
1895 return kvm_handle_page_fault(vcpu, error_code, fault_address,
1896 static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
1897 svm->vmcb->control.insn_bytes : NULL,
1898 svm->vmcb->control.insn_len);
1899}
1900
1901static int npf_interception(struct kvm_vcpu *vcpu)
1902{
1903 struct vcpu_svm *svm = to_svm(vcpu);
1904
1905 u64 fault_address = svm->vmcb->control.exit_info_2;
1906 u64 error_code = svm->vmcb->control.exit_info_1;
1907
1908 trace_kvm_page_fault(fault_address, error_code);
1909 return kvm_mmu_page_fault(vcpu, fault_address, error_code,
1910 static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
1911 svm->vmcb->control.insn_bytes : NULL,
1912 svm->vmcb->control.insn_len);
1913}
1914
1915static int db_interception(struct kvm_vcpu *vcpu)
1916{
1917 struct kvm_run *kvm_run = vcpu->run;
1918 struct vcpu_svm *svm = to_svm(vcpu);
1919
1920 if (!(vcpu->guest_debug &
1921 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
1922 !svm->nmi_singlestep) {
1923 u32 payload = svm->vmcb->save.dr6 ^ DR6_ACTIVE_LOW;
1924 kvm_queue_exception_p(vcpu, DB_VECTOR, payload);
1925 return 1;
1926 }
1927
1928 if (svm->nmi_singlestep) {
1929 disable_nmi_singlestep(svm);
1930
1931 kvm_make_request(KVM_REQ_EVENT, vcpu);
1932 }
1933
1934 if (vcpu->guest_debug &
1935 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
1936 kvm_run->exit_reason = KVM_EXIT_DEBUG;
1937 kvm_run->debug.arch.dr6 = svm->vmcb->save.dr6;
1938 kvm_run->debug.arch.dr7 = svm->vmcb->save.dr7;
1939 kvm_run->debug.arch.pc =
1940 svm->vmcb->save.cs.base + svm->vmcb->save.rip;
1941 kvm_run->debug.arch.exception = DB_VECTOR;
1942 return 0;
1943 }
1944
1945 return 1;
1946}
1947
1948static int bp_interception(struct kvm_vcpu *vcpu)
1949{
1950 struct vcpu_svm *svm = to_svm(vcpu);
1951 struct kvm_run *kvm_run = vcpu->run;
1952
1953 kvm_run->exit_reason = KVM_EXIT_DEBUG;
1954 kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
1955 kvm_run->debug.arch.exception = BP_VECTOR;
1956 return 0;
1957}
1958
1959static int ud_interception(struct kvm_vcpu *vcpu)
1960{
1961 return handle_ud(vcpu);
1962}
1963
1964static int ac_interception(struct kvm_vcpu *vcpu)
1965{
1966 kvm_queue_exception_e(vcpu, AC_VECTOR, 0);
1967 return 1;
1968}
1969
1970static bool is_erratum_383(void)
1971{
1972 int err, i;
1973 u64 value;
1974
1975 if (!erratum_383_found)
1976 return false;
1977
1978 value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
1979 if (err)
1980 return false;
1981
1982
1983 value &= ~(1ULL << 62);
1984
1985 if (value != 0xb600000000010015ULL)
1986 return false;
1987
1988
1989 for (i = 0; i < 6; ++i)
1990 native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);
1991
1992 value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
1993 if (!err) {
1994 u32 low, high;
1995
1996 value &= ~(1ULL << 2);
1997 low = lower_32_bits(value);
1998 high = upper_32_bits(value);
1999
2000 native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
2001 }
2002
2003
2004 __flush_tlb_all();
2005
2006 return true;
2007}
2008
2009static void svm_handle_mce(struct kvm_vcpu *vcpu)
2010{
2011 if (is_erratum_383()) {
2012
2013
2014
2015
2016 pr_err("KVM: Guest triggered AMD Erratum 383\n");
2017
2018 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
2019
2020 return;
2021 }
2022
2023
2024
2025
2026
2027 kvm_machine_check();
2028}
2029
2030static int mc_interception(struct kvm_vcpu *vcpu)
2031{
2032 return 1;
2033}
2034
2035static int shutdown_interception(struct kvm_vcpu *vcpu)
2036{
2037 struct kvm_run *kvm_run = vcpu->run;
2038 struct vcpu_svm *svm = to_svm(vcpu);
2039
2040
2041
2042
2043
2044 if (sev_es_guest(vcpu->kvm))
2045 return -EINVAL;
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055 clear_page(svm->vmcb);
2056 kvm_vcpu_reset(vcpu, true);
2057
2058 kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
2059 return 0;
2060}
2061
2062static int io_interception(struct kvm_vcpu *vcpu)
2063{
2064 struct vcpu_svm *svm = to_svm(vcpu);
2065 u32 io_info = svm->vmcb->control.exit_info_1;
2066 int size, in, string;
2067 unsigned port;
2068
2069 ++vcpu->stat.io_exits;
2070 string = (io_info & SVM_IOIO_STR_MASK) != 0;
2071 in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
2072 port = io_info >> 16;
2073 size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
2074
2075 if (string) {
2076 if (sev_es_guest(vcpu->kvm))
2077 return sev_es_string_io(svm, size, port, in);
2078 else
2079 return kvm_emulate_instruction(vcpu, 0);
2080 }
2081
2082 svm->next_rip = svm->vmcb->control.exit_info_2;
2083
2084 return kvm_fast_pio(vcpu, size, port, in);
2085}
2086
2087static int nmi_interception(struct kvm_vcpu *vcpu)
2088{
2089 return 1;
2090}
2091
2092static int smi_interception(struct kvm_vcpu *vcpu)
2093{
2094 return 1;
2095}
2096
2097static int intr_interception(struct kvm_vcpu *vcpu)
2098{
2099 ++vcpu->stat.irq_exits;
2100 return 1;
2101}
2102
2103static int vmload_vmsave_interception(struct kvm_vcpu *vcpu, bool vmload)
2104{
2105 struct vcpu_svm *svm = to_svm(vcpu);
2106 struct vmcb *vmcb12;
2107 struct kvm_host_map map;
2108 int ret;
2109
2110 if (nested_svm_check_permissions(vcpu))
2111 return 1;
2112
2113 ret = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map);
2114 if (ret) {
2115 if (ret == -EINVAL)
2116 kvm_inject_gp(vcpu, 0);
2117 return 1;
2118 }
2119
2120 vmcb12 = map.hva;
2121
2122 ret = kvm_skip_emulated_instruction(vcpu);
2123
2124 if (vmload) {
2125 svm_copy_vmloadsave_state(svm->vmcb, vmcb12);
2126 svm->sysenter_eip_hi = 0;
2127 svm->sysenter_esp_hi = 0;
2128 } else {
2129 svm_copy_vmloadsave_state(vmcb12, svm->vmcb);
2130 }
2131
2132 kvm_vcpu_unmap(vcpu, &map, true);
2133
2134 return ret;
2135}
2136
2137static int vmload_interception(struct kvm_vcpu *vcpu)
2138{
2139 return vmload_vmsave_interception(vcpu, true);
2140}
2141
2142static int vmsave_interception(struct kvm_vcpu *vcpu)
2143{
2144 return vmload_vmsave_interception(vcpu, false);
2145}
2146
2147static int vmrun_interception(struct kvm_vcpu *vcpu)
2148{
2149 if (nested_svm_check_permissions(vcpu))
2150 return 1;
2151
2152 return nested_svm_vmrun(vcpu);
2153}
2154
2155enum {
2156 NONE_SVM_INSTR,
2157 SVM_INSTR_VMRUN,
2158 SVM_INSTR_VMLOAD,
2159 SVM_INSTR_VMSAVE,
2160};
2161
2162
2163static int svm_instr_opcode(struct kvm_vcpu *vcpu)
2164{
2165 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
2166
2167 if (ctxt->b != 0x1 || ctxt->opcode_len != 2)
2168 return NONE_SVM_INSTR;
2169
2170 switch (ctxt->modrm) {
2171 case 0xd8:
2172 return SVM_INSTR_VMRUN;
2173 case 0xda:
2174 return SVM_INSTR_VMLOAD;
2175 case 0xdb:
2176 return SVM_INSTR_VMSAVE;
2177 default:
2178 break;
2179 }
2180
2181 return NONE_SVM_INSTR;
2182}
2183
2184static int emulate_svm_instr(struct kvm_vcpu *vcpu, int opcode)
2185{
2186 const int guest_mode_exit_codes[] = {
2187 [SVM_INSTR_VMRUN] = SVM_EXIT_VMRUN,
2188 [SVM_INSTR_VMLOAD] = SVM_EXIT_VMLOAD,
2189 [SVM_INSTR_VMSAVE] = SVM_EXIT_VMSAVE,
2190 };
2191 int (*const svm_instr_handlers[])(struct kvm_vcpu *vcpu) = {
2192 [SVM_INSTR_VMRUN] = vmrun_interception,
2193 [SVM_INSTR_VMLOAD] = vmload_interception,
2194 [SVM_INSTR_VMSAVE] = vmsave_interception,
2195 };
2196 struct vcpu_svm *svm = to_svm(vcpu);
2197 int ret;
2198
2199 if (is_guest_mode(vcpu)) {
2200
2201 ret = nested_svm_simple_vmexit(svm, guest_mode_exit_codes[opcode]);
2202 if (ret)
2203 return ret;
2204 return 1;
2205 }
2206 return svm_instr_handlers[opcode](vcpu);
2207}
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217static int gp_interception(struct kvm_vcpu *vcpu)
2218{
2219 struct vcpu_svm *svm = to_svm(vcpu);
2220 u32 error_code = svm->vmcb->control.exit_info_1;
2221 int opcode;
2222
2223
2224 if (error_code)
2225 goto reinject;
2226
2227
2228 if (svm->vmcb->save.rax & ~PAGE_MASK)
2229 goto reinject;
2230
2231
2232 if (x86_decode_emulated_instruction(vcpu, 0, NULL, 0) != EMULATION_OK)
2233 goto reinject;
2234
2235 opcode = svm_instr_opcode(vcpu);
2236
2237 if (opcode == NONE_SVM_INSTR) {
2238 if (!enable_vmware_backdoor)
2239 goto reinject;
2240
2241
2242
2243
2244
2245 if (!is_guest_mode(vcpu))
2246 return kvm_emulate_instruction(vcpu,
2247 EMULTYPE_VMWARE_GP | EMULTYPE_NO_DECODE);
2248 } else
2249 return emulate_svm_instr(vcpu, opcode);
2250
2251reinject:
2252 kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
2253 return 1;
2254}
2255
2256void svm_set_gif(struct vcpu_svm *svm, bool value)
2257{
2258 if (value) {
2259
2260
2261
2262
2263
2264
2265 if (vgif_enabled(svm))
2266 svm_clr_intercept(svm, INTERCEPT_STGI);
2267 if (svm_is_intercept(svm, INTERCEPT_VINTR))
2268 svm_clear_vintr(svm);
2269
2270 enable_gif(svm);
2271 if (svm->vcpu.arch.smi_pending ||
2272 svm->vcpu.arch.nmi_pending ||
2273 kvm_cpu_has_injectable_intr(&svm->vcpu))
2274 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2275 } else {
2276 disable_gif(svm);
2277
2278
2279
2280
2281
2282
2283 if (!vgif_enabled(svm))
2284 svm_clear_vintr(svm);
2285 }
2286}
2287
2288static int stgi_interception(struct kvm_vcpu *vcpu)
2289{
2290 int ret;
2291
2292 if (nested_svm_check_permissions(vcpu))
2293 return 1;
2294
2295 ret = kvm_skip_emulated_instruction(vcpu);
2296 svm_set_gif(to_svm(vcpu), true);
2297 return ret;
2298}
2299
2300static int clgi_interception(struct kvm_vcpu *vcpu)
2301{
2302 int ret;
2303
2304 if (nested_svm_check_permissions(vcpu))
2305 return 1;
2306
2307 ret = kvm_skip_emulated_instruction(vcpu);
2308 svm_set_gif(to_svm(vcpu), false);
2309 return ret;
2310}
2311
2312static int invlpga_interception(struct kvm_vcpu *vcpu)
2313{
2314 gva_t gva = kvm_rax_read(vcpu);
2315 u32 asid = kvm_rcx_read(vcpu);
2316
2317
2318 if (!is_long_mode(vcpu))
2319 gva = (u32)gva;
2320
2321 trace_kvm_invlpga(to_svm(vcpu)->vmcb->save.rip, asid, gva);
2322
2323
2324 kvm_mmu_invlpg(vcpu, gva);
2325
2326 return kvm_skip_emulated_instruction(vcpu);
2327}
2328
2329static int skinit_interception(struct kvm_vcpu *vcpu)
2330{
2331 trace_kvm_skinit(to_svm(vcpu)->vmcb->save.rip, kvm_rax_read(vcpu));
2332
2333 kvm_queue_exception(vcpu, UD_VECTOR);
2334 return 1;
2335}
2336
2337static int task_switch_interception(struct kvm_vcpu *vcpu)
2338{
2339 struct vcpu_svm *svm = to_svm(vcpu);
2340 u16 tss_selector;
2341 int reason;
2342 int int_type = svm->vmcb->control.exit_int_info &
2343 SVM_EXITINTINFO_TYPE_MASK;
2344 int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
2345 uint32_t type =
2346 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
2347 uint32_t idt_v =
2348 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
2349 bool has_error_code = false;
2350 u32 error_code = 0;
2351
2352 tss_selector = (u16)svm->vmcb->control.exit_info_1;
2353
2354 if (svm->vmcb->control.exit_info_2 &
2355 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
2356 reason = TASK_SWITCH_IRET;
2357 else if (svm->vmcb->control.exit_info_2 &
2358 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
2359 reason = TASK_SWITCH_JMP;
2360 else if (idt_v)
2361 reason = TASK_SWITCH_GATE;
2362 else
2363 reason = TASK_SWITCH_CALL;
2364
2365 if (reason == TASK_SWITCH_GATE) {
2366 switch (type) {
2367 case SVM_EXITINTINFO_TYPE_NMI:
2368 vcpu->arch.nmi_injected = false;
2369 break;
2370 case SVM_EXITINTINFO_TYPE_EXEPT:
2371 if (svm->vmcb->control.exit_info_2 &
2372 (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
2373 has_error_code = true;
2374 error_code =
2375 (u32)svm->vmcb->control.exit_info_2;
2376 }
2377 kvm_clear_exception_queue(vcpu);
2378 break;
2379 case SVM_EXITINTINFO_TYPE_INTR:
2380 kvm_clear_interrupt_queue(vcpu);
2381 break;
2382 default:
2383 break;
2384 }
2385 }
2386
2387 if (reason != TASK_SWITCH_GATE ||
2388 int_type == SVM_EXITINTINFO_TYPE_SOFT ||
2389 (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
2390 (int_vec == OF_VECTOR || int_vec == BP_VECTOR))) {
2391 if (!skip_emulated_instruction(vcpu))
2392 return 0;
2393 }
2394
2395 if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
2396 int_vec = -1;
2397
2398 return kvm_task_switch(vcpu, tss_selector, int_vec, reason,
2399 has_error_code, error_code);
2400}
2401
2402static int iret_interception(struct kvm_vcpu *vcpu)
2403{
2404 struct vcpu_svm *svm = to_svm(vcpu);
2405
2406 ++vcpu->stat.nmi_window_exits;
2407 vcpu->arch.hflags |= HF_IRET_MASK;
2408 if (!sev_es_guest(vcpu->kvm)) {
2409 svm_clr_intercept(svm, INTERCEPT_IRET);
2410 svm->nmi_iret_rip = kvm_rip_read(vcpu);
2411 }
2412 kvm_make_request(KVM_REQ_EVENT, vcpu);
2413 return 1;
2414}
2415
2416static int invlpg_interception(struct kvm_vcpu *vcpu)
2417{
2418 if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2419 return kvm_emulate_instruction(vcpu, 0);
2420
2421 kvm_mmu_invlpg(vcpu, to_svm(vcpu)->vmcb->control.exit_info_1);
2422 return kvm_skip_emulated_instruction(vcpu);
2423}
2424
2425static int emulate_on_interception(struct kvm_vcpu *vcpu)
2426{
2427 return kvm_emulate_instruction(vcpu, 0);
2428}
2429
2430static int rsm_interception(struct kvm_vcpu *vcpu)
2431{
2432 return kvm_emulate_instruction_from_buffer(vcpu, rsm_ins_bytes, 2);
2433}
2434
2435static bool check_selective_cr0_intercepted(struct kvm_vcpu *vcpu,
2436 unsigned long val)
2437{
2438 struct vcpu_svm *svm = to_svm(vcpu);
2439 unsigned long cr0 = vcpu->arch.cr0;
2440 bool ret = false;
2441
2442 if (!is_guest_mode(vcpu) ||
2443 (!(vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_SELECTIVE_CR0))))
2444 return false;
2445
2446 cr0 &= ~SVM_CR0_SELECTIVE_MASK;
2447 val &= ~SVM_CR0_SELECTIVE_MASK;
2448
2449 if (cr0 ^ val) {
2450 svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
2451 ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE);
2452 }
2453
2454 return ret;
2455}
2456
2457#define CR_VALID (1ULL << 63)
2458
2459static int cr_interception(struct kvm_vcpu *vcpu)
2460{
2461 struct vcpu_svm *svm = to_svm(vcpu);
2462 int reg, cr;
2463 unsigned long val;
2464 int err;
2465
2466 if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2467 return emulate_on_interception(vcpu);
2468
2469 if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
2470 return emulate_on_interception(vcpu);
2471
2472 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2473 if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
2474 cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
2475 else
2476 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
2477
2478 err = 0;
2479 if (cr >= 16) {
2480 cr -= 16;
2481 val = kvm_register_read(vcpu, reg);
2482 trace_kvm_cr_write(cr, val);
2483 switch (cr) {
2484 case 0:
2485 if (!check_selective_cr0_intercepted(vcpu, val))
2486 err = kvm_set_cr0(vcpu, val);
2487 else
2488 return 1;
2489
2490 break;
2491 case 3:
2492 err = kvm_set_cr3(vcpu, val);
2493 break;
2494 case 4:
2495 err = kvm_set_cr4(vcpu, val);
2496 break;
2497 case 8:
2498 err = kvm_set_cr8(vcpu, val);
2499 break;
2500 default:
2501 WARN(1, "unhandled write to CR%d", cr);
2502 kvm_queue_exception(vcpu, UD_VECTOR);
2503 return 1;
2504 }
2505 } else {
2506 switch (cr) {
2507 case 0:
2508 val = kvm_read_cr0(vcpu);
2509 break;
2510 case 2:
2511 val = vcpu->arch.cr2;
2512 break;
2513 case 3:
2514 val = kvm_read_cr3(vcpu);
2515 break;
2516 case 4:
2517 val = kvm_read_cr4(vcpu);
2518 break;
2519 case 8:
2520 val = kvm_get_cr8(vcpu);
2521 break;
2522 default:
2523 WARN(1, "unhandled read from CR%d", cr);
2524 kvm_queue_exception(vcpu, UD_VECTOR);
2525 return 1;
2526 }
2527 kvm_register_write(vcpu, reg, val);
2528 trace_kvm_cr_read(cr, val);
2529 }
2530 return kvm_complete_insn_gp(vcpu, err);
2531}
2532
2533static int cr_trap(struct kvm_vcpu *vcpu)
2534{
2535 struct vcpu_svm *svm = to_svm(vcpu);
2536 unsigned long old_value, new_value;
2537 unsigned int cr;
2538 int ret = 0;
2539
2540 new_value = (unsigned long)svm->vmcb->control.exit_info_1;
2541
2542 cr = svm->vmcb->control.exit_code - SVM_EXIT_CR0_WRITE_TRAP;
2543 switch (cr) {
2544 case 0:
2545 old_value = kvm_read_cr0(vcpu);
2546 svm_set_cr0(vcpu, new_value);
2547
2548 kvm_post_set_cr0(vcpu, old_value, new_value);
2549 break;
2550 case 4:
2551 old_value = kvm_read_cr4(vcpu);
2552 svm_set_cr4(vcpu, new_value);
2553
2554 kvm_post_set_cr4(vcpu, old_value, new_value);
2555 break;
2556 case 8:
2557 ret = kvm_set_cr8(vcpu, new_value);
2558 break;
2559 default:
2560 WARN(1, "unhandled CR%d write trap", cr);
2561 kvm_queue_exception(vcpu, UD_VECTOR);
2562 return 1;
2563 }
2564
2565 return kvm_complete_insn_gp(vcpu, ret);
2566}
2567
2568static int dr_interception(struct kvm_vcpu *vcpu)
2569{
2570 struct vcpu_svm *svm = to_svm(vcpu);
2571 int reg, dr;
2572 unsigned long val;
2573 int err = 0;
2574
2575 if (vcpu->guest_debug == 0) {
2576
2577
2578
2579
2580
2581 clr_dr_intercepts(svm);
2582 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
2583 return 1;
2584 }
2585
2586 if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
2587 return emulate_on_interception(vcpu);
2588
2589 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2590 dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
2591 if (dr >= 16) {
2592 dr -= 16;
2593 val = kvm_register_read(vcpu, reg);
2594 err = kvm_set_dr(vcpu, dr, val);
2595 } else {
2596 kvm_get_dr(vcpu, dr, &val);
2597 kvm_register_write(vcpu, reg, val);
2598 }
2599
2600 return kvm_complete_insn_gp(vcpu, err);
2601}
2602
2603static int cr8_write_interception(struct kvm_vcpu *vcpu)
2604{
2605 int r;
2606
2607 u8 cr8_prev = kvm_get_cr8(vcpu);
2608
2609 r = cr_interception(vcpu);
2610 if (lapic_in_kernel(vcpu))
2611 return r;
2612 if (cr8_prev <= kvm_get_cr8(vcpu))
2613 return r;
2614 vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
2615 return 0;
2616}
2617
2618static int efer_trap(struct kvm_vcpu *vcpu)
2619{
2620 struct msr_data msr_info;
2621 int ret;
2622
2623
2624
2625
2626
2627
2628
2629 msr_info.host_initiated = false;
2630 msr_info.index = MSR_EFER;
2631 msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & ~EFER_SVME;
2632 ret = kvm_set_msr_common(vcpu, &msr_info);
2633
2634 return kvm_complete_insn_gp(vcpu, ret);
2635}
2636
2637static int svm_get_msr_feature(struct kvm_msr_entry *msr)
2638{
2639 msr->data = 0;
2640
2641 switch (msr->index) {
2642 case MSR_F10H_DECFG:
2643 if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC))
2644 msr->data |= MSR_F10H_DECFG_LFENCE_SERIALIZE;
2645 break;
2646 case MSR_IA32_PERF_CAPABILITIES:
2647 return 0;
2648 default:
2649 return KVM_MSR_RET_INVALID;
2650 }
2651
2652 return 0;
2653}
2654
2655static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2656{
2657 struct vcpu_svm *svm = to_svm(vcpu);
2658
2659 switch (msr_info->index) {
2660 case MSR_STAR:
2661 msr_info->data = svm->vmcb01.ptr->save.star;
2662 break;
2663#ifdef CONFIG_X86_64
2664 case MSR_LSTAR:
2665 msr_info->data = svm->vmcb01.ptr->save.lstar;
2666 break;
2667 case MSR_CSTAR:
2668 msr_info->data = svm->vmcb01.ptr->save.cstar;
2669 break;
2670 case MSR_KERNEL_GS_BASE:
2671 msr_info->data = svm->vmcb01.ptr->save.kernel_gs_base;
2672 break;
2673 case MSR_SYSCALL_MASK:
2674 msr_info->data = svm->vmcb01.ptr->save.sfmask;
2675 break;
2676#endif
2677 case MSR_IA32_SYSENTER_CS:
2678 msr_info->data = svm->vmcb01.ptr->save.sysenter_cs;
2679 break;
2680 case MSR_IA32_SYSENTER_EIP:
2681 msr_info->data = (u32)svm->vmcb01.ptr->save.sysenter_eip;
2682 if (guest_cpuid_is_intel(vcpu))
2683 msr_info->data |= (u64)svm->sysenter_eip_hi << 32;
2684 break;
2685 case MSR_IA32_SYSENTER_ESP:
2686 msr_info->data = svm->vmcb01.ptr->save.sysenter_esp;
2687 if (guest_cpuid_is_intel(vcpu))
2688 msr_info->data |= (u64)svm->sysenter_esp_hi << 32;
2689 break;
2690 case MSR_TSC_AUX:
2691 msr_info->data = svm->tsc_aux;
2692 break;
2693
2694
2695
2696
2697
2698 case MSR_IA32_DEBUGCTLMSR:
2699 msr_info->data = svm->vmcb->save.dbgctl;
2700 break;
2701 case MSR_IA32_LASTBRANCHFROMIP:
2702 msr_info->data = svm->vmcb->save.br_from;
2703 break;
2704 case MSR_IA32_LASTBRANCHTOIP:
2705 msr_info->data = svm->vmcb->save.br_to;
2706 break;
2707 case MSR_IA32_LASTINTFROMIP:
2708 msr_info->data = svm->vmcb->save.last_excp_from;
2709 break;
2710 case MSR_IA32_LASTINTTOIP:
2711 msr_info->data = svm->vmcb->save.last_excp_to;
2712 break;
2713 case MSR_VM_HSAVE_PA:
2714 msr_info->data = svm->nested.hsave_msr;
2715 break;
2716 case MSR_VM_CR:
2717 msr_info->data = svm->nested.vm_cr_msr;
2718 break;
2719 case MSR_IA32_SPEC_CTRL:
2720 if (!msr_info->host_initiated &&
2721 !guest_has_spec_ctrl_msr(vcpu))
2722 return 1;
2723
2724 if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
2725 msr_info->data = svm->vmcb->save.spec_ctrl;
2726 else
2727 msr_info->data = svm->spec_ctrl;
2728 break;
2729 case MSR_AMD64_VIRT_SPEC_CTRL:
2730 if (!msr_info->host_initiated &&
2731 !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
2732 return 1;
2733
2734 msr_info->data = svm->virt_spec_ctrl;
2735 break;
2736 case MSR_F15H_IC_CFG: {
2737
2738 int family, model;
2739
2740 family = guest_cpuid_family(vcpu);
2741 model = guest_cpuid_model(vcpu);
2742
2743 if (family < 0 || model < 0)
2744 return kvm_get_msr_common(vcpu, msr_info);
2745
2746 msr_info->data = 0;
2747
2748 if (family == 0x15 &&
2749 (model >= 0x2 && model < 0x20))
2750 msr_info->data = 0x1E;
2751 }
2752 break;
2753 case MSR_F10H_DECFG:
2754 msr_info->data = svm->msr_decfg;
2755 break;
2756 default:
2757 return kvm_get_msr_common(vcpu, msr_info);
2758 }
2759 return 0;
2760}
2761
2762static int svm_complete_emulated_msr(struct kvm_vcpu *vcpu, int err)
2763{
2764 struct vcpu_svm *svm = to_svm(vcpu);
2765 if (!err || !sev_es_guest(vcpu->kvm) || WARN_ON_ONCE(!svm->ghcb))
2766 return kvm_complete_insn_gp(vcpu, err);
2767
2768 ghcb_set_sw_exit_info_1(svm->ghcb, 1);
2769 ghcb_set_sw_exit_info_2(svm->ghcb,
2770 X86_TRAP_GP |
2771 SVM_EVTINJ_TYPE_EXEPT |
2772 SVM_EVTINJ_VALID);
2773 return 1;
2774}
2775
2776static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
2777{
2778 struct vcpu_svm *svm = to_svm(vcpu);
2779 int svm_dis, chg_mask;
2780
2781 if (data & ~SVM_VM_CR_VALID_MASK)
2782 return 1;
2783
2784 chg_mask = SVM_VM_CR_VALID_MASK;
2785
2786 if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
2787 chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
2788
2789 svm->nested.vm_cr_msr &= ~chg_mask;
2790 svm->nested.vm_cr_msr |= (data & chg_mask);
2791
2792 svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
2793
2794
2795 if (svm_dis && (vcpu->arch.efer & EFER_SVME))
2796 return 1;
2797
2798 return 0;
2799}
2800
2801static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
2802{
2803 struct vcpu_svm *svm = to_svm(vcpu);
2804 int r;
2805
2806 u32 ecx = msr->index;
2807 u64 data = msr->data;
2808 switch (ecx) {
2809 case MSR_IA32_CR_PAT:
2810 if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data))
2811 return 1;
2812 vcpu->arch.pat = data;
2813 svm->vmcb01.ptr->save.g_pat = data;
2814 if (is_guest_mode(vcpu))
2815 nested_vmcb02_compute_g_pat(svm);
2816 vmcb_mark_dirty(svm->vmcb, VMCB_NPT);
2817 break;
2818 case MSR_IA32_SPEC_CTRL:
2819 if (!msr->host_initiated &&
2820 !guest_has_spec_ctrl_msr(vcpu))
2821 return 1;
2822
2823 if (kvm_spec_ctrl_test_value(data))
2824 return 1;
2825
2826 if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
2827 svm->vmcb->save.spec_ctrl = data;
2828 else
2829 svm->spec_ctrl = data;
2830 if (!data)
2831 break;
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
2845 break;
2846 case MSR_IA32_PRED_CMD:
2847 if (!msr->host_initiated &&
2848 !guest_has_pred_cmd_msr(vcpu))
2849 return 1;
2850
2851 if (data & ~PRED_CMD_IBPB)
2852 return 1;
2853 if (!boot_cpu_has(X86_FEATURE_IBPB))
2854 return 1;
2855 if (!data)
2856 break;
2857
2858 wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB);
2859 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_PRED_CMD, 0, 1);
2860 break;
2861 case MSR_AMD64_VIRT_SPEC_CTRL:
2862 if (!msr->host_initiated &&
2863 !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
2864 return 1;
2865
2866 if (data & ~SPEC_CTRL_SSBD)
2867 return 1;
2868
2869 svm->virt_spec_ctrl = data;
2870 break;
2871 case MSR_STAR:
2872 svm->vmcb01.ptr->save.star = data;
2873 break;
2874#ifdef CONFIG_X86_64
2875 case MSR_LSTAR:
2876 svm->vmcb01.ptr->save.lstar = data;
2877 break;
2878 case MSR_CSTAR:
2879 svm->vmcb01.ptr->save.cstar = data;
2880 break;
2881 case MSR_KERNEL_GS_BASE:
2882 svm->vmcb01.ptr->save.kernel_gs_base = data;
2883 break;
2884 case MSR_SYSCALL_MASK:
2885 svm->vmcb01.ptr->save.sfmask = data;
2886 break;
2887#endif
2888 case MSR_IA32_SYSENTER_CS:
2889 svm->vmcb01.ptr->save.sysenter_cs = data;
2890 break;
2891 case MSR_IA32_SYSENTER_EIP:
2892 svm->vmcb01.ptr->save.sysenter_eip = (u32)data;
2893
2894
2895
2896
2897
2898
2899
2900 svm->sysenter_eip_hi = guest_cpuid_is_intel(vcpu) ? (data >> 32) : 0;
2901 break;
2902 case MSR_IA32_SYSENTER_ESP:
2903 svm->vmcb01.ptr->save.sysenter_esp = (u32)data;
2904 svm->sysenter_esp_hi = guest_cpuid_is_intel(vcpu) ? (data >> 32) : 0;
2905 break;
2906 case MSR_TSC_AUX:
2907
2908
2909
2910
2911
2912 preempt_disable();
2913 r = kvm_set_user_return_msr(tsc_aux_uret_slot, data, -1ull);
2914 preempt_enable();
2915 if (r)
2916 return 1;
2917
2918 svm->tsc_aux = data;
2919 break;
2920 case MSR_IA32_DEBUGCTLMSR:
2921 if (!boot_cpu_has(X86_FEATURE_LBRV)) {
2922 vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
2923 __func__, data);
2924 break;
2925 }
2926 if (data & DEBUGCTL_RESERVED_BITS)
2927 return 1;
2928
2929 svm->vmcb->save.dbgctl = data;
2930 vmcb_mark_dirty(svm->vmcb, VMCB_LBR);
2931 if (data & (1ULL<<0))
2932 svm_enable_lbrv(vcpu);
2933 else
2934 svm_disable_lbrv(vcpu);
2935 break;
2936 case MSR_VM_HSAVE_PA:
2937
2938
2939
2940
2941
2942
2943 if (!msr->host_initiated && !page_address_valid(vcpu, data))
2944 return 1;
2945
2946 svm->nested.hsave_msr = data & PAGE_MASK;
2947 break;
2948 case MSR_VM_CR:
2949 return svm_set_vm_cr(vcpu, data);
2950 case MSR_VM_IGNNE:
2951 vcpu_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
2952 break;
2953 case MSR_F10H_DECFG: {
2954 struct kvm_msr_entry msr_entry;
2955
2956 msr_entry.index = msr->index;
2957 if (svm_get_msr_feature(&msr_entry))
2958 return 1;
2959
2960
2961 if (data & ~msr_entry.data)
2962 return 1;
2963
2964
2965 if (!msr->host_initiated && (data ^ msr_entry.data))
2966 return 1;
2967
2968 svm->msr_decfg = data;
2969 break;
2970 }
2971 default:
2972 return kvm_set_msr_common(vcpu, msr);
2973 }
2974 return 0;
2975}
2976
2977static int msr_interception(struct kvm_vcpu *vcpu)
2978{
2979 if (to_svm(vcpu)->vmcb->control.exit_info_1)
2980 return kvm_emulate_wrmsr(vcpu);
2981 else
2982 return kvm_emulate_rdmsr(vcpu);
2983}
2984
2985static int interrupt_window_interception(struct kvm_vcpu *vcpu)
2986{
2987 kvm_make_request(KVM_REQ_EVENT, vcpu);
2988 svm_clear_vintr(to_svm(vcpu));
2989
2990
2991
2992
2993
2994
2995 kvm_request_apicv_update(vcpu->kvm, true, APICV_INHIBIT_REASON_IRQWIN);
2996
2997 ++vcpu->stat.irq_window_exits;
2998 return 1;
2999}
3000
3001static int pause_interception(struct kvm_vcpu *vcpu)
3002{
3003 bool in_kernel;
3004
3005
3006
3007
3008
3009
3010 in_kernel = !sev_es_guest(vcpu->kvm) && svm_get_cpl(vcpu) == 0;
3011
3012 if (!kvm_pause_in_guest(vcpu->kvm))
3013 grow_ple_window(vcpu);
3014
3015 kvm_vcpu_on_spin(vcpu, in_kernel);
3016 return kvm_skip_emulated_instruction(vcpu);
3017}
3018
3019static int invpcid_interception(struct kvm_vcpu *vcpu)
3020{
3021 struct vcpu_svm *svm = to_svm(vcpu);
3022 unsigned long type;
3023 gva_t gva;
3024
3025 if (!guest_cpuid_has(vcpu, X86_FEATURE_INVPCID)) {
3026 kvm_queue_exception(vcpu, UD_VECTOR);
3027 return 1;
3028 }
3029
3030
3031
3032
3033
3034
3035 type = svm->vmcb->control.exit_info_2;
3036 gva = svm->vmcb->control.exit_info_1;
3037
3038 if (type > 3) {
3039 kvm_inject_gp(vcpu, 0);
3040 return 1;
3041 }
3042
3043 return kvm_handle_invpcid(vcpu, type, gva);
3044}
3045
3046static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = {
3047 [SVM_EXIT_READ_CR0] = cr_interception,
3048 [SVM_EXIT_READ_CR3] = cr_interception,
3049 [SVM_EXIT_READ_CR4] = cr_interception,
3050 [SVM_EXIT_READ_CR8] = cr_interception,
3051 [SVM_EXIT_CR0_SEL_WRITE] = cr_interception,
3052 [SVM_EXIT_WRITE_CR0] = cr_interception,
3053 [SVM_EXIT_WRITE_CR3] = cr_interception,
3054 [SVM_EXIT_WRITE_CR4] = cr_interception,
3055 [SVM_EXIT_WRITE_CR8] = cr8_write_interception,
3056 [SVM_EXIT_READ_DR0] = dr_interception,
3057 [SVM_EXIT_READ_DR1] = dr_interception,
3058 [SVM_EXIT_READ_DR2] = dr_interception,
3059 [SVM_EXIT_READ_DR3] = dr_interception,
3060 [SVM_EXIT_READ_DR4] = dr_interception,
3061 [SVM_EXIT_READ_DR5] = dr_interception,
3062 [SVM_EXIT_READ_DR6] = dr_interception,
3063 [SVM_EXIT_READ_DR7] = dr_interception,
3064 [SVM_EXIT_WRITE_DR0] = dr_interception,
3065 [SVM_EXIT_WRITE_DR1] = dr_interception,
3066 [SVM_EXIT_WRITE_DR2] = dr_interception,
3067 [SVM_EXIT_WRITE_DR3] = dr_interception,
3068 [SVM_EXIT_WRITE_DR4] = dr_interception,
3069 [SVM_EXIT_WRITE_DR5] = dr_interception,
3070 [SVM_EXIT_WRITE_DR6] = dr_interception,
3071 [SVM_EXIT_WRITE_DR7] = dr_interception,
3072 [SVM_EXIT_EXCP_BASE + DB_VECTOR] = db_interception,
3073 [SVM_EXIT_EXCP_BASE + BP_VECTOR] = bp_interception,
3074 [SVM_EXIT_EXCP_BASE + UD_VECTOR] = ud_interception,
3075 [SVM_EXIT_EXCP_BASE + PF_VECTOR] = pf_interception,
3076 [SVM_EXIT_EXCP_BASE + MC_VECTOR] = mc_interception,
3077 [SVM_EXIT_EXCP_BASE + AC_VECTOR] = ac_interception,
3078 [SVM_EXIT_EXCP_BASE + GP_VECTOR] = gp_interception,
3079 [SVM_EXIT_INTR] = intr_interception,
3080 [SVM_EXIT_NMI] = nmi_interception,
3081 [SVM_EXIT_SMI] = smi_interception,
3082 [SVM_EXIT_VINTR] = interrupt_window_interception,
3083 [SVM_EXIT_RDPMC] = kvm_emulate_rdpmc,
3084 [SVM_EXIT_CPUID] = kvm_emulate_cpuid,
3085 [SVM_EXIT_IRET] = iret_interception,
3086 [SVM_EXIT_INVD] = kvm_emulate_invd,
3087 [SVM_EXIT_PAUSE] = pause_interception,
3088 [SVM_EXIT_HLT] = kvm_emulate_halt,
3089 [SVM_EXIT_INVLPG] = invlpg_interception,
3090 [SVM_EXIT_INVLPGA] = invlpga_interception,
3091 [SVM_EXIT_IOIO] = io_interception,
3092 [SVM_EXIT_MSR] = msr_interception,
3093 [SVM_EXIT_TASK_SWITCH] = task_switch_interception,
3094 [SVM_EXIT_SHUTDOWN] = shutdown_interception,
3095 [SVM_EXIT_VMRUN] = vmrun_interception,
3096 [SVM_EXIT_VMMCALL] = kvm_emulate_hypercall,
3097 [SVM_EXIT_VMLOAD] = vmload_interception,
3098 [SVM_EXIT_VMSAVE] = vmsave_interception,
3099 [SVM_EXIT_STGI] = stgi_interception,
3100 [SVM_EXIT_CLGI] = clgi_interception,
3101 [SVM_EXIT_SKINIT] = skinit_interception,
3102 [SVM_EXIT_RDTSCP] = kvm_handle_invalid_op,
3103 [SVM_EXIT_WBINVD] = kvm_emulate_wbinvd,
3104 [SVM_EXIT_MONITOR] = kvm_emulate_monitor,
3105 [SVM_EXIT_MWAIT] = kvm_emulate_mwait,
3106 [SVM_EXIT_XSETBV] = kvm_emulate_xsetbv,
3107 [SVM_EXIT_RDPRU] = kvm_handle_invalid_op,
3108 [SVM_EXIT_EFER_WRITE_TRAP] = efer_trap,
3109 [SVM_EXIT_CR0_WRITE_TRAP] = cr_trap,
3110 [SVM_EXIT_CR4_WRITE_TRAP] = cr_trap,
3111 [SVM_EXIT_CR8_WRITE_TRAP] = cr_trap,
3112 [SVM_EXIT_INVPCID] = invpcid_interception,
3113 [SVM_EXIT_NPF] = npf_interception,
3114 [SVM_EXIT_RSM] = rsm_interception,
3115 [SVM_EXIT_AVIC_INCOMPLETE_IPI] = avic_incomplete_ipi_interception,
3116 [SVM_EXIT_AVIC_UNACCELERATED_ACCESS] = avic_unaccelerated_access_interception,
3117 [SVM_EXIT_VMGEXIT] = sev_handle_vmgexit,
3118};
3119
3120static void dump_vmcb(struct kvm_vcpu *vcpu)
3121{
3122 struct vcpu_svm *svm = to_svm(vcpu);
3123 struct vmcb_control_area *control = &svm->vmcb->control;
3124 struct vmcb_save_area *save = &svm->vmcb->save;
3125 struct vmcb_save_area *save01 = &svm->vmcb01.ptr->save;
3126
3127 if (!dump_invalid_vmcb) {
3128 pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n");
3129 return;
3130 }
3131
3132 pr_err("VMCB %p, last attempted VMRUN on CPU %d\n",
3133 svm->current_vmcb->ptr, vcpu->arch.last_vmentry_cpu);
3134 pr_err("VMCB Control Area:\n");
3135 pr_err("%-20s%04x\n", "cr_read:", control->intercepts[INTERCEPT_CR] & 0xffff);
3136 pr_err("%-20s%04x\n", "cr_write:", control->intercepts[INTERCEPT_CR] >> 16);
3137 pr_err("%-20s%04x\n", "dr_read:", control->intercepts[INTERCEPT_DR] & 0xffff);
3138 pr_err("%-20s%04x\n", "dr_write:", control->intercepts[INTERCEPT_DR] >> 16);
3139 pr_err("%-20s%08x\n", "exceptions:", control->intercepts[INTERCEPT_EXCEPTION]);
3140 pr_err("%-20s%08x %08x\n", "intercepts:",
3141 control->intercepts[INTERCEPT_WORD3],
3142 control->intercepts[INTERCEPT_WORD4]);
3143 pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count);
3144 pr_err("%-20s%d\n", "pause filter threshold:",
3145 control->pause_filter_thresh);
3146 pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa);
3147 pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa);
3148 pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset);
3149 pr_err("%-20s%d\n", "asid:", control->asid);
3150 pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl);
3151 pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl);
3152 pr_err("%-20s%08x\n", "int_vector:", control->int_vector);
3153 pr_err("%-20s%08x\n", "int_state:", control->int_state);
3154 pr_err("%-20s%08x\n", "exit_code:", control->exit_code);
3155 pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1);
3156 pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
3157 pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
3158 pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
3159 pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl);
3160 pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
3161 pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar);
3162 pr_err("%-20s%016llx\n", "ghcb:", control->ghcb_gpa);
3163 pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
3164 pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err);
3165 pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext);
3166 pr_err("%-20s%016llx\n", "next_rip:", control->next_rip);
3167 pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page);
3168 pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id);
3169 pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id);
3170 pr_err("%-20s%016llx\n", "vmsa_pa:", control->vmsa_pa);
3171 pr_err("VMCB State Save Area:\n");
3172 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3173 "es:",
3174 save->es.selector, save->es.attrib,
3175 save->es.limit, save->es.base);
3176 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3177 "cs:",
3178 save->cs.selector, save->cs.attrib,
3179 save->cs.limit, save->cs.base);
3180 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3181 "ss:",
3182 save->ss.selector, save->ss.attrib,
3183 save->ss.limit, save->ss.base);
3184 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3185 "ds:",
3186 save->ds.selector, save->ds.attrib,
3187 save->ds.limit, save->ds.base);
3188 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3189 "fs:",
3190 save01->fs.selector, save01->fs.attrib,
3191 save01->fs.limit, save01->fs.base);
3192 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3193 "gs:",
3194 save01->gs.selector, save01->gs.attrib,
3195 save01->gs.limit, save01->gs.base);
3196 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3197 "gdtr:",
3198 save->gdtr.selector, save->gdtr.attrib,
3199 save->gdtr.limit, save->gdtr.base);
3200 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3201 "ldtr:",
3202 save01->ldtr.selector, save01->ldtr.attrib,
3203 save01->ldtr.limit, save01->ldtr.base);
3204 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3205 "idtr:",
3206 save->idtr.selector, save->idtr.attrib,
3207 save->idtr.limit, save->idtr.base);
3208 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3209 "tr:",
3210 save01->tr.selector, save01->tr.attrib,
3211 save01->tr.limit, save01->tr.base);
3212 pr_err("cpl: %d efer: %016llx\n",
3213 save->cpl, save->efer);
3214 pr_err("%-15s %016llx %-13s %016llx\n",
3215 "cr0:", save->cr0, "cr2:", save->cr2);
3216 pr_err("%-15s %016llx %-13s %016llx\n",
3217 "cr3:", save->cr3, "cr4:", save->cr4);
3218 pr_err("%-15s %016llx %-13s %016llx\n",
3219 "dr6:", save->dr6, "dr7:", save->dr7);
3220 pr_err("%-15s %016llx %-13s %016llx\n",
3221 "rip:", save->rip, "rflags:", save->rflags);
3222 pr_err("%-15s %016llx %-13s %016llx\n",
3223 "rsp:", save->rsp, "rax:", save->rax);
3224 pr_err("%-15s %016llx %-13s %016llx\n",
3225 "star:", save01->star, "lstar:", save01->lstar);
3226 pr_err("%-15s %016llx %-13s %016llx\n",
3227 "cstar:", save01->cstar, "sfmask:", save01->sfmask);
3228 pr_err("%-15s %016llx %-13s %016llx\n",
3229 "kernel_gs_base:", save01->kernel_gs_base,
3230 "sysenter_cs:", save01->sysenter_cs);
3231 pr_err("%-15s %016llx %-13s %016llx\n",
3232 "sysenter_esp:", save01->sysenter_esp,
3233 "sysenter_eip:", save01->sysenter_eip);
3234 pr_err("%-15s %016llx %-13s %016llx\n",
3235 "gpat:", save->g_pat, "dbgctl:", save->dbgctl);
3236 pr_err("%-15s %016llx %-13s %016llx\n",
3237 "br_from:", save->br_from, "br_to:", save->br_to);
3238 pr_err("%-15s %016llx %-13s %016llx\n",
3239 "excp_from:", save->last_excp_from,
3240 "excp_to:", save->last_excp_to);
3241}
3242
3243static bool svm_check_exit_valid(struct kvm_vcpu *vcpu, u64 exit_code)
3244{
3245 return (exit_code < ARRAY_SIZE(svm_exit_handlers) &&
3246 svm_exit_handlers[exit_code]);
3247}
3248
3249static int svm_handle_invalid_exit(struct kvm_vcpu *vcpu, u64 exit_code)
3250{
3251 vcpu_unimpl(vcpu, "svm: unexpected exit reason 0x%llx\n", exit_code);
3252 dump_vmcb(vcpu);
3253 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
3254 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
3255 vcpu->run->internal.ndata = 2;
3256 vcpu->run->internal.data[0] = exit_code;
3257 vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu;
3258 return 0;
3259}
3260
3261int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 exit_code)
3262{
3263 if (!svm_check_exit_valid(vcpu, exit_code))
3264 return svm_handle_invalid_exit(vcpu, exit_code);
3265
3266#ifdef CONFIG_RETPOLINE
3267 if (exit_code == SVM_EXIT_MSR)
3268 return msr_interception(vcpu);
3269 else if (exit_code == SVM_EXIT_VINTR)
3270 return interrupt_window_interception(vcpu);
3271 else if (exit_code == SVM_EXIT_INTR)
3272 return intr_interception(vcpu);
3273 else if (exit_code == SVM_EXIT_HLT)
3274 return kvm_emulate_halt(vcpu);
3275 else if (exit_code == SVM_EXIT_NPF)
3276 return npf_interception(vcpu);
3277#endif
3278 return svm_exit_handlers[exit_code](vcpu);
3279}
3280
3281static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2,
3282 u32 *intr_info, u32 *error_code)
3283{
3284 struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
3285
3286 *info1 = control->exit_info_1;
3287 *info2 = control->exit_info_2;
3288 *intr_info = control->exit_int_info;
3289 if ((*intr_info & SVM_EXITINTINFO_VALID) &&
3290 (*intr_info & SVM_EXITINTINFO_VALID_ERR))
3291 *error_code = control->exit_int_info_err;
3292 else
3293 *error_code = 0;
3294}
3295
3296static int handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
3297{
3298 struct vcpu_svm *svm = to_svm(vcpu);
3299 struct kvm_run *kvm_run = vcpu->run;
3300 u32 exit_code = svm->vmcb->control.exit_code;
3301
3302 trace_kvm_exit(exit_code, vcpu, KVM_ISA_SVM);
3303
3304
3305 if (!sev_es_guest(vcpu->kvm)) {
3306 if (!svm_is_intercept(svm, INTERCEPT_CR0_WRITE))
3307 vcpu->arch.cr0 = svm->vmcb->save.cr0;
3308 if (npt_enabled)
3309 vcpu->arch.cr3 = svm->vmcb->save.cr3;
3310 }
3311
3312 if (is_guest_mode(vcpu)) {
3313 int vmexit;
3314
3315 trace_kvm_nested_vmexit(exit_code, vcpu, KVM_ISA_SVM);
3316
3317 vmexit = nested_svm_exit_special(svm);
3318
3319 if (vmexit == NESTED_EXIT_CONTINUE)
3320 vmexit = nested_svm_exit_handled(svm);
3321
3322 if (vmexit == NESTED_EXIT_DONE)
3323 return 1;
3324 }
3325
3326 if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
3327 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
3328 kvm_run->fail_entry.hardware_entry_failure_reason
3329 = svm->vmcb->control.exit_code;
3330 kvm_run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu;
3331 dump_vmcb(vcpu);
3332 return 0;
3333 }
3334
3335 if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
3336 exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
3337 exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH &&
3338 exit_code != SVM_EXIT_INTR && exit_code != SVM_EXIT_NMI)
3339 printk(KERN_ERR "%s: unexpected exit_int_info 0x%x "
3340 "exit_code 0x%x\n",
3341 __func__, svm->vmcb->control.exit_int_info,
3342 exit_code);
3343
3344 if (exit_fastpath != EXIT_FASTPATH_NONE)
3345 return 1;
3346
3347 return svm_invoke_exit_handler(vcpu, exit_code);
3348}
3349
3350static void reload_tss(struct kvm_vcpu *vcpu)
3351{
3352 struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu);
3353
3354 sd->tss_desc->type = 9;
3355 load_TR_desc();
3356}
3357
3358static void pre_svm_run(struct kvm_vcpu *vcpu)
3359{
3360 struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu);
3361 struct vcpu_svm *svm = to_svm(vcpu);
3362
3363
3364
3365
3366
3367
3368 if (unlikely(svm->current_vmcb->cpu != vcpu->cpu)) {
3369 svm->current_vmcb->asid_generation = 0;
3370 vmcb_mark_all_dirty(svm->vmcb);
3371 svm->current_vmcb->cpu = vcpu->cpu;
3372 }
3373
3374 if (sev_guest(vcpu->kvm))
3375 return pre_sev_run(svm, vcpu->cpu);
3376
3377
3378 if (svm->current_vmcb->asid_generation != sd->asid_generation)
3379 new_asid(svm, sd);
3380}
3381
3382static void svm_inject_nmi(struct kvm_vcpu *vcpu)
3383{
3384 struct vcpu_svm *svm = to_svm(vcpu);
3385
3386 svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
3387 vcpu->arch.hflags |= HF_NMI_MASK;
3388 if (!sev_es_guest(vcpu->kvm))
3389 svm_set_intercept(svm, INTERCEPT_IRET);
3390 ++vcpu->stat.nmi_injections;
3391}
3392
3393static void svm_set_irq(struct kvm_vcpu *vcpu)
3394{
3395 struct vcpu_svm *svm = to_svm(vcpu);
3396
3397 BUG_ON(!(gif_set(svm)));
3398
3399 trace_kvm_inj_virq(vcpu->arch.interrupt.nr);
3400 ++vcpu->stat.irq_injections;
3401
3402 svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
3403 SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
3404}
3405
3406static void svm_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
3407{
3408 struct vcpu_svm *svm = to_svm(vcpu);
3409
3410
3411
3412
3413
3414 if (sev_es_guest(vcpu->kvm))
3415 return;
3416
3417 if (nested_svm_virtualize_tpr(vcpu))
3418 return;
3419
3420 svm_clr_intercept(svm, INTERCEPT_CR8_WRITE);
3421
3422 if (irr == -1)
3423 return;
3424
3425 if (tpr >= irr)
3426 svm_set_intercept(svm, INTERCEPT_CR8_WRITE);
3427}
3428
3429bool svm_nmi_blocked(struct kvm_vcpu *vcpu)
3430{
3431 struct vcpu_svm *svm = to_svm(vcpu);
3432 struct vmcb *vmcb = svm->vmcb;
3433 bool ret;
3434
3435 if (!gif_set(svm))
3436 return true;
3437
3438 if (is_guest_mode(vcpu) && nested_exit_on_nmi(svm))
3439 return false;
3440
3441 ret = (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) ||
3442 (vcpu->arch.hflags & HF_NMI_MASK);
3443
3444 return ret;
3445}
3446
3447static int svm_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
3448{
3449 struct vcpu_svm *svm = to_svm(vcpu);
3450 if (svm->nested.nested_run_pending)
3451 return -EBUSY;
3452
3453
3454 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(svm))
3455 return -EBUSY;
3456
3457 return !svm_nmi_blocked(vcpu);
3458}
3459
3460static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
3461{
3462 return !!(vcpu->arch.hflags & HF_NMI_MASK);
3463}
3464
3465static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
3466{
3467 struct vcpu_svm *svm = to_svm(vcpu);
3468
3469 if (masked) {
3470 vcpu->arch.hflags |= HF_NMI_MASK;
3471 if (!sev_es_guest(vcpu->kvm))
3472 svm_set_intercept(svm, INTERCEPT_IRET);
3473 } else {
3474 vcpu->arch.hflags &= ~HF_NMI_MASK;
3475 if (!sev_es_guest(vcpu->kvm))
3476 svm_clr_intercept(svm, INTERCEPT_IRET);
3477 }
3478}
3479
3480bool svm_interrupt_blocked(struct kvm_vcpu *vcpu)
3481{
3482 struct vcpu_svm *svm = to_svm(vcpu);
3483 struct vmcb *vmcb = svm->vmcb;
3484
3485 if (!gif_set(svm))
3486 return true;
3487
3488 if (sev_es_guest(vcpu->kvm)) {
3489
3490
3491
3492
3493 if (!(vmcb->control.int_state & SVM_GUEST_INTERRUPT_MASK))
3494 return true;
3495 } else if (is_guest_mode(vcpu)) {
3496
3497 if ((svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK)
3498 ? !(svm->vmcb01.ptr->save.rflags & X86_EFLAGS_IF)
3499 : !(kvm_get_rflags(vcpu) & X86_EFLAGS_IF))
3500 return true;
3501
3502
3503 if (nested_exit_on_intr(svm))
3504 return false;
3505 } else {
3506 if (!(kvm_get_rflags(vcpu) & X86_EFLAGS_IF))
3507 return true;
3508 }
3509
3510 return (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK);
3511}
3512
3513static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection)
3514{
3515 struct vcpu_svm *svm = to_svm(vcpu);
3516 if (svm->nested.nested_run_pending)
3517 return -EBUSY;
3518
3519
3520
3521
3522
3523 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(svm))
3524 return -EBUSY;
3525
3526 return !svm_interrupt_blocked(vcpu);
3527}
3528
3529static void svm_enable_irq_window(struct kvm_vcpu *vcpu)
3530{
3531 struct vcpu_svm *svm = to_svm(vcpu);
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541 if (vgif_enabled(svm) || gif_set(svm)) {
3542
3543
3544
3545
3546
3547
3548 kvm_request_apicv_update(vcpu->kvm, false, APICV_INHIBIT_REASON_IRQWIN);
3549 svm_set_vintr(svm);
3550 }
3551}
3552
3553static void svm_enable_nmi_window(struct kvm_vcpu *vcpu)
3554{
3555 struct vcpu_svm *svm = to_svm(vcpu);
3556
3557 if ((vcpu->arch.hflags & (HF_NMI_MASK | HF_IRET_MASK)) == HF_NMI_MASK)
3558 return;
3559
3560 if (!gif_set(svm)) {
3561 if (vgif_enabled(svm))
3562 svm_set_intercept(svm, INTERCEPT_STGI);
3563 return;
3564 }
3565
3566
3567
3568
3569
3570 svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu);
3571 svm->nmi_singlestep = true;
3572 svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
3573}
3574
3575static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
3576{
3577 return 0;
3578}
3579
3580static int svm_set_identity_map_addr(struct kvm *kvm, u64 ident_addr)
3581{
3582 return 0;
3583}
3584
3585void svm_flush_tlb(struct kvm_vcpu *vcpu)
3586{
3587 struct vcpu_svm *svm = to_svm(vcpu);
3588
3589
3590
3591
3592
3593
3594
3595
3596 if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
3597 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
3598 else
3599 svm->current_vmcb->asid_generation--;
3600}
3601
3602static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva)
3603{
3604 struct vcpu_svm *svm = to_svm(vcpu);
3605
3606 invlpga(gva, svm->vmcb->control.asid);
3607}
3608
3609static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
3610{
3611 struct vcpu_svm *svm = to_svm(vcpu);
3612
3613 if (nested_svm_virtualize_tpr(vcpu))
3614 return;
3615
3616 if (!svm_is_intercept(svm, INTERCEPT_CR8_WRITE)) {
3617 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
3618 kvm_set_cr8(vcpu, cr8);
3619 }
3620}
3621
3622static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
3623{
3624 struct vcpu_svm *svm = to_svm(vcpu);
3625 u64 cr8;
3626
3627 if (nested_svm_virtualize_tpr(vcpu) ||
3628 kvm_vcpu_apicv_active(vcpu))
3629 return;
3630
3631 cr8 = kvm_get_cr8(vcpu);
3632 svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
3633 svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
3634}
3635
3636static void svm_complete_interrupts(struct kvm_vcpu *vcpu)
3637{
3638 struct vcpu_svm *svm = to_svm(vcpu);
3639 u8 vector;
3640 int type;
3641 u32 exitintinfo = svm->vmcb->control.exit_int_info;
3642 unsigned int3_injected = svm->int3_injected;
3643
3644 svm->int3_injected = 0;
3645
3646
3647
3648
3649
3650 if ((vcpu->arch.hflags & HF_IRET_MASK) &&
3651 (sev_es_guest(vcpu->kvm) ||
3652 kvm_rip_read(vcpu) != svm->nmi_iret_rip)) {
3653 vcpu->arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK);
3654 kvm_make_request(KVM_REQ_EVENT, vcpu);
3655 }
3656
3657 vcpu->arch.nmi_injected = false;
3658 kvm_clear_exception_queue(vcpu);
3659 kvm_clear_interrupt_queue(vcpu);
3660
3661 if (!(exitintinfo & SVM_EXITINTINFO_VALID))
3662 return;
3663
3664 kvm_make_request(KVM_REQ_EVENT, vcpu);
3665
3666 vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
3667 type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
3668
3669 switch (type) {
3670 case SVM_EXITINTINFO_TYPE_NMI:
3671 vcpu->arch.nmi_injected = true;
3672 break;
3673 case SVM_EXITINTINFO_TYPE_EXEPT:
3674
3675
3676
3677 if (vector == X86_TRAP_VC)
3678 break;
3679
3680
3681
3682
3683
3684
3685 if (kvm_exception_is_soft(vector)) {
3686 if (vector == BP_VECTOR && int3_injected &&
3687 kvm_is_linear_rip(vcpu, svm->int3_rip))
3688 kvm_rip_write(vcpu,
3689 kvm_rip_read(vcpu) - int3_injected);
3690 break;
3691 }
3692 if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
3693 u32 err = svm->vmcb->control.exit_int_info_err;
3694 kvm_requeue_exception_e(vcpu, vector, err);
3695
3696 } else
3697 kvm_requeue_exception(vcpu, vector);
3698 break;
3699 case SVM_EXITINTINFO_TYPE_INTR:
3700 kvm_queue_interrupt(vcpu, vector, false);
3701 break;
3702 default:
3703 break;
3704 }
3705}
3706
3707static void svm_cancel_injection(struct kvm_vcpu *vcpu)
3708{
3709 struct vcpu_svm *svm = to_svm(vcpu);
3710 struct vmcb_control_area *control = &svm->vmcb->control;
3711
3712 control->exit_int_info = control->event_inj;
3713 control->exit_int_info_err = control->event_inj_err;
3714 control->event_inj = 0;
3715 svm_complete_interrupts(vcpu);
3716}
3717
3718static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu)
3719{
3720 if (to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_MSR &&
3721 to_svm(vcpu)->vmcb->control.exit_info_1)
3722 return handle_fastpath_set_msr_irqoff(vcpu);
3723
3724 return EXIT_FASTPATH_NONE;
3725}
3726
3727static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu)
3728{
3729 struct vcpu_svm *svm = to_svm(vcpu);
3730 unsigned long vmcb_pa = svm->current_vmcb->pa;
3731
3732 kvm_guest_enter_irqoff();
3733
3734 if (sev_es_guest(vcpu->kvm)) {
3735 __svm_sev_es_vcpu_run(vmcb_pa);
3736 } else {
3737 struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu);
3738
3739
3740
3741
3742
3743
3744
3745 vmload(svm->vmcb01.pa);
3746 __svm_vcpu_run(vmcb_pa, (unsigned long *)&vcpu->arch.regs);
3747 vmsave(svm->vmcb01.pa);
3748
3749 vmload(__sme_page_pa(sd->save_area));
3750 }
3751
3752 kvm_guest_exit_irqoff();
3753}
3754
3755static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu)
3756{
3757 struct vcpu_svm *svm = to_svm(vcpu);
3758
3759 trace_kvm_entry(vcpu);
3760
3761 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
3762 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
3763 svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
3764
3765
3766
3767
3768
3769
3770
3771 if (svm->nmi_singlestep && svm->vmcb->control.event_inj) {
3772
3773
3774
3775
3776
3777 disable_nmi_singlestep(svm);
3778 smp_send_reschedule(vcpu->cpu);
3779 }
3780
3781 pre_svm_run(vcpu);
3782
3783 WARN_ON_ONCE(kvm_apicv_activated(vcpu->kvm) != kvm_vcpu_apicv_active(vcpu));
3784
3785 sync_lapic_to_cr8(vcpu);
3786
3787 if (unlikely(svm->asid != svm->vmcb->control.asid)) {
3788 svm->vmcb->control.asid = svm->asid;
3789 vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
3790 }
3791 svm->vmcb->save.cr2 = vcpu->arch.cr2;
3792
3793 svm_hv_update_vp_id(svm->vmcb, vcpu);
3794
3795
3796
3797
3798
3799 if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT))
3800 svm_set_dr6(svm, vcpu->arch.dr6);
3801 else
3802 svm_set_dr6(svm, DR6_ACTIVE_LOW);
3803
3804 clgi();
3805 kvm_load_guest_xsave_state(vcpu);
3806
3807 kvm_wait_lapic_expire(vcpu);
3808
3809
3810
3811
3812
3813
3814
3815 if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL))
3816 x86_spec_ctrl_set_guest(svm->spec_ctrl, svm->virt_spec_ctrl);
3817
3818 svm_vcpu_enter_exit(vcpu);
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835 if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL) &&
3836 unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL)))
3837 svm->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL);
3838
3839 if (!sev_es_guest(vcpu->kvm))
3840 reload_tss(vcpu);
3841
3842 if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL))
3843 x86_spec_ctrl_restore_host(svm->spec_ctrl, svm->virt_spec_ctrl);
3844
3845 if (!sev_es_guest(vcpu->kvm)) {
3846 vcpu->arch.cr2 = svm->vmcb->save.cr2;
3847 vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
3848 vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
3849 vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
3850 }
3851
3852 if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
3853 kvm_before_interrupt(vcpu);
3854
3855 kvm_load_host_xsave_state(vcpu);
3856 stgi();
3857
3858
3859
3860 if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
3861 kvm_after_interrupt(vcpu);
3862
3863 sync_cr8_to_lapic(vcpu);
3864
3865 svm->next_rip = 0;
3866 if (is_guest_mode(vcpu)) {
3867 nested_sync_control_from_vmcb02(svm);
3868
3869
3870 if (svm->nested.nested_run_pending &&
3871 svm->vmcb->control.exit_code != SVM_EXIT_ERR)
3872 ++vcpu->stat.nested_run;
3873
3874 svm->nested.nested_run_pending = 0;
3875 }
3876
3877 svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
3878 vmcb_mark_all_clean(svm->vmcb);
3879
3880
3881 if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
3882 vcpu->arch.apf.host_apf_flags =
3883 kvm_read_and_reset_apf_flags();
3884
3885 if (npt_enabled)
3886 kvm_register_clear_available(vcpu, VCPU_EXREG_PDPTR);
3887
3888
3889
3890
3891
3892 if (unlikely(svm->vmcb->control.exit_code ==
3893 SVM_EXIT_EXCP_BASE + MC_VECTOR))
3894 svm_handle_mce(vcpu);
3895
3896 svm_complete_interrupts(vcpu);
3897
3898 if (is_guest_mode(vcpu))
3899 return EXIT_FASTPATH_NONE;
3900
3901 return svm_exit_handlers_fastpath(vcpu);
3902}
3903
3904static void svm_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa,
3905 int root_level)
3906{
3907 struct vcpu_svm *svm = to_svm(vcpu);
3908 unsigned long cr3;
3909
3910 if (npt_enabled) {
3911 svm->vmcb->control.nested_cr3 = __sme_set(root_hpa);
3912 vmcb_mark_dirty(svm->vmcb, VMCB_NPT);
3913
3914 hv_track_root_tdp(vcpu, root_hpa);
3915
3916
3917 if (!test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail))
3918 return;
3919 cr3 = vcpu->arch.cr3;
3920 } else if (vcpu->arch.mmu->shadow_root_level >= PT64_ROOT_4LEVEL) {
3921 cr3 = __sme_set(root_hpa) | kvm_get_active_pcid(vcpu);
3922 } else {
3923
3924 WARN_ON_ONCE(kvm_get_active_pcid(vcpu));
3925 cr3 = root_hpa;
3926 }
3927
3928 svm->vmcb->save.cr3 = cr3;
3929 vmcb_mark_dirty(svm->vmcb, VMCB_CR);
3930}
3931
3932static int is_disabled(void)
3933{
3934 u64 vm_cr;
3935
3936 rdmsrl(MSR_VM_CR, vm_cr);
3937 if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
3938 return 1;
3939
3940 return 0;
3941}
3942
3943static void
3944svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
3945{
3946
3947
3948
3949 hypercall[0] = 0x0f;
3950 hypercall[1] = 0x01;
3951 hypercall[2] = 0xd9;
3952}
3953
3954static int __init svm_check_processor_compat(void)
3955{
3956 return 0;
3957}
3958
3959static bool svm_cpu_has_accelerated_tpr(void)
3960{
3961 return false;
3962}
3963
3964
3965
3966
3967
3968static bool svm_has_emulated_msr(struct kvm *kvm, u32 index)
3969{
3970 switch (index) {
3971 case MSR_IA32_MCG_EXT_CTL:
3972 case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
3973 return false;
3974 case MSR_IA32_SMBASE:
3975
3976 if (kvm && sev_es_guest(kvm))
3977 return false;
3978 break;
3979 default:
3980 break;
3981 }
3982
3983 return true;
3984}
3985
3986static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
3987{
3988 return 0;
3989}
3990
3991static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
3992{
3993 struct vcpu_svm *svm = to_svm(vcpu);
3994 struct kvm_cpuid_entry2 *best;
3995
3996 vcpu->arch.xsaves_enabled = guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) &&
3997 boot_cpu_has(X86_FEATURE_XSAVE) &&
3998 boot_cpu_has(X86_FEATURE_XSAVES);
3999
4000
4001 svm->nrips_enabled = kvm_cpu_cap_has(X86_FEATURE_NRIPS) &&
4002 guest_cpuid_has(vcpu, X86_FEATURE_NRIPS);
4003
4004 svm_recalc_instruction_intercepts(vcpu, svm);
4005
4006
4007 if (sev_guest(vcpu->kvm)) {
4008 best = kvm_find_cpuid_entry(vcpu, 0x8000001F, 0);
4009 if (best)
4010 vcpu->arch.reserved_gpa_bits &= ~(1UL << (best->ebx & 0x3f));
4011 }
4012
4013 if (kvm_vcpu_apicv_active(vcpu)) {
4014
4015
4016
4017
4018 if (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC))
4019 kvm_request_apicv_update(vcpu->kvm, false,
4020 APICV_INHIBIT_REASON_X2APIC);
4021
4022
4023
4024
4025
4026 if (nested && guest_cpuid_has(vcpu, X86_FEATURE_SVM))
4027 kvm_request_apicv_update(vcpu->kvm, false,
4028 APICV_INHIBIT_REASON_NESTED);
4029 }
4030
4031 if (guest_cpuid_is_intel(vcpu)) {
4032
4033
4034
4035
4036
4037 svm_set_intercept(svm, INTERCEPT_VMLOAD);
4038 svm_set_intercept(svm, INTERCEPT_VMSAVE);
4039 svm->vmcb->control.virt_ext &= ~VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
4040
4041 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 0, 0);
4042 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 0, 0);
4043 } else {
4044
4045
4046
4047
4048 if (vls) {
4049 svm_clr_intercept(svm, INTERCEPT_VMLOAD);
4050 svm_clr_intercept(svm, INTERCEPT_VMSAVE);
4051 svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
4052 }
4053
4054 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 1, 1);
4055 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 1, 1);
4056 }
4057}
4058
4059static bool svm_has_wbinvd_exit(void)
4060{
4061 return true;
4062}
4063
4064#define PRE_EX(exit) { .exit_code = (exit), \
4065 .stage = X86_ICPT_PRE_EXCEPT, }
4066#define POST_EX(exit) { .exit_code = (exit), \
4067 .stage = X86_ICPT_POST_EXCEPT, }
4068#define POST_MEM(exit) { .exit_code = (exit), \
4069 .stage = X86_ICPT_POST_MEMACCESS, }
4070
4071static const struct __x86_intercept {
4072 u32 exit_code;
4073 enum x86_intercept_stage stage;
4074} x86_intercept_map[] = {
4075 [x86_intercept_cr_read] = POST_EX(SVM_EXIT_READ_CR0),
4076 [x86_intercept_cr_write] = POST_EX(SVM_EXIT_WRITE_CR0),
4077 [x86_intercept_clts] = POST_EX(SVM_EXIT_WRITE_CR0),
4078 [x86_intercept_lmsw] = POST_EX(SVM_EXIT_WRITE_CR0),
4079 [x86_intercept_smsw] = POST_EX(SVM_EXIT_READ_CR0),
4080 [x86_intercept_dr_read] = POST_EX(SVM_EXIT_READ_DR0),
4081 [x86_intercept_dr_write] = POST_EX(SVM_EXIT_WRITE_DR0),
4082 [x86_intercept_sldt] = POST_EX(SVM_EXIT_LDTR_READ),
4083 [x86_intercept_str] = POST_EX(SVM_EXIT_TR_READ),
4084 [x86_intercept_lldt] = POST_EX(SVM_EXIT_LDTR_WRITE),
4085 [x86_intercept_ltr] = POST_EX(SVM_EXIT_TR_WRITE),
4086 [x86_intercept_sgdt] = POST_EX(SVM_EXIT_GDTR_READ),
4087 [x86_intercept_sidt] = POST_EX(SVM_EXIT_IDTR_READ),
4088 [x86_intercept_lgdt] = POST_EX(SVM_EXIT_GDTR_WRITE),
4089 [x86_intercept_lidt] = POST_EX(SVM_EXIT_IDTR_WRITE),
4090 [x86_intercept_vmrun] = POST_EX(SVM_EXIT_VMRUN),
4091 [x86_intercept_vmmcall] = POST_EX(SVM_EXIT_VMMCALL),
4092 [x86_intercept_vmload] = POST_EX(SVM_EXIT_VMLOAD),
4093 [x86_intercept_vmsave] = POST_EX(SVM_EXIT_VMSAVE),
4094 [x86_intercept_stgi] = POST_EX(SVM_EXIT_STGI),
4095 [x86_intercept_clgi] = POST_EX(SVM_EXIT_CLGI),
4096 [x86_intercept_skinit] = POST_EX(SVM_EXIT_SKINIT),
4097 [x86_intercept_invlpga] = POST_EX(SVM_EXIT_INVLPGA),
4098 [x86_intercept_rdtscp] = POST_EX(SVM_EXIT_RDTSCP),
4099 [x86_intercept_monitor] = POST_MEM(SVM_EXIT_MONITOR),
4100 [x86_intercept_mwait] = POST_EX(SVM_EXIT_MWAIT),
4101 [x86_intercept_invlpg] = POST_EX(SVM_EXIT_INVLPG),
4102 [x86_intercept_invd] = POST_EX(SVM_EXIT_INVD),
4103 [x86_intercept_wbinvd] = POST_EX(SVM_EXIT_WBINVD),
4104 [x86_intercept_wrmsr] = POST_EX(SVM_EXIT_MSR),
4105 [x86_intercept_rdtsc] = POST_EX(SVM_EXIT_RDTSC),
4106 [x86_intercept_rdmsr] = POST_EX(SVM_EXIT_MSR),
4107 [x86_intercept_rdpmc] = POST_EX(SVM_EXIT_RDPMC),
4108 [x86_intercept_cpuid] = PRE_EX(SVM_EXIT_CPUID),
4109 [x86_intercept_rsm] = PRE_EX(SVM_EXIT_RSM),
4110 [x86_intercept_pause] = PRE_EX(SVM_EXIT_PAUSE),
4111 [x86_intercept_pushf] = PRE_EX(SVM_EXIT_PUSHF),
4112 [x86_intercept_popf] = PRE_EX(SVM_EXIT_POPF),
4113 [x86_intercept_intn] = PRE_EX(SVM_EXIT_SWINT),
4114 [x86_intercept_iret] = PRE_EX(SVM_EXIT_IRET),
4115 [x86_intercept_icebp] = PRE_EX(SVM_EXIT_ICEBP),
4116 [x86_intercept_hlt] = POST_EX(SVM_EXIT_HLT),
4117 [x86_intercept_in] = POST_EX(SVM_EXIT_IOIO),
4118 [x86_intercept_ins] = POST_EX(SVM_EXIT_IOIO),
4119 [x86_intercept_out] = POST_EX(SVM_EXIT_IOIO),
4120 [x86_intercept_outs] = POST_EX(SVM_EXIT_IOIO),
4121 [x86_intercept_xsetbv] = PRE_EX(SVM_EXIT_XSETBV),
4122};
4123
4124#undef PRE_EX
4125#undef POST_EX
4126#undef POST_MEM
4127
4128static int svm_check_intercept(struct kvm_vcpu *vcpu,
4129 struct x86_instruction_info *info,
4130 enum x86_intercept_stage stage,
4131 struct x86_exception *exception)
4132{
4133 struct vcpu_svm *svm = to_svm(vcpu);
4134 int vmexit, ret = X86EMUL_CONTINUE;
4135 struct __x86_intercept icpt_info;
4136 struct vmcb *vmcb = svm->vmcb;
4137
4138 if (info->intercept >= ARRAY_SIZE(x86_intercept_map))
4139 goto out;
4140
4141 icpt_info = x86_intercept_map[info->intercept];
4142
4143 if (stage != icpt_info.stage)
4144 goto out;
4145
4146 switch (icpt_info.exit_code) {
4147 case SVM_EXIT_READ_CR0:
4148 if (info->intercept == x86_intercept_cr_read)
4149 icpt_info.exit_code += info->modrm_reg;
4150 break;
4151 case SVM_EXIT_WRITE_CR0: {
4152 unsigned long cr0, val;
4153
4154 if (info->intercept == x86_intercept_cr_write)
4155 icpt_info.exit_code += info->modrm_reg;
4156
4157 if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 ||
4158 info->intercept == x86_intercept_clts)
4159 break;
4160
4161 if (!(vmcb_is_intercept(&svm->nested.ctl,
4162 INTERCEPT_SELECTIVE_CR0)))
4163 break;
4164
4165 cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
4166 val = info->src_val & ~SVM_CR0_SELECTIVE_MASK;
4167
4168 if (info->intercept == x86_intercept_lmsw) {
4169 cr0 &= 0xfUL;
4170 val &= 0xfUL;
4171
4172 if (cr0 & X86_CR0_PE)
4173 val |= X86_CR0_PE;
4174 }
4175
4176 if (cr0 ^ val)
4177 icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
4178
4179 break;
4180 }
4181 case SVM_EXIT_READ_DR0:
4182 case SVM_EXIT_WRITE_DR0:
4183 icpt_info.exit_code += info->modrm_reg;
4184 break;
4185 case SVM_EXIT_MSR:
4186 if (info->intercept == x86_intercept_wrmsr)
4187 vmcb->control.exit_info_1 = 1;
4188 else
4189 vmcb->control.exit_info_1 = 0;
4190 break;
4191 case SVM_EXIT_PAUSE:
4192
4193
4194
4195
4196 if (info->rep_prefix != REPE_PREFIX)
4197 goto out;
4198 break;
4199 case SVM_EXIT_IOIO: {
4200 u64 exit_info;
4201 u32 bytes;
4202
4203 if (info->intercept == x86_intercept_in ||
4204 info->intercept == x86_intercept_ins) {
4205 exit_info = ((info->src_val & 0xffff) << 16) |
4206 SVM_IOIO_TYPE_MASK;
4207 bytes = info->dst_bytes;
4208 } else {
4209 exit_info = (info->dst_val & 0xffff) << 16;
4210 bytes = info->src_bytes;
4211 }
4212
4213 if (info->intercept == x86_intercept_outs ||
4214 info->intercept == x86_intercept_ins)
4215 exit_info |= SVM_IOIO_STR_MASK;
4216
4217 if (info->rep_prefix)
4218 exit_info |= SVM_IOIO_REP_MASK;
4219
4220 bytes = min(bytes, 4u);
4221
4222 exit_info |= bytes << SVM_IOIO_SIZE_SHIFT;
4223
4224 exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1);
4225
4226 vmcb->control.exit_info_1 = exit_info;
4227 vmcb->control.exit_info_2 = info->next_rip;
4228
4229 break;
4230 }
4231 default:
4232 break;
4233 }
4234
4235
4236 if (static_cpu_has(X86_FEATURE_NRIPS))
4237 vmcb->control.next_rip = info->next_rip;
4238 vmcb->control.exit_code = icpt_info.exit_code;
4239 vmexit = nested_svm_exit_handled(svm);
4240
4241 ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED
4242 : X86EMUL_CONTINUE;
4243
4244out:
4245 return ret;
4246}
4247
4248static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu)
4249{
4250}
4251
4252static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu)
4253{
4254 if (!kvm_pause_in_guest(vcpu->kvm))
4255 shrink_ple_window(vcpu);
4256}
4257
4258static void svm_setup_mce(struct kvm_vcpu *vcpu)
4259{
4260
4261 vcpu->arch.mcg_cap &= 0x1ff;
4262}
4263
4264bool svm_smi_blocked(struct kvm_vcpu *vcpu)
4265{
4266 struct vcpu_svm *svm = to_svm(vcpu);
4267
4268
4269 if (!gif_set(svm))
4270 return true;
4271
4272 return is_smm(vcpu);
4273}
4274
4275static int svm_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
4276{
4277 struct vcpu_svm *svm = to_svm(vcpu);
4278 if (svm->nested.nested_run_pending)
4279 return -EBUSY;
4280
4281
4282 if (for_injection && is_guest_mode(vcpu) && nested_exit_on_smi(svm))
4283 return -EBUSY;
4284
4285 return !svm_smi_blocked(vcpu);
4286}
4287
4288static int svm_enter_smm(struct kvm_vcpu *vcpu, char *smstate)
4289{
4290 struct vcpu_svm *svm = to_svm(vcpu);
4291 struct kvm_host_map map_save;
4292 int ret;
4293
4294 if (!is_guest_mode(vcpu))
4295 return 0;
4296
4297
4298 put_smstate(u64, smstate, 0x7ed8, 1);
4299
4300 put_smstate(u64, smstate, 0x7ee0, svm->nested.vmcb12_gpa);
4301
4302 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
4303 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
4304 svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
4305
4306 ret = nested_svm_vmexit(svm);
4307 if (ret)
4308 return ret;
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322 if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr),
4323 &map_save) == -EINVAL)
4324 return 1;
4325
4326 BUILD_BUG_ON(offsetof(struct vmcb, save) != 0x400);
4327
4328 svm_copy_vmrun_state(map_save.hva + 0x400,
4329 &svm->vmcb01.ptr->save);
4330
4331 kvm_vcpu_unmap(vcpu, &map_save, true);
4332 return 0;
4333}
4334
4335static int svm_leave_smm(struct kvm_vcpu *vcpu, const char *smstate)
4336{
4337 struct vcpu_svm *svm = to_svm(vcpu);
4338 struct kvm_host_map map, map_save;
4339 u64 saved_efer, vmcb12_gpa;
4340 struct vmcb *vmcb12;
4341 int ret;
4342
4343 if (!guest_cpuid_has(vcpu, X86_FEATURE_LM))
4344 return 0;
4345
4346
4347 if (!GET_SMSTATE(u64, smstate, 0x7ed8))
4348 return 0;
4349
4350 if (!guest_cpuid_has(vcpu, X86_FEATURE_SVM))
4351 return 1;
4352
4353 saved_efer = GET_SMSTATE(u64, smstate, 0x7ed0);
4354 if (!(saved_efer & EFER_SVME))
4355 return 1;
4356
4357 vmcb12_gpa = GET_SMSTATE(u64, smstate, 0x7ee0);
4358 if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcb12_gpa), &map) == -EINVAL)
4359 return 1;
4360
4361 ret = 1;
4362 if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save) == -EINVAL)
4363 goto unmap_map;
4364
4365 if (svm_allocate_nested(svm))
4366 goto unmap_save;
4367
4368
4369
4370
4371
4372
4373 svm_copy_vmrun_state(&svm->vmcb01.ptr->save, map_save.hva + 0x400);
4374
4375
4376
4377
4378
4379 vmcb12 = map.hva;
4380 nested_load_control_from_vmcb12(svm, &vmcb12->control);
4381 ret = enter_svm_guest_mode(vcpu, vmcb12_gpa, vmcb12, false);
4382
4383unmap_save:
4384 kvm_vcpu_unmap(vcpu, &map_save, true);
4385unmap_map:
4386 kvm_vcpu_unmap(vcpu, &map, true);
4387 return ret;
4388}
4389
4390static void svm_enable_smi_window(struct kvm_vcpu *vcpu)
4391{
4392 struct vcpu_svm *svm = to_svm(vcpu);
4393
4394 if (!gif_set(svm)) {
4395 if (vgif_enabled(svm))
4396 svm_set_intercept(svm, INTERCEPT_STGI);
4397
4398 } else {
4399
4400 }
4401}
4402
4403static bool svm_can_emulate_instruction(struct kvm_vcpu *vcpu, void *insn, int insn_len)
4404{
4405 bool smep, smap, is_user;
4406 unsigned long cr4;
4407
4408
4409
4410
4411 if (sev_es_guest(vcpu->kvm))
4412 return false;
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454 if (likely(!insn || insn_len))
4455 return true;
4456
4457
4458
4459
4460
4461 if (!kvm_vcpu_gfn_to_memslot(vcpu, kvm_rip_read(vcpu) >> PAGE_SHIFT))
4462 return true;
4463
4464 cr4 = kvm_read_cr4(vcpu);
4465 smep = cr4 & X86_CR4_SMEP;
4466 smap = cr4 & X86_CR4_SMAP;
4467 is_user = svm_get_cpl(vcpu) == 3;
4468 if (smap && (!smep || is_user)) {
4469 if (!sev_guest(vcpu->kvm))
4470 return true;
4471
4472 pr_err_ratelimited("KVM: SEV Guest triggered AMD Erratum 1096\n");
4473 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
4474 }
4475
4476 return false;
4477}
4478
4479static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu)
4480{
4481 struct vcpu_svm *svm = to_svm(vcpu);
4482
4483
4484
4485
4486
4487
4488
4489
4490 return !gif_set(svm) ||
4491 (vmcb_is_intercept(&svm->vmcb->control, INTERCEPT_INIT));
4492}
4493
4494static void svm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
4495{
4496 if (!sev_es_guest(vcpu->kvm))
4497 return kvm_vcpu_deliver_sipi_vector(vcpu, vector);
4498
4499 sev_vcpu_deliver_sipi_vector(vcpu, vector);
4500}
4501
4502static void svm_vm_destroy(struct kvm *kvm)
4503{
4504 avic_vm_destroy(kvm);
4505 sev_vm_destroy(kvm);
4506}
4507
4508static int svm_vm_init(struct kvm *kvm)
4509{
4510 if (!pause_filter_count || !pause_filter_thresh)
4511 kvm->arch.pause_in_guest = true;
4512
4513 if (enable_apicv) {
4514 int ret = avic_vm_init(kvm);
4515 if (ret)
4516 return ret;
4517 }
4518
4519 return 0;
4520}
4521
4522static struct kvm_x86_ops svm_x86_ops __initdata = {
4523 .hardware_unsetup = svm_hardware_teardown,
4524 .hardware_enable = svm_hardware_enable,
4525 .hardware_disable = svm_hardware_disable,
4526 .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
4527 .has_emulated_msr = svm_has_emulated_msr,
4528
4529 .vcpu_create = svm_create_vcpu,
4530 .vcpu_free = svm_free_vcpu,
4531 .vcpu_reset = svm_vcpu_reset,
4532
4533 .vm_size = sizeof(struct kvm_svm),
4534 .vm_init = svm_vm_init,
4535 .vm_destroy = svm_vm_destroy,
4536
4537 .prepare_guest_switch = svm_prepare_guest_switch,
4538 .vcpu_load = svm_vcpu_load,
4539 .vcpu_put = svm_vcpu_put,
4540 .vcpu_blocking = svm_vcpu_blocking,
4541 .vcpu_unblocking = svm_vcpu_unblocking,
4542
4543 .update_exception_bitmap = svm_update_exception_bitmap,
4544 .get_msr_feature = svm_get_msr_feature,
4545 .get_msr = svm_get_msr,
4546 .set_msr = svm_set_msr,
4547 .get_segment_base = svm_get_segment_base,
4548 .get_segment = svm_get_segment,
4549 .set_segment = svm_set_segment,
4550 .get_cpl = svm_get_cpl,
4551 .get_cs_db_l_bits = kvm_get_cs_db_l_bits,
4552 .set_cr0 = svm_set_cr0,
4553 .is_valid_cr4 = svm_is_valid_cr4,
4554 .set_cr4 = svm_set_cr4,
4555 .set_efer = svm_set_efer,
4556 .get_idt = svm_get_idt,
4557 .set_idt = svm_set_idt,
4558 .get_gdt = svm_get_gdt,
4559 .set_gdt = svm_set_gdt,
4560 .set_dr7 = svm_set_dr7,
4561 .sync_dirty_debug_regs = svm_sync_dirty_debug_regs,
4562 .cache_reg = svm_cache_reg,
4563 .get_rflags = svm_get_rflags,
4564 .set_rflags = svm_set_rflags,
4565
4566 .tlb_flush_all = svm_flush_tlb,
4567 .tlb_flush_current = svm_flush_tlb,
4568 .tlb_flush_gva = svm_flush_tlb_gva,
4569 .tlb_flush_guest = svm_flush_tlb,
4570
4571 .run = svm_vcpu_run,
4572 .handle_exit = handle_exit,
4573 .skip_emulated_instruction = skip_emulated_instruction,
4574 .update_emulated_instruction = NULL,
4575 .set_interrupt_shadow = svm_set_interrupt_shadow,
4576 .get_interrupt_shadow = svm_get_interrupt_shadow,
4577 .patch_hypercall = svm_patch_hypercall,
4578 .set_irq = svm_set_irq,
4579 .set_nmi = svm_inject_nmi,
4580 .queue_exception = svm_queue_exception,
4581 .cancel_injection = svm_cancel_injection,
4582 .interrupt_allowed = svm_interrupt_allowed,
4583 .nmi_allowed = svm_nmi_allowed,
4584 .get_nmi_mask = svm_get_nmi_mask,
4585 .set_nmi_mask = svm_set_nmi_mask,
4586 .enable_nmi_window = svm_enable_nmi_window,
4587 .enable_irq_window = svm_enable_irq_window,
4588 .update_cr8_intercept = svm_update_cr8_intercept,
4589 .set_virtual_apic_mode = svm_set_virtual_apic_mode,
4590 .refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl,
4591 .check_apicv_inhibit_reasons = svm_check_apicv_inhibit_reasons,
4592 .load_eoi_exitmap = svm_load_eoi_exitmap,
4593 .hwapic_irr_update = svm_hwapic_irr_update,
4594 .hwapic_isr_update = svm_hwapic_isr_update,
4595 .sync_pir_to_irr = kvm_lapic_find_highest_irr,
4596 .apicv_post_state_restore = avic_post_state_restore,
4597
4598 .set_tss_addr = svm_set_tss_addr,
4599 .set_identity_map_addr = svm_set_identity_map_addr,
4600 .get_mt_mask = svm_get_mt_mask,
4601
4602 .get_exit_info = svm_get_exit_info,
4603
4604 .vcpu_after_set_cpuid = svm_vcpu_after_set_cpuid,
4605
4606 .has_wbinvd_exit = svm_has_wbinvd_exit,
4607
4608 .get_l2_tsc_offset = svm_get_l2_tsc_offset,
4609 .get_l2_tsc_multiplier = svm_get_l2_tsc_multiplier,
4610 .write_tsc_offset = svm_write_tsc_offset,
4611 .write_tsc_multiplier = svm_write_tsc_multiplier,
4612
4613 .load_mmu_pgd = svm_load_mmu_pgd,
4614
4615 .check_intercept = svm_check_intercept,
4616 .handle_exit_irqoff = svm_handle_exit_irqoff,
4617
4618 .request_immediate_exit = __kvm_request_immediate_exit,
4619
4620 .sched_in = svm_sched_in,
4621
4622 .pmu_ops = &amd_pmu_ops,
4623 .nested_ops = &svm_nested_ops,
4624
4625 .deliver_posted_interrupt = svm_deliver_avic_intr,
4626 .dy_apicv_has_pending_interrupt = svm_dy_apicv_has_pending_interrupt,
4627 .update_pi_irte = svm_update_pi_irte,
4628 .setup_mce = svm_setup_mce,
4629
4630 .smi_allowed = svm_smi_allowed,
4631 .enter_smm = svm_enter_smm,
4632 .leave_smm = svm_leave_smm,
4633 .enable_smi_window = svm_enable_smi_window,
4634
4635 .mem_enc_op = svm_mem_enc_op,
4636 .mem_enc_reg_region = svm_register_enc_region,
4637 .mem_enc_unreg_region = svm_unregister_enc_region,
4638
4639 .vm_copy_enc_context_from = svm_vm_copy_asid_from,
4640
4641 .can_emulate_instruction = svm_can_emulate_instruction,
4642
4643 .apic_init_signal_blocked = svm_apic_init_signal_blocked,
4644
4645 .msr_filter_changed = svm_msr_filter_changed,
4646 .complete_emulated_msr = svm_complete_emulated_msr,
4647
4648 .vcpu_deliver_sipi_vector = svm_vcpu_deliver_sipi_vector,
4649};
4650
4651static struct kvm_x86_init_ops svm_init_ops __initdata = {
4652 .cpu_has_kvm_support = has_svm,
4653 .disabled_by_bios = is_disabled,
4654 .hardware_setup = svm_hardware_setup,
4655 .check_processor_compatibility = svm_check_processor_compat,
4656
4657 .runtime_ops = &svm_x86_ops,
4658};
4659
4660static int __init svm_init(void)
4661{
4662 __unused_size_checks();
4663
4664 return kvm_init(&svm_init_ops, sizeof(struct vcpu_svm),
4665 __alignof__(struct vcpu_svm), THIS_MODULE);
4666}
4667
4668static void __exit svm_exit(void)
4669{
4670 kvm_exit();
4671}
4672
4673module_init(svm_init)
4674module_exit(svm_exit)
4675