1
2
3#include <linux/frame.h>
4#include <linux/percpu.h>
5
6#include <asm/debugreg.h>
7#include <asm/mmu_context.h>
8
9#include "cpuid.h"
10#include "hyperv.h"
11#include "mmu.h"
12#include "nested.h"
13#include "pmu.h"
14#include "trace.h"
15#include "x86.h"
16
17static bool __read_mostly enable_shadow_vmcs = 1;
18module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO);
19
20static bool __read_mostly nested_early_check = 0;
21module_param(nested_early_check, bool, S_IRUGO);
22
23#define CC(consistency_check) \
24({ \
25 bool failed = (consistency_check); \
26 if (failed) \
27 trace_kvm_nested_vmenter_failed(#consistency_check, 0); \
28 failed; \
29})
30
31
32
33
34
35#define VMX_VPID_EXTENT_SUPPORTED_MASK \
36 (VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT | \
37 VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT | \
38 VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT | \
39 VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT)
40
41#define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5
42
43enum {
44 VMX_VMREAD_BITMAP,
45 VMX_VMWRITE_BITMAP,
46 VMX_BITMAP_NR
47};
48static unsigned long *vmx_bitmap[VMX_BITMAP_NR];
49
50#define vmx_vmread_bitmap (vmx_bitmap[VMX_VMREAD_BITMAP])
51#define vmx_vmwrite_bitmap (vmx_bitmap[VMX_VMWRITE_BITMAP])
52
53struct shadow_vmcs_field {
54 u16 encoding;
55 u16 offset;
56};
57static struct shadow_vmcs_field shadow_read_only_fields[] = {
58#define SHADOW_FIELD_RO(x, y) { x, offsetof(struct vmcs12, y) },
59#include "vmcs_shadow_fields.h"
60};
61static int max_shadow_read_only_fields =
62 ARRAY_SIZE(shadow_read_only_fields);
63
64static struct shadow_vmcs_field shadow_read_write_fields[] = {
65#define SHADOW_FIELD_RW(x, y) { x, offsetof(struct vmcs12, y) },
66#include "vmcs_shadow_fields.h"
67};
68static int max_shadow_read_write_fields =
69 ARRAY_SIZE(shadow_read_write_fields);
70
71static void init_vmcs_shadow_fields(void)
72{
73 int i, j;
74
75 memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE);
76 memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE);
77
78 for (i = j = 0; i < max_shadow_read_only_fields; i++) {
79 struct shadow_vmcs_field entry = shadow_read_only_fields[i];
80 u16 field = entry.encoding;
81
82 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 &&
83 (i + 1 == max_shadow_read_only_fields ||
84 shadow_read_only_fields[i + 1].encoding != field + 1))
85 pr_err("Missing field from shadow_read_only_field %x\n",
86 field + 1);
87
88 clear_bit(field, vmx_vmread_bitmap);
89 if (field & 1)
90#ifdef CONFIG_X86_64
91 continue;
92#else
93 entry.offset += sizeof(u32);
94#endif
95 shadow_read_only_fields[j++] = entry;
96 }
97 max_shadow_read_only_fields = j;
98
99 for (i = j = 0; i < max_shadow_read_write_fields; i++) {
100 struct shadow_vmcs_field entry = shadow_read_write_fields[i];
101 u16 field = entry.encoding;
102
103 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 &&
104 (i + 1 == max_shadow_read_write_fields ||
105 shadow_read_write_fields[i + 1].encoding != field + 1))
106 pr_err("Missing field from shadow_read_write_field %x\n",
107 field + 1);
108
109 WARN_ONCE(field >= GUEST_ES_AR_BYTES &&
110 field <= GUEST_TR_AR_BYTES,
111 "Update vmcs12_write_any() to drop reserved bits from AR_BYTES");
112
113
114
115
116
117
118 switch (field) {
119 case GUEST_PML_INDEX:
120 if (!cpu_has_vmx_pml())
121 continue;
122 break;
123 case VMX_PREEMPTION_TIMER_VALUE:
124 if (!cpu_has_vmx_preemption_timer())
125 continue;
126 break;
127 case GUEST_INTR_STATUS:
128 if (!cpu_has_vmx_apicv())
129 continue;
130 break;
131 default:
132 break;
133 }
134
135 clear_bit(field, vmx_vmwrite_bitmap);
136 clear_bit(field, vmx_vmread_bitmap);
137 if (field & 1)
138#ifdef CONFIG_X86_64
139 continue;
140#else
141 entry.offset += sizeof(u32);
142#endif
143 shadow_read_write_fields[j++] = entry;
144 }
145 max_shadow_read_write_fields = j;
146}
147
148
149
150
151
152
153
154static int nested_vmx_succeed(struct kvm_vcpu *vcpu)
155{
156 vmx_set_rflags(vcpu, vmx_get_rflags(vcpu)
157 & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
158 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF));
159 return kvm_skip_emulated_instruction(vcpu);
160}
161
162static int nested_vmx_failInvalid(struct kvm_vcpu *vcpu)
163{
164 vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
165 & ~(X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
166 X86_EFLAGS_SF | X86_EFLAGS_OF))
167 | X86_EFLAGS_CF);
168 return kvm_skip_emulated_instruction(vcpu);
169}
170
171static int nested_vmx_failValid(struct kvm_vcpu *vcpu,
172 u32 vm_instruction_error)
173{
174 struct vcpu_vmx *vmx = to_vmx(vcpu);
175
176
177
178
179
180 if (vmx->nested.current_vmptr == -1ull && !vmx->nested.hv_evmcs)
181 return nested_vmx_failInvalid(vcpu);
182
183 vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
184 & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
185 X86_EFLAGS_SF | X86_EFLAGS_OF))
186 | X86_EFLAGS_ZF);
187 get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error;
188
189
190
191
192 return kvm_skip_emulated_instruction(vcpu);
193}
194
195static void nested_vmx_abort(struct kvm_vcpu *vcpu, u32 indicator)
196{
197
198 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
199 pr_debug_ratelimited("kvm: nested vmx abort, indicator %d\n", indicator);
200}
201
202static inline bool vmx_control_verify(u32 control, u32 low, u32 high)
203{
204 return fixed_bits_valid(control, low, high);
205}
206
207static inline u64 vmx_control_msr(u32 low, u32 high)
208{
209 return low | ((u64)high << 32);
210}
211
212static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx)
213{
214 secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_SHADOW_VMCS);
215 vmcs_write64(VMCS_LINK_POINTER, -1ull);
216 vmx->nested.need_vmcs12_to_shadow_sync = false;
217}
218
219static inline void nested_release_evmcs(struct kvm_vcpu *vcpu)
220{
221 struct vcpu_vmx *vmx = to_vmx(vcpu);
222
223 if (!vmx->nested.hv_evmcs)
224 return;
225
226 kvm_vcpu_unmap(vcpu, &vmx->nested.hv_evmcs_map, true);
227 vmx->nested.hv_evmcs_vmptr = 0;
228 vmx->nested.hv_evmcs = NULL;
229}
230
231
232
233
234
235static void free_nested(struct kvm_vcpu *vcpu)
236{
237 struct vcpu_vmx *vmx = to_vmx(vcpu);
238
239 if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon)
240 return;
241
242 kvm_clear_request(KVM_REQ_GET_VMCS12_PAGES, vcpu);
243
244 vmx->nested.vmxon = false;
245 vmx->nested.smm.vmxon = false;
246 free_vpid(vmx->nested.vpid02);
247 vmx->nested.posted_intr_nv = -1;
248 vmx->nested.current_vmptr = -1ull;
249 if (enable_shadow_vmcs) {
250 vmx_disable_shadow_vmcs(vmx);
251 vmcs_clear(vmx->vmcs01.shadow_vmcs);
252 free_vmcs(vmx->vmcs01.shadow_vmcs);
253 vmx->vmcs01.shadow_vmcs = NULL;
254 }
255 kfree(vmx->nested.cached_vmcs12);
256 vmx->nested.cached_vmcs12 = NULL;
257 kfree(vmx->nested.cached_shadow_vmcs12);
258 vmx->nested.cached_shadow_vmcs12 = NULL;
259
260 if (vmx->nested.apic_access_page) {
261 kvm_release_page_clean(vmx->nested.apic_access_page);
262 vmx->nested.apic_access_page = NULL;
263 }
264 kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map, true);
265 kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map, true);
266 vmx->nested.pi_desc = NULL;
267
268 kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
269
270 nested_release_evmcs(vcpu);
271
272 free_loaded_vmcs(&vmx->nested.vmcs02);
273}
274
275static void vmx_sync_vmcs_host_state(struct vcpu_vmx *vmx,
276 struct loaded_vmcs *prev)
277{
278 struct vmcs_host_state *dest, *src;
279
280 if (unlikely(!vmx->guest_state_loaded))
281 return;
282
283 src = &prev->host_state;
284 dest = &vmx->loaded_vmcs->host_state;
285
286 vmx_set_host_fs_gs(dest, src->fs_sel, src->gs_sel, src->fs_base, src->gs_base);
287 dest->ldt_sel = src->ldt_sel;
288#ifdef CONFIG_X86_64
289 dest->ds_sel = src->ds_sel;
290 dest->es_sel = src->es_sel;
291#endif
292}
293
294static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs)
295{
296 struct vcpu_vmx *vmx = to_vmx(vcpu);
297 struct loaded_vmcs *prev;
298 int cpu;
299
300 if (vmx->loaded_vmcs == vmcs)
301 return;
302
303 cpu = get_cpu();
304 prev = vmx->loaded_vmcs;
305 vmx->loaded_vmcs = vmcs;
306 vmx_vcpu_load_vmcs(vcpu, cpu);
307 vmx_sync_vmcs_host_state(vmx, prev);
308 put_cpu();
309
310 vmx_segment_cache_clear(vmx);
311}
312
313
314
315
316
317void nested_vmx_free_vcpu(struct kvm_vcpu *vcpu)
318{
319 vcpu_load(vcpu);
320 vmx_leave_nested(vcpu);
321 vmx_switch_vmcs(vcpu, &to_vmx(vcpu)->vmcs01);
322 free_nested(vcpu);
323 vcpu_put(vcpu);
324}
325
326static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
327 struct x86_exception *fault)
328{
329 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
330 struct vcpu_vmx *vmx = to_vmx(vcpu);
331 u32 exit_reason;
332 unsigned long exit_qualification = vcpu->arch.exit_qualification;
333
334 if (vmx->nested.pml_full) {
335 exit_reason = EXIT_REASON_PML_FULL;
336 vmx->nested.pml_full = false;
337 exit_qualification &= INTR_INFO_UNBLOCK_NMI;
338 } else if (fault->error_code & PFERR_RSVD_MASK)
339 exit_reason = EXIT_REASON_EPT_MISCONFIG;
340 else
341 exit_reason = EXIT_REASON_EPT_VIOLATION;
342
343 nested_vmx_vmexit(vcpu, exit_reason, 0, exit_qualification);
344 vmcs12->guest_physical_address = fault->address;
345}
346
347static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
348{
349 WARN_ON(mmu_is_nested(vcpu));
350
351 vcpu->arch.mmu = &vcpu->arch.guest_mmu;
352 kvm_init_shadow_ept_mmu(vcpu,
353 to_vmx(vcpu)->nested.msrs.ept_caps &
354 VMX_EPT_EXECUTE_ONLY_BIT,
355 nested_ept_ad_enabled(vcpu),
356 nested_ept_get_cr3(vcpu));
357 vcpu->arch.mmu->set_cr3 = vmx_set_cr3;
358 vcpu->arch.mmu->get_cr3 = nested_ept_get_cr3;
359 vcpu->arch.mmu->inject_page_fault = nested_ept_inject_page_fault;
360 vcpu->arch.mmu->get_pdptr = kvm_pdptr_read;
361
362 vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu;
363}
364
365static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu)
366{
367 vcpu->arch.mmu = &vcpu->arch.root_mmu;
368 vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
369}
370
371static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12,
372 u16 error_code)
373{
374 bool inequality, bit;
375
376 bit = (vmcs12->exception_bitmap & (1u << PF_VECTOR)) != 0;
377 inequality =
378 (error_code & vmcs12->page_fault_error_code_mask) !=
379 vmcs12->page_fault_error_code_match;
380 return inequality ^ bit;
381}
382
383
384
385
386
387
388static int nested_vmx_check_exception(struct kvm_vcpu *vcpu, unsigned long *exit_qual)
389{
390 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
391 unsigned int nr = vcpu->arch.exception.nr;
392 bool has_payload = vcpu->arch.exception.has_payload;
393 unsigned long payload = vcpu->arch.exception.payload;
394
395 if (nr == PF_VECTOR) {
396 if (vcpu->arch.exception.nested_apf) {
397 *exit_qual = vcpu->arch.apf.nested_apf_token;
398 return 1;
399 }
400 if (nested_vmx_is_page_fault_vmexit(vmcs12,
401 vcpu->arch.exception.error_code)) {
402 *exit_qual = has_payload ? payload : vcpu->arch.cr2;
403 return 1;
404 }
405 } else if (vmcs12->exception_bitmap & (1u << nr)) {
406 if (nr == DB_VECTOR) {
407 if (!has_payload) {
408 payload = vcpu->arch.dr6;
409 payload &= ~(DR6_FIXED_1 | DR6_BT);
410 payload ^= DR6_RTM;
411 }
412 *exit_qual = payload;
413 } else
414 *exit_qual = 0;
415 return 1;
416 }
417
418 return 0;
419}
420
421
422static void vmx_inject_page_fault_nested(struct kvm_vcpu *vcpu,
423 struct x86_exception *fault)
424{
425 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
426
427 WARN_ON(!is_guest_mode(vcpu));
428
429 if (nested_vmx_is_page_fault_vmexit(vmcs12, fault->error_code) &&
430 !to_vmx(vcpu)->nested.nested_run_pending) {
431 vmcs12->vm_exit_intr_error_code = fault->error_code;
432 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI,
433 PF_VECTOR | INTR_TYPE_HARD_EXCEPTION |
434 INTR_INFO_DELIVER_CODE_MASK | INTR_INFO_VALID_MASK,
435 fault->address);
436 } else {
437 kvm_inject_page_fault(vcpu, fault);
438 }
439}
440
441static bool page_address_valid(struct kvm_vcpu *vcpu, gpa_t gpa)
442{
443 return PAGE_ALIGNED(gpa) && !(gpa >> cpuid_maxphyaddr(vcpu));
444}
445
446static int nested_vmx_check_io_bitmap_controls(struct kvm_vcpu *vcpu,
447 struct vmcs12 *vmcs12)
448{
449 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
450 return 0;
451
452 if (CC(!page_address_valid(vcpu, vmcs12->io_bitmap_a)) ||
453 CC(!page_address_valid(vcpu, vmcs12->io_bitmap_b)))
454 return -EINVAL;
455
456 return 0;
457}
458
459static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu,
460 struct vmcs12 *vmcs12)
461{
462 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
463 return 0;
464
465 if (CC(!page_address_valid(vcpu, vmcs12->msr_bitmap)))
466 return -EINVAL;
467
468 return 0;
469}
470
471static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu,
472 struct vmcs12 *vmcs12)
473{
474 if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
475 return 0;
476
477 if (CC(!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr)))
478 return -EINVAL;
479
480 return 0;
481}
482
483
484
485
486static bool msr_write_intercepted_l01(struct kvm_vcpu *vcpu, u32 msr)
487{
488 unsigned long *msr_bitmap;
489 int f = sizeof(unsigned long);
490
491 if (!cpu_has_vmx_msr_bitmap())
492 return true;
493
494 msr_bitmap = to_vmx(vcpu)->vmcs01.msr_bitmap;
495
496 if (msr <= 0x1fff) {
497 return !!test_bit(msr, msr_bitmap + 0x800 / f);
498 } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
499 msr &= 0x1fff;
500 return !!test_bit(msr, msr_bitmap + 0xc00 / f);
501 }
502
503 return true;
504}
505
506
507
508
509
510static void nested_vmx_disable_intercept_for_msr(unsigned long *msr_bitmap_l1,
511 unsigned long *msr_bitmap_nested,
512 u32 msr, int type)
513{
514 int f = sizeof(unsigned long);
515
516
517
518
519
520
521 if (msr <= 0x1fff) {
522 if (type & MSR_TYPE_R &&
523 !test_bit(msr, msr_bitmap_l1 + 0x000 / f))
524
525 __clear_bit(msr, msr_bitmap_nested + 0x000 / f);
526
527 if (type & MSR_TYPE_W &&
528 !test_bit(msr, msr_bitmap_l1 + 0x800 / f))
529
530 __clear_bit(msr, msr_bitmap_nested + 0x800 / f);
531
532 } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
533 msr &= 0x1fff;
534 if (type & MSR_TYPE_R &&
535 !test_bit(msr, msr_bitmap_l1 + 0x400 / f))
536
537 __clear_bit(msr, msr_bitmap_nested + 0x400 / f);
538
539 if (type & MSR_TYPE_W &&
540 !test_bit(msr, msr_bitmap_l1 + 0xc00 / f))
541
542 __clear_bit(msr, msr_bitmap_nested + 0xc00 / f);
543
544 }
545}
546
547static inline void enable_x2apic_msr_intercepts(unsigned long *msr_bitmap)
548{
549 int msr;
550
551 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) {
552 unsigned word = msr / BITS_PER_LONG;
553
554 msr_bitmap[word] = ~0;
555 msr_bitmap[word + (0x800 / sizeof(long))] = ~0;
556 }
557}
558
559
560
561
562
563static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu,
564 struct vmcs12 *vmcs12)
565{
566 int msr;
567 unsigned long *msr_bitmap_l1;
568 unsigned long *msr_bitmap_l0 = to_vmx(vcpu)->nested.vmcs02.msr_bitmap;
569 struct kvm_host_map *map = &to_vmx(vcpu)->nested.msr_bitmap_map;
570
571
572 if (!cpu_has_vmx_msr_bitmap() ||
573 !nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
574 return false;
575
576 if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->msr_bitmap), map))
577 return false;
578
579 msr_bitmap_l1 = (unsigned long *)map->hva;
580
581
582
583
584
585
586 enable_x2apic_msr_intercepts(msr_bitmap_l0);
587
588 if (nested_cpu_has_virt_x2apic_mode(vmcs12)) {
589 if (nested_cpu_has_apic_reg_virt(vmcs12)) {
590
591
592
593
594
595
596 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) {
597 unsigned word = msr / BITS_PER_LONG;
598
599 msr_bitmap_l0[word] = msr_bitmap_l1[word];
600 }
601 }
602
603 nested_vmx_disable_intercept_for_msr(
604 msr_bitmap_l1, msr_bitmap_l0,
605 X2APIC_MSR(APIC_TASKPRI),
606 MSR_TYPE_R | MSR_TYPE_W);
607
608 if (nested_cpu_has_vid(vmcs12)) {
609 nested_vmx_disable_intercept_for_msr(
610 msr_bitmap_l1, msr_bitmap_l0,
611 X2APIC_MSR(APIC_EOI),
612 MSR_TYPE_W);
613 nested_vmx_disable_intercept_for_msr(
614 msr_bitmap_l1, msr_bitmap_l0,
615 X2APIC_MSR(APIC_SELF_IPI),
616 MSR_TYPE_W);
617 }
618 }
619
620
621 nested_vmx_disable_intercept_for_msr(msr_bitmap_l1, msr_bitmap_l0,
622 MSR_FS_BASE, MSR_TYPE_RW);
623
624 nested_vmx_disable_intercept_for_msr(msr_bitmap_l1, msr_bitmap_l0,
625 MSR_GS_BASE, MSR_TYPE_RW);
626
627 nested_vmx_disable_intercept_for_msr(msr_bitmap_l1, msr_bitmap_l0,
628 MSR_KERNEL_GS_BASE, MSR_TYPE_RW);
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643 if (!msr_write_intercepted_l01(vcpu, MSR_IA32_SPEC_CTRL))
644 nested_vmx_disable_intercept_for_msr(
645 msr_bitmap_l1, msr_bitmap_l0,
646 MSR_IA32_SPEC_CTRL,
647 MSR_TYPE_R | MSR_TYPE_W);
648
649 if (!msr_write_intercepted_l01(vcpu, MSR_IA32_PRED_CMD))
650 nested_vmx_disable_intercept_for_msr(
651 msr_bitmap_l1, msr_bitmap_l0,
652 MSR_IA32_PRED_CMD,
653 MSR_TYPE_W);
654
655 kvm_vcpu_unmap(vcpu, &to_vmx(vcpu)->nested.msr_bitmap_map, false);
656
657 return true;
658}
659
660static void nested_cache_shadow_vmcs12(struct kvm_vcpu *vcpu,
661 struct vmcs12 *vmcs12)
662{
663 struct kvm_host_map map;
664 struct vmcs12 *shadow;
665
666 if (!nested_cpu_has_shadow_vmcs(vmcs12) ||
667 vmcs12->vmcs_link_pointer == -1ull)
668 return;
669
670 shadow = get_shadow_vmcs12(vcpu);
671
672 if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->vmcs_link_pointer), &map))
673 return;
674
675 memcpy(shadow, map.hva, VMCS12_SIZE);
676 kvm_vcpu_unmap(vcpu, &map, false);
677}
678
679static void nested_flush_cached_shadow_vmcs12(struct kvm_vcpu *vcpu,
680 struct vmcs12 *vmcs12)
681{
682 struct vcpu_vmx *vmx = to_vmx(vcpu);
683
684 if (!nested_cpu_has_shadow_vmcs(vmcs12) ||
685 vmcs12->vmcs_link_pointer == -1ull)
686 return;
687
688 kvm_write_guest(vmx->vcpu.kvm, vmcs12->vmcs_link_pointer,
689 get_shadow_vmcs12(vcpu), VMCS12_SIZE);
690}
691
692
693
694
695
696static bool nested_exit_intr_ack_set(struct kvm_vcpu *vcpu)
697{
698 return get_vmcs12(vcpu)->vm_exit_controls &
699 VM_EXIT_ACK_INTR_ON_EXIT;
700}
701
702static bool nested_exit_on_nmi(struct kvm_vcpu *vcpu)
703{
704 return nested_cpu_has_nmi_exiting(get_vmcs12(vcpu));
705}
706
707static int nested_vmx_check_apic_access_controls(struct kvm_vcpu *vcpu,
708 struct vmcs12 *vmcs12)
709{
710 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) &&
711 CC(!page_address_valid(vcpu, vmcs12->apic_access_addr)))
712 return -EINVAL;
713 else
714 return 0;
715}
716
717static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu,
718 struct vmcs12 *vmcs12)
719{
720 if (!nested_cpu_has_virt_x2apic_mode(vmcs12) &&
721 !nested_cpu_has_apic_reg_virt(vmcs12) &&
722 !nested_cpu_has_vid(vmcs12) &&
723 !nested_cpu_has_posted_intr(vmcs12))
724 return 0;
725
726
727
728
729
730 if (CC(nested_cpu_has_virt_x2apic_mode(vmcs12) &&
731 nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)))
732 return -EINVAL;
733
734
735
736
737
738 if (CC(nested_cpu_has_vid(vmcs12) && !nested_exit_on_intr(vcpu)))
739 return -EINVAL;
740
741
742
743
744
745
746
747
748 if (nested_cpu_has_posted_intr(vmcs12) &&
749 (CC(!nested_cpu_has_vid(vmcs12)) ||
750 CC(!nested_exit_intr_ack_set(vcpu)) ||
751 CC((vmcs12->posted_intr_nv & 0xff00)) ||
752 CC((vmcs12->posted_intr_desc_addr & 0x3f)) ||
753 CC((vmcs12->posted_intr_desc_addr >> cpuid_maxphyaddr(vcpu)))))
754 return -EINVAL;
755
756
757 if (CC(!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)))
758 return -EINVAL;
759
760 return 0;
761}
762
763static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu,
764 u32 count, u64 addr)
765{
766 int maxphyaddr;
767
768 if (count == 0)
769 return 0;
770 maxphyaddr = cpuid_maxphyaddr(vcpu);
771 if (!IS_ALIGNED(addr, 16) || addr >> maxphyaddr ||
772 (addr + count * sizeof(struct vmx_msr_entry) - 1) >> maxphyaddr)
773 return -EINVAL;
774
775 return 0;
776}
777
778static int nested_vmx_check_exit_msr_switch_controls(struct kvm_vcpu *vcpu,
779 struct vmcs12 *vmcs12)
780{
781 if (CC(nested_vmx_check_msr_switch(vcpu,
782 vmcs12->vm_exit_msr_load_count,
783 vmcs12->vm_exit_msr_load_addr)) ||
784 CC(nested_vmx_check_msr_switch(vcpu,
785 vmcs12->vm_exit_msr_store_count,
786 vmcs12->vm_exit_msr_store_addr)))
787 return -EINVAL;
788
789 return 0;
790}
791
792static int nested_vmx_check_entry_msr_switch_controls(struct kvm_vcpu *vcpu,
793 struct vmcs12 *vmcs12)
794{
795 if (CC(nested_vmx_check_msr_switch(vcpu,
796 vmcs12->vm_entry_msr_load_count,
797 vmcs12->vm_entry_msr_load_addr)))
798 return -EINVAL;
799
800 return 0;
801}
802
803static int nested_vmx_check_pml_controls(struct kvm_vcpu *vcpu,
804 struct vmcs12 *vmcs12)
805{
806 if (!nested_cpu_has_pml(vmcs12))
807 return 0;
808
809 if (CC(!nested_cpu_has_ept(vmcs12)) ||
810 CC(!page_address_valid(vcpu, vmcs12->pml_address)))
811 return -EINVAL;
812
813 return 0;
814}
815
816static int nested_vmx_check_unrestricted_guest_controls(struct kvm_vcpu *vcpu,
817 struct vmcs12 *vmcs12)
818{
819 if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST) &&
820 !nested_cpu_has_ept(vmcs12)))
821 return -EINVAL;
822 return 0;
823}
824
825static int nested_vmx_check_mode_based_ept_exec_controls(struct kvm_vcpu *vcpu,
826 struct vmcs12 *vmcs12)
827{
828 if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_MODE_BASED_EPT_EXEC) &&
829 !nested_cpu_has_ept(vmcs12)))
830 return -EINVAL;
831 return 0;
832}
833
834static int nested_vmx_check_shadow_vmcs_controls(struct kvm_vcpu *vcpu,
835 struct vmcs12 *vmcs12)
836{
837 if (!nested_cpu_has_shadow_vmcs(vmcs12))
838 return 0;
839
840 if (CC(!page_address_valid(vcpu, vmcs12->vmread_bitmap)) ||
841 CC(!page_address_valid(vcpu, vmcs12->vmwrite_bitmap)))
842 return -EINVAL;
843
844 return 0;
845}
846
847static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu,
848 struct vmx_msr_entry *e)
849{
850
851 if (CC(vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8))
852 return -EINVAL;
853 if (CC(e->index == MSR_IA32_UCODE_WRITE) ||
854 CC(e->index == MSR_IA32_UCODE_REV))
855 return -EINVAL;
856 if (CC(e->reserved != 0))
857 return -EINVAL;
858 return 0;
859}
860
861static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu,
862 struct vmx_msr_entry *e)
863{
864 if (CC(e->index == MSR_FS_BASE) ||
865 CC(e->index == MSR_GS_BASE) ||
866 CC(e->index == MSR_IA32_SMM_MONITOR_CTL) ||
867 nested_vmx_msr_check_common(vcpu, e))
868 return -EINVAL;
869 return 0;
870}
871
872static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu,
873 struct vmx_msr_entry *e)
874{
875 if (CC(e->index == MSR_IA32_SMBASE) ||
876 nested_vmx_msr_check_common(vcpu, e))
877 return -EINVAL;
878 return 0;
879}
880
881static u32 nested_vmx_max_atomic_switch_msrs(struct kvm_vcpu *vcpu)
882{
883 struct vcpu_vmx *vmx = to_vmx(vcpu);
884 u64 vmx_misc = vmx_control_msr(vmx->nested.msrs.misc_low,
885 vmx->nested.msrs.misc_high);
886
887 return (vmx_misc_max_msr(vmx_misc) + 1) * VMX_MISC_MSR_LIST_MULTIPLIER;
888}
889
890
891
892
893
894
895
896
897
898
899static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
900{
901 u32 i;
902 struct vmx_msr_entry e;
903 u32 max_msr_list_size = nested_vmx_max_atomic_switch_msrs(vcpu);
904
905 for (i = 0; i < count; i++) {
906 if (unlikely(i >= max_msr_list_size))
907 goto fail;
908
909 if (kvm_vcpu_read_guest(vcpu, gpa + i * sizeof(e),
910 &e, sizeof(e))) {
911 pr_debug_ratelimited(
912 "%s cannot read MSR entry (%u, 0x%08llx)\n",
913 __func__, i, gpa + i * sizeof(e));
914 goto fail;
915 }
916 if (nested_vmx_load_msr_check(vcpu, &e)) {
917 pr_debug_ratelimited(
918 "%s check failed (%u, 0x%x, 0x%x)\n",
919 __func__, i, e.index, e.reserved);
920 goto fail;
921 }
922 if (kvm_set_msr(vcpu, e.index, e.value)) {
923 pr_debug_ratelimited(
924 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
925 __func__, i, e.index, e.value);
926 goto fail;
927 }
928 }
929 return 0;
930fail:
931 return i + 1;
932}
933
934static bool nested_vmx_get_vmexit_msr_value(struct kvm_vcpu *vcpu,
935 u32 msr_index,
936 u64 *data)
937{
938 struct vcpu_vmx *vmx = to_vmx(vcpu);
939
940
941
942
943
944
945 if (msr_index == MSR_IA32_TSC) {
946 int index = vmx_find_msr_index(&vmx->msr_autostore.guest,
947 MSR_IA32_TSC);
948
949 if (index >= 0) {
950 u64 val = vmx->msr_autostore.guest.val[index].value;
951
952 *data = kvm_read_l1_tsc(vcpu, val);
953 return true;
954 }
955 }
956
957 if (kvm_get_msr(vcpu, msr_index, data)) {
958 pr_debug_ratelimited("%s cannot read MSR (0x%x)\n", __func__,
959 msr_index);
960 return false;
961 }
962 return true;
963}
964
965static bool read_and_check_msr_entry(struct kvm_vcpu *vcpu, u64 gpa, int i,
966 struct vmx_msr_entry *e)
967{
968 if (kvm_vcpu_read_guest(vcpu,
969 gpa + i * sizeof(*e),
970 e, 2 * sizeof(u32))) {
971 pr_debug_ratelimited(
972 "%s cannot read MSR entry (%u, 0x%08llx)\n",
973 __func__, i, gpa + i * sizeof(*e));
974 return false;
975 }
976 if (nested_vmx_store_msr_check(vcpu, e)) {
977 pr_debug_ratelimited(
978 "%s check failed (%u, 0x%x, 0x%x)\n",
979 __func__, i, e->index, e->reserved);
980 return false;
981 }
982 return true;
983}
984
985static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
986{
987 u64 data;
988 u32 i;
989 struct vmx_msr_entry e;
990 u32 max_msr_list_size = nested_vmx_max_atomic_switch_msrs(vcpu);
991
992 for (i = 0; i < count; i++) {
993 if (unlikely(i >= max_msr_list_size))
994 return -EINVAL;
995
996 if (!read_and_check_msr_entry(vcpu, gpa, i, &e))
997 return -EINVAL;
998
999 if (!nested_vmx_get_vmexit_msr_value(vcpu, e.index, &data))
1000 return -EINVAL;
1001
1002 if (kvm_vcpu_write_guest(vcpu,
1003 gpa + i * sizeof(e) +
1004 offsetof(struct vmx_msr_entry, value),
1005 &data, sizeof(data))) {
1006 pr_debug_ratelimited(
1007 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
1008 __func__, i, e.index, data);
1009 return -EINVAL;
1010 }
1011 }
1012 return 0;
1013}
1014
1015static bool nested_msr_store_list_has_msr(struct kvm_vcpu *vcpu, u32 msr_index)
1016{
1017 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
1018 u32 count = vmcs12->vm_exit_msr_store_count;
1019 u64 gpa = vmcs12->vm_exit_msr_store_addr;
1020 struct vmx_msr_entry e;
1021 u32 i;
1022
1023 for (i = 0; i < count; i++) {
1024 if (!read_and_check_msr_entry(vcpu, gpa, i, &e))
1025 return false;
1026
1027 if (e.index == msr_index)
1028 return true;
1029 }
1030 return false;
1031}
1032
1033static void prepare_vmx_msr_autostore_list(struct kvm_vcpu *vcpu,
1034 u32 msr_index)
1035{
1036 struct vcpu_vmx *vmx = to_vmx(vcpu);
1037 struct vmx_msrs *autostore = &vmx->msr_autostore.guest;
1038 bool in_vmcs12_store_list;
1039 int msr_autostore_index;
1040 bool in_autostore_list;
1041 int last;
1042
1043 msr_autostore_index = vmx_find_msr_index(autostore, msr_index);
1044 in_autostore_list = msr_autostore_index >= 0;
1045 in_vmcs12_store_list = nested_msr_store_list_has_msr(vcpu, msr_index);
1046
1047 if (in_vmcs12_store_list && !in_autostore_list) {
1048 if (autostore->nr == NR_LOADSTORE_MSRS) {
1049
1050
1051
1052
1053
1054
1055
1056 pr_warn_ratelimited(
1057 "Not enough msr entries in msr_autostore. Can't add msr %x\n",
1058 msr_index);
1059 return;
1060 }
1061 last = autostore->nr++;
1062 autostore->val[last].index = msr_index;
1063 } else if (!in_vmcs12_store_list && in_autostore_list) {
1064 last = --autostore->nr;
1065 autostore->val[msr_autostore_index] = autostore->val[last];
1066 }
1067}
1068
1069static bool nested_cr3_valid(struct kvm_vcpu *vcpu, unsigned long val)
1070{
1071 unsigned long invalid_mask;
1072
1073 invalid_mask = (~0ULL) << cpuid_maxphyaddr(vcpu);
1074 return (val & invalid_mask) == 0;
1075}
1076
1077
1078
1079
1080
1081
1082
1083static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool nested_ept,
1084 u32 *entry_failure_code)
1085{
1086 if (cr3 != kvm_read_cr3(vcpu) || (!nested_ept && pdptrs_changed(vcpu))) {
1087 if (CC(!nested_cr3_valid(vcpu, cr3))) {
1088 *entry_failure_code = ENTRY_FAIL_DEFAULT;
1089 return -EINVAL;
1090 }
1091
1092
1093
1094
1095
1096 if (is_pae_paging(vcpu) && !nested_ept) {
1097 if (CC(!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))) {
1098 *entry_failure_code = ENTRY_FAIL_PDPTE;
1099 return -EINVAL;
1100 }
1101 }
1102 }
1103
1104 if (!nested_ept)
1105 kvm_mmu_new_cr3(vcpu, cr3, false);
1106
1107 vcpu->arch.cr3 = cr3;
1108 kvm_register_mark_available(vcpu, VCPU_EXREG_CR3);
1109
1110 kvm_init_mmu(vcpu, false);
1111
1112 return 0;
1113}
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128static bool nested_has_guest_tlb_tag(struct kvm_vcpu *vcpu)
1129{
1130 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
1131
1132 return enable_ept ||
1133 (nested_cpu_has_vpid(vmcs12) && to_vmx(vcpu)->nested.vpid02);
1134}
1135
1136static u16 nested_get_vpid02(struct kvm_vcpu *vcpu)
1137{
1138 struct vcpu_vmx *vmx = to_vmx(vcpu);
1139
1140 return vmx->nested.vpid02 ? vmx->nested.vpid02 : vmx->vpid;
1141}
1142
1143static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask)
1144{
1145 superset &= mask;
1146 subset &= mask;
1147
1148 return (superset | subset) == superset;
1149}
1150
1151static int vmx_restore_vmx_basic(struct vcpu_vmx *vmx, u64 data)
1152{
1153 const u64 feature_and_reserved =
1154
1155 BIT_ULL(49) | BIT_ULL(54) | BIT_ULL(55) |
1156
1157 BIT_ULL(31) | GENMASK_ULL(47, 45) | GENMASK_ULL(63, 56);
1158 u64 vmx_basic = vmx->nested.msrs.basic;
1159
1160 if (!is_bitwise_subset(vmx_basic, data, feature_and_reserved))
1161 return -EINVAL;
1162
1163
1164
1165
1166
1167 if (data & BIT_ULL(48))
1168 return -EINVAL;
1169
1170 if (vmx_basic_vmcs_revision_id(vmx_basic) !=
1171 vmx_basic_vmcs_revision_id(data))
1172 return -EINVAL;
1173
1174 if (vmx_basic_vmcs_size(vmx_basic) > vmx_basic_vmcs_size(data))
1175 return -EINVAL;
1176
1177 vmx->nested.msrs.basic = data;
1178 return 0;
1179}
1180
1181static int
1182vmx_restore_control_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data)
1183{
1184 u64 supported;
1185 u32 *lowp, *highp;
1186
1187 switch (msr_index) {
1188 case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
1189 lowp = &vmx->nested.msrs.pinbased_ctls_low;
1190 highp = &vmx->nested.msrs.pinbased_ctls_high;
1191 break;
1192 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
1193 lowp = &vmx->nested.msrs.procbased_ctls_low;
1194 highp = &vmx->nested.msrs.procbased_ctls_high;
1195 break;
1196 case MSR_IA32_VMX_TRUE_EXIT_CTLS:
1197 lowp = &vmx->nested.msrs.exit_ctls_low;
1198 highp = &vmx->nested.msrs.exit_ctls_high;
1199 break;
1200 case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
1201 lowp = &vmx->nested.msrs.entry_ctls_low;
1202 highp = &vmx->nested.msrs.entry_ctls_high;
1203 break;
1204 case MSR_IA32_VMX_PROCBASED_CTLS2:
1205 lowp = &vmx->nested.msrs.secondary_ctls_low;
1206 highp = &vmx->nested.msrs.secondary_ctls_high;
1207 break;
1208 default:
1209 BUG();
1210 }
1211
1212 supported = vmx_control_msr(*lowp, *highp);
1213
1214
1215 if (!is_bitwise_subset(data, supported, GENMASK_ULL(31, 0)))
1216 return -EINVAL;
1217
1218
1219 if (!is_bitwise_subset(supported, data, GENMASK_ULL(63, 32)))
1220 return -EINVAL;
1221
1222 *lowp = data;
1223 *highp = data >> 32;
1224 return 0;
1225}
1226
1227static int vmx_restore_vmx_misc(struct vcpu_vmx *vmx, u64 data)
1228{
1229 const u64 feature_and_reserved_bits =
1230
1231 BIT_ULL(5) | GENMASK_ULL(8, 6) | BIT_ULL(14) | BIT_ULL(15) |
1232 BIT_ULL(28) | BIT_ULL(29) | BIT_ULL(30) |
1233
1234 GENMASK_ULL(13, 9) | BIT_ULL(31);
1235 u64 vmx_misc;
1236
1237 vmx_misc = vmx_control_msr(vmx->nested.msrs.misc_low,
1238 vmx->nested.msrs.misc_high);
1239
1240 if (!is_bitwise_subset(vmx_misc, data, feature_and_reserved_bits))
1241 return -EINVAL;
1242
1243 if ((vmx->nested.msrs.pinbased_ctls_high &
1244 PIN_BASED_VMX_PREEMPTION_TIMER) &&
1245 vmx_misc_preemption_timer_rate(data) !=
1246 vmx_misc_preemption_timer_rate(vmx_misc))
1247 return -EINVAL;
1248
1249 if (vmx_misc_cr3_count(data) > vmx_misc_cr3_count(vmx_misc))
1250 return -EINVAL;
1251
1252 if (vmx_misc_max_msr(data) > vmx_misc_max_msr(vmx_misc))
1253 return -EINVAL;
1254
1255 if (vmx_misc_mseg_revid(data) != vmx_misc_mseg_revid(vmx_misc))
1256 return -EINVAL;
1257
1258 vmx->nested.msrs.misc_low = data;
1259 vmx->nested.msrs.misc_high = data >> 32;
1260
1261 return 0;
1262}
1263
1264static int vmx_restore_vmx_ept_vpid_cap(struct vcpu_vmx *vmx, u64 data)
1265{
1266 u64 vmx_ept_vpid_cap;
1267
1268 vmx_ept_vpid_cap = vmx_control_msr(vmx->nested.msrs.ept_caps,
1269 vmx->nested.msrs.vpid_caps);
1270
1271
1272 if (!is_bitwise_subset(vmx_ept_vpid_cap, data, -1ULL))
1273 return -EINVAL;
1274
1275 vmx->nested.msrs.ept_caps = data;
1276 vmx->nested.msrs.vpid_caps = data >> 32;
1277 return 0;
1278}
1279
1280static int vmx_restore_fixed0_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data)
1281{
1282 u64 *msr;
1283
1284 switch (msr_index) {
1285 case MSR_IA32_VMX_CR0_FIXED0:
1286 msr = &vmx->nested.msrs.cr0_fixed0;
1287 break;
1288 case MSR_IA32_VMX_CR4_FIXED0:
1289 msr = &vmx->nested.msrs.cr4_fixed0;
1290 break;
1291 default:
1292 BUG();
1293 }
1294
1295
1296
1297
1298
1299 if (!is_bitwise_subset(data, *msr, -1ULL))
1300 return -EINVAL;
1301
1302 *msr = data;
1303 return 0;
1304}
1305
1306
1307
1308
1309
1310
1311int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
1312{
1313 struct vcpu_vmx *vmx = to_vmx(vcpu);
1314
1315
1316
1317
1318
1319 if (vmx->nested.vmxon)
1320 return -EBUSY;
1321
1322 switch (msr_index) {
1323 case MSR_IA32_VMX_BASIC:
1324 return vmx_restore_vmx_basic(vmx, data);
1325 case MSR_IA32_VMX_PINBASED_CTLS:
1326 case MSR_IA32_VMX_PROCBASED_CTLS:
1327 case MSR_IA32_VMX_EXIT_CTLS:
1328 case MSR_IA32_VMX_ENTRY_CTLS:
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338 return -EINVAL;
1339 case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
1340 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
1341 case MSR_IA32_VMX_TRUE_EXIT_CTLS:
1342 case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
1343 case MSR_IA32_VMX_PROCBASED_CTLS2:
1344 return vmx_restore_control_msr(vmx, msr_index, data);
1345 case MSR_IA32_VMX_MISC:
1346 return vmx_restore_vmx_misc(vmx, data);
1347 case MSR_IA32_VMX_CR0_FIXED0:
1348 case MSR_IA32_VMX_CR4_FIXED0:
1349 return vmx_restore_fixed0_msr(vmx, msr_index, data);
1350 case MSR_IA32_VMX_CR0_FIXED1:
1351 case MSR_IA32_VMX_CR4_FIXED1:
1352
1353
1354
1355
1356 return -EINVAL;
1357 case MSR_IA32_VMX_EPT_VPID_CAP:
1358 return vmx_restore_vmx_ept_vpid_cap(vmx, data);
1359 case MSR_IA32_VMX_VMCS_ENUM:
1360 vmx->nested.msrs.vmcs_enum = data;
1361 return 0;
1362 case MSR_IA32_VMX_VMFUNC:
1363 if (data & ~vmx->nested.msrs.vmfunc_controls)
1364 return -EINVAL;
1365 vmx->nested.msrs.vmfunc_controls = data;
1366 return 0;
1367 default:
1368
1369
1370
1371 return -EINVAL;
1372 }
1373}
1374
1375
1376int vmx_get_vmx_msr(struct nested_vmx_msrs *msrs, u32 msr_index, u64 *pdata)
1377{
1378 switch (msr_index) {
1379 case MSR_IA32_VMX_BASIC:
1380 *pdata = msrs->basic;
1381 break;
1382 case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
1383 case MSR_IA32_VMX_PINBASED_CTLS:
1384 *pdata = vmx_control_msr(
1385 msrs->pinbased_ctls_low,
1386 msrs->pinbased_ctls_high);
1387 if (msr_index == MSR_IA32_VMX_PINBASED_CTLS)
1388 *pdata |= PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
1389 break;
1390 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
1391 case MSR_IA32_VMX_PROCBASED_CTLS:
1392 *pdata = vmx_control_msr(
1393 msrs->procbased_ctls_low,
1394 msrs->procbased_ctls_high);
1395 if (msr_index == MSR_IA32_VMX_PROCBASED_CTLS)
1396 *pdata |= CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
1397 break;
1398 case MSR_IA32_VMX_TRUE_EXIT_CTLS:
1399 case MSR_IA32_VMX_EXIT_CTLS:
1400 *pdata = vmx_control_msr(
1401 msrs->exit_ctls_low,
1402 msrs->exit_ctls_high);
1403 if (msr_index == MSR_IA32_VMX_EXIT_CTLS)
1404 *pdata |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR;
1405 break;
1406 case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
1407 case MSR_IA32_VMX_ENTRY_CTLS:
1408 *pdata = vmx_control_msr(
1409 msrs->entry_ctls_low,
1410 msrs->entry_ctls_high);
1411 if (msr_index == MSR_IA32_VMX_ENTRY_CTLS)
1412 *pdata |= VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR;
1413 break;
1414 case MSR_IA32_VMX_MISC:
1415 *pdata = vmx_control_msr(
1416 msrs->misc_low,
1417 msrs->misc_high);
1418 break;
1419 case MSR_IA32_VMX_CR0_FIXED0:
1420 *pdata = msrs->cr0_fixed0;
1421 break;
1422 case MSR_IA32_VMX_CR0_FIXED1:
1423 *pdata = msrs->cr0_fixed1;
1424 break;
1425 case MSR_IA32_VMX_CR4_FIXED0:
1426 *pdata = msrs->cr4_fixed0;
1427 break;
1428 case MSR_IA32_VMX_CR4_FIXED1:
1429 *pdata = msrs->cr4_fixed1;
1430 break;
1431 case MSR_IA32_VMX_VMCS_ENUM:
1432 *pdata = msrs->vmcs_enum;
1433 break;
1434 case MSR_IA32_VMX_PROCBASED_CTLS2:
1435 *pdata = vmx_control_msr(
1436 msrs->secondary_ctls_low,
1437 msrs->secondary_ctls_high);
1438 break;
1439 case MSR_IA32_VMX_EPT_VPID_CAP:
1440 *pdata = msrs->ept_caps |
1441 ((u64)msrs->vpid_caps << 32);
1442 break;
1443 case MSR_IA32_VMX_VMFUNC:
1444 *pdata = msrs->vmfunc_controls;
1445 break;
1446 default:
1447 return 1;
1448 }
1449
1450 return 0;
1451}
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx)
1462{
1463 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs;
1464 struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu);
1465 struct shadow_vmcs_field field;
1466 unsigned long val;
1467 int i;
1468
1469 if (WARN_ON(!shadow_vmcs))
1470 return;
1471
1472 preempt_disable();
1473
1474 vmcs_load(shadow_vmcs);
1475
1476 for (i = 0; i < max_shadow_read_write_fields; i++) {
1477 field = shadow_read_write_fields[i];
1478 val = __vmcs_readl(field.encoding);
1479 vmcs12_write_any(vmcs12, field.encoding, field.offset, val);
1480 }
1481
1482 vmcs_clear(shadow_vmcs);
1483 vmcs_load(vmx->loaded_vmcs->vmcs);
1484
1485 preempt_enable();
1486}
1487
1488static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx)
1489{
1490 const struct shadow_vmcs_field *fields[] = {
1491 shadow_read_write_fields,
1492 shadow_read_only_fields
1493 };
1494 const int max_fields[] = {
1495 max_shadow_read_write_fields,
1496 max_shadow_read_only_fields
1497 };
1498 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs;
1499 struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu);
1500 struct shadow_vmcs_field field;
1501 unsigned long val;
1502 int i, q;
1503
1504 if (WARN_ON(!shadow_vmcs))
1505 return;
1506
1507 vmcs_load(shadow_vmcs);
1508
1509 for (q = 0; q < ARRAY_SIZE(fields); q++) {
1510 for (i = 0; i < max_fields[q]; i++) {
1511 field = fields[q][i];
1512 val = vmcs12_read_any(vmcs12, field.encoding,
1513 field.offset);
1514 __vmcs_writel(field.encoding, val);
1515 }
1516 }
1517
1518 vmcs_clear(shadow_vmcs);
1519 vmcs_load(vmx->loaded_vmcs->vmcs);
1520}
1521
1522static int copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx)
1523{
1524 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12;
1525 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs;
1526
1527
1528 vmcs12->tpr_threshold = evmcs->tpr_threshold;
1529 vmcs12->guest_rip = evmcs->guest_rip;
1530
1531 if (unlikely(!(evmcs->hv_clean_fields &
1532 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC))) {
1533 vmcs12->guest_rsp = evmcs->guest_rsp;
1534 vmcs12->guest_rflags = evmcs->guest_rflags;
1535 vmcs12->guest_interruptibility_info =
1536 evmcs->guest_interruptibility_info;
1537 }
1538
1539 if (unlikely(!(evmcs->hv_clean_fields &
1540 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC))) {
1541 vmcs12->cpu_based_vm_exec_control =
1542 evmcs->cpu_based_vm_exec_control;
1543 }
1544
1545 if (unlikely(!(evmcs->hv_clean_fields &
1546 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EXCPN))) {
1547 vmcs12->exception_bitmap = evmcs->exception_bitmap;
1548 }
1549
1550 if (unlikely(!(evmcs->hv_clean_fields &
1551 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY))) {
1552 vmcs12->vm_entry_controls = evmcs->vm_entry_controls;
1553 }
1554
1555 if (unlikely(!(evmcs->hv_clean_fields &
1556 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT))) {
1557 vmcs12->vm_entry_intr_info_field =
1558 evmcs->vm_entry_intr_info_field;
1559 vmcs12->vm_entry_exception_error_code =
1560 evmcs->vm_entry_exception_error_code;
1561 vmcs12->vm_entry_instruction_len =
1562 evmcs->vm_entry_instruction_len;
1563 }
1564
1565 if (unlikely(!(evmcs->hv_clean_fields &
1566 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1))) {
1567 vmcs12->host_ia32_pat = evmcs->host_ia32_pat;
1568 vmcs12->host_ia32_efer = evmcs->host_ia32_efer;
1569 vmcs12->host_cr0 = evmcs->host_cr0;
1570 vmcs12->host_cr3 = evmcs->host_cr3;
1571 vmcs12->host_cr4 = evmcs->host_cr4;
1572 vmcs12->host_ia32_sysenter_esp = evmcs->host_ia32_sysenter_esp;
1573 vmcs12->host_ia32_sysenter_eip = evmcs->host_ia32_sysenter_eip;
1574 vmcs12->host_rip = evmcs->host_rip;
1575 vmcs12->host_ia32_sysenter_cs = evmcs->host_ia32_sysenter_cs;
1576 vmcs12->host_es_selector = evmcs->host_es_selector;
1577 vmcs12->host_cs_selector = evmcs->host_cs_selector;
1578 vmcs12->host_ss_selector = evmcs->host_ss_selector;
1579 vmcs12->host_ds_selector = evmcs->host_ds_selector;
1580 vmcs12->host_fs_selector = evmcs->host_fs_selector;
1581 vmcs12->host_gs_selector = evmcs->host_gs_selector;
1582 vmcs12->host_tr_selector = evmcs->host_tr_selector;
1583 }
1584
1585 if (unlikely(!(evmcs->hv_clean_fields &
1586 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1))) {
1587 vmcs12->pin_based_vm_exec_control =
1588 evmcs->pin_based_vm_exec_control;
1589 vmcs12->vm_exit_controls = evmcs->vm_exit_controls;
1590 vmcs12->secondary_vm_exec_control =
1591 evmcs->secondary_vm_exec_control;
1592 }
1593
1594 if (unlikely(!(evmcs->hv_clean_fields &
1595 HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP))) {
1596 vmcs12->io_bitmap_a = evmcs->io_bitmap_a;
1597 vmcs12->io_bitmap_b = evmcs->io_bitmap_b;
1598 }
1599
1600 if (unlikely(!(evmcs->hv_clean_fields &
1601 HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP))) {
1602 vmcs12->msr_bitmap = evmcs->msr_bitmap;
1603 }
1604
1605 if (unlikely(!(evmcs->hv_clean_fields &
1606 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2))) {
1607 vmcs12->guest_es_base = evmcs->guest_es_base;
1608 vmcs12->guest_cs_base = evmcs->guest_cs_base;
1609 vmcs12->guest_ss_base = evmcs->guest_ss_base;
1610 vmcs12->guest_ds_base = evmcs->guest_ds_base;
1611 vmcs12->guest_fs_base = evmcs->guest_fs_base;
1612 vmcs12->guest_gs_base = evmcs->guest_gs_base;
1613 vmcs12->guest_ldtr_base = evmcs->guest_ldtr_base;
1614 vmcs12->guest_tr_base = evmcs->guest_tr_base;
1615 vmcs12->guest_gdtr_base = evmcs->guest_gdtr_base;
1616 vmcs12->guest_idtr_base = evmcs->guest_idtr_base;
1617 vmcs12->guest_es_limit = evmcs->guest_es_limit;
1618 vmcs12->guest_cs_limit = evmcs->guest_cs_limit;
1619 vmcs12->guest_ss_limit = evmcs->guest_ss_limit;
1620 vmcs12->guest_ds_limit = evmcs->guest_ds_limit;
1621 vmcs12->guest_fs_limit = evmcs->guest_fs_limit;
1622 vmcs12->guest_gs_limit = evmcs->guest_gs_limit;
1623 vmcs12->guest_ldtr_limit = evmcs->guest_ldtr_limit;
1624 vmcs12->guest_tr_limit = evmcs->guest_tr_limit;
1625 vmcs12->guest_gdtr_limit = evmcs->guest_gdtr_limit;
1626 vmcs12->guest_idtr_limit = evmcs->guest_idtr_limit;
1627 vmcs12->guest_es_ar_bytes = evmcs->guest_es_ar_bytes;
1628 vmcs12->guest_cs_ar_bytes = evmcs->guest_cs_ar_bytes;
1629 vmcs12->guest_ss_ar_bytes = evmcs->guest_ss_ar_bytes;
1630 vmcs12->guest_ds_ar_bytes = evmcs->guest_ds_ar_bytes;
1631 vmcs12->guest_fs_ar_bytes = evmcs->guest_fs_ar_bytes;
1632 vmcs12->guest_gs_ar_bytes = evmcs->guest_gs_ar_bytes;
1633 vmcs12->guest_ldtr_ar_bytes = evmcs->guest_ldtr_ar_bytes;
1634 vmcs12->guest_tr_ar_bytes = evmcs->guest_tr_ar_bytes;
1635 vmcs12->guest_es_selector = evmcs->guest_es_selector;
1636 vmcs12->guest_cs_selector = evmcs->guest_cs_selector;
1637 vmcs12->guest_ss_selector = evmcs->guest_ss_selector;
1638 vmcs12->guest_ds_selector = evmcs->guest_ds_selector;
1639 vmcs12->guest_fs_selector = evmcs->guest_fs_selector;
1640 vmcs12->guest_gs_selector = evmcs->guest_gs_selector;
1641 vmcs12->guest_ldtr_selector = evmcs->guest_ldtr_selector;
1642 vmcs12->guest_tr_selector = evmcs->guest_tr_selector;
1643 }
1644
1645 if (unlikely(!(evmcs->hv_clean_fields &
1646 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2))) {
1647 vmcs12->tsc_offset = evmcs->tsc_offset;
1648 vmcs12->virtual_apic_page_addr = evmcs->virtual_apic_page_addr;
1649 vmcs12->xss_exit_bitmap = evmcs->xss_exit_bitmap;
1650 }
1651
1652 if (unlikely(!(evmcs->hv_clean_fields &
1653 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR))) {
1654 vmcs12->cr0_guest_host_mask = evmcs->cr0_guest_host_mask;
1655 vmcs12->cr4_guest_host_mask = evmcs->cr4_guest_host_mask;
1656 vmcs12->cr0_read_shadow = evmcs->cr0_read_shadow;
1657 vmcs12->cr4_read_shadow = evmcs->cr4_read_shadow;
1658 vmcs12->guest_cr0 = evmcs->guest_cr0;
1659 vmcs12->guest_cr3 = evmcs->guest_cr3;
1660 vmcs12->guest_cr4 = evmcs->guest_cr4;
1661 vmcs12->guest_dr7 = evmcs->guest_dr7;
1662 }
1663
1664 if (unlikely(!(evmcs->hv_clean_fields &
1665 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER))) {
1666 vmcs12->host_fs_base = evmcs->host_fs_base;
1667 vmcs12->host_gs_base = evmcs->host_gs_base;
1668 vmcs12->host_tr_base = evmcs->host_tr_base;
1669 vmcs12->host_gdtr_base = evmcs->host_gdtr_base;
1670 vmcs12->host_idtr_base = evmcs->host_idtr_base;
1671 vmcs12->host_rsp = evmcs->host_rsp;
1672 }
1673
1674 if (unlikely(!(evmcs->hv_clean_fields &
1675 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT))) {
1676 vmcs12->ept_pointer = evmcs->ept_pointer;
1677 vmcs12->virtual_processor_id = evmcs->virtual_processor_id;
1678 }
1679
1680 if (unlikely(!(evmcs->hv_clean_fields &
1681 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1))) {
1682 vmcs12->vmcs_link_pointer = evmcs->vmcs_link_pointer;
1683 vmcs12->guest_ia32_debugctl = evmcs->guest_ia32_debugctl;
1684 vmcs12->guest_ia32_pat = evmcs->guest_ia32_pat;
1685 vmcs12->guest_ia32_efer = evmcs->guest_ia32_efer;
1686 vmcs12->guest_pdptr0 = evmcs->guest_pdptr0;
1687 vmcs12->guest_pdptr1 = evmcs->guest_pdptr1;
1688 vmcs12->guest_pdptr2 = evmcs->guest_pdptr2;
1689 vmcs12->guest_pdptr3 = evmcs->guest_pdptr3;
1690 vmcs12->guest_pending_dbg_exceptions =
1691 evmcs->guest_pending_dbg_exceptions;
1692 vmcs12->guest_sysenter_esp = evmcs->guest_sysenter_esp;
1693 vmcs12->guest_sysenter_eip = evmcs->guest_sysenter_eip;
1694 vmcs12->guest_bndcfgs = evmcs->guest_bndcfgs;
1695 vmcs12->guest_activity_state = evmcs->guest_activity_state;
1696 vmcs12->guest_sysenter_cs = evmcs->guest_sysenter_cs;
1697 }
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739 return 0;
1740}
1741
1742static int copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx)
1743{
1744 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12;
1745 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs;
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815 evmcs->guest_es_selector = vmcs12->guest_es_selector;
1816 evmcs->guest_cs_selector = vmcs12->guest_cs_selector;
1817 evmcs->guest_ss_selector = vmcs12->guest_ss_selector;
1818 evmcs->guest_ds_selector = vmcs12->guest_ds_selector;
1819 evmcs->guest_fs_selector = vmcs12->guest_fs_selector;
1820 evmcs->guest_gs_selector = vmcs12->guest_gs_selector;
1821 evmcs->guest_ldtr_selector = vmcs12->guest_ldtr_selector;
1822 evmcs->guest_tr_selector = vmcs12->guest_tr_selector;
1823
1824 evmcs->guest_es_limit = vmcs12->guest_es_limit;
1825 evmcs->guest_cs_limit = vmcs12->guest_cs_limit;
1826 evmcs->guest_ss_limit = vmcs12->guest_ss_limit;
1827 evmcs->guest_ds_limit = vmcs12->guest_ds_limit;
1828 evmcs->guest_fs_limit = vmcs12->guest_fs_limit;
1829 evmcs->guest_gs_limit = vmcs12->guest_gs_limit;
1830 evmcs->guest_ldtr_limit = vmcs12->guest_ldtr_limit;
1831 evmcs->guest_tr_limit = vmcs12->guest_tr_limit;
1832 evmcs->guest_gdtr_limit = vmcs12->guest_gdtr_limit;
1833 evmcs->guest_idtr_limit = vmcs12->guest_idtr_limit;
1834
1835 evmcs->guest_es_ar_bytes = vmcs12->guest_es_ar_bytes;
1836 evmcs->guest_cs_ar_bytes = vmcs12->guest_cs_ar_bytes;
1837 evmcs->guest_ss_ar_bytes = vmcs12->guest_ss_ar_bytes;
1838 evmcs->guest_ds_ar_bytes = vmcs12->guest_ds_ar_bytes;
1839 evmcs->guest_fs_ar_bytes = vmcs12->guest_fs_ar_bytes;
1840 evmcs->guest_gs_ar_bytes = vmcs12->guest_gs_ar_bytes;
1841 evmcs->guest_ldtr_ar_bytes = vmcs12->guest_ldtr_ar_bytes;
1842 evmcs->guest_tr_ar_bytes = vmcs12->guest_tr_ar_bytes;
1843
1844 evmcs->guest_es_base = vmcs12->guest_es_base;
1845 evmcs->guest_cs_base = vmcs12->guest_cs_base;
1846 evmcs->guest_ss_base = vmcs12->guest_ss_base;
1847 evmcs->guest_ds_base = vmcs12->guest_ds_base;
1848 evmcs->guest_fs_base = vmcs12->guest_fs_base;
1849 evmcs->guest_gs_base = vmcs12->guest_gs_base;
1850 evmcs->guest_ldtr_base = vmcs12->guest_ldtr_base;
1851 evmcs->guest_tr_base = vmcs12->guest_tr_base;
1852 evmcs->guest_gdtr_base = vmcs12->guest_gdtr_base;
1853 evmcs->guest_idtr_base = vmcs12->guest_idtr_base;
1854
1855 evmcs->guest_ia32_pat = vmcs12->guest_ia32_pat;
1856 evmcs->guest_ia32_efer = vmcs12->guest_ia32_efer;
1857
1858 evmcs->guest_pdptr0 = vmcs12->guest_pdptr0;
1859 evmcs->guest_pdptr1 = vmcs12->guest_pdptr1;
1860 evmcs->guest_pdptr2 = vmcs12->guest_pdptr2;
1861 evmcs->guest_pdptr3 = vmcs12->guest_pdptr3;
1862
1863 evmcs->guest_pending_dbg_exceptions =
1864 vmcs12->guest_pending_dbg_exceptions;
1865 evmcs->guest_sysenter_esp = vmcs12->guest_sysenter_esp;
1866 evmcs->guest_sysenter_eip = vmcs12->guest_sysenter_eip;
1867
1868 evmcs->guest_activity_state = vmcs12->guest_activity_state;
1869 evmcs->guest_sysenter_cs = vmcs12->guest_sysenter_cs;
1870
1871 evmcs->guest_cr0 = vmcs12->guest_cr0;
1872 evmcs->guest_cr3 = vmcs12->guest_cr3;
1873 evmcs->guest_cr4 = vmcs12->guest_cr4;
1874 evmcs->guest_dr7 = vmcs12->guest_dr7;
1875
1876 evmcs->guest_physical_address = vmcs12->guest_physical_address;
1877
1878 evmcs->vm_instruction_error = vmcs12->vm_instruction_error;
1879 evmcs->vm_exit_reason = vmcs12->vm_exit_reason;
1880 evmcs->vm_exit_intr_info = vmcs12->vm_exit_intr_info;
1881 evmcs->vm_exit_intr_error_code = vmcs12->vm_exit_intr_error_code;
1882 evmcs->idt_vectoring_info_field = vmcs12->idt_vectoring_info_field;
1883 evmcs->idt_vectoring_error_code = vmcs12->idt_vectoring_error_code;
1884 evmcs->vm_exit_instruction_len = vmcs12->vm_exit_instruction_len;
1885 evmcs->vmx_instruction_info = vmcs12->vmx_instruction_info;
1886
1887 evmcs->exit_qualification = vmcs12->exit_qualification;
1888
1889 evmcs->guest_linear_address = vmcs12->guest_linear_address;
1890 evmcs->guest_rsp = vmcs12->guest_rsp;
1891 evmcs->guest_rflags = vmcs12->guest_rflags;
1892
1893 evmcs->guest_interruptibility_info =
1894 vmcs12->guest_interruptibility_info;
1895 evmcs->cpu_based_vm_exec_control = vmcs12->cpu_based_vm_exec_control;
1896 evmcs->vm_entry_controls = vmcs12->vm_entry_controls;
1897 evmcs->vm_entry_intr_info_field = vmcs12->vm_entry_intr_info_field;
1898 evmcs->vm_entry_exception_error_code =
1899 vmcs12->vm_entry_exception_error_code;
1900 evmcs->vm_entry_instruction_len = vmcs12->vm_entry_instruction_len;
1901
1902 evmcs->guest_rip = vmcs12->guest_rip;
1903
1904 evmcs->guest_bndcfgs = vmcs12->guest_bndcfgs;
1905
1906 return 0;
1907}
1908
1909
1910
1911
1912
1913static int nested_vmx_handle_enlightened_vmptrld(struct kvm_vcpu *vcpu,
1914 bool from_launch)
1915{
1916 struct vcpu_vmx *vmx = to_vmx(vcpu);
1917 bool evmcs_gpa_changed = false;
1918 u64 evmcs_gpa;
1919
1920 if (likely(!vmx->nested.enlightened_vmcs_enabled))
1921 return 1;
1922
1923 if (!nested_enlightened_vmentry(vcpu, &evmcs_gpa))
1924 return 1;
1925
1926 if (unlikely(!vmx->nested.hv_evmcs ||
1927 evmcs_gpa != vmx->nested.hv_evmcs_vmptr)) {
1928 if (!vmx->nested.hv_evmcs)
1929 vmx->nested.current_vmptr = -1ull;
1930
1931 nested_release_evmcs(vcpu);
1932
1933 if (kvm_vcpu_map(vcpu, gpa_to_gfn(evmcs_gpa),
1934 &vmx->nested.hv_evmcs_map))
1935 return 0;
1936
1937 vmx->nested.hv_evmcs = vmx->nested.hv_evmcs_map.hva;
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961 if ((vmx->nested.hv_evmcs->revision_id != KVM_EVMCS_VERSION) &&
1962 (vmx->nested.hv_evmcs->revision_id != VMCS12_REVISION)) {
1963 nested_release_evmcs(vcpu);
1964 return 0;
1965 }
1966
1967 vmx->nested.dirty_vmcs12 = true;
1968 vmx->nested.hv_evmcs_vmptr = evmcs_gpa;
1969
1970 evmcs_gpa_changed = true;
1971
1972
1973
1974
1975
1976
1977 if (from_launch) {
1978 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
1979 memset(vmcs12, 0, sizeof(*vmcs12));
1980 vmcs12->hdr.revision_id = VMCS12_REVISION;
1981 }
1982
1983 }
1984
1985
1986
1987
1988
1989 if (from_launch || evmcs_gpa_changed)
1990 vmx->nested.hv_evmcs->hv_clean_fields &=
1991 ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL;
1992
1993 return 1;
1994}
1995
1996void nested_sync_vmcs12_to_shadow(struct kvm_vcpu *vcpu)
1997{
1998 struct vcpu_vmx *vmx = to_vmx(vcpu);
1999
2000
2001
2002
2003
2004
2005 if (vmx->nested.enlightened_vmcs_enabled && !vmx->nested.hv_evmcs)
2006 nested_vmx_handle_enlightened_vmptrld(vcpu, false);
2007
2008 if (vmx->nested.hv_evmcs) {
2009 copy_vmcs12_to_enlightened(vmx);
2010
2011 vmx->nested.hv_evmcs->hv_clean_fields |=
2012 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL;
2013 } else {
2014 copy_vmcs12_to_shadow(vmx);
2015 }
2016
2017 vmx->nested.need_vmcs12_to_shadow_sync = false;
2018}
2019
2020static enum hrtimer_restart vmx_preemption_timer_fn(struct hrtimer *timer)
2021{
2022 struct vcpu_vmx *vmx =
2023 container_of(timer, struct vcpu_vmx, nested.preemption_timer);
2024
2025 vmx->nested.preemption_timer_expired = true;
2026 kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu);
2027 kvm_vcpu_kick(&vmx->vcpu);
2028
2029 return HRTIMER_NORESTART;
2030}
2031
2032static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu)
2033{
2034 u64 preemption_timeout = get_vmcs12(vcpu)->vmx_preemption_timer_value;
2035 struct vcpu_vmx *vmx = to_vmx(vcpu);
2036
2037
2038
2039
2040
2041 if (preemption_timeout == 0) {
2042 vmx_preemption_timer_fn(&vmx->nested.preemption_timer);
2043 return;
2044 }
2045
2046 if (vcpu->arch.virtual_tsc_khz == 0)
2047 return;
2048
2049 preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
2050 preemption_timeout *= 1000000;
2051 do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz);
2052 hrtimer_start(&vmx->nested.preemption_timer,
2053 ns_to_ktime(preemption_timeout), HRTIMER_MODE_REL);
2054}
2055
2056static u64 nested_vmx_calc_efer(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12)
2057{
2058 if (vmx->nested.nested_run_pending &&
2059 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER))
2060 return vmcs12->guest_ia32_efer;
2061 else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE)
2062 return vmx->vcpu.arch.efer | (EFER_LMA | EFER_LME);
2063 else
2064 return vmx->vcpu.arch.efer & ~(EFER_LMA | EFER_LME);
2065}
2066
2067static void prepare_vmcs02_constant_state(struct vcpu_vmx *vmx)
2068{
2069
2070
2071
2072
2073
2074
2075 if (vmx->nested.vmcs02_initialized)
2076 return;
2077 vmx->nested.vmcs02_initialized = true;
2078
2079
2080
2081
2082
2083
2084 if (enable_ept && nested_early_check)
2085 vmcs_write64(EPT_POINTER, construct_eptp(&vmx->vcpu, 0));
2086
2087
2088 if (cpu_has_vmx_vmfunc())
2089 vmcs_write64(VM_FUNCTION_CONTROL, 0);
2090
2091 if (cpu_has_vmx_posted_intr())
2092 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_NESTED_VECTOR);
2093
2094 if (cpu_has_vmx_msr_bitmap())
2095 vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap));
2096
2097
2098
2099
2100
2101
2102
2103
2104 if (enable_pml) {
2105 vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg));
2106 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
2107 }
2108
2109 if (cpu_has_vmx_encls_vmexit())
2110 vmcs_write64(ENCLS_EXITING_BITMAP, -1ull);
2111
2112
2113
2114
2115
2116
2117 vmcs_write64(VM_EXIT_MSR_STORE_ADDR, __pa(vmx->msr_autostore.guest.val));
2118 vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val));
2119 vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val));
2120
2121 vmx_set_constant_host_state(vmx);
2122}
2123
2124static void prepare_vmcs02_early_rare(struct vcpu_vmx *vmx,
2125 struct vmcs12 *vmcs12)
2126{
2127 prepare_vmcs02_constant_state(vmx);
2128
2129 vmcs_write64(VMCS_LINK_POINTER, -1ull);
2130
2131 if (enable_vpid) {
2132 if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02)
2133 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02);
2134 else
2135 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
2136 }
2137}
2138
2139static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12)
2140{
2141 u32 exec_control, vmcs12_exec_ctrl;
2142 u64 guest_efer = nested_vmx_calc_efer(vmx, vmcs12);
2143
2144 if (vmx->nested.dirty_vmcs12 || vmx->nested.hv_evmcs)
2145 prepare_vmcs02_early_rare(vmx, vmcs12);
2146
2147
2148
2149
2150 exec_control = vmx_pin_based_exec_ctrl(vmx);
2151 exec_control |= (vmcs12->pin_based_vm_exec_control &
2152 ~PIN_BASED_VMX_PREEMPTION_TIMER);
2153
2154
2155 if (nested_cpu_has_posted_intr(vmcs12)) {
2156 vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv;
2157 vmx->nested.pi_pending = false;
2158 } else {
2159 exec_control &= ~PIN_BASED_POSTED_INTR;
2160 }
2161 pin_controls_set(vmx, exec_control);
2162
2163
2164
2165
2166 exec_control = vmx_exec_control(vmx);
2167 exec_control &= ~CPU_BASED_INTR_WINDOW_EXITING;
2168 exec_control &= ~CPU_BASED_NMI_WINDOW_EXITING;
2169 exec_control &= ~CPU_BASED_TPR_SHADOW;
2170 exec_control |= vmcs12->cpu_based_vm_exec_control;
2171
2172 vmx->nested.l1_tpr_threshold = -1;
2173 if (exec_control & CPU_BASED_TPR_SHADOW)
2174 vmcs_write32(TPR_THRESHOLD, vmcs12->tpr_threshold);
2175#ifdef CONFIG_X86_64
2176 else
2177 exec_control |= CPU_BASED_CR8_LOAD_EXITING |
2178 CPU_BASED_CR8_STORE_EXITING;
2179#endif
2180
2181
2182
2183
2184
2185 exec_control |= CPU_BASED_UNCOND_IO_EXITING;
2186 exec_control &= ~CPU_BASED_USE_IO_BITMAPS;
2187
2188
2189
2190
2191
2192
2193
2194 exec_control &= ~CPU_BASED_USE_MSR_BITMAPS;
2195 exec_control |= exec_controls_get(vmx) & CPU_BASED_USE_MSR_BITMAPS;
2196
2197 exec_controls_set(vmx, exec_control);
2198
2199
2200
2201
2202 if (cpu_has_secondary_exec_ctrls()) {
2203 exec_control = vmx->secondary_exec_control;
2204
2205
2206 exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
2207 SECONDARY_EXEC_ENABLE_INVPCID |
2208 SECONDARY_EXEC_RDTSCP |
2209 SECONDARY_EXEC_XSAVES |
2210 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE |
2211 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
2212 SECONDARY_EXEC_APIC_REGISTER_VIRT |
2213 SECONDARY_EXEC_ENABLE_VMFUNC);
2214 if (nested_cpu_has(vmcs12,
2215 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) {
2216 vmcs12_exec_ctrl = vmcs12->secondary_vm_exec_control &
2217 ~SECONDARY_EXEC_ENABLE_PML;
2218 exec_control |= vmcs12_exec_ctrl;
2219 }
2220
2221
2222 exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS;
2223
2224
2225
2226
2227
2228 if (!boot_cpu_has(X86_FEATURE_UMIP) && vmx_umip_emulated() &&
2229 (vmcs12->guest_cr4 & X86_CR4_UMIP))
2230 exec_control |= SECONDARY_EXEC_DESC;
2231
2232 if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY)
2233 vmcs_write16(GUEST_INTR_STATUS,
2234 vmcs12->guest_intr_status);
2235
2236 secondary_exec_controls_set(vmx, exec_control);
2237 }
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247 exec_control = (vmcs12->vm_entry_controls | vmx_vmentry_ctrl()) &
2248 ~VM_ENTRY_IA32E_MODE & ~VM_ENTRY_LOAD_IA32_EFER;
2249 if (cpu_has_load_ia32_efer()) {
2250 if (guest_efer & EFER_LMA)
2251 exec_control |= VM_ENTRY_IA32E_MODE;
2252 if (guest_efer != host_efer)
2253 exec_control |= VM_ENTRY_LOAD_IA32_EFER;
2254 }
2255 vm_entry_controls_set(vmx, exec_control);
2256
2257
2258
2259
2260
2261
2262
2263
2264 exec_control = vmx_vmexit_ctrl();
2265 if (cpu_has_load_ia32_efer() && guest_efer != host_efer)
2266 exec_control |= VM_EXIT_LOAD_IA32_EFER;
2267 vm_exit_controls_set(vmx, exec_control);
2268
2269
2270
2271
2272 if (vmx->nested.nested_run_pending) {
2273 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
2274 vmcs12->vm_entry_intr_info_field);
2275 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE,
2276 vmcs12->vm_entry_exception_error_code);
2277 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
2278 vmcs12->vm_entry_instruction_len);
2279 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO,
2280 vmcs12->guest_interruptibility_info);
2281 vmx->loaded_vmcs->nmi_known_unmasked =
2282 !(vmcs12->guest_interruptibility_info & GUEST_INTR_STATE_NMI);
2283 } else {
2284 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);
2285 }
2286}
2287
2288static void prepare_vmcs02_rare(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12)
2289{
2290 struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs;
2291
2292 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields &
2293 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2)) {
2294 vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector);
2295 vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector);
2296 vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector);
2297 vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector);
2298 vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector);
2299 vmcs_write16(GUEST_GS_SELECTOR, vmcs12->guest_gs_selector);
2300 vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector);
2301 vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector);
2302 vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit);
2303 vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit);
2304 vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit);
2305 vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit);
2306 vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit);
2307 vmcs_write32(GUEST_GS_LIMIT, vmcs12->guest_gs_limit);
2308 vmcs_write32(GUEST_LDTR_LIMIT, vmcs12->guest_ldtr_limit);
2309 vmcs_write32(GUEST_TR_LIMIT, vmcs12->guest_tr_limit);
2310 vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit);
2311 vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit);
2312 vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes);
2313 vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes);
2314 vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes);
2315 vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes);
2316 vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes);
2317 vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes);
2318 vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes);
2319 vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes);
2320 vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base);
2321 vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base);
2322 vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base);
2323 vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base);
2324 vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base);
2325 vmcs_writel(GUEST_GS_BASE, vmcs12->guest_gs_base);
2326 vmcs_writel(GUEST_LDTR_BASE, vmcs12->guest_ldtr_base);
2327 vmcs_writel(GUEST_TR_BASE, vmcs12->guest_tr_base);
2328 vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base);
2329 vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base);
2330 }
2331
2332 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields &
2333 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1)) {
2334 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs);
2335 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS,
2336 vmcs12->guest_pending_dbg_exceptions);
2337 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp);
2338 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip);
2339
2340
2341
2342
2343
2344 if (enable_ept) {
2345 vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0);
2346 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1);
2347 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2);
2348 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3);
2349 }
2350
2351 if (kvm_mpx_supported() && vmx->nested.nested_run_pending &&
2352 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS))
2353 vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs);
2354 }
2355
2356 if (nested_cpu_has_xsaves(vmcs12))
2357 vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap);
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK,
2374 enable_ept ? vmcs12->page_fault_error_code_mask : 0);
2375 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH,
2376 enable_ept ? vmcs12->page_fault_error_code_match : 0);
2377
2378 if (cpu_has_vmx_apicv()) {
2379 vmcs_write64(EOI_EXIT_BITMAP0, vmcs12->eoi_exit_bitmap0);
2380 vmcs_write64(EOI_EXIT_BITMAP1, vmcs12->eoi_exit_bitmap1);
2381 vmcs_write64(EOI_EXIT_BITMAP2, vmcs12->eoi_exit_bitmap2);
2382 vmcs_write64(EOI_EXIT_BITMAP3, vmcs12->eoi_exit_bitmap3);
2383 }
2384
2385
2386
2387
2388
2389 prepare_vmx_msr_autostore_list(&vmx->vcpu, MSR_IA32_TSC);
2390
2391 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, vmx->msr_autostore.guest.nr);
2392 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
2393 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
2394
2395 set_cr4_guest_host_mask(vmx);
2396}
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
2410 u32 *entry_failure_code)
2411{
2412 struct vcpu_vmx *vmx = to_vmx(vcpu);
2413 struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs;
2414 bool load_guest_pdptrs_vmcs12 = false;
2415
2416 if (vmx->nested.dirty_vmcs12 || hv_evmcs) {
2417 prepare_vmcs02_rare(vmx, vmcs12);
2418 vmx->nested.dirty_vmcs12 = false;
2419
2420 load_guest_pdptrs_vmcs12 = !hv_evmcs ||
2421 !(hv_evmcs->hv_clean_fields &
2422 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1);
2423 }
2424
2425 if (vmx->nested.nested_run_pending &&
2426 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) {
2427 kvm_set_dr(vcpu, 7, vmcs12->guest_dr7);
2428 vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl);
2429 } else {
2430 kvm_set_dr(vcpu, 7, vcpu->arch.dr7);
2431 vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.vmcs01_debugctl);
2432 }
2433 if (kvm_mpx_supported() && (!vmx->nested.nested_run_pending ||
2434 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)))
2435 vmcs_write64(GUEST_BNDCFGS, vmx->nested.vmcs01_guest_bndcfgs);
2436 vmx_set_rflags(vcpu, vmcs12->guest_rflags);
2437
2438
2439
2440
2441
2442 update_exception_bitmap(vcpu);
2443 vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask;
2444 vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
2445
2446 if (vmx->nested.nested_run_pending &&
2447 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT)) {
2448 vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat);
2449 vcpu->arch.pat = vmcs12->guest_ia32_pat;
2450 } else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
2451 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);
2452 }
2453
2454 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset);
2455
2456 if (kvm_has_tsc_control)
2457 decache_tsc_multiplier(vmx);
2458
2459 if (enable_vpid) {
2460
2461
2462
2463
2464
2465
2466
2467
2468 if (nested_cpu_has_vpid(vmcs12) && nested_has_guest_tlb_tag(vcpu)) {
2469 if (vmcs12->virtual_processor_id != vmx->nested.last_vpid) {
2470 vmx->nested.last_vpid = vmcs12->virtual_processor_id;
2471 __vmx_flush_tlb(vcpu, nested_get_vpid02(vcpu), false);
2472 }
2473 } else {
2474
2475
2476
2477
2478
2479
2480
2481
2482 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2483 }
2484 }
2485
2486 if (nested_cpu_has_ept(vmcs12))
2487 nested_ept_init_mmu_context(vcpu);
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497 vmx_set_cr0(vcpu, vmcs12->guest_cr0);
2498 vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12));
2499
2500 vmx_set_cr4(vcpu, vmcs12->guest_cr4);
2501 vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12));
2502
2503 vcpu->arch.efer = nested_vmx_calc_efer(vmx, vmcs12);
2504
2505 vmx_set_efer(vcpu, vcpu->arch.efer);
2506
2507
2508
2509
2510
2511
2512 if (vmx->emulation_required) {
2513 *entry_failure_code = ENTRY_FAIL_DEFAULT;
2514 return -EINVAL;
2515 }
2516
2517
2518 if (nested_vmx_load_cr3(vcpu, vmcs12->guest_cr3, nested_cpu_has_ept(vmcs12),
2519 entry_failure_code))
2520 return -EINVAL;
2521
2522
2523
2524
2525
2526
2527
2528
2529 if (enable_ept)
2530 vmcs_writel(GUEST_CR3, vmcs12->guest_cr3);
2531
2532
2533 if (load_guest_pdptrs_vmcs12 && nested_cpu_has_ept(vmcs12) &&
2534 is_pae_paging(vcpu)) {
2535 vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0);
2536 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1);
2537 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2);
2538 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3);
2539 }
2540
2541 if (!enable_ept)
2542 vcpu->arch.walk_mmu->inject_page_fault = vmx_inject_page_fault_nested;
2543
2544 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) &&
2545 WARN_ON_ONCE(kvm_set_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL,
2546 vmcs12->guest_ia32_perf_global_ctrl)))
2547 return -EINVAL;
2548
2549 kvm_rsp_write(vcpu, vmcs12->guest_rsp);
2550 kvm_rip_write(vcpu, vmcs12->guest_rip);
2551 return 0;
2552}
2553
2554static int nested_vmx_check_nmi_controls(struct vmcs12 *vmcs12)
2555{
2556 if (CC(!nested_cpu_has_nmi_exiting(vmcs12) &&
2557 nested_cpu_has_virtual_nmis(vmcs12)))
2558 return -EINVAL;
2559
2560 if (CC(!nested_cpu_has_virtual_nmis(vmcs12) &&
2561 nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING)))
2562 return -EINVAL;
2563
2564 return 0;
2565}
2566
2567static bool valid_ept_address(struct kvm_vcpu *vcpu, u64 address)
2568{
2569 struct vcpu_vmx *vmx = to_vmx(vcpu);
2570 int maxphyaddr = cpuid_maxphyaddr(vcpu);
2571
2572
2573 switch (address & VMX_EPTP_MT_MASK) {
2574 case VMX_EPTP_MT_UC:
2575 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT)))
2576 return false;
2577 break;
2578 case VMX_EPTP_MT_WB:
2579 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_WB_BIT)))
2580 return false;
2581 break;
2582 default:
2583 return false;
2584 }
2585
2586
2587 if (CC((address & VMX_EPTP_PWL_MASK) != VMX_EPTP_PWL_4))
2588 return false;
2589
2590
2591 if (CC(address >> maxphyaddr || ((address >> 7) & 0x1f)))
2592 return false;
2593
2594
2595 if (address & VMX_EPTP_AD_ENABLE_BIT) {
2596 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_AD_BIT)))
2597 return false;
2598 }
2599
2600 return true;
2601}
2602
2603
2604
2605
2606static int nested_check_vm_execution_controls(struct kvm_vcpu *vcpu,
2607 struct vmcs12 *vmcs12)
2608{
2609 struct vcpu_vmx *vmx = to_vmx(vcpu);
2610
2611 if (CC(!vmx_control_verify(vmcs12->pin_based_vm_exec_control,
2612 vmx->nested.msrs.pinbased_ctls_low,
2613 vmx->nested.msrs.pinbased_ctls_high)) ||
2614 CC(!vmx_control_verify(vmcs12->cpu_based_vm_exec_control,
2615 vmx->nested.msrs.procbased_ctls_low,
2616 vmx->nested.msrs.procbased_ctls_high)))
2617 return -EINVAL;
2618
2619 if (nested_cpu_has(vmcs12, CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) &&
2620 CC(!vmx_control_verify(vmcs12->secondary_vm_exec_control,
2621 vmx->nested.msrs.secondary_ctls_low,
2622 vmx->nested.msrs.secondary_ctls_high)))
2623 return -EINVAL;
2624
2625 if (CC(vmcs12->cr3_target_count > nested_cpu_vmx_misc_cr3_count(vcpu)) ||
2626 nested_vmx_check_io_bitmap_controls(vcpu, vmcs12) ||
2627 nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12) ||
2628 nested_vmx_check_tpr_shadow_controls(vcpu, vmcs12) ||
2629 nested_vmx_check_apic_access_controls(vcpu, vmcs12) ||
2630 nested_vmx_check_apicv_controls(vcpu, vmcs12) ||
2631 nested_vmx_check_nmi_controls(vmcs12) ||
2632 nested_vmx_check_pml_controls(vcpu, vmcs12) ||
2633 nested_vmx_check_unrestricted_guest_controls(vcpu, vmcs12) ||
2634 nested_vmx_check_mode_based_ept_exec_controls(vcpu, vmcs12) ||
2635 nested_vmx_check_shadow_vmcs_controls(vcpu, vmcs12) ||
2636 CC(nested_cpu_has_vpid(vmcs12) && !vmcs12->virtual_processor_id))
2637 return -EINVAL;
2638
2639 if (!nested_cpu_has_preemption_timer(vmcs12) &&
2640 nested_cpu_has_save_preemption_timer(vmcs12))
2641 return -EINVAL;
2642
2643 if (nested_cpu_has_ept(vmcs12) &&
2644 CC(!valid_ept_address(vcpu, vmcs12->ept_pointer)))
2645 return -EINVAL;
2646
2647 if (nested_cpu_has_vmfunc(vmcs12)) {
2648 if (CC(vmcs12->vm_function_control &
2649 ~vmx->nested.msrs.vmfunc_controls))
2650 return -EINVAL;
2651
2652 if (nested_cpu_has_eptp_switching(vmcs12)) {
2653 if (CC(!nested_cpu_has_ept(vmcs12)) ||
2654 CC(!page_address_valid(vcpu, vmcs12->eptp_list_address)))
2655 return -EINVAL;
2656 }
2657 }
2658
2659 return 0;
2660}
2661
2662
2663
2664
2665static int nested_check_vm_exit_controls(struct kvm_vcpu *vcpu,
2666 struct vmcs12 *vmcs12)
2667{
2668 struct vcpu_vmx *vmx = to_vmx(vcpu);
2669
2670 if (CC(!vmx_control_verify(vmcs12->vm_exit_controls,
2671 vmx->nested.msrs.exit_ctls_low,
2672 vmx->nested.msrs.exit_ctls_high)) ||
2673 CC(nested_vmx_check_exit_msr_switch_controls(vcpu, vmcs12)))
2674 return -EINVAL;
2675
2676 return 0;
2677}
2678
2679
2680
2681
2682static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu,
2683 struct vmcs12 *vmcs12)
2684{
2685 struct vcpu_vmx *vmx = to_vmx(vcpu);
2686
2687 if (CC(!vmx_control_verify(vmcs12->vm_entry_controls,
2688 vmx->nested.msrs.entry_ctls_low,
2689 vmx->nested.msrs.entry_ctls_high)))
2690 return -EINVAL;
2691
2692
2693
2694
2695
2696
2697
2698 if (vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) {
2699 u32 intr_info = vmcs12->vm_entry_intr_info_field;
2700 u8 vector = intr_info & INTR_INFO_VECTOR_MASK;
2701 u32 intr_type = intr_info & INTR_INFO_INTR_TYPE_MASK;
2702 bool has_error_code = intr_info & INTR_INFO_DELIVER_CODE_MASK;
2703 bool should_have_error_code;
2704 bool urg = nested_cpu_has2(vmcs12,
2705 SECONDARY_EXEC_UNRESTRICTED_GUEST);
2706 bool prot_mode = !urg || vmcs12->guest_cr0 & X86_CR0_PE;
2707
2708
2709 if (CC(intr_type == INTR_TYPE_RESERVED) ||
2710 CC(intr_type == INTR_TYPE_OTHER_EVENT &&
2711 !nested_cpu_supports_monitor_trap_flag(vcpu)))
2712 return -EINVAL;
2713
2714
2715 if (CC(intr_type == INTR_TYPE_NMI_INTR && vector != NMI_VECTOR) ||
2716 CC(intr_type == INTR_TYPE_HARD_EXCEPTION && vector > 31) ||
2717 CC(intr_type == INTR_TYPE_OTHER_EVENT && vector != 0))
2718 return -EINVAL;
2719
2720
2721 should_have_error_code =
2722 intr_type == INTR_TYPE_HARD_EXCEPTION && prot_mode &&
2723 x86_exception_has_error_code(vector);
2724 if (CC(has_error_code != should_have_error_code))
2725 return -EINVAL;
2726
2727
2728 if (CC(has_error_code &&
2729 vmcs12->vm_entry_exception_error_code & GENMASK(31, 16)))
2730 return -EINVAL;
2731
2732
2733 if (CC(intr_info & INTR_INFO_RESVD_BITS_MASK))
2734 return -EINVAL;
2735
2736
2737 switch (intr_type) {
2738 case INTR_TYPE_SOFT_EXCEPTION:
2739 case INTR_TYPE_SOFT_INTR:
2740 case INTR_TYPE_PRIV_SW_EXCEPTION:
2741 if (CC(vmcs12->vm_entry_instruction_len > 15) ||
2742 CC(vmcs12->vm_entry_instruction_len == 0 &&
2743 CC(!nested_cpu_has_zero_length_injection(vcpu))))
2744 return -EINVAL;
2745 }
2746 }
2747
2748 if (nested_vmx_check_entry_msr_switch_controls(vcpu, vmcs12))
2749 return -EINVAL;
2750
2751 return 0;
2752}
2753
2754static int nested_vmx_check_controls(struct kvm_vcpu *vcpu,
2755 struct vmcs12 *vmcs12)
2756{
2757 if (nested_check_vm_execution_controls(vcpu, vmcs12) ||
2758 nested_check_vm_exit_controls(vcpu, vmcs12) ||
2759 nested_check_vm_entry_controls(vcpu, vmcs12))
2760 return -EINVAL;
2761
2762 if (to_vmx(vcpu)->nested.enlightened_vmcs_enabled)
2763 return nested_evmcs_check_controls(vmcs12);
2764
2765 return 0;
2766}
2767
2768static int nested_vmx_check_host_state(struct kvm_vcpu *vcpu,
2769 struct vmcs12 *vmcs12)
2770{
2771 bool ia32e;
2772
2773 if (CC(!nested_host_cr0_valid(vcpu, vmcs12->host_cr0)) ||
2774 CC(!nested_host_cr4_valid(vcpu, vmcs12->host_cr4)) ||
2775 CC(!nested_cr3_valid(vcpu, vmcs12->host_cr3)))
2776 return -EINVAL;
2777
2778 if (CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_esp, vcpu)) ||
2779 CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_eip, vcpu)))
2780 return -EINVAL;
2781
2782 if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) &&
2783 CC(!kvm_pat_valid(vmcs12->host_ia32_pat)))
2784 return -EINVAL;
2785
2786 if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) &&
2787 CC(!kvm_valid_perf_global_ctrl(vcpu_to_pmu(vcpu),
2788 vmcs12->host_ia32_perf_global_ctrl)))
2789 return -EINVAL;
2790
2791#ifdef CONFIG_X86_64
2792 ia32e = !!(vcpu->arch.efer & EFER_LMA);
2793#else
2794 ia32e = false;
2795#endif
2796
2797 if (ia32e) {
2798 if (CC(!(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)) ||
2799 CC(!(vmcs12->host_cr4 & X86_CR4_PAE)))
2800 return -EINVAL;
2801 } else {
2802 if (CC(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) ||
2803 CC(vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) ||
2804 CC(vmcs12->host_cr4 & X86_CR4_PCIDE) ||
2805 CC((vmcs12->host_rip) >> 32))
2806 return -EINVAL;
2807 }
2808
2809 if (CC(vmcs12->host_cs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
2810 CC(vmcs12->host_ss_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
2811 CC(vmcs12->host_ds_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
2812 CC(vmcs12->host_es_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
2813 CC(vmcs12->host_fs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
2814 CC(vmcs12->host_gs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
2815 CC(vmcs12->host_tr_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
2816 CC(vmcs12->host_cs_selector == 0) ||
2817 CC(vmcs12->host_tr_selector == 0) ||
2818 CC(vmcs12->host_ss_selector == 0 && !ia32e))
2819 return -EINVAL;
2820
2821 if (CC(is_noncanonical_address(vmcs12->host_fs_base, vcpu)) ||
2822 CC(is_noncanonical_address(vmcs12->host_gs_base, vcpu)) ||
2823 CC(is_noncanonical_address(vmcs12->host_gdtr_base, vcpu)) ||
2824 CC(is_noncanonical_address(vmcs12->host_idtr_base, vcpu)) ||
2825 CC(is_noncanonical_address(vmcs12->host_tr_base, vcpu)) ||
2826 CC(is_noncanonical_address(vmcs12->host_rip, vcpu)))
2827 return -EINVAL;
2828
2829
2830
2831
2832
2833
2834
2835 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) {
2836 if (CC(!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer)) ||
2837 CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA)) ||
2838 CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LME)))
2839 return -EINVAL;
2840 }
2841
2842 return 0;
2843}
2844
2845static int nested_vmx_check_vmcs_link_ptr(struct kvm_vcpu *vcpu,
2846 struct vmcs12 *vmcs12)
2847{
2848 int r = 0;
2849 struct vmcs12 *shadow;
2850 struct kvm_host_map map;
2851
2852 if (vmcs12->vmcs_link_pointer == -1ull)
2853 return 0;
2854
2855 if (CC(!page_address_valid(vcpu, vmcs12->vmcs_link_pointer)))
2856 return -EINVAL;
2857
2858 if (CC(kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->vmcs_link_pointer), &map)))
2859 return -EINVAL;
2860
2861 shadow = map.hva;
2862
2863 if (CC(shadow->hdr.revision_id != VMCS12_REVISION) ||
2864 CC(shadow->hdr.shadow_vmcs != nested_cpu_has_shadow_vmcs(vmcs12)))
2865 r = -EINVAL;
2866
2867 kvm_vcpu_unmap(vcpu, &map, false);
2868 return r;
2869}
2870
2871
2872
2873
2874static int nested_check_guest_non_reg_state(struct vmcs12 *vmcs12)
2875{
2876 if (CC(vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE &&
2877 vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT))
2878 return -EINVAL;
2879
2880 return 0;
2881}
2882
2883static int nested_vmx_check_guest_state(struct kvm_vcpu *vcpu,
2884 struct vmcs12 *vmcs12,
2885 u32 *exit_qual)
2886{
2887 bool ia32e;
2888
2889 *exit_qual = ENTRY_FAIL_DEFAULT;
2890
2891 if (CC(!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0)) ||
2892 CC(!nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4)))
2893 return -EINVAL;
2894
2895 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) &&
2896 CC(!kvm_dr7_valid(vmcs12->guest_dr7)))
2897 return -EINVAL;
2898
2899 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT) &&
2900 CC(!kvm_pat_valid(vmcs12->guest_ia32_pat)))
2901 return -EINVAL;
2902
2903 if (nested_vmx_check_vmcs_link_ptr(vcpu, vmcs12)) {
2904 *exit_qual = ENTRY_FAIL_VMCS_LINK_PTR;
2905 return -EINVAL;
2906 }
2907
2908 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) &&
2909 CC(!kvm_valid_perf_global_ctrl(vcpu_to_pmu(vcpu),
2910 vmcs12->guest_ia32_perf_global_ctrl)))
2911 return -EINVAL;
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922 if (to_vmx(vcpu)->nested.nested_run_pending &&
2923 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) {
2924 ia32e = (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) != 0;
2925 if (CC(!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer)) ||
2926 CC(ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA)) ||
2927 CC(((vmcs12->guest_cr0 & X86_CR0_PG) &&
2928 ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME))))
2929 return -EINVAL;
2930 }
2931
2932 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS) &&
2933 (CC(is_noncanonical_address(vmcs12->guest_bndcfgs & PAGE_MASK, vcpu)) ||
2934 CC((vmcs12->guest_bndcfgs & MSR_IA32_BNDCFGS_RSVD))))
2935 return -EINVAL;
2936
2937 if (nested_check_guest_non_reg_state(vmcs12))
2938 return -EINVAL;
2939
2940 return 0;
2941}
2942
2943static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu)
2944{
2945 struct vcpu_vmx *vmx = to_vmx(vcpu);
2946 unsigned long cr3, cr4;
2947 bool vm_fail;
2948
2949 if (!nested_early_check)
2950 return 0;
2951
2952 if (vmx->msr_autoload.host.nr)
2953 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
2954 if (vmx->msr_autoload.guest.nr)
2955 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
2956
2957 preempt_disable();
2958
2959 vmx_prepare_switch_to_guest(vcpu);
2960
2961
2962
2963
2964
2965
2966
2967 vmcs_writel(GUEST_RFLAGS, 0);
2968
2969 cr3 = __get_current_cr3_fast();
2970 if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) {
2971 vmcs_writel(HOST_CR3, cr3);
2972 vmx->loaded_vmcs->host_state.cr3 = cr3;
2973 }
2974
2975 cr4 = cr4_read_shadow();
2976 if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) {
2977 vmcs_writel(HOST_CR4, cr4);
2978 vmx->loaded_vmcs->host_state.cr4 = cr4;
2979 }
2980
2981 asm(
2982 "sub $%c[wordsize], %%" _ASM_SP "\n\t"
2983 "cmp %%" _ASM_SP ", %c[host_state_rsp](%[loaded_vmcs]) \n\t"
2984 "je 1f \n\t"
2985 __ex("vmwrite %%" _ASM_SP ", %[HOST_RSP]") "\n\t"
2986 "mov %%" _ASM_SP ", %c[host_state_rsp](%[loaded_vmcs]) \n\t"
2987 "1: \n\t"
2988 "add $%c[wordsize], %%" _ASM_SP "\n\t"
2989
2990
2991 "cmpb $0, %c[launched](%[loaded_vmcs])\n\t"
2992
2993
2994
2995
2996
2997
2998
2999 "call vmx_vmenter\n\t"
3000
3001 CC_SET(be)
3002 : ASM_CALL_CONSTRAINT, CC_OUT(be) (vm_fail)
3003 : [HOST_RSP]"r"((unsigned long)HOST_RSP),
3004 [loaded_vmcs]"r"(vmx->loaded_vmcs),
3005 [launched]"i"(offsetof(struct loaded_vmcs, launched)),
3006 [host_state_rsp]"i"(offsetof(struct loaded_vmcs, host_state.rsp)),
3007 [wordsize]"i"(sizeof(ulong))
3008 : "memory"
3009 );
3010
3011 if (vmx->msr_autoload.host.nr)
3012 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
3013 if (vmx->msr_autoload.guest.nr)
3014 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
3015
3016 if (vm_fail) {
3017 u32 error = vmcs_read32(VM_INSTRUCTION_ERROR);
3018
3019 preempt_enable();
3020
3021 trace_kvm_nested_vmenter_failed(
3022 "early hardware check VM-instruction error: ", error);
3023 WARN_ON_ONCE(error != VMXERR_ENTRY_INVALID_CONTROL_FIELD);
3024 return 1;
3025 }
3026
3027
3028
3029
3030 local_irq_enable();
3031 if (hw_breakpoint_active())
3032 set_debugreg(__this_cpu_read(cpu_dr7), 7);
3033 preempt_enable();
3034
3035
3036
3037
3038
3039
3040
3041
3042 WARN_ON(!(vmcs_read32(VM_EXIT_REASON) &
3043 VMX_EXIT_REASONS_FAILED_VMENTRY));
3044
3045 return 0;
3046}
3047
3048static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu)
3049{
3050 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3051 struct vcpu_vmx *vmx = to_vmx(vcpu);
3052 struct kvm_host_map *map;
3053 struct page *page;
3054 u64 hpa;
3055
3056 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
3057
3058
3059
3060
3061
3062
3063 if (vmx->nested.apic_access_page) {
3064 kvm_release_page_clean(vmx->nested.apic_access_page);
3065 vmx->nested.apic_access_page = NULL;
3066 }
3067 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->apic_access_addr);
3068 if (!is_error_page(page)) {
3069 vmx->nested.apic_access_page = page;
3070 hpa = page_to_phys(vmx->nested.apic_access_page);
3071 vmcs_write64(APIC_ACCESS_ADDR, hpa);
3072 } else {
3073 pr_debug_ratelimited("%s: no backing 'struct page' for APIC-access address in vmcs12\n",
3074 __func__);
3075 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
3076 vcpu->run->internal.suberror =
3077 KVM_INTERNAL_ERROR_EMULATION;
3078 vcpu->run->internal.ndata = 0;
3079 return false;
3080 }
3081 }
3082
3083 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) {
3084 map = &vmx->nested.virtual_apic_map;
3085
3086 if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->virtual_apic_page_addr), map)) {
3087 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, pfn_to_hpa(map->pfn));
3088 } else if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING) &&
3089 nested_cpu_has(vmcs12, CPU_BASED_CR8_STORE_EXITING) &&
3090 !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
3091
3092
3093
3094
3095
3096
3097
3098
3099 exec_controls_clearbit(vmx, CPU_BASED_TPR_SHADOW);
3100 } else {
3101
3102
3103
3104
3105 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, -1ull);
3106 }
3107 }
3108
3109 if (nested_cpu_has_posted_intr(vmcs12)) {
3110 map = &vmx->nested.pi_desc_map;
3111
3112 if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->posted_intr_desc_addr), map)) {
3113 vmx->nested.pi_desc =
3114 (struct pi_desc *)(((void *)map->hva) +
3115 offset_in_page(vmcs12->posted_intr_desc_addr));
3116 vmcs_write64(POSTED_INTR_DESC_ADDR,
3117 pfn_to_hpa(map->pfn) + offset_in_page(vmcs12->posted_intr_desc_addr));
3118 }
3119 }
3120 if (nested_vmx_prepare_msr_bitmap(vcpu, vmcs12))
3121 exec_controls_setbit(vmx, CPU_BASED_USE_MSR_BITMAPS);
3122 else
3123 exec_controls_clearbit(vmx, CPU_BASED_USE_MSR_BITMAPS);
3124 return true;
3125}
3126
3127
3128
3129
3130
3131
3132
3133
3134static int nested_vmx_check_permission(struct kvm_vcpu *vcpu)
3135{
3136 if (!to_vmx(vcpu)->nested.vmxon) {
3137 kvm_queue_exception(vcpu, UD_VECTOR);
3138 return 0;
3139 }
3140
3141 if (vmx_get_cpl(vcpu)) {
3142 kvm_inject_gp(vcpu, 0);
3143 return 0;
3144 }
3145
3146 return 1;
3147}
3148
3149static u8 vmx_has_apicv_interrupt(struct kvm_vcpu *vcpu)
3150{
3151 u8 rvi = vmx_get_rvi();
3152 u8 vppr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_PROCPRI);
3153
3154 return ((rvi & 0xf0) > (vppr & 0xf0));
3155}
3156
3157static void load_vmcs12_host_state(struct kvm_vcpu *vcpu,
3158 struct vmcs12 *vmcs12);
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu,
3171 bool from_vmentry)
3172{
3173 struct vcpu_vmx *vmx = to_vmx(vcpu);
3174 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3175 bool evaluate_pending_interrupts;
3176 u32 exit_reason = EXIT_REASON_INVALID_STATE;
3177 u32 exit_qual;
3178
3179 evaluate_pending_interrupts = exec_controls_get(vmx) &
3180 (CPU_BASED_INTR_WINDOW_EXITING | CPU_BASED_NMI_WINDOW_EXITING);
3181 if (likely(!evaluate_pending_interrupts) && kvm_vcpu_apicv_active(vcpu))
3182 evaluate_pending_interrupts |= vmx_has_apicv_interrupt(vcpu);
3183
3184 if (!(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS))
3185 vmx->nested.vmcs01_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL);
3186 if (kvm_mpx_supported() &&
3187 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS))
3188 vmx->nested.vmcs01_guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS);
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206 if (!enable_ept && !nested_early_check)
3207 vmcs_writel(GUEST_CR3, vcpu->arch.cr3);
3208
3209 vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02);
3210
3211 prepare_vmcs02_early(vmx, vmcs12);
3212
3213 if (from_vmentry) {
3214 if (unlikely(!nested_get_vmcs12_pages(vcpu)))
3215 return NVMX_VMENTRY_KVM_INTERNAL_ERROR;
3216
3217 if (nested_vmx_check_vmentry_hw(vcpu)) {
3218 vmx_switch_vmcs(vcpu, &vmx->vmcs01);
3219 return NVMX_VMENTRY_VMFAIL;
3220 }
3221
3222 if (nested_vmx_check_guest_state(vcpu, vmcs12, &exit_qual))
3223 goto vmentry_fail_vmexit;
3224 }
3225
3226 enter_guest_mode(vcpu);
3227 if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING)
3228 vcpu->arch.tsc_offset += vmcs12->tsc_offset;
3229
3230 if (prepare_vmcs02(vcpu, vmcs12, &exit_qual))
3231 goto vmentry_fail_vmexit_guest_mode;
3232
3233 if (from_vmentry) {
3234 exit_reason = EXIT_REASON_MSR_LOAD_FAIL;
3235 exit_qual = nested_vmx_load_msr(vcpu,
3236 vmcs12->vm_entry_msr_load_addr,
3237 vmcs12->vm_entry_msr_load_count);
3238 if (exit_qual)
3239 goto vmentry_fail_vmexit_guest_mode;
3240 } else {
3241
3242
3243
3244
3245
3246
3247
3248 kvm_make_request(KVM_REQ_GET_VMCS12_PAGES, vcpu);
3249 }
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265 if (unlikely(evaluate_pending_interrupts))
3266 kvm_make_request(KVM_REQ_EVENT, vcpu);
3267
3268
3269
3270
3271
3272
3273 vmx->nested.preemption_timer_expired = false;
3274 if (nested_cpu_has_preemption_timer(vmcs12))
3275 vmx_start_preemption_timer(vcpu);
3276
3277
3278
3279
3280
3281
3282
3283 return NVMX_VMENTRY_SUCCESS;
3284
3285
3286
3287
3288
3289
3290vmentry_fail_vmexit_guest_mode:
3291 if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING)
3292 vcpu->arch.tsc_offset -= vmcs12->tsc_offset;
3293 leave_guest_mode(vcpu);
3294
3295vmentry_fail_vmexit:
3296 vmx_switch_vmcs(vcpu, &vmx->vmcs01);
3297
3298 if (!from_vmentry)
3299 return NVMX_VMENTRY_VMEXIT;
3300
3301 load_vmcs12_host_state(vcpu, vmcs12);
3302 vmcs12->vm_exit_reason = exit_reason | VMX_EXIT_REASONS_FAILED_VMENTRY;
3303 vmcs12->exit_qualification = exit_qual;
3304 if (enable_shadow_vmcs || vmx->nested.hv_evmcs)
3305 vmx->nested.need_vmcs12_to_shadow_sync = true;
3306 return NVMX_VMENTRY_VMEXIT;
3307}
3308
3309
3310
3311
3312
3313static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
3314{
3315 struct vmcs12 *vmcs12;
3316 enum nvmx_vmentry_status status;
3317 struct vcpu_vmx *vmx = to_vmx(vcpu);
3318 u32 interrupt_shadow = vmx_get_interrupt_shadow(vcpu);
3319
3320 if (!nested_vmx_check_permission(vcpu))
3321 return 1;
3322
3323 if (!nested_vmx_handle_enlightened_vmptrld(vcpu, launch))
3324 return 1;
3325
3326 if (!vmx->nested.hv_evmcs && vmx->nested.current_vmptr == -1ull)
3327 return nested_vmx_failInvalid(vcpu);
3328
3329 vmcs12 = get_vmcs12(vcpu);
3330
3331
3332
3333
3334
3335
3336
3337 if (vmcs12->hdr.shadow_vmcs)
3338 return nested_vmx_failInvalid(vcpu);
3339
3340 if (vmx->nested.hv_evmcs) {
3341 copy_enlightened_to_vmcs12(vmx);
3342
3343 vmcs12->launch_state = !launch;
3344 } else if (enable_shadow_vmcs) {
3345 copy_shadow_to_vmcs12(vmx);
3346 }
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358 if (interrupt_shadow & KVM_X86_SHADOW_INT_MOV_SS)
3359 return nested_vmx_failValid(vcpu,
3360 VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS);
3361
3362 if (vmcs12->launch_state == launch)
3363 return nested_vmx_failValid(vcpu,
3364 launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS
3365 : VMXERR_VMRESUME_NONLAUNCHED_VMCS);
3366
3367 if (nested_vmx_check_controls(vcpu, vmcs12))
3368 return nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
3369
3370 if (nested_vmx_check_host_state(vcpu, vmcs12))
3371 return nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_HOST_STATE_FIELD);
3372
3373
3374
3375
3376
3377 vmx->nested.nested_run_pending = 1;
3378 status = nested_vmx_enter_non_root_mode(vcpu, true);
3379 if (unlikely(status != NVMX_VMENTRY_SUCCESS))
3380 goto vmentry_failed;
3381
3382
3383 vmx->vcpu.arch.l1tf_flush_l1d = true;
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395 nested_cache_shadow_vmcs12(vcpu, vmcs12);
3396
3397
3398
3399
3400
3401
3402 if ((vmcs12->guest_activity_state == GUEST_ACTIVITY_HLT) &&
3403 !(vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) &&
3404 !(vmcs12->cpu_based_vm_exec_control & CPU_BASED_NMI_WINDOW_EXITING) &&
3405 !((vmcs12->cpu_based_vm_exec_control & CPU_BASED_INTR_WINDOW_EXITING) &&
3406 (vmcs12->guest_rflags & X86_EFLAGS_IF))) {
3407 vmx->nested.nested_run_pending = 0;
3408 return kvm_vcpu_halt(vcpu);
3409 }
3410 return 1;
3411
3412vmentry_failed:
3413 vmx->nested.nested_run_pending = 0;
3414 if (status == NVMX_VMENTRY_KVM_INTERNAL_ERROR)
3415 return 0;
3416 if (status == NVMX_VMENTRY_VMEXIT)
3417 return 1;
3418 WARN_ON_ONCE(status != NVMX_VMENTRY_VMFAIL);
3419 return nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
3420}
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439static inline unsigned long
3440vmcs12_guest_cr0(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
3441{
3442 return
3443 (vmcs_readl(GUEST_CR0) & vcpu->arch.cr0_guest_owned_bits) |
3444 (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask) |
3445 (vmcs_readl(CR0_READ_SHADOW) & ~(vmcs12->cr0_guest_host_mask |
3446 vcpu->arch.cr0_guest_owned_bits));
3447}
3448
3449static inline unsigned long
3450vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
3451{
3452 return
3453 (vmcs_readl(GUEST_CR4) & vcpu->arch.cr4_guest_owned_bits) |
3454 (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask) |
3455 (vmcs_readl(CR4_READ_SHADOW) & ~(vmcs12->cr4_guest_host_mask |
3456 vcpu->arch.cr4_guest_owned_bits));
3457}
3458
3459static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu,
3460 struct vmcs12 *vmcs12)
3461{
3462 u32 idt_vectoring;
3463 unsigned int nr;
3464
3465 if (vcpu->arch.exception.injected) {
3466 nr = vcpu->arch.exception.nr;
3467 idt_vectoring = nr | VECTORING_INFO_VALID_MASK;
3468
3469 if (kvm_exception_is_soft(nr)) {
3470 vmcs12->vm_exit_instruction_len =
3471 vcpu->arch.event_exit_inst_len;
3472 idt_vectoring |= INTR_TYPE_SOFT_EXCEPTION;
3473 } else
3474 idt_vectoring |= INTR_TYPE_HARD_EXCEPTION;
3475
3476 if (vcpu->arch.exception.has_error_code) {
3477 idt_vectoring |= VECTORING_INFO_DELIVER_CODE_MASK;
3478 vmcs12->idt_vectoring_error_code =
3479 vcpu->arch.exception.error_code;
3480 }
3481
3482 vmcs12->idt_vectoring_info_field = idt_vectoring;
3483 } else if (vcpu->arch.nmi_injected) {
3484 vmcs12->idt_vectoring_info_field =
3485 INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR;
3486 } else if (vcpu->arch.interrupt.injected) {
3487 nr = vcpu->arch.interrupt.nr;
3488 idt_vectoring = nr | VECTORING_INFO_VALID_MASK;
3489
3490 if (vcpu->arch.interrupt.soft) {
3491 idt_vectoring |= INTR_TYPE_SOFT_INTR;
3492 vmcs12->vm_entry_instruction_len =
3493 vcpu->arch.event_exit_inst_len;
3494 } else
3495 idt_vectoring |= INTR_TYPE_EXT_INTR;
3496
3497 vmcs12->idt_vectoring_info_field = idt_vectoring;
3498 }
3499}
3500
3501
3502static void nested_mark_vmcs12_pages_dirty(struct kvm_vcpu *vcpu)
3503{
3504 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3505 gfn_t gfn;
3506
3507
3508
3509
3510
3511
3512 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) {
3513 gfn = vmcs12->virtual_apic_page_addr >> PAGE_SHIFT;
3514 kvm_vcpu_mark_page_dirty(vcpu, gfn);
3515 }
3516
3517 if (nested_cpu_has_posted_intr(vmcs12)) {
3518 gfn = vmcs12->posted_intr_desc_addr >> PAGE_SHIFT;
3519 kvm_vcpu_mark_page_dirty(vcpu, gfn);
3520 }
3521}
3522
3523static void vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu)
3524{
3525 struct vcpu_vmx *vmx = to_vmx(vcpu);
3526 int max_irr;
3527 void *vapic_page;
3528 u16 status;
3529
3530 if (!vmx->nested.pi_desc || !vmx->nested.pi_pending)
3531 return;
3532
3533 vmx->nested.pi_pending = false;
3534 if (!pi_test_and_clear_on(vmx->nested.pi_desc))
3535 return;
3536
3537 max_irr = find_last_bit((unsigned long *)vmx->nested.pi_desc->pir, 256);
3538 if (max_irr != 256) {
3539 vapic_page = vmx->nested.virtual_apic_map.hva;
3540 if (!vapic_page)
3541 return;
3542
3543 __kvm_apic_update_irr(vmx->nested.pi_desc->pir,
3544 vapic_page, &max_irr);
3545 status = vmcs_read16(GUEST_INTR_STATUS);
3546 if ((u8)max_irr > ((u8)status & 0xff)) {
3547 status &= ~0xff;
3548 status |= (u8)max_irr;
3549 vmcs_write16(GUEST_INTR_STATUS, status);
3550 }
3551 }
3552
3553 nested_mark_vmcs12_pages_dirty(vcpu);
3554}
3555
3556static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu,
3557 unsigned long exit_qual)
3558{
3559 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3560 unsigned int nr = vcpu->arch.exception.nr;
3561 u32 intr_info = nr | INTR_INFO_VALID_MASK;
3562
3563 if (vcpu->arch.exception.has_error_code) {
3564 vmcs12->vm_exit_intr_error_code = vcpu->arch.exception.error_code;
3565 intr_info |= INTR_INFO_DELIVER_CODE_MASK;
3566 }
3567
3568 if (kvm_exception_is_soft(nr))
3569 intr_info |= INTR_TYPE_SOFT_EXCEPTION;
3570 else
3571 intr_info |= INTR_TYPE_HARD_EXCEPTION;
3572
3573 if (!(vmcs12->idt_vectoring_info_field & VECTORING_INFO_VALID_MASK) &&
3574 vmx_get_nmi_mask(vcpu))
3575 intr_info |= INTR_INFO_UNBLOCK_NMI;
3576
3577 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, intr_info, exit_qual);
3578}
3579
3580
3581
3582
3583
3584
3585
3586static inline bool vmx_pending_dbg_trap(struct kvm_vcpu *vcpu)
3587{
3588 return vcpu->arch.exception.pending &&
3589 vcpu->arch.exception.nr == DB_VECTOR &&
3590 vcpu->arch.exception.payload;
3591}
3592
3593
3594
3595
3596
3597
3598
3599
3600static void nested_vmx_update_pending_dbg(struct kvm_vcpu *vcpu)
3601{
3602 if (vmx_pending_dbg_trap(vcpu))
3603 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS,
3604 vcpu->arch.exception.payload);
3605}
3606
3607static int vmx_check_nested_events(struct kvm_vcpu *vcpu, bool external_intr)
3608{
3609 struct vcpu_vmx *vmx = to_vmx(vcpu);
3610 unsigned long exit_qual;
3611 bool block_nested_events =
3612 vmx->nested.nested_run_pending || kvm_event_needs_reinjection(vcpu);
3613 bool mtf_pending = vmx->nested.mtf_pending;
3614 struct kvm_lapic *apic = vcpu->arch.apic;
3615
3616
3617
3618
3619
3620 vmx->nested.mtf_pending = false;
3621
3622 if (lapic_in_kernel(vcpu) &&
3623 test_bit(KVM_APIC_INIT, &apic->pending_events)) {
3624 if (block_nested_events)
3625 return -EBUSY;
3626 nested_vmx_update_pending_dbg(vcpu);
3627 clear_bit(KVM_APIC_INIT, &apic->pending_events);
3628 nested_vmx_vmexit(vcpu, EXIT_REASON_INIT_SIGNAL, 0, 0);
3629 return 0;
3630 }
3631
3632
3633
3634
3635 if (vcpu->arch.exception.pending &&
3636 !vmx_pending_dbg_trap(vcpu) &&
3637 nested_vmx_check_exception(vcpu, &exit_qual)) {
3638 if (block_nested_events)
3639 return -EBUSY;
3640 nested_vmx_inject_exception_vmexit(vcpu, exit_qual);
3641 return 0;
3642 }
3643
3644 if (mtf_pending) {
3645 if (block_nested_events)
3646 return -EBUSY;
3647 nested_vmx_update_pending_dbg(vcpu);
3648 nested_vmx_vmexit(vcpu, EXIT_REASON_MONITOR_TRAP_FLAG, 0, 0);
3649 return 0;
3650 }
3651
3652 if (vcpu->arch.exception.pending &&
3653 nested_vmx_check_exception(vcpu, &exit_qual)) {
3654 if (block_nested_events)
3655 return -EBUSY;
3656 nested_vmx_inject_exception_vmexit(vcpu, exit_qual);
3657 return 0;
3658 }
3659
3660 if (nested_cpu_has_preemption_timer(get_vmcs12(vcpu)) &&
3661 vmx->nested.preemption_timer_expired) {
3662 if (block_nested_events)
3663 return -EBUSY;
3664 nested_vmx_vmexit(vcpu, EXIT_REASON_PREEMPTION_TIMER, 0, 0);
3665 return 0;
3666 }
3667
3668 if (vcpu->arch.nmi_pending && nested_exit_on_nmi(vcpu)) {
3669 if (block_nested_events)
3670 return -EBUSY;
3671 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI,
3672 NMI_VECTOR | INTR_TYPE_NMI_INTR |
3673 INTR_INFO_VALID_MASK, 0);
3674
3675
3676
3677
3678 vcpu->arch.nmi_pending = 0;
3679 vmx_set_nmi_mask(vcpu, true);
3680 return 0;
3681 }
3682
3683 if ((kvm_cpu_has_interrupt(vcpu) || external_intr) &&
3684 nested_exit_on_intr(vcpu)) {
3685 if (block_nested_events)
3686 return -EBUSY;
3687 nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 0, 0);
3688 return 0;
3689 }
3690
3691 vmx_complete_nested_posted_interrupt(vcpu);
3692 return 0;
3693}
3694
3695static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu)
3696{
3697 ktime_t remaining =
3698 hrtimer_get_remaining(&to_vmx(vcpu)->nested.preemption_timer);
3699 u64 value;
3700
3701 if (ktime_to_ns(remaining) <= 0)
3702 return 0;
3703
3704 value = ktime_to_ns(remaining) * vcpu->arch.virtual_tsc_khz;
3705 do_div(value, 1000000);
3706 return value >> VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
3707}
3708
3709static bool is_vmcs12_ext_field(unsigned long field)
3710{
3711 switch (field) {
3712 case GUEST_ES_SELECTOR:
3713 case GUEST_CS_SELECTOR:
3714 case GUEST_SS_SELECTOR:
3715 case GUEST_DS_SELECTOR:
3716 case GUEST_FS_SELECTOR:
3717 case GUEST_GS_SELECTOR:
3718 case GUEST_LDTR_SELECTOR:
3719 case GUEST_TR_SELECTOR:
3720 case GUEST_ES_LIMIT:
3721 case GUEST_CS_LIMIT:
3722 case GUEST_SS_LIMIT:
3723 case GUEST_DS_LIMIT:
3724 case GUEST_FS_LIMIT:
3725 case GUEST_GS_LIMIT:
3726 case GUEST_LDTR_LIMIT:
3727 case GUEST_TR_LIMIT:
3728 case GUEST_GDTR_LIMIT:
3729 case GUEST_IDTR_LIMIT:
3730 case GUEST_ES_AR_BYTES:
3731 case GUEST_DS_AR_BYTES:
3732 case GUEST_FS_AR_BYTES:
3733 case GUEST_GS_AR_BYTES:
3734 case GUEST_LDTR_AR_BYTES:
3735 case GUEST_TR_AR_BYTES:
3736 case GUEST_ES_BASE:
3737 case GUEST_CS_BASE:
3738 case GUEST_SS_BASE:
3739 case GUEST_DS_BASE:
3740 case GUEST_FS_BASE:
3741 case GUEST_GS_BASE:
3742 case GUEST_LDTR_BASE:
3743 case GUEST_TR_BASE:
3744 case GUEST_GDTR_BASE:
3745 case GUEST_IDTR_BASE:
3746 case GUEST_PENDING_DBG_EXCEPTIONS:
3747 case GUEST_BNDCFGS:
3748 return true;
3749 default:
3750 break;
3751 }
3752
3753 return false;
3754}
3755
3756static void sync_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu,
3757 struct vmcs12 *vmcs12)
3758{
3759 struct vcpu_vmx *vmx = to_vmx(vcpu);
3760
3761 vmcs12->guest_es_selector = vmcs_read16(GUEST_ES_SELECTOR);
3762 vmcs12->guest_cs_selector = vmcs_read16(GUEST_CS_SELECTOR);
3763 vmcs12->guest_ss_selector = vmcs_read16(GUEST_SS_SELECTOR);
3764 vmcs12->guest_ds_selector = vmcs_read16(GUEST_DS_SELECTOR);
3765 vmcs12->guest_fs_selector = vmcs_read16(GUEST_FS_SELECTOR);
3766 vmcs12->guest_gs_selector = vmcs_read16(GUEST_GS_SELECTOR);
3767 vmcs12->guest_ldtr_selector = vmcs_read16(GUEST_LDTR_SELECTOR);
3768 vmcs12->guest_tr_selector = vmcs_read16(GUEST_TR_SELECTOR);
3769 vmcs12->guest_es_limit = vmcs_read32(GUEST_ES_LIMIT);
3770 vmcs12->guest_cs_limit = vmcs_read32(GUEST_CS_LIMIT);
3771 vmcs12->guest_ss_limit = vmcs_read32(GUEST_SS_LIMIT);
3772 vmcs12->guest_ds_limit = vmcs_read32(GUEST_DS_LIMIT);
3773 vmcs12->guest_fs_limit = vmcs_read32(GUEST_FS_LIMIT);
3774 vmcs12->guest_gs_limit = vmcs_read32(GUEST_GS_LIMIT);
3775 vmcs12->guest_ldtr_limit = vmcs_read32(GUEST_LDTR_LIMIT);
3776 vmcs12->guest_tr_limit = vmcs_read32(GUEST_TR_LIMIT);
3777 vmcs12->guest_gdtr_limit = vmcs_read32(GUEST_GDTR_LIMIT);
3778 vmcs12->guest_idtr_limit = vmcs_read32(GUEST_IDTR_LIMIT);
3779 vmcs12->guest_es_ar_bytes = vmcs_read32(GUEST_ES_AR_BYTES);
3780 vmcs12->guest_ds_ar_bytes = vmcs_read32(GUEST_DS_AR_BYTES);
3781 vmcs12->guest_fs_ar_bytes = vmcs_read32(GUEST_FS_AR_BYTES);
3782 vmcs12->guest_gs_ar_bytes = vmcs_read32(GUEST_GS_AR_BYTES);
3783 vmcs12->guest_ldtr_ar_bytes = vmcs_read32(GUEST_LDTR_AR_BYTES);
3784 vmcs12->guest_tr_ar_bytes = vmcs_read32(GUEST_TR_AR_BYTES);
3785 vmcs12->guest_es_base = vmcs_readl(GUEST_ES_BASE);
3786 vmcs12->guest_cs_base = vmcs_readl(GUEST_CS_BASE);
3787 vmcs12->guest_ss_base = vmcs_readl(GUEST_SS_BASE);
3788 vmcs12->guest_ds_base = vmcs_readl(GUEST_DS_BASE);
3789 vmcs12->guest_fs_base = vmcs_readl(GUEST_FS_BASE);
3790 vmcs12->guest_gs_base = vmcs_readl(GUEST_GS_BASE);
3791 vmcs12->guest_ldtr_base = vmcs_readl(GUEST_LDTR_BASE);
3792 vmcs12->guest_tr_base = vmcs_readl(GUEST_TR_BASE);
3793 vmcs12->guest_gdtr_base = vmcs_readl(GUEST_GDTR_BASE);
3794 vmcs12->guest_idtr_base = vmcs_readl(GUEST_IDTR_BASE);
3795 vmcs12->guest_pending_dbg_exceptions =
3796 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS);
3797 if (kvm_mpx_supported())
3798 vmcs12->guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS);
3799
3800 vmx->nested.need_sync_vmcs02_to_vmcs12_rare = false;
3801}
3802
3803static void copy_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu,
3804 struct vmcs12 *vmcs12)
3805{
3806 struct vcpu_vmx *vmx = to_vmx(vcpu);
3807 int cpu;
3808
3809 if (!vmx->nested.need_sync_vmcs02_to_vmcs12_rare)
3810 return;
3811
3812
3813 WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01);
3814
3815 cpu = get_cpu();
3816 vmx->loaded_vmcs = &vmx->nested.vmcs02;
3817 vmx_vcpu_load(&vmx->vcpu, cpu);
3818
3819 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
3820
3821 vmx->loaded_vmcs = &vmx->vmcs01;
3822 vmx_vcpu_load(&vmx->vcpu, cpu);
3823 put_cpu();
3824}
3825
3826
3827
3828
3829
3830
3831
3832static void sync_vmcs02_to_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
3833{
3834 struct vcpu_vmx *vmx = to_vmx(vcpu);
3835
3836 if (vmx->nested.hv_evmcs)
3837 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
3838
3839 vmx->nested.need_sync_vmcs02_to_vmcs12_rare = !vmx->nested.hv_evmcs;
3840
3841 vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12);
3842 vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12);
3843
3844 vmcs12->guest_rsp = kvm_rsp_read(vcpu);
3845 vmcs12->guest_rip = kvm_rip_read(vcpu);
3846 vmcs12->guest_rflags = vmcs_readl(GUEST_RFLAGS);
3847
3848 vmcs12->guest_cs_ar_bytes = vmcs_read32(GUEST_CS_AR_BYTES);
3849 vmcs12->guest_ss_ar_bytes = vmcs_read32(GUEST_SS_AR_BYTES);
3850
3851 vmcs12->guest_sysenter_cs = vmcs_read32(GUEST_SYSENTER_CS);
3852 vmcs12->guest_sysenter_esp = vmcs_readl(GUEST_SYSENTER_ESP);
3853 vmcs12->guest_sysenter_eip = vmcs_readl(GUEST_SYSENTER_EIP);
3854
3855 vmcs12->guest_interruptibility_info =
3856 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
3857
3858 if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED)
3859 vmcs12->guest_activity_state = GUEST_ACTIVITY_HLT;
3860 else
3861 vmcs12->guest_activity_state = GUEST_ACTIVITY_ACTIVE;
3862
3863 if (nested_cpu_has_preemption_timer(vmcs12) &&
3864 vmcs12->vm_exit_controls & VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)
3865 vmcs12->vmx_preemption_timer_value =
3866 vmx_get_preemption_timer_value(vcpu);
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876 if (enable_ept) {
3877 vmcs12->guest_cr3 = vmcs_readl(GUEST_CR3);
3878 if (nested_cpu_has_ept(vmcs12) && is_pae_paging(vcpu)) {
3879 vmcs12->guest_pdptr0 = vmcs_read64(GUEST_PDPTR0);
3880 vmcs12->guest_pdptr1 = vmcs_read64(GUEST_PDPTR1);
3881 vmcs12->guest_pdptr2 = vmcs_read64(GUEST_PDPTR2);
3882 vmcs12->guest_pdptr3 = vmcs_read64(GUEST_PDPTR3);
3883 }
3884 }
3885
3886 vmcs12->guest_linear_address = vmcs_readl(GUEST_LINEAR_ADDRESS);
3887
3888 if (nested_cpu_has_vid(vmcs12))
3889 vmcs12->guest_intr_status = vmcs_read16(GUEST_INTR_STATUS);
3890
3891 vmcs12->vm_entry_controls =
3892 (vmcs12->vm_entry_controls & ~VM_ENTRY_IA32E_MODE) |
3893 (vm_entry_controls_get(to_vmx(vcpu)) & VM_ENTRY_IA32E_MODE);
3894
3895 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_DEBUG_CONTROLS)
3896 kvm_get_dr(vcpu, 7, (unsigned long *)&vmcs12->guest_dr7);
3897
3898 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_EFER)
3899 vmcs12->guest_ia32_efer = vcpu->arch.efer;
3900}
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
3914 u32 exit_reason, u32 exit_intr_info,
3915 unsigned long exit_qualification)
3916{
3917
3918 vmcs12->vm_exit_reason = exit_reason;
3919 vmcs12->exit_qualification = exit_qualification;
3920 vmcs12->vm_exit_intr_info = exit_intr_info;
3921
3922 vmcs12->idt_vectoring_info_field = 0;
3923 vmcs12->vm_exit_instruction_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
3924 vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
3925
3926 if (!(vmcs12->vm_exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) {
3927 vmcs12->launch_state = 1;
3928
3929
3930
3931 vmcs12->vm_entry_intr_info_field &= ~INTR_INFO_VALID_MASK;
3932
3933
3934
3935
3936
3937 vmcs12_save_pending_event(vcpu, vmcs12);
3938
3939
3940
3941
3942
3943
3944
3945 if (nested_vmx_store_msr(vcpu,
3946 vmcs12->vm_exit_msr_store_addr,
3947 vmcs12->vm_exit_msr_store_count))
3948 nested_vmx_abort(vcpu,
3949 VMX_ABORT_SAVE_GUEST_MSR_FAIL);
3950 }
3951
3952
3953
3954
3955
3956 vcpu->arch.nmi_injected = false;
3957 kvm_clear_exception_queue(vcpu);
3958 kvm_clear_interrupt_queue(vcpu);
3959}
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970static void load_vmcs12_host_state(struct kvm_vcpu *vcpu,
3971 struct vmcs12 *vmcs12)
3972{
3973 struct kvm_segment seg;
3974 u32 entry_failure_code;
3975
3976 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER)
3977 vcpu->arch.efer = vmcs12->host_ia32_efer;
3978 else if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)
3979 vcpu->arch.efer |= (EFER_LMA | EFER_LME);
3980 else
3981 vcpu->arch.efer &= ~(EFER_LMA | EFER_LME);
3982 vmx_set_efer(vcpu, vcpu->arch.efer);
3983
3984 kvm_rsp_write(vcpu, vmcs12->host_rsp);
3985 kvm_rip_write(vcpu, vmcs12->host_rip);
3986 vmx_set_rflags(vcpu, X86_EFLAGS_FIXED);
3987 vmx_set_interrupt_shadow(vcpu, 0);
3988
3989
3990
3991
3992
3993
3994
3995
3996 vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS;
3997 vmx_set_cr0(vcpu, vmcs12->host_cr0);
3998
3999
4000 vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK);
4001 vmx_set_cr4(vcpu, vmcs12->host_cr4);
4002
4003 nested_ept_uninit_mmu_context(vcpu);
4004
4005
4006
4007
4008
4009 if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, &entry_failure_code))
4010 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL);
4011
4012 if (!enable_ept)
4013 vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault;
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029 if (enable_vpid &&
4030 (!nested_cpu_has_vpid(vmcs12) || !nested_has_guest_tlb_tag(vcpu))) {
4031 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
4032 }
4033
4034 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs);
4035 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp);
4036 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->host_ia32_sysenter_eip);
4037 vmcs_writel(GUEST_IDTR_BASE, vmcs12->host_idtr_base);
4038 vmcs_writel(GUEST_GDTR_BASE, vmcs12->host_gdtr_base);
4039 vmcs_write32(GUEST_IDTR_LIMIT, 0xFFFF);
4040 vmcs_write32(GUEST_GDTR_LIMIT, 0xFFFF);
4041
4042
4043 if (vmcs12->vm_exit_controls & VM_EXIT_CLEAR_BNDCFGS)
4044 vmcs_write64(GUEST_BNDCFGS, 0);
4045
4046 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) {
4047 vmcs_write64(GUEST_IA32_PAT, vmcs12->host_ia32_pat);
4048 vcpu->arch.pat = vmcs12->host_ia32_pat;
4049 }
4050 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL)
4051 WARN_ON_ONCE(kvm_set_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL,
4052 vmcs12->host_ia32_perf_global_ctrl));
4053
4054
4055
4056 seg = (struct kvm_segment) {
4057 .base = 0,
4058 .limit = 0xFFFFFFFF,
4059 .selector = vmcs12->host_cs_selector,
4060 .type = 11,
4061 .present = 1,
4062 .s = 1,
4063 .g = 1
4064 };
4065 if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)
4066 seg.l = 1;
4067 else
4068 seg.db = 1;
4069 vmx_set_segment(vcpu, &seg, VCPU_SREG_CS);
4070 seg = (struct kvm_segment) {
4071 .base = 0,
4072 .limit = 0xFFFFFFFF,
4073 .type = 3,
4074 .present = 1,
4075 .s = 1,
4076 .db = 1,
4077 .g = 1
4078 };
4079 seg.selector = vmcs12->host_ds_selector;
4080 vmx_set_segment(vcpu, &seg, VCPU_SREG_DS);
4081 seg.selector = vmcs12->host_es_selector;
4082 vmx_set_segment(vcpu, &seg, VCPU_SREG_ES);
4083 seg.selector = vmcs12->host_ss_selector;
4084 vmx_set_segment(vcpu, &seg, VCPU_SREG_SS);
4085 seg.selector = vmcs12->host_fs_selector;
4086 seg.base = vmcs12->host_fs_base;
4087 vmx_set_segment(vcpu, &seg, VCPU_SREG_FS);
4088 seg.selector = vmcs12->host_gs_selector;
4089 seg.base = vmcs12->host_gs_base;
4090 vmx_set_segment(vcpu, &seg, VCPU_SREG_GS);
4091 seg = (struct kvm_segment) {
4092 .base = vmcs12->host_tr_base,
4093 .limit = 0x67,
4094 .selector = vmcs12->host_tr_selector,
4095 .type = 11,
4096 .present = 1
4097 };
4098 vmx_set_segment(vcpu, &seg, VCPU_SREG_TR);
4099
4100 kvm_set_dr(vcpu, 7, 0x400);
4101 vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
4102
4103 if (cpu_has_vmx_msr_bitmap())
4104 vmx_update_msr_bitmap(vcpu);
4105
4106 if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr,
4107 vmcs12->vm_exit_msr_load_count))
4108 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL);
4109}
4110
4111static inline u64 nested_vmx_get_vmcs01_guest_efer(struct vcpu_vmx *vmx)
4112{
4113 struct shared_msr_entry *efer_msr;
4114 unsigned int i;
4115
4116 if (vm_entry_controls_get(vmx) & VM_ENTRY_LOAD_IA32_EFER)
4117 return vmcs_read64(GUEST_IA32_EFER);
4118
4119 if (cpu_has_load_ia32_efer())
4120 return host_efer;
4121
4122 for (i = 0; i < vmx->msr_autoload.guest.nr; ++i) {
4123 if (vmx->msr_autoload.guest.val[i].index == MSR_EFER)
4124 return vmx->msr_autoload.guest.val[i].value;
4125 }
4126
4127 efer_msr = find_msr_entry(vmx, MSR_EFER);
4128 if (efer_msr)
4129 return efer_msr->data;
4130
4131 return host_efer;
4132}
4133
4134static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu)
4135{
4136 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
4137 struct vcpu_vmx *vmx = to_vmx(vcpu);
4138 struct vmx_msr_entry g, h;
4139 gpa_t gpa;
4140 u32 i, j;
4141
4142 vcpu->arch.pat = vmcs_read64(GUEST_IA32_PAT);
4143
4144 if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) {
4145
4146
4147
4148
4149
4150
4151 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
4152 kvm_set_dr(vcpu, 7, DR7_FIXED_1);
4153 else
4154 WARN_ON(kvm_set_dr(vcpu, 7, vmcs_readl(GUEST_DR7)));
4155 }
4156
4157
4158
4159
4160
4161 vmx_set_efer(vcpu, nested_vmx_get_vmcs01_guest_efer(vmx));
4162
4163 vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS;
4164 vmx_set_cr0(vcpu, vmcs_readl(CR0_READ_SHADOW));
4165
4166 vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK);
4167 vmx_set_cr4(vcpu, vmcs_readl(CR4_READ_SHADOW));
4168
4169 nested_ept_uninit_mmu_context(vcpu);
4170 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
4171 kvm_register_mark_available(vcpu, VCPU_EXREG_CR3);
4172
4173
4174
4175
4176
4177
4178
4179 if (enable_ept)
4180 ept_save_pdptrs(vcpu);
4181
4182 kvm_mmu_reset_context(vcpu);
4183
4184 if (cpu_has_vmx_msr_bitmap())
4185 vmx_update_msr_bitmap(vcpu);
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198 for (i = 0; i < vmcs12->vm_entry_msr_load_count; i++) {
4199 gpa = vmcs12->vm_entry_msr_load_addr + (i * sizeof(g));
4200 if (kvm_vcpu_read_guest(vcpu, gpa, &g, sizeof(g))) {
4201 pr_debug_ratelimited(
4202 "%s read MSR index failed (%u, 0x%08llx)\n",
4203 __func__, i, gpa);
4204 goto vmabort;
4205 }
4206
4207 for (j = 0; j < vmcs12->vm_exit_msr_load_count; j++) {
4208 gpa = vmcs12->vm_exit_msr_load_addr + (j * sizeof(h));
4209 if (kvm_vcpu_read_guest(vcpu, gpa, &h, sizeof(h))) {
4210 pr_debug_ratelimited(
4211 "%s read MSR failed (%u, 0x%08llx)\n",
4212 __func__, j, gpa);
4213 goto vmabort;
4214 }
4215 if (h.index != g.index)
4216 continue;
4217 if (h.value == g.value)
4218 break;
4219
4220 if (nested_vmx_load_msr_check(vcpu, &h)) {
4221 pr_debug_ratelimited(
4222 "%s check failed (%u, 0x%x, 0x%x)\n",
4223 __func__, j, h.index, h.reserved);
4224 goto vmabort;
4225 }
4226
4227 if (kvm_set_msr(vcpu, h.index, h.value)) {
4228 pr_debug_ratelimited(
4229 "%s WRMSR failed (%u, 0x%x, 0x%llx)\n",
4230 __func__, j, h.index, h.value);
4231 goto vmabort;
4232 }
4233 }
4234 }
4235
4236 return;
4237
4238vmabort:
4239 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL);
4240}
4241
4242
4243
4244
4245
4246
4247void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason,
4248 u32 exit_intr_info, unsigned long exit_qualification)
4249{
4250 struct vcpu_vmx *vmx = to_vmx(vcpu);
4251 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
4252
4253
4254 WARN_ON_ONCE(vmx->nested.nested_run_pending);
4255
4256 leave_guest_mode(vcpu);
4257
4258 if (nested_cpu_has_preemption_timer(vmcs12))
4259 hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer);
4260
4261 if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING)
4262 vcpu->arch.tsc_offset -= vmcs12->tsc_offset;
4263
4264 if (likely(!vmx->fail)) {
4265 sync_vmcs02_to_vmcs12(vcpu, vmcs12);
4266
4267 if (exit_reason != -1)
4268 prepare_vmcs12(vcpu, vmcs12, exit_reason, exit_intr_info,
4269 exit_qualification);
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280 nested_flush_cached_shadow_vmcs12(vcpu, vmcs12);
4281 } else {
4282
4283
4284
4285
4286
4287
4288 WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) !=
4289 VMXERR_ENTRY_INVALID_CONTROL_FIELD);
4290 WARN_ON_ONCE(nested_early_check);
4291 }
4292
4293 vmx_switch_vmcs(vcpu, &vmx->vmcs01);
4294
4295
4296 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
4297 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
4298 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset);
4299 if (vmx->nested.l1_tpr_threshold != -1)
4300 vmcs_write32(TPR_THRESHOLD, vmx->nested.l1_tpr_threshold);
4301
4302 if (kvm_has_tsc_control)
4303 decache_tsc_multiplier(vmx);
4304
4305 if (vmx->nested.change_vmcs01_virtual_apic_mode) {
4306 vmx->nested.change_vmcs01_virtual_apic_mode = false;
4307 vmx_set_virtual_apic_mode(vcpu);
4308 }
4309
4310
4311 if (vmx->nested.apic_access_page) {
4312 kvm_release_page_clean(vmx->nested.apic_access_page);
4313 vmx->nested.apic_access_page = NULL;
4314 }
4315 kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map, true);
4316 kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map, true);
4317 vmx->nested.pi_desc = NULL;
4318
4319
4320
4321
4322
4323 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
4324
4325 if ((exit_reason != -1) && (enable_shadow_vmcs || vmx->nested.hv_evmcs))
4326 vmx->nested.need_vmcs12_to_shadow_sync = true;
4327
4328
4329 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
4330
4331 if (likely(!vmx->fail)) {
4332
4333
4334
4335
4336
4337
4338
4339
4340 if (nested_exit_intr_ack_set(vcpu) &&
4341 exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT &&
4342 kvm_cpu_has_interrupt(vcpu)) {
4343 int irq = kvm_cpu_get_interrupt(vcpu);
4344 WARN_ON(irq < 0);
4345 vmcs12->vm_exit_intr_info = irq |
4346 INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR;
4347 }
4348
4349 if (exit_reason != -1)
4350 trace_kvm_nested_vmexit_inject(vmcs12->vm_exit_reason,
4351 vmcs12->exit_qualification,
4352 vmcs12->idt_vectoring_info_field,
4353 vmcs12->vm_exit_intr_info,
4354 vmcs12->vm_exit_intr_error_code,
4355 KVM_ISA_VMX);
4356
4357 load_vmcs12_host_state(vcpu, vmcs12);
4358
4359 return;
4360 }
4361
4362
4363
4364
4365
4366
4367
4368
4369 (void)nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
4370
4371
4372
4373
4374
4375
4376
4377 nested_vmx_restore_host_state(vcpu);
4378
4379 vmx->fail = 0;
4380}
4381
4382
4383
4384
4385
4386
4387
4388int get_vmx_mem_address(struct kvm_vcpu *vcpu, unsigned long exit_qualification,
4389 u32 vmx_instruction_info, bool wr, int len, gva_t *ret)
4390{
4391 gva_t off;
4392 bool exn;
4393 struct kvm_segment s;
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403 int scaling = vmx_instruction_info & 3;
4404 int addr_size = (vmx_instruction_info >> 7) & 7;
4405 bool is_reg = vmx_instruction_info & (1u << 10);
4406 int seg_reg = (vmx_instruction_info >> 15) & 7;
4407 int index_reg = (vmx_instruction_info >> 18) & 0xf;
4408 bool index_is_valid = !(vmx_instruction_info & (1u << 22));
4409 int base_reg = (vmx_instruction_info >> 23) & 0xf;
4410 bool base_is_valid = !(vmx_instruction_info & (1u << 27));
4411
4412 if (is_reg) {
4413 kvm_queue_exception(vcpu, UD_VECTOR);
4414 return 1;
4415 }
4416
4417
4418
4419 off = exit_qualification;
4420 if (addr_size == 1)
4421 off = (gva_t)sign_extend64(off, 31);
4422 else if (addr_size == 0)
4423 off = (gva_t)sign_extend64(off, 15);
4424 if (base_is_valid)
4425 off += kvm_register_read(vcpu, base_reg);
4426 if (index_is_valid)
4427 off += kvm_register_read(vcpu, index_reg)<<scaling;
4428 vmx_get_segment(vcpu, &s, seg_reg);
4429
4430
4431
4432
4433
4434
4435
4436 if (addr_size == 1)
4437 off &= 0xffffffff;
4438 else if (addr_size == 0)
4439 off &= 0xffff;
4440
4441
4442 exn = false;
4443 if (is_long_mode(vcpu)) {
4444
4445
4446
4447
4448
4449 if (seg_reg == VCPU_SREG_FS || seg_reg == VCPU_SREG_GS)
4450 *ret = s.base + off;
4451 else
4452 *ret = off;
4453
4454
4455
4456
4457
4458 exn = is_noncanonical_address(*ret, vcpu);
4459 } else {
4460
4461
4462
4463
4464
4465 *ret = (s.base + off) & 0xffffffff;
4466
4467
4468
4469
4470
4471
4472
4473 if (wr)
4474
4475
4476
4477 exn = ((s.type & 0xa) == 0 || (s.type & 8));
4478 else
4479
4480
4481
4482 exn = ((s.type & 0xa) == 8);
4483 if (exn) {
4484 kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
4485 return 1;
4486 }
4487
4488
4489 exn = (s.unusable != 0);
4490
4491
4492
4493
4494
4495
4496
4497 if (!(s.base == 0 && s.limit == 0xffffffff &&
4498 ((s.type & 8) || !(s.type & 4))))
4499 exn = exn || ((u64)off + len - 1 > s.limit);
4500 }
4501 if (exn) {
4502 kvm_queue_exception_e(vcpu,
4503 seg_reg == VCPU_SREG_SS ?
4504 SS_VECTOR : GP_VECTOR,
4505 0);
4506 return 1;
4507 }
4508
4509 return 0;
4510}
4511
4512void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu)
4513{
4514 struct vcpu_vmx *vmx;
4515
4516 if (!nested_vmx_allowed(vcpu))
4517 return;
4518
4519 vmx = to_vmx(vcpu);
4520 if (kvm_x86_ops->pmu_ops->is_valid_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {
4521 vmx->nested.msrs.entry_ctls_high |=
4522 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
4523 vmx->nested.msrs.exit_ctls_high |=
4524 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL;
4525 } else {
4526 vmx->nested.msrs.entry_ctls_high &=
4527 ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
4528 vmx->nested.msrs.exit_ctls_high &=
4529 ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
4530 }
4531}
4532
4533static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer)
4534{
4535 gva_t gva;
4536 struct x86_exception e;
4537
4538 if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
4539 vmcs_read32(VMX_INSTRUCTION_INFO), false,
4540 sizeof(*vmpointer), &gva))
4541 return 1;
4542
4543 if (kvm_read_guest_virt(vcpu, gva, vmpointer, sizeof(*vmpointer), &e)) {
4544 kvm_inject_page_fault(vcpu, &e);
4545 return 1;
4546 }
4547
4548 return 0;
4549}
4550
4551
4552
4553
4554
4555
4556static struct vmcs *alloc_shadow_vmcs(struct kvm_vcpu *vcpu)
4557{
4558 struct vcpu_vmx *vmx = to_vmx(vcpu);
4559 struct loaded_vmcs *loaded_vmcs = vmx->loaded_vmcs;
4560
4561
4562
4563
4564
4565
4566
4567 WARN_ON(loaded_vmcs == &vmx->vmcs01 && loaded_vmcs->shadow_vmcs);
4568
4569 if (!loaded_vmcs->shadow_vmcs) {
4570 loaded_vmcs->shadow_vmcs = alloc_vmcs(true);
4571 if (loaded_vmcs->shadow_vmcs)
4572 vmcs_clear(loaded_vmcs->shadow_vmcs);
4573 }
4574 return loaded_vmcs->shadow_vmcs;
4575}
4576
4577static int enter_vmx_operation(struct kvm_vcpu *vcpu)
4578{
4579 struct vcpu_vmx *vmx = to_vmx(vcpu);
4580 int r;
4581
4582 r = alloc_loaded_vmcs(&vmx->nested.vmcs02);
4583 if (r < 0)
4584 goto out_vmcs02;
4585
4586 vmx->nested.cached_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT);
4587 if (!vmx->nested.cached_vmcs12)
4588 goto out_cached_vmcs12;
4589
4590 vmx->nested.cached_shadow_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT);
4591 if (!vmx->nested.cached_shadow_vmcs12)
4592 goto out_cached_shadow_vmcs12;
4593
4594 if (enable_shadow_vmcs && !alloc_shadow_vmcs(vcpu))
4595 goto out_shadow_vmcs;
4596
4597 hrtimer_init(&vmx->nested.preemption_timer, CLOCK_MONOTONIC,
4598 HRTIMER_MODE_REL_PINNED);
4599 vmx->nested.preemption_timer.function = vmx_preemption_timer_fn;
4600
4601 vmx->nested.vpid02 = allocate_vpid();
4602
4603 vmx->nested.vmcs02_initialized = false;
4604 vmx->nested.vmxon = true;
4605
4606 if (pt_mode == PT_MODE_HOST_GUEST) {
4607 vmx->pt_desc.guest.ctl = 0;
4608 pt_update_intercept_for_msr(vmx);
4609 }
4610
4611 return 0;
4612
4613out_shadow_vmcs:
4614 kfree(vmx->nested.cached_shadow_vmcs12);
4615
4616out_cached_shadow_vmcs12:
4617 kfree(vmx->nested.cached_vmcs12);
4618
4619out_cached_vmcs12:
4620 free_loaded_vmcs(&vmx->nested.vmcs02);
4621
4622out_vmcs02:
4623 return -ENOMEM;
4624}
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634static int handle_vmon(struct kvm_vcpu *vcpu)
4635{
4636 int ret;
4637 gpa_t vmptr;
4638 uint32_t revision;
4639 struct vcpu_vmx *vmx = to_vmx(vcpu);
4640 const u64 VMXON_NEEDED_FEATURES = FEAT_CTL_LOCKED
4641 | FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX;
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652 if (!kvm_read_cr4_bits(vcpu, X86_CR4_VMXE)) {
4653 kvm_queue_exception(vcpu, UD_VECTOR);
4654 return 1;
4655 }
4656
4657
4658 if (vmx_get_cpl(vcpu)) {
4659 kvm_inject_gp(vcpu, 0);
4660 return 1;
4661 }
4662
4663 if (vmx->nested.vmxon)
4664 return nested_vmx_failValid(vcpu,
4665 VMXERR_VMXON_IN_VMX_ROOT_OPERATION);
4666
4667 if ((vmx->msr_ia32_feature_control & VMXON_NEEDED_FEATURES)
4668 != VMXON_NEEDED_FEATURES) {
4669 kvm_inject_gp(vcpu, 0);
4670 return 1;
4671 }
4672
4673 if (nested_vmx_get_vmptr(vcpu, &vmptr))
4674 return 1;
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684 if (!page_address_valid(vcpu, vmptr))
4685 return nested_vmx_failInvalid(vcpu);
4686
4687 if (kvm_read_guest(vcpu->kvm, vmptr, &revision, sizeof(revision)) ||
4688 revision != VMCS12_REVISION)
4689 return nested_vmx_failInvalid(vcpu);
4690
4691 vmx->nested.vmxon_ptr = vmptr;
4692 ret = enter_vmx_operation(vcpu);
4693 if (ret)
4694 return ret;
4695
4696 return nested_vmx_succeed(vcpu);
4697}
4698
4699static inline void nested_release_vmcs12(struct kvm_vcpu *vcpu)
4700{
4701 struct vcpu_vmx *vmx = to_vmx(vcpu);
4702
4703 if (vmx->nested.current_vmptr == -1ull)
4704 return;
4705
4706 copy_vmcs02_to_vmcs12_rare(vcpu, get_vmcs12(vcpu));
4707
4708 if (enable_shadow_vmcs) {
4709
4710
4711 copy_shadow_to_vmcs12(vmx);
4712 vmx_disable_shadow_vmcs(vmx);
4713 }
4714 vmx->nested.posted_intr_nv = -1;
4715
4716
4717 kvm_vcpu_write_guest_page(vcpu,
4718 vmx->nested.current_vmptr >> PAGE_SHIFT,
4719 vmx->nested.cached_vmcs12, 0, VMCS12_SIZE);
4720
4721 kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
4722
4723 vmx->nested.current_vmptr = -1ull;
4724}
4725
4726
4727static int handle_vmoff(struct kvm_vcpu *vcpu)
4728{
4729 if (!nested_vmx_check_permission(vcpu))
4730 return 1;
4731
4732 free_nested(vcpu);
4733
4734
4735 kvm_make_request(KVM_REQ_EVENT, vcpu);
4736
4737 return nested_vmx_succeed(vcpu);
4738}
4739
4740
4741static int handle_vmclear(struct kvm_vcpu *vcpu)
4742{
4743 struct vcpu_vmx *vmx = to_vmx(vcpu);
4744 u32 zero = 0;
4745 gpa_t vmptr;
4746 u64 evmcs_gpa;
4747
4748 if (!nested_vmx_check_permission(vcpu))
4749 return 1;
4750
4751 if (nested_vmx_get_vmptr(vcpu, &vmptr))
4752 return 1;
4753
4754 if (!page_address_valid(vcpu, vmptr))
4755 return nested_vmx_failValid(vcpu,
4756 VMXERR_VMCLEAR_INVALID_ADDRESS);
4757
4758 if (vmptr == vmx->nested.vmxon_ptr)
4759 return nested_vmx_failValid(vcpu,
4760 VMXERR_VMCLEAR_VMXON_POINTER);
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772 if (likely(!vmx->nested.enlightened_vmcs_enabled ||
4773 !nested_enlightened_vmentry(vcpu, &evmcs_gpa))) {
4774 if (vmptr == vmx->nested.current_vmptr)
4775 nested_release_vmcs12(vcpu);
4776
4777 kvm_vcpu_write_guest(vcpu,
4778 vmptr + offsetof(struct vmcs12,
4779 launch_state),
4780 &zero, sizeof(zero));
4781 }
4782
4783 return nested_vmx_succeed(vcpu);
4784}
4785
4786
4787static int handle_vmlaunch(struct kvm_vcpu *vcpu)
4788{
4789 return nested_vmx_run(vcpu, true);
4790}
4791
4792
4793static int handle_vmresume(struct kvm_vcpu *vcpu)
4794{
4795
4796 return nested_vmx_run(vcpu, false);
4797}
4798
4799static int handle_vmread(struct kvm_vcpu *vcpu)
4800{
4801 struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu)
4802 : get_vmcs12(vcpu);
4803 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
4804 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO);
4805 struct vcpu_vmx *vmx = to_vmx(vcpu);
4806 struct x86_exception e;
4807 unsigned long field;
4808 u64 value;
4809 gva_t gva = 0;
4810 short offset;
4811 int len;
4812
4813 if (!nested_vmx_check_permission(vcpu))
4814 return 1;
4815
4816
4817
4818
4819
4820 if (vmx->nested.current_vmptr == -1ull ||
4821 (is_guest_mode(vcpu) &&
4822 get_vmcs12(vcpu)->vmcs_link_pointer == -1ull))
4823 return nested_vmx_failInvalid(vcpu);
4824
4825
4826 field = kvm_register_readl(vcpu, (((instr_info) >> 28) & 0xf));
4827
4828 offset = vmcs_field_to_offset(field);
4829 if (offset < 0)
4830 return nested_vmx_failValid(vcpu,
4831 VMXERR_UNSUPPORTED_VMCS_COMPONENT);
4832
4833 if (!is_guest_mode(vcpu) && is_vmcs12_ext_field(field))
4834 copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
4835
4836
4837 value = vmcs12_read_any(vmcs12, field, offset);
4838
4839
4840
4841
4842
4843
4844 if (instr_info & BIT(10)) {
4845 kvm_register_writel(vcpu, (((instr_info) >> 3) & 0xf), value);
4846 } else {
4847 len = is_64_bit_mode(vcpu) ? 8 : 4;
4848 if (get_vmx_mem_address(vcpu, exit_qualification,
4849 instr_info, true, len, &gva))
4850 return 1;
4851
4852 if (kvm_write_guest_virt_system(vcpu, gva, &value, len, &e)) {
4853 kvm_inject_page_fault(vcpu, &e);
4854 return 1;
4855 }
4856 }
4857
4858 return nested_vmx_succeed(vcpu);
4859}
4860
4861static bool is_shadow_field_rw(unsigned long field)
4862{
4863 switch (field) {
4864#define SHADOW_FIELD_RW(x, y) case x:
4865#include "vmcs_shadow_fields.h"
4866 return true;
4867 default:
4868 break;
4869 }
4870 return false;
4871}
4872
4873static bool is_shadow_field_ro(unsigned long field)
4874{
4875 switch (field) {
4876#define SHADOW_FIELD_RO(x, y) case x:
4877#include "vmcs_shadow_fields.h"
4878 return true;
4879 default:
4880 break;
4881 }
4882 return false;
4883}
4884
4885static int handle_vmwrite(struct kvm_vcpu *vcpu)
4886{
4887 struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu)
4888 : get_vmcs12(vcpu);
4889 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
4890 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO);
4891 struct vcpu_vmx *vmx = to_vmx(vcpu);
4892 struct x86_exception e;
4893 unsigned long field;
4894 short offset;
4895 gva_t gva;
4896 int len;
4897
4898
4899
4900
4901
4902
4903
4904
4905 u64 value = 0;
4906
4907 if (!nested_vmx_check_permission(vcpu))
4908 return 1;
4909
4910
4911
4912
4913
4914 if (vmx->nested.current_vmptr == -1ull ||
4915 (is_guest_mode(vcpu) &&
4916 get_vmcs12(vcpu)->vmcs_link_pointer == -1ull))
4917 return nested_vmx_failInvalid(vcpu);
4918
4919 if (instr_info & BIT(10))
4920 value = kvm_register_readl(vcpu, (((instr_info) >> 3) & 0xf));
4921 else {
4922 len = is_64_bit_mode(vcpu) ? 8 : 4;
4923 if (get_vmx_mem_address(vcpu, exit_qualification,
4924 instr_info, false, len, &gva))
4925 return 1;
4926 if (kvm_read_guest_virt(vcpu, gva, &value, len, &e)) {
4927 kvm_inject_page_fault(vcpu, &e);
4928 return 1;
4929 }
4930 }
4931
4932 field = kvm_register_readl(vcpu, (((instr_info) >> 28) & 0xf));
4933
4934 offset = vmcs_field_to_offset(field);
4935 if (offset < 0)
4936 return nested_vmx_failValid(vcpu,
4937 VMXERR_UNSUPPORTED_VMCS_COMPONENT);
4938
4939
4940
4941
4942
4943 if (vmcs_field_readonly(field) &&
4944 !nested_cpu_has_vmwrite_any_field(vcpu))
4945 return nested_vmx_failValid(vcpu,
4946 VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT);
4947
4948
4949
4950
4951
4952 if (!is_guest_mode(vcpu) && !is_shadow_field_rw(field))
4953 copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963 if (field >= GUEST_ES_AR_BYTES && field <= GUEST_TR_AR_BYTES)
4964 value &= 0x1f0ff;
4965
4966 vmcs12_write_any(vmcs12, field, offset, value);
4967
4968
4969
4970
4971
4972
4973
4974 if (!is_guest_mode(vcpu) && !is_shadow_field_rw(field)) {
4975
4976
4977
4978
4979 if (enable_shadow_vmcs && is_shadow_field_ro(field)) {
4980 preempt_disable();
4981 vmcs_load(vmx->vmcs01.shadow_vmcs);
4982
4983 __vmcs_writel(field, value);
4984
4985 vmcs_clear(vmx->vmcs01.shadow_vmcs);
4986 vmcs_load(vmx->loaded_vmcs->vmcs);
4987 preempt_enable();
4988 }
4989 vmx->nested.dirty_vmcs12 = true;
4990 }
4991
4992 return nested_vmx_succeed(vcpu);
4993}
4994
4995static void set_current_vmptr(struct vcpu_vmx *vmx, gpa_t vmptr)
4996{
4997 vmx->nested.current_vmptr = vmptr;
4998 if (enable_shadow_vmcs) {
4999 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_SHADOW_VMCS);
5000 vmcs_write64(VMCS_LINK_POINTER,
5001 __pa(vmx->vmcs01.shadow_vmcs));
5002 vmx->nested.need_vmcs12_to_shadow_sync = true;
5003 }
5004 vmx->nested.dirty_vmcs12 = true;
5005}
5006
5007
5008static int handle_vmptrld(struct kvm_vcpu *vcpu)
5009{
5010 struct vcpu_vmx *vmx = to_vmx(vcpu);
5011 gpa_t vmptr;
5012
5013 if (!nested_vmx_check_permission(vcpu))
5014 return 1;
5015
5016 if (nested_vmx_get_vmptr(vcpu, &vmptr))
5017 return 1;
5018
5019 if (!page_address_valid(vcpu, vmptr))
5020 return nested_vmx_failValid(vcpu,
5021 VMXERR_VMPTRLD_INVALID_ADDRESS);
5022
5023 if (vmptr == vmx->nested.vmxon_ptr)
5024 return nested_vmx_failValid(vcpu,
5025 VMXERR_VMPTRLD_VMXON_POINTER);
5026
5027
5028 if (vmx->nested.hv_evmcs)
5029 return 1;
5030
5031 if (vmx->nested.current_vmptr != vmptr) {
5032 struct kvm_host_map map;
5033 struct vmcs12 *new_vmcs12;
5034
5035 if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmptr), &map)) {
5036
5037
5038
5039
5040
5041
5042 return nested_vmx_failValid(vcpu,
5043 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
5044 }
5045
5046 new_vmcs12 = map.hva;
5047
5048 if (new_vmcs12->hdr.revision_id != VMCS12_REVISION ||
5049 (new_vmcs12->hdr.shadow_vmcs &&
5050 !nested_cpu_has_vmx_shadow_vmcs(vcpu))) {
5051 kvm_vcpu_unmap(vcpu, &map, false);
5052 return nested_vmx_failValid(vcpu,
5053 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
5054 }
5055
5056 nested_release_vmcs12(vcpu);
5057
5058
5059
5060
5061
5062 memcpy(vmx->nested.cached_vmcs12, new_vmcs12, VMCS12_SIZE);
5063 kvm_vcpu_unmap(vcpu, &map, false);
5064
5065 set_current_vmptr(vmx, vmptr);
5066 }
5067
5068 return nested_vmx_succeed(vcpu);
5069}
5070
5071
5072static int handle_vmptrst(struct kvm_vcpu *vcpu)
5073{
5074 unsigned long exit_qual = vmcs_readl(EXIT_QUALIFICATION);
5075 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO);
5076 gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr;
5077 struct x86_exception e;
5078 gva_t gva;
5079
5080 if (!nested_vmx_check_permission(vcpu))
5081 return 1;
5082
5083 if (unlikely(to_vmx(vcpu)->nested.hv_evmcs))
5084 return 1;
5085
5086 if (get_vmx_mem_address(vcpu, exit_qual, instr_info,
5087 true, sizeof(gpa_t), &gva))
5088 return 1;
5089
5090 if (kvm_write_guest_virt_system(vcpu, gva, (void *)¤t_vmptr,
5091 sizeof(gpa_t), &e)) {
5092 kvm_inject_page_fault(vcpu, &e);
5093 return 1;
5094 }
5095 return nested_vmx_succeed(vcpu);
5096}
5097
5098
5099static int handle_invept(struct kvm_vcpu *vcpu)
5100{
5101 struct vcpu_vmx *vmx = to_vmx(vcpu);
5102 u32 vmx_instruction_info, types;
5103 unsigned long type;
5104 gva_t gva;
5105 struct x86_exception e;
5106 struct {
5107 u64 eptp, gpa;
5108 } operand;
5109
5110 if (!(vmx->nested.msrs.secondary_ctls_high &
5111 SECONDARY_EXEC_ENABLE_EPT) ||
5112 !(vmx->nested.msrs.ept_caps & VMX_EPT_INVEPT_BIT)) {
5113 kvm_queue_exception(vcpu, UD_VECTOR);
5114 return 1;
5115 }
5116
5117 if (!nested_vmx_check_permission(vcpu))
5118 return 1;
5119
5120 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
5121 type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf);
5122
5123 types = (vmx->nested.msrs.ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6;
5124
5125 if (type >= 32 || !(types & (1 << type)))
5126 return nested_vmx_failValid(vcpu,
5127 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
5128
5129
5130
5131
5132 if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
5133 vmx_instruction_info, false, sizeof(operand), &gva))
5134 return 1;
5135 if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) {
5136 kvm_inject_page_fault(vcpu, &e);
5137 return 1;
5138 }
5139
5140 switch (type) {
5141 case VMX_EPT_EXTENT_GLOBAL:
5142 case VMX_EPT_EXTENT_CONTEXT:
5143
5144
5145
5146
5147 break;
5148 default:
5149 BUG_ON(1);
5150 break;
5151 }
5152
5153 return nested_vmx_succeed(vcpu);
5154}
5155
5156static int handle_invvpid(struct kvm_vcpu *vcpu)
5157{
5158 struct vcpu_vmx *vmx = to_vmx(vcpu);
5159 u32 vmx_instruction_info;
5160 unsigned long type, types;
5161 gva_t gva;
5162 struct x86_exception e;
5163 struct {
5164 u64 vpid;
5165 u64 gla;
5166 } operand;
5167 u16 vpid02;
5168
5169 if (!(vmx->nested.msrs.secondary_ctls_high &
5170 SECONDARY_EXEC_ENABLE_VPID) ||
5171 !(vmx->nested.msrs.vpid_caps & VMX_VPID_INVVPID_BIT)) {
5172 kvm_queue_exception(vcpu, UD_VECTOR);
5173 return 1;
5174 }
5175
5176 if (!nested_vmx_check_permission(vcpu))
5177 return 1;
5178
5179 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
5180 type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf);
5181
5182 types = (vmx->nested.msrs.vpid_caps &
5183 VMX_VPID_EXTENT_SUPPORTED_MASK) >> 8;
5184
5185 if (type >= 32 || !(types & (1 << type)))
5186 return nested_vmx_failValid(vcpu,
5187 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
5188
5189
5190
5191
5192 if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
5193 vmx_instruction_info, false, sizeof(operand), &gva))
5194 return 1;
5195 if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) {
5196 kvm_inject_page_fault(vcpu, &e);
5197 return 1;
5198 }
5199 if (operand.vpid >> 16)
5200 return nested_vmx_failValid(vcpu,
5201 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
5202
5203 vpid02 = nested_get_vpid02(vcpu);
5204 switch (type) {
5205 case VMX_VPID_EXTENT_INDIVIDUAL_ADDR:
5206 if (!operand.vpid ||
5207 is_noncanonical_address(operand.gla, vcpu))
5208 return nested_vmx_failValid(vcpu,
5209 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
5210 if (cpu_has_vmx_invvpid_individual_addr()) {
5211 __invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR,
5212 vpid02, operand.gla);
5213 } else
5214 __vmx_flush_tlb(vcpu, vpid02, false);
5215 break;
5216 case VMX_VPID_EXTENT_SINGLE_CONTEXT:
5217 case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL:
5218 if (!operand.vpid)
5219 return nested_vmx_failValid(vcpu,
5220 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
5221 __vmx_flush_tlb(vcpu, vpid02, false);
5222 break;
5223 case VMX_VPID_EXTENT_ALL_CONTEXT:
5224 __vmx_flush_tlb(vcpu, vpid02, false);
5225 break;
5226 default:
5227 WARN_ON_ONCE(1);
5228 return kvm_skip_emulated_instruction(vcpu);
5229 }
5230
5231 return nested_vmx_succeed(vcpu);
5232}
5233
5234static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu,
5235 struct vmcs12 *vmcs12)
5236{
5237 u32 index = kvm_rcx_read(vcpu);
5238 u64 address;
5239 bool accessed_dirty;
5240 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
5241
5242 if (!nested_cpu_has_eptp_switching(vmcs12) ||
5243 !nested_cpu_has_ept(vmcs12))
5244 return 1;
5245
5246 if (index >= VMFUNC_EPTP_ENTRIES)
5247 return 1;
5248
5249
5250 if (kvm_vcpu_read_guest_page(vcpu, vmcs12->eptp_list_address >> PAGE_SHIFT,
5251 &address, index * 8, 8))
5252 return 1;
5253
5254 accessed_dirty = !!(address & VMX_EPTP_AD_ENABLE_BIT);
5255
5256
5257
5258
5259
5260 if (vmcs12->ept_pointer != address) {
5261 if (!valid_ept_address(vcpu, address))
5262 return 1;
5263
5264 kvm_mmu_unload(vcpu);
5265 mmu->ept_ad = accessed_dirty;
5266 mmu->mmu_role.base.ad_disabled = !accessed_dirty;
5267 vmcs12->ept_pointer = address;
5268
5269
5270
5271
5272
5273 kvm_mmu_reload(vcpu);
5274 }
5275
5276 return 0;
5277}
5278
5279static int handle_vmfunc(struct kvm_vcpu *vcpu)
5280{
5281 struct vcpu_vmx *vmx = to_vmx(vcpu);
5282 struct vmcs12 *vmcs12;
5283 u32 function = kvm_rax_read(vcpu);
5284
5285
5286
5287
5288
5289
5290 if (!is_guest_mode(vcpu)) {
5291 kvm_queue_exception(vcpu, UD_VECTOR);
5292 return 1;
5293 }
5294
5295 vmcs12 = get_vmcs12(vcpu);
5296 if ((vmcs12->vm_function_control & (1 << function)) == 0)
5297 goto fail;
5298
5299 switch (function) {
5300 case 0:
5301 if (nested_vmx_eptp_switching(vcpu, vmcs12))
5302 goto fail;
5303 break;
5304 default:
5305 goto fail;
5306 }
5307 return kvm_skip_emulated_instruction(vcpu);
5308
5309fail:
5310 nested_vmx_vmexit(vcpu, vmx->exit_reason,
5311 vmcs_read32(VM_EXIT_INTR_INFO),
5312 vmcs_readl(EXIT_QUALIFICATION));
5313 return 1;
5314}
5315
5316
5317
5318
5319
5320bool nested_vmx_check_io_bitmaps(struct kvm_vcpu *vcpu, unsigned int port,
5321 int size)
5322{
5323 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
5324 gpa_t bitmap, last_bitmap;
5325 u8 b;
5326
5327 last_bitmap = (gpa_t)-1;
5328 b = -1;
5329
5330 while (size > 0) {
5331 if (port < 0x8000)
5332 bitmap = vmcs12->io_bitmap_a;
5333 else if (port < 0x10000)
5334 bitmap = vmcs12->io_bitmap_b;
5335 else
5336 return true;
5337 bitmap += (port & 0x7fff) / 8;
5338
5339 if (last_bitmap != bitmap)
5340 if (kvm_vcpu_read_guest(vcpu, bitmap, &b, 1))
5341 return true;
5342 if (b & (1 << (port & 7)))
5343 return true;
5344
5345 port++;
5346 size--;
5347 last_bitmap = bitmap;
5348 }
5349
5350 return false;
5351}
5352
5353static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu,
5354 struct vmcs12 *vmcs12)
5355{
5356 unsigned long exit_qualification;
5357 unsigned short port;
5358 int size;
5359
5360 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
5361 return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING);
5362
5363 exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
5364
5365 port = exit_qualification >> 16;
5366 size = (exit_qualification & 7) + 1;
5367
5368 return nested_vmx_check_io_bitmaps(vcpu, port, size);
5369}
5370
5371
5372
5373
5374
5375
5376
5377static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu,
5378 struct vmcs12 *vmcs12, u32 exit_reason)
5379{
5380 u32 msr_index = kvm_rcx_read(vcpu);
5381 gpa_t bitmap;
5382
5383 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
5384 return true;
5385
5386
5387
5388
5389
5390
5391 bitmap = vmcs12->msr_bitmap;
5392 if (exit_reason == EXIT_REASON_MSR_WRITE)
5393 bitmap += 2048;
5394 if (msr_index >= 0xc0000000) {
5395 msr_index -= 0xc0000000;
5396 bitmap += 1024;
5397 }
5398
5399
5400 if (msr_index < 1024*8) {
5401 unsigned char b;
5402 if (kvm_vcpu_read_guest(vcpu, bitmap + msr_index/8, &b, 1))
5403 return true;
5404 return 1 & (b >> (msr_index & 7));
5405 } else
5406 return true;
5407}
5408
5409
5410
5411
5412
5413
5414static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu,
5415 struct vmcs12 *vmcs12)
5416{
5417 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
5418 int cr = exit_qualification & 15;
5419 int reg;
5420 unsigned long val;
5421
5422 switch ((exit_qualification >> 4) & 3) {
5423 case 0:
5424 reg = (exit_qualification >> 8) & 15;
5425 val = kvm_register_readl(vcpu, reg);
5426 switch (cr) {
5427 case 0:
5428 if (vmcs12->cr0_guest_host_mask &
5429 (val ^ vmcs12->cr0_read_shadow))
5430 return true;
5431 break;
5432 case 3:
5433 if ((vmcs12->cr3_target_count >= 1 &&
5434 vmcs12->cr3_target_value0 == val) ||
5435 (vmcs12->cr3_target_count >= 2 &&
5436 vmcs12->cr3_target_value1 == val) ||
5437 (vmcs12->cr3_target_count >= 3 &&
5438 vmcs12->cr3_target_value2 == val) ||
5439 (vmcs12->cr3_target_count >= 4 &&
5440 vmcs12->cr3_target_value3 == val))
5441 return false;
5442 if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING))
5443 return true;
5444 break;
5445 case 4:
5446 if (vmcs12->cr4_guest_host_mask &
5447 (vmcs12->cr4_read_shadow ^ val))
5448 return true;
5449 break;
5450 case 8:
5451 if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING))
5452 return true;
5453 break;
5454 }
5455 break;
5456 case 2:
5457 if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) &&
5458 (vmcs12->cr0_read_shadow & X86_CR0_TS))
5459 return true;
5460 break;
5461 case 1:
5462 switch (cr) {
5463 case 3:
5464 if (vmcs12->cpu_based_vm_exec_control &
5465 CPU_BASED_CR3_STORE_EXITING)
5466 return true;
5467 break;
5468 case 8:
5469 if (vmcs12->cpu_based_vm_exec_control &
5470 CPU_BASED_CR8_STORE_EXITING)
5471 return true;
5472 break;
5473 }
5474 break;
5475 case 3:
5476
5477
5478
5479
5480 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f;
5481 if (vmcs12->cr0_guest_host_mask & 0xe &
5482 (val ^ vmcs12->cr0_read_shadow))
5483 return true;
5484 if ((vmcs12->cr0_guest_host_mask & 0x1) &&
5485 !(vmcs12->cr0_read_shadow & 0x1) &&
5486 (val & 0x1))
5487 return true;
5488 break;
5489 }
5490 return false;
5491}
5492
5493static bool nested_vmx_exit_handled_vmcs_access(struct kvm_vcpu *vcpu,
5494 struct vmcs12 *vmcs12, gpa_t bitmap)
5495{
5496 u32 vmx_instruction_info;
5497 unsigned long field;
5498 u8 b;
5499
5500 if (!nested_cpu_has_shadow_vmcs(vmcs12))
5501 return true;
5502
5503
5504 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
5505 field = kvm_register_read(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
5506
5507
5508 if (field >> 15)
5509 return true;
5510
5511 if (kvm_vcpu_read_guest(vcpu, bitmap + field/8, &b, 1))
5512 return true;
5513
5514 return 1 & (b >> (field & 7));
5515}
5516
5517
5518
5519
5520
5521
5522bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason)
5523{
5524 u32 intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
5525 struct vcpu_vmx *vmx = to_vmx(vcpu);
5526 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
5527
5528 if (vmx->nested.nested_run_pending)
5529 return false;
5530
5531 if (unlikely(vmx->fail)) {
5532 trace_kvm_nested_vmenter_failed(
5533 "hardware VM-instruction error: ",
5534 vmcs_read32(VM_INSTRUCTION_ERROR));
5535 return true;
5536 }
5537
5538
5539
5540
5541
5542
5543
5544
5545
5546
5547
5548
5549 nested_mark_vmcs12_pages_dirty(vcpu);
5550
5551 trace_kvm_nested_vmexit(kvm_rip_read(vcpu), exit_reason,
5552 vmcs_readl(EXIT_QUALIFICATION),
5553 vmx->idt_vectoring_info,
5554 intr_info,
5555 vmcs_read32(VM_EXIT_INTR_ERROR_CODE),
5556 KVM_ISA_VMX);
5557
5558 switch (exit_reason) {
5559 case EXIT_REASON_EXCEPTION_NMI:
5560 if (is_nmi(intr_info))
5561 return false;
5562 else if (is_page_fault(intr_info))
5563 return !vmx->vcpu.arch.apf.host_apf_reason && enable_ept;
5564 else if (is_debug(intr_info) &&
5565 vcpu->guest_debug &
5566 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
5567 return false;
5568 else if (is_breakpoint(intr_info) &&
5569 vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
5570 return false;
5571 return vmcs12->exception_bitmap &
5572 (1u << (intr_info & INTR_INFO_VECTOR_MASK));
5573 case EXIT_REASON_EXTERNAL_INTERRUPT:
5574 return false;
5575 case EXIT_REASON_TRIPLE_FAULT:
5576 return true;
5577 case EXIT_REASON_INTERRUPT_WINDOW:
5578 return nested_cpu_has(vmcs12, CPU_BASED_INTR_WINDOW_EXITING);
5579 case EXIT_REASON_NMI_WINDOW:
5580 return nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING);
5581 case EXIT_REASON_TASK_SWITCH:
5582 return true;
5583 case EXIT_REASON_CPUID:
5584 return true;
5585 case EXIT_REASON_HLT:
5586 return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING);
5587 case EXIT_REASON_INVD:
5588 return true;
5589 case EXIT_REASON_INVLPG:
5590 return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING);
5591 case EXIT_REASON_RDPMC:
5592 return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING);
5593 case EXIT_REASON_RDRAND:
5594 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDRAND_EXITING);
5595 case EXIT_REASON_RDSEED:
5596 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDSEED_EXITING);
5597 case EXIT_REASON_RDTSC: case EXIT_REASON_RDTSCP:
5598 return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING);
5599 case EXIT_REASON_VMREAD:
5600 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12,
5601 vmcs12->vmread_bitmap);
5602 case EXIT_REASON_VMWRITE:
5603 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12,
5604 vmcs12->vmwrite_bitmap);
5605 case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR:
5606 case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD:
5607 case EXIT_REASON_VMPTRST: case EXIT_REASON_VMRESUME:
5608 case EXIT_REASON_VMOFF: case EXIT_REASON_VMON:
5609 case EXIT_REASON_INVEPT: case EXIT_REASON_INVVPID:
5610
5611
5612
5613
5614 return true;
5615 case EXIT_REASON_CR_ACCESS:
5616 return nested_vmx_exit_handled_cr(vcpu, vmcs12);
5617 case EXIT_REASON_DR_ACCESS:
5618 return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING);
5619 case EXIT_REASON_IO_INSTRUCTION:
5620 return nested_vmx_exit_handled_io(vcpu, vmcs12);
5621 case EXIT_REASON_GDTR_IDTR: case EXIT_REASON_LDTR_TR:
5622 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC);
5623 case EXIT_REASON_MSR_READ:
5624 case EXIT_REASON_MSR_WRITE:
5625 return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason);
5626 case EXIT_REASON_INVALID_STATE:
5627 return true;
5628 case EXIT_REASON_MWAIT_INSTRUCTION:
5629 return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING);
5630 case EXIT_REASON_MONITOR_TRAP_FLAG:
5631 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_TRAP_FLAG);
5632 case EXIT_REASON_MONITOR_INSTRUCTION:
5633 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_EXITING);
5634 case EXIT_REASON_PAUSE_INSTRUCTION:
5635 return nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING) ||
5636 nested_cpu_has2(vmcs12,
5637 SECONDARY_EXEC_PAUSE_LOOP_EXITING);
5638 case EXIT_REASON_MCE_DURING_VMENTRY:
5639 return false;
5640 case EXIT_REASON_TPR_BELOW_THRESHOLD:
5641 return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW);
5642 case EXIT_REASON_APIC_ACCESS:
5643 case EXIT_REASON_APIC_WRITE:
5644 case EXIT_REASON_EOI_INDUCED:
5645
5646
5647
5648
5649
5650 return true;
5651 case EXIT_REASON_EPT_VIOLATION:
5652
5653
5654
5655
5656
5657
5658 return false;
5659 case EXIT_REASON_EPT_MISCONFIG:
5660
5661
5662
5663
5664
5665
5666 return false;
5667 case EXIT_REASON_INVPCID:
5668 return
5669 nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_INVPCID) &&
5670 nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING);
5671 case EXIT_REASON_WBINVD:
5672 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING);
5673 case EXIT_REASON_XSETBV:
5674 return true;
5675 case EXIT_REASON_XSAVES: case EXIT_REASON_XRSTORS:
5676
5677
5678
5679
5680
5681
5682 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES);
5683 case EXIT_REASON_PREEMPTION_TIMER:
5684 return false;
5685 case EXIT_REASON_PML_FULL:
5686
5687 return false;
5688 case EXIT_REASON_VMFUNC:
5689
5690 return false;
5691 case EXIT_REASON_ENCLS:
5692
5693 return false;
5694 case EXIT_REASON_UMWAIT:
5695 case EXIT_REASON_TPAUSE:
5696 return nested_cpu_has2(vmcs12,
5697 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE);
5698 default:
5699 return true;
5700 }
5701}
5702
5703
5704static int vmx_get_nested_state(struct kvm_vcpu *vcpu,
5705 struct kvm_nested_state __user *user_kvm_nested_state,
5706 u32 user_data_size)
5707{
5708 struct vcpu_vmx *vmx;
5709 struct vmcs12 *vmcs12;
5710 struct kvm_nested_state kvm_state = {
5711 .flags = 0,
5712 .format = KVM_STATE_NESTED_FORMAT_VMX,
5713 .size = sizeof(kvm_state),
5714 .hdr.vmx.vmxon_pa = -1ull,
5715 .hdr.vmx.vmcs12_pa = -1ull,
5716 };
5717 struct kvm_vmx_nested_state_data __user *user_vmx_nested_state =
5718 &user_kvm_nested_state->data.vmx[0];
5719
5720 if (!vcpu)
5721 return kvm_state.size + sizeof(*user_vmx_nested_state);
5722
5723 vmx = to_vmx(vcpu);
5724 vmcs12 = get_vmcs12(vcpu);
5725
5726 if (nested_vmx_allowed(vcpu) &&
5727 (vmx->nested.vmxon || vmx->nested.smm.vmxon)) {
5728 kvm_state.hdr.vmx.vmxon_pa = vmx->nested.vmxon_ptr;
5729 kvm_state.hdr.vmx.vmcs12_pa = vmx->nested.current_vmptr;
5730
5731 if (vmx_has_valid_vmcs12(vcpu)) {
5732 kvm_state.size += sizeof(user_vmx_nested_state->vmcs12);
5733
5734 if (vmx->nested.hv_evmcs)
5735 kvm_state.flags |= KVM_STATE_NESTED_EVMCS;
5736
5737 if (is_guest_mode(vcpu) &&
5738 nested_cpu_has_shadow_vmcs(vmcs12) &&
5739 vmcs12->vmcs_link_pointer != -1ull)
5740 kvm_state.size += sizeof(user_vmx_nested_state->shadow_vmcs12);
5741 }
5742
5743 if (vmx->nested.smm.vmxon)
5744 kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_VMXON;
5745
5746 if (vmx->nested.smm.guest_mode)
5747 kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_GUEST_MODE;
5748
5749 if (is_guest_mode(vcpu)) {
5750 kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE;
5751
5752 if (vmx->nested.nested_run_pending)
5753 kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING;
5754
5755 if (vmx->nested.mtf_pending)
5756 kvm_state.flags |= KVM_STATE_NESTED_MTF_PENDING;
5757 }
5758 }
5759
5760 if (user_data_size < kvm_state.size)
5761 goto out;
5762
5763 if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state)))
5764 return -EFAULT;
5765
5766 if (!vmx_has_valid_vmcs12(vcpu))
5767 goto out;
5768
5769
5770
5771
5772
5773
5774
5775
5776 if (is_guest_mode(vcpu)) {
5777 sync_vmcs02_to_vmcs12(vcpu, vmcs12);
5778 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
5779 } else if (!vmx->nested.need_vmcs12_to_shadow_sync) {
5780 if (vmx->nested.hv_evmcs)
5781 copy_enlightened_to_vmcs12(vmx);
5782 else if (enable_shadow_vmcs)
5783 copy_shadow_to_vmcs12(vmx);
5784 }
5785
5786 BUILD_BUG_ON(sizeof(user_vmx_nested_state->vmcs12) < VMCS12_SIZE);
5787 BUILD_BUG_ON(sizeof(user_vmx_nested_state->shadow_vmcs12) < VMCS12_SIZE);
5788
5789
5790
5791
5792
5793 if (copy_to_user(user_vmx_nested_state->vmcs12, vmcs12, VMCS12_SIZE))
5794 return -EFAULT;
5795
5796 if (nested_cpu_has_shadow_vmcs(vmcs12) &&
5797 vmcs12->vmcs_link_pointer != -1ull) {
5798 if (copy_to_user(user_vmx_nested_state->shadow_vmcs12,
5799 get_shadow_vmcs12(vcpu), VMCS12_SIZE))
5800 return -EFAULT;
5801 }
5802
5803out:
5804 return kvm_state.size;
5805}
5806
5807
5808
5809
5810void vmx_leave_nested(struct kvm_vcpu *vcpu)
5811{
5812 if (is_guest_mode(vcpu)) {
5813 to_vmx(vcpu)->nested.nested_run_pending = 0;
5814 nested_vmx_vmexit(vcpu, -1, 0, 0);
5815 }
5816 free_nested(vcpu);
5817}
5818
5819static int vmx_set_nested_state(struct kvm_vcpu *vcpu,
5820 struct kvm_nested_state __user *user_kvm_nested_state,
5821 struct kvm_nested_state *kvm_state)
5822{
5823 struct vcpu_vmx *vmx = to_vmx(vcpu);
5824 struct vmcs12 *vmcs12;
5825 u32 exit_qual;
5826 struct kvm_vmx_nested_state_data __user *user_vmx_nested_state =
5827 &user_kvm_nested_state->data.vmx[0];
5828 int ret;
5829
5830 if (kvm_state->format != KVM_STATE_NESTED_FORMAT_VMX)
5831 return -EINVAL;
5832
5833 if (kvm_state->hdr.vmx.vmxon_pa == -1ull) {
5834 if (kvm_state->hdr.vmx.smm.flags)
5835 return -EINVAL;
5836
5837 if (kvm_state->hdr.vmx.vmcs12_pa != -1ull)
5838 return -EINVAL;
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849 if (kvm_state->flags & ~KVM_STATE_NESTED_EVMCS)
5850 return -EINVAL;
5851 } else {
5852 if (!nested_vmx_allowed(vcpu))
5853 return -EINVAL;
5854
5855 if (!page_address_valid(vcpu, kvm_state->hdr.vmx.vmxon_pa))
5856 return -EINVAL;
5857 }
5858
5859 if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) &&
5860 (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE))
5861 return -EINVAL;
5862
5863 if (kvm_state->hdr.vmx.smm.flags &
5864 ~(KVM_STATE_NESTED_SMM_GUEST_MODE | KVM_STATE_NESTED_SMM_VMXON))
5865 return -EINVAL;
5866
5867
5868
5869
5870
5871
5872 if (is_smm(vcpu) ?
5873 (kvm_state->flags &
5874 (KVM_STATE_NESTED_GUEST_MODE | KVM_STATE_NESTED_RUN_PENDING))
5875 : kvm_state->hdr.vmx.smm.flags)
5876 return -EINVAL;
5877
5878 if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) &&
5879 !(kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON))
5880 return -EINVAL;
5881
5882 if ((kvm_state->flags & KVM_STATE_NESTED_EVMCS) &&
5883 (!nested_vmx_allowed(vcpu) || !vmx->nested.enlightened_vmcs_enabled))
5884 return -EINVAL;
5885
5886 vmx_leave_nested(vcpu);
5887
5888 if (kvm_state->hdr.vmx.vmxon_pa == -1ull)
5889 return 0;
5890
5891 vmx->nested.vmxon_ptr = kvm_state->hdr.vmx.vmxon_pa;
5892 ret = enter_vmx_operation(vcpu);
5893 if (ret)
5894 return ret;
5895
5896
5897 if (kvm_state->size < sizeof(*kvm_state) + sizeof(*vmcs12))
5898 return 0;
5899
5900 if (kvm_state->hdr.vmx.vmcs12_pa != -1ull) {
5901 if (kvm_state->hdr.vmx.vmcs12_pa == kvm_state->hdr.vmx.vmxon_pa ||
5902 !page_address_valid(vcpu, kvm_state->hdr.vmx.vmcs12_pa))
5903 return -EINVAL;
5904
5905 set_current_vmptr(vmx, kvm_state->hdr.vmx.vmcs12_pa);
5906 } else if (kvm_state->flags & KVM_STATE_NESTED_EVMCS) {
5907
5908
5909
5910
5911 vmx->nested.need_vmcs12_to_shadow_sync = true;
5912 } else {
5913 return -EINVAL;
5914 }
5915
5916 if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON) {
5917 vmx->nested.smm.vmxon = true;
5918 vmx->nested.vmxon = false;
5919
5920 if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE)
5921 vmx->nested.smm.guest_mode = true;
5922 }
5923
5924 vmcs12 = get_vmcs12(vcpu);
5925 if (copy_from_user(vmcs12, user_vmx_nested_state->vmcs12, sizeof(*vmcs12)))
5926 return -EFAULT;
5927
5928 if (vmcs12->hdr.revision_id != VMCS12_REVISION)
5929 return -EINVAL;
5930
5931 if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE))
5932 return 0;
5933
5934 vmx->nested.nested_run_pending =
5935 !!(kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING);
5936
5937 vmx->nested.mtf_pending =
5938 !!(kvm_state->flags & KVM_STATE_NESTED_MTF_PENDING);
5939
5940 ret = -EINVAL;
5941 if (nested_cpu_has_shadow_vmcs(vmcs12) &&
5942 vmcs12->vmcs_link_pointer != -1ull) {
5943 struct vmcs12 *shadow_vmcs12 = get_shadow_vmcs12(vcpu);
5944
5945 if (kvm_state->size <
5946 sizeof(*kvm_state) +
5947 sizeof(user_vmx_nested_state->vmcs12) + sizeof(*shadow_vmcs12))
5948 goto error_guest_mode;
5949
5950 if (copy_from_user(shadow_vmcs12,
5951 user_vmx_nested_state->shadow_vmcs12,
5952 sizeof(*shadow_vmcs12))) {
5953 ret = -EFAULT;
5954 goto error_guest_mode;
5955 }
5956
5957 if (shadow_vmcs12->hdr.revision_id != VMCS12_REVISION ||
5958 !shadow_vmcs12->hdr.shadow_vmcs)
5959 goto error_guest_mode;
5960 }
5961
5962 if (nested_vmx_check_controls(vcpu, vmcs12) ||
5963 nested_vmx_check_host_state(vcpu, vmcs12) ||
5964 nested_vmx_check_guest_state(vcpu, vmcs12, &exit_qual))
5965 goto error_guest_mode;
5966
5967 vmx->nested.dirty_vmcs12 = true;
5968 ret = nested_vmx_enter_non_root_mode(vcpu, false);
5969 if (ret)
5970 goto error_guest_mode;
5971
5972 return 0;
5973
5974error_guest_mode:
5975 vmx->nested.nested_run_pending = 0;
5976 return ret;
5977}
5978
5979void nested_vmx_set_vmcs_shadowing_bitmap(void)
5980{
5981 if (enable_shadow_vmcs) {
5982 vmcs_write64(VMREAD_BITMAP, __pa(vmx_vmread_bitmap));
5983 vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap));
5984 }
5985}
5986
5987
5988
5989
5990
5991
5992
5993
5994
5995
5996
5997void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, u32 ept_caps)
5998{
5999
6000
6001
6002
6003
6004
6005
6006
6007
6008
6009
6010
6011
6012
6013
6014
6015 rdmsr(MSR_IA32_VMX_PINBASED_CTLS,
6016 msrs->pinbased_ctls_low,
6017 msrs->pinbased_ctls_high);
6018 msrs->pinbased_ctls_low |=
6019 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
6020 msrs->pinbased_ctls_high &=
6021 PIN_BASED_EXT_INTR_MASK |
6022 PIN_BASED_NMI_EXITING |
6023 PIN_BASED_VIRTUAL_NMIS |
6024 (enable_apicv ? PIN_BASED_POSTED_INTR : 0);
6025 msrs->pinbased_ctls_high |=
6026 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
6027 PIN_BASED_VMX_PREEMPTION_TIMER;
6028
6029
6030 rdmsr(MSR_IA32_VMX_EXIT_CTLS,
6031 msrs->exit_ctls_low,
6032 msrs->exit_ctls_high);
6033 msrs->exit_ctls_low =
6034 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR;
6035
6036 msrs->exit_ctls_high &=
6037#ifdef CONFIG_X86_64
6038 VM_EXIT_HOST_ADDR_SPACE_SIZE |
6039#endif
6040 VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT;
6041 msrs->exit_ctls_high |=
6042 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR |
6043 VM_EXIT_LOAD_IA32_EFER | VM_EXIT_SAVE_IA32_EFER |
6044 VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT;
6045
6046
6047 msrs->exit_ctls_low &= ~VM_EXIT_SAVE_DEBUG_CONTROLS;
6048
6049
6050 rdmsr(MSR_IA32_VMX_ENTRY_CTLS,
6051 msrs->entry_ctls_low,
6052 msrs->entry_ctls_high);
6053 msrs->entry_ctls_low =
6054 VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR;
6055 msrs->entry_ctls_high &=
6056#ifdef CONFIG_X86_64
6057 VM_ENTRY_IA32E_MODE |
6058#endif
6059 VM_ENTRY_LOAD_IA32_PAT;
6060 msrs->entry_ctls_high |=
6061 (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | VM_ENTRY_LOAD_IA32_EFER);
6062
6063
6064 msrs->entry_ctls_low &= ~VM_ENTRY_LOAD_DEBUG_CONTROLS;
6065
6066
6067 rdmsr(MSR_IA32_VMX_PROCBASED_CTLS,
6068 msrs->procbased_ctls_low,
6069 msrs->procbased_ctls_high);
6070 msrs->procbased_ctls_low =
6071 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
6072 msrs->procbased_ctls_high &=
6073 CPU_BASED_INTR_WINDOW_EXITING |
6074 CPU_BASED_NMI_WINDOW_EXITING | CPU_BASED_USE_TSC_OFFSETTING |
6075 CPU_BASED_HLT_EXITING | CPU_BASED_INVLPG_EXITING |
6076 CPU_BASED_MWAIT_EXITING | CPU_BASED_CR3_LOAD_EXITING |
6077 CPU_BASED_CR3_STORE_EXITING |
6078#ifdef CONFIG_X86_64
6079 CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING |
6080#endif
6081 CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING |
6082 CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_TRAP_FLAG |
6083 CPU_BASED_MONITOR_EXITING | CPU_BASED_RDPMC_EXITING |
6084 CPU_BASED_RDTSC_EXITING | CPU_BASED_PAUSE_EXITING |
6085 CPU_BASED_TPR_SHADOW | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
6086
6087
6088
6089
6090
6091
6092 msrs->procbased_ctls_high |=
6093 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
6094 CPU_BASED_USE_MSR_BITMAPS;
6095
6096
6097 msrs->procbased_ctls_low &=
6098 ~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING);
6099
6100
6101
6102
6103
6104 if (msrs->procbased_ctls_high & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)
6105 rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2,
6106 msrs->secondary_ctls_low,
6107 msrs->secondary_ctls_high);
6108
6109 msrs->secondary_ctls_low = 0;
6110 msrs->secondary_ctls_high &=
6111 SECONDARY_EXEC_DESC |
6112 SECONDARY_EXEC_RDTSCP |
6113 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
6114 SECONDARY_EXEC_WBINVD_EXITING |
6115 SECONDARY_EXEC_APIC_REGISTER_VIRT |
6116 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
6117 SECONDARY_EXEC_RDRAND_EXITING |
6118 SECONDARY_EXEC_ENABLE_INVPCID |
6119 SECONDARY_EXEC_RDSEED_EXITING |
6120 SECONDARY_EXEC_XSAVES;
6121
6122
6123
6124
6125
6126 msrs->secondary_ctls_high |=
6127 SECONDARY_EXEC_SHADOW_VMCS;
6128
6129 if (enable_ept) {
6130
6131 msrs->secondary_ctls_high |=
6132 SECONDARY_EXEC_ENABLE_EPT;
6133 msrs->ept_caps = VMX_EPT_PAGE_WALK_4_BIT |
6134 VMX_EPTP_WB_BIT | VMX_EPT_INVEPT_BIT;
6135 if (cpu_has_vmx_ept_execute_only())
6136 msrs->ept_caps |=
6137 VMX_EPT_EXECUTE_ONLY_BIT;
6138 msrs->ept_caps &= ept_caps;
6139 msrs->ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT |
6140 VMX_EPT_EXTENT_CONTEXT_BIT | VMX_EPT_2MB_PAGE_BIT |
6141 VMX_EPT_1GB_PAGE_BIT;
6142 if (enable_ept_ad_bits) {
6143 msrs->secondary_ctls_high |=
6144 SECONDARY_EXEC_ENABLE_PML;
6145 msrs->ept_caps |= VMX_EPT_AD_BIT;
6146 }
6147 }
6148
6149 if (cpu_has_vmx_vmfunc()) {
6150 msrs->secondary_ctls_high |=
6151 SECONDARY_EXEC_ENABLE_VMFUNC;
6152
6153
6154
6155
6156 if (enable_ept)
6157 msrs->vmfunc_controls =
6158 VMX_VMFUNC_EPTP_SWITCHING;
6159 }
6160
6161
6162
6163
6164
6165
6166
6167 if (enable_vpid) {
6168 msrs->secondary_ctls_high |=
6169 SECONDARY_EXEC_ENABLE_VPID;
6170 msrs->vpid_caps = VMX_VPID_INVVPID_BIT |
6171 VMX_VPID_EXTENT_SUPPORTED_MASK;
6172 }
6173
6174 if (enable_unrestricted_guest)
6175 msrs->secondary_ctls_high |=
6176 SECONDARY_EXEC_UNRESTRICTED_GUEST;
6177
6178 if (flexpriority_enabled)
6179 msrs->secondary_ctls_high |=
6180 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
6181
6182
6183 rdmsr(MSR_IA32_VMX_MISC,
6184 msrs->misc_low,
6185 msrs->misc_high);
6186 msrs->misc_low &= VMX_MISC_SAVE_EFER_LMA;
6187 msrs->misc_low |=
6188 MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS |
6189 VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE |
6190 VMX_MISC_ACTIVITY_HLT;
6191 msrs->misc_high = 0;
6192
6193
6194
6195
6196
6197
6198
6199 msrs->basic =
6200 VMCS12_REVISION |
6201 VMX_BASIC_TRUE_CTLS |
6202 ((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) |
6203 (VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT);
6204
6205 if (cpu_has_vmx_basic_inout())
6206 msrs->basic |= VMX_BASIC_INOUT;
6207
6208
6209
6210
6211
6212
6213#define VMXON_CR0_ALWAYSON (X86_CR0_PE | X86_CR0_PG | X86_CR0_NE)
6214#define VMXON_CR4_ALWAYSON X86_CR4_VMXE
6215 msrs->cr0_fixed0 = VMXON_CR0_ALWAYSON;
6216 msrs->cr4_fixed0 = VMXON_CR4_ALWAYSON;
6217
6218
6219 rdmsrl(MSR_IA32_VMX_CR0_FIXED1, msrs->cr0_fixed1);
6220 rdmsrl(MSR_IA32_VMX_CR4_FIXED1, msrs->cr4_fixed1);
6221
6222
6223 msrs->vmcs_enum = VMCS12_MAX_FIELD_INDEX << 1;
6224}
6225
6226void nested_vmx_hardware_unsetup(void)
6227{
6228 int i;
6229
6230 if (enable_shadow_vmcs) {
6231 for (i = 0; i < VMX_BITMAP_NR; i++)
6232 free_page((unsigned long)vmx_bitmap[i]);
6233 }
6234}
6235
6236__init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *))
6237{
6238 int i;
6239
6240 if (!cpu_has_vmx_shadow_vmcs())
6241 enable_shadow_vmcs = 0;
6242 if (enable_shadow_vmcs) {
6243 for (i = 0; i < VMX_BITMAP_NR; i++) {
6244
6245
6246
6247
6248 vmx_bitmap[i] = (unsigned long *)
6249 __get_free_page(GFP_KERNEL);
6250 if (!vmx_bitmap[i]) {
6251 nested_vmx_hardware_unsetup();
6252 return -ENOMEM;
6253 }
6254 }
6255
6256 init_vmcs_shadow_fields();
6257 }
6258
6259 exit_handlers[EXIT_REASON_VMCLEAR] = handle_vmclear;
6260 exit_handlers[EXIT_REASON_VMLAUNCH] = handle_vmlaunch;
6261 exit_handlers[EXIT_REASON_VMPTRLD] = handle_vmptrld;
6262 exit_handlers[EXIT_REASON_VMPTRST] = handle_vmptrst;
6263 exit_handlers[EXIT_REASON_VMREAD] = handle_vmread;
6264 exit_handlers[EXIT_REASON_VMRESUME] = handle_vmresume;
6265 exit_handlers[EXIT_REASON_VMWRITE] = handle_vmwrite;
6266 exit_handlers[EXIT_REASON_VMOFF] = handle_vmoff;
6267 exit_handlers[EXIT_REASON_VMON] = handle_vmon;
6268 exit_handlers[EXIT_REASON_INVEPT] = handle_invept;
6269 exit_handlers[EXIT_REASON_INVVPID] = handle_invvpid;
6270 exit_handlers[EXIT_REASON_VMFUNC] = handle_vmfunc;
6271
6272 kvm_x86_ops->check_nested_events = vmx_check_nested_events;
6273 kvm_x86_ops->get_nested_state = vmx_get_nested_state;
6274 kvm_x86_ops->set_nested_state = vmx_set_nested_state;
6275 kvm_x86_ops->get_vmcs12_pages = nested_get_vmcs12_pages;
6276 kvm_x86_ops->nested_enable_evmcs = nested_enable_evmcs;
6277 kvm_x86_ops->nested_get_evmcs_version = nested_get_evmcs_version;
6278
6279 return 0;
6280}
6281