1
2
3
4
5
6
7
8#ifndef _ASM_X86_KVM_HOST_H
9#define _ASM_X86_KVM_HOST_H
10
11#include <linux/types.h>
12#include <linux/mm.h>
13#include <linux/mmu_notifier.h>
14#include <linux/tracepoint.h>
15#include <linux/cpumask.h>
16#include <linux/irq_work.h>
17#include <linux/irq.h>
18
19#include <linux/kvm.h>
20#include <linux/kvm_para.h>
21#include <linux/kvm_types.h>
22#include <linux/perf_event.h>
23#include <linux/pvclock_gtod.h>
24#include <linux/clocksource.h>
25#include <linux/irqbypass.h>
26#include <linux/hyperv.h>
27
28#include <asm/apic.h>
29#include <asm/pvclock-abi.h>
30#include <asm/desc.h>
31#include <asm/mtrr.h>
32#include <asm/msr-index.h>
33#include <asm/asm.h>
34#include <asm/kvm_page_track.h>
35#include <asm/kvm_vcpu_regs.h>
36#include <asm/hyperv-tlfs.h>
37
38#define __KVM_HAVE_ARCH_VCPU_DEBUGFS
39
40#define KVM_MAX_VCPUS 1024
41#define KVM_SOFT_MAX_VCPUS 710
42
43
44
45
46
47
48
49
50
51
52#define KVM_VCPU_ID_RATIO 4
53#define KVM_MAX_VCPU_ID (KVM_MAX_VCPUS * KVM_VCPU_ID_RATIO)
54
55
56#define KVM_PRIVATE_MEM_SLOTS 3
57
58#define KVM_HALT_POLL_NS_DEFAULT 200000
59
60#define KVM_IRQCHIP_NUM_PINS KVM_IOAPIC_NUM_PINS
61
62#define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \
63 KVM_DIRTY_LOG_INITIALLY_SET)
64
65#define KVM_BUS_LOCK_DETECTION_VALID_MODE (KVM_BUS_LOCK_DETECTION_OFF | \
66 KVM_BUS_LOCK_DETECTION_EXIT)
67
68
69#define KVM_REQ_MIGRATE_TIMER KVM_ARCH_REQ(0)
70#define KVM_REQ_REPORT_TPR_ACCESS KVM_ARCH_REQ(1)
71#define KVM_REQ_TRIPLE_FAULT KVM_ARCH_REQ(2)
72#define KVM_REQ_MMU_SYNC KVM_ARCH_REQ(3)
73#define KVM_REQ_CLOCK_UPDATE KVM_ARCH_REQ(4)
74#define KVM_REQ_LOAD_MMU_PGD KVM_ARCH_REQ(5)
75#define KVM_REQ_EVENT KVM_ARCH_REQ(6)
76#define KVM_REQ_APF_HALT KVM_ARCH_REQ(7)
77#define KVM_REQ_STEAL_UPDATE KVM_ARCH_REQ(8)
78#define KVM_REQ_NMI KVM_ARCH_REQ(9)
79#define KVM_REQ_PMU KVM_ARCH_REQ(10)
80#define KVM_REQ_PMI KVM_ARCH_REQ(11)
81#define KVM_REQ_SMI KVM_ARCH_REQ(12)
82#define KVM_REQ_MASTERCLOCK_UPDATE KVM_ARCH_REQ(13)
83#define KVM_REQ_MCLOCK_INPROGRESS \
84 KVM_ARCH_REQ_FLAGS(14, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
85#define KVM_REQ_SCAN_IOAPIC \
86 KVM_ARCH_REQ_FLAGS(15, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
87#define KVM_REQ_GLOBAL_CLOCK_UPDATE KVM_ARCH_REQ(16)
88#define KVM_REQ_APIC_PAGE_RELOAD \
89 KVM_ARCH_REQ_FLAGS(17, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
90#define KVM_REQ_HV_CRASH KVM_ARCH_REQ(18)
91#define KVM_REQ_IOAPIC_EOI_EXIT KVM_ARCH_REQ(19)
92#define KVM_REQ_HV_RESET KVM_ARCH_REQ(20)
93#define KVM_REQ_HV_EXIT KVM_ARCH_REQ(21)
94#define KVM_REQ_HV_STIMER KVM_ARCH_REQ(22)
95#define KVM_REQ_LOAD_EOI_EXITMAP KVM_ARCH_REQ(23)
96#define KVM_REQ_GET_NESTED_STATE_PAGES KVM_ARCH_REQ(24)
97#define KVM_REQ_APICV_UPDATE \
98 KVM_ARCH_REQ_FLAGS(25, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
99#define KVM_REQ_TLB_FLUSH_CURRENT KVM_ARCH_REQ(26)
100#define KVM_REQ_TLB_FLUSH_GUEST \
101 KVM_ARCH_REQ_FLAGS(27, KVM_REQUEST_NO_WAKEUP)
102#define KVM_REQ_APF_READY KVM_ARCH_REQ(28)
103#define KVM_REQ_MSR_FILTER_CHANGED KVM_ARCH_REQ(29)
104#define KVM_REQ_UPDATE_CPU_DIRTY_LOGGING \
105 KVM_ARCH_REQ_FLAGS(30, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
106
107#define CR0_RESERVED_BITS \
108 (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \
109 | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \
110 | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG))
111
112#define CR4_RESERVED_BITS \
113 (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\
114 | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE \
115 | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR | X86_CR4_PCIDE \
116 | X86_CR4_OSXSAVE | X86_CR4_SMEP | X86_CR4_FSGSBASE \
117 | X86_CR4_OSXMMEXCPT | X86_CR4_LA57 | X86_CR4_VMXE \
118 | X86_CR4_SMAP | X86_CR4_PKE | X86_CR4_UMIP))
119
120#define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
121
122
123
124#define INVALID_PAGE (~(hpa_t)0)
125#define VALID_PAGE(x) ((x) != INVALID_PAGE)
126
127#define UNMAPPED_GVA (~(gpa_t)0)
128#define INVALID_GPA (~(gpa_t)0)
129
130
131#define KVM_MAX_HUGEPAGE_LEVEL PG_LEVEL_1G
132#define KVM_NR_PAGE_SIZES (KVM_MAX_HUGEPAGE_LEVEL - PG_LEVEL_4K + 1)
133#define KVM_HPAGE_GFN_SHIFT(x) (((x) - 1) * 9)
134#define KVM_HPAGE_SHIFT(x) (PAGE_SHIFT + KVM_HPAGE_GFN_SHIFT(x))
135#define KVM_HPAGE_SIZE(x) (1UL << KVM_HPAGE_SHIFT(x))
136#define KVM_HPAGE_MASK(x) (~(KVM_HPAGE_SIZE(x) - 1))
137#define KVM_PAGES_PER_HPAGE(x) (KVM_HPAGE_SIZE(x) / PAGE_SIZE)
138
139#define KVM_PERMILLE_MMU_PAGES 20
140#define KVM_MIN_ALLOC_MMU_PAGES 64UL
141#define KVM_MMU_HASH_SHIFT 12
142#define KVM_NUM_MMU_PAGES (1 << KVM_MMU_HASH_SHIFT)
143#define KVM_MIN_FREE_MMU_PAGES 5
144#define KVM_REFILL_PAGES 25
145#define KVM_MAX_CPUID_ENTRIES 256
146#define KVM_NR_FIXED_MTRR_REGION 88
147#define KVM_NR_VAR_MTRR 8
148
149#define ASYNC_PF_PER_VCPU 64
150
151enum kvm_reg {
152 VCPU_REGS_RAX = __VCPU_REGS_RAX,
153 VCPU_REGS_RCX = __VCPU_REGS_RCX,
154 VCPU_REGS_RDX = __VCPU_REGS_RDX,
155 VCPU_REGS_RBX = __VCPU_REGS_RBX,
156 VCPU_REGS_RSP = __VCPU_REGS_RSP,
157 VCPU_REGS_RBP = __VCPU_REGS_RBP,
158 VCPU_REGS_RSI = __VCPU_REGS_RSI,
159 VCPU_REGS_RDI = __VCPU_REGS_RDI,
160#ifdef CONFIG_X86_64
161 VCPU_REGS_R8 = __VCPU_REGS_R8,
162 VCPU_REGS_R9 = __VCPU_REGS_R9,
163 VCPU_REGS_R10 = __VCPU_REGS_R10,
164 VCPU_REGS_R11 = __VCPU_REGS_R11,
165 VCPU_REGS_R12 = __VCPU_REGS_R12,
166 VCPU_REGS_R13 = __VCPU_REGS_R13,
167 VCPU_REGS_R14 = __VCPU_REGS_R14,
168 VCPU_REGS_R15 = __VCPU_REGS_R15,
169#endif
170 VCPU_REGS_RIP,
171 NR_VCPU_REGS,
172
173 VCPU_EXREG_PDPTR = NR_VCPU_REGS,
174 VCPU_EXREG_CR0,
175 VCPU_EXREG_CR3,
176 VCPU_EXREG_CR4,
177 VCPU_EXREG_RFLAGS,
178 VCPU_EXREG_SEGMENTS,
179 VCPU_EXREG_EXIT_INFO_1,
180 VCPU_EXREG_EXIT_INFO_2,
181};
182
183enum {
184 VCPU_SREG_ES,
185 VCPU_SREG_CS,
186 VCPU_SREG_SS,
187 VCPU_SREG_DS,
188 VCPU_SREG_FS,
189 VCPU_SREG_GS,
190 VCPU_SREG_TR,
191 VCPU_SREG_LDTR,
192};
193
194enum exit_fastpath_completion {
195 EXIT_FASTPATH_NONE,
196 EXIT_FASTPATH_REENTER_GUEST,
197 EXIT_FASTPATH_EXIT_HANDLED,
198};
199typedef enum exit_fastpath_completion fastpath_t;
200
201struct x86_emulate_ctxt;
202struct x86_exception;
203enum x86_intercept;
204enum x86_intercept_stage;
205
206#define KVM_NR_DB_REGS 4
207
208#define DR6_BUS_LOCK (1 << 11)
209#define DR6_BD (1 << 13)
210#define DR6_BS (1 << 14)
211#define DR6_BT (1 << 15)
212#define DR6_RTM (1 << 16)
213
214
215
216
217
218
219
220
221#define DR6_ACTIVE_LOW 0xffff0ff0
222#define DR6_VOLATILE 0x0001e80f
223#define DR6_FIXED_1 (DR6_ACTIVE_LOW & ~DR6_VOLATILE)
224
225#define DR7_BP_EN_MASK 0x000000ff
226#define DR7_GE (1 << 9)
227#define DR7_GD (1 << 13)
228#define DR7_FIXED_1 0x00000400
229#define DR7_VOLATILE 0xffff2bff
230
231#define KVM_GUESTDBG_VALID_MASK \
232 (KVM_GUESTDBG_ENABLE | \
233 KVM_GUESTDBG_SINGLESTEP | \
234 KVM_GUESTDBG_USE_HW_BP | \
235 KVM_GUESTDBG_USE_SW_BP | \
236 KVM_GUESTDBG_INJECT_BP | \
237 KVM_GUESTDBG_INJECT_DB | \
238 KVM_GUESTDBG_BLOCKIRQ)
239
240
241#define PFERR_PRESENT_BIT 0
242#define PFERR_WRITE_BIT 1
243#define PFERR_USER_BIT 2
244#define PFERR_RSVD_BIT 3
245#define PFERR_FETCH_BIT 4
246#define PFERR_PK_BIT 5
247#define PFERR_SGX_BIT 15
248#define PFERR_GUEST_FINAL_BIT 32
249#define PFERR_GUEST_PAGE_BIT 33
250
251#define PFERR_PRESENT_MASK (1U << PFERR_PRESENT_BIT)
252#define PFERR_WRITE_MASK (1U << PFERR_WRITE_BIT)
253#define PFERR_USER_MASK (1U << PFERR_USER_BIT)
254#define PFERR_RSVD_MASK (1U << PFERR_RSVD_BIT)
255#define PFERR_FETCH_MASK (1U << PFERR_FETCH_BIT)
256#define PFERR_PK_MASK (1U << PFERR_PK_BIT)
257#define PFERR_SGX_MASK (1U << PFERR_SGX_BIT)
258#define PFERR_GUEST_FINAL_MASK (1ULL << PFERR_GUEST_FINAL_BIT)
259#define PFERR_GUEST_PAGE_MASK (1ULL << PFERR_GUEST_PAGE_BIT)
260
261#define PFERR_NESTED_GUEST_PAGE (PFERR_GUEST_PAGE_MASK | \
262 PFERR_WRITE_MASK | \
263 PFERR_PRESENT_MASK)
264
265
266#define KVM_APIC_CHECK_VAPIC 0
267
268
269
270
271
272
273#define KVM_APIC_PV_EOI_PENDING 1
274
275struct kvm_kernel_irq_routing_entry;
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309union kvm_mmu_page_role {
310 u32 word;
311 struct {
312 unsigned level:4;
313 unsigned gpte_is_8_bytes:1;
314 unsigned quadrant:2;
315 unsigned direct:1;
316 unsigned access:3;
317 unsigned invalid:1;
318 unsigned efer_nx:1;
319 unsigned cr0_wp:1;
320 unsigned smep_andnot_wp:1;
321 unsigned smap_andnot_wp:1;
322 unsigned ad_disabled:1;
323 unsigned guest_mode:1;
324 unsigned :6;
325
326
327
328
329
330
331
332 unsigned smm:8;
333 };
334};
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355union kvm_mmu_extended_role {
356 u32 word;
357 struct {
358 unsigned int valid:1;
359 unsigned int execonly:1;
360 unsigned int cr0_pg:1;
361 unsigned int cr4_pae:1;
362 unsigned int cr4_pse:1;
363 unsigned int cr4_pke:1;
364 unsigned int cr4_smap:1;
365 unsigned int cr4_smep:1;
366 unsigned int cr4_la57:1;
367 };
368};
369
370union kvm_mmu_role {
371 u64 as_u64;
372 struct {
373 union kvm_mmu_page_role base;
374 union kvm_mmu_extended_role ext;
375 };
376};
377
378struct kvm_rmap_head {
379 unsigned long val;
380};
381
382struct kvm_pio_request {
383 unsigned long linear_rip;
384 unsigned long count;
385 int in;
386 int port;
387 int size;
388};
389
390#define PT64_ROOT_MAX_LEVEL 5
391
392struct rsvd_bits_validate {
393 u64 rsvd_bits_mask[2][PT64_ROOT_MAX_LEVEL];
394 u64 bad_mt_xwr;
395};
396
397struct kvm_mmu_root_info {
398 gpa_t pgd;
399 hpa_t hpa;
400};
401
402#define KVM_MMU_ROOT_INFO_INVALID \
403 ((struct kvm_mmu_root_info) { .pgd = INVALID_PAGE, .hpa = INVALID_PAGE })
404
405#define KVM_MMU_NUM_PREV_ROOTS 3
406
407#define KVM_HAVE_MMU_RWLOCK
408
409struct kvm_mmu_page;
410
411
412
413
414
415
416struct kvm_mmu {
417 unsigned long (*get_guest_pgd)(struct kvm_vcpu *vcpu);
418 u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index);
419 int (*page_fault)(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u32 err,
420 bool prefault);
421 void (*inject_page_fault)(struct kvm_vcpu *vcpu,
422 struct x86_exception *fault);
423 gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, gpa_t gva_or_gpa,
424 u32 access, struct x86_exception *exception);
425 gpa_t (*translate_gpa)(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
426 struct x86_exception *exception);
427 int (*sync_page)(struct kvm_vcpu *vcpu,
428 struct kvm_mmu_page *sp);
429 void (*invlpg)(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root_hpa);
430 hpa_t root_hpa;
431 gpa_t root_pgd;
432 union kvm_mmu_role mmu_role;
433 u8 root_level;
434 u8 shadow_root_level;
435 u8 ept_ad;
436 bool direct_map;
437 struct kvm_mmu_root_info prev_roots[KVM_MMU_NUM_PREV_ROOTS];
438
439
440
441
442
443
444 u8 permissions[16];
445
446
447
448
449
450
451
452 u32 pkru_mask;
453
454 u64 *pae_root;
455 u64 *pml4_root;
456 u64 *pml5_root;
457
458
459
460
461
462
463 struct rsvd_bits_validate shadow_zero_check;
464
465 struct rsvd_bits_validate guest_rsvd_check;
466
467 u64 pdptrs[4];
468};
469
470struct kvm_tlb_range {
471 u64 start_gfn;
472 u64 pages;
473};
474
475enum pmc_type {
476 KVM_PMC_GP = 0,
477 KVM_PMC_FIXED,
478};
479
480struct kvm_pmc {
481 enum pmc_type type;
482 u8 idx;
483 u64 counter;
484 u64 eventsel;
485 struct perf_event *perf_event;
486 struct kvm_vcpu *vcpu;
487
488
489
490
491 u64 current_config;
492 bool is_paused;
493};
494
495struct kvm_pmu {
496 unsigned nr_arch_gp_counters;
497 unsigned nr_arch_fixed_counters;
498 unsigned available_event_types;
499 u64 fixed_ctr_ctrl;
500 u64 global_ctrl;
501 u64 global_status;
502 u64 global_ovf_ctrl;
503 u64 counter_bitmask[2];
504 u64 global_ctrl_mask;
505 u64 global_ovf_ctrl_mask;
506 u64 reserved_bits;
507 u8 version;
508 struct kvm_pmc gp_counters[INTEL_PMC_MAX_GENERIC];
509 struct kvm_pmc fixed_counters[INTEL_PMC_MAX_FIXED];
510 struct irq_work irq_work;
511 DECLARE_BITMAP(reprogram_pmi, X86_PMC_IDX_MAX);
512 DECLARE_BITMAP(all_valid_pmc_idx, X86_PMC_IDX_MAX);
513 DECLARE_BITMAP(pmc_in_use, X86_PMC_IDX_MAX);
514
515
516
517
518
519 bool need_cleanup;
520
521
522
523
524
525 u8 event_count;
526};
527
528struct kvm_pmu_ops;
529
530enum {
531 KVM_DEBUGREG_BP_ENABLED = 1,
532 KVM_DEBUGREG_WONT_EXIT = 2,
533};
534
535struct kvm_mtrr_range {
536 u64 base;
537 u64 mask;
538 struct list_head node;
539};
540
541struct kvm_mtrr {
542 struct kvm_mtrr_range var_ranges[KVM_NR_VAR_MTRR];
543 mtrr_type fixed_ranges[KVM_NR_FIXED_MTRR_REGION];
544 u64 deftype;
545
546 struct list_head head;
547};
548
549
550struct kvm_vcpu_hv_stimer {
551 struct hrtimer timer;
552 int index;
553 union hv_stimer_config config;
554 u64 count;
555 u64 exp_time;
556 struct hv_message msg;
557 bool msg_pending;
558};
559
560
561struct kvm_vcpu_hv_synic {
562 u64 version;
563 u64 control;
564 u64 msg_page;
565 u64 evt_page;
566 atomic64_t sint[HV_SYNIC_SINT_COUNT];
567 atomic_t sint_to_gsi[HV_SYNIC_SINT_COUNT];
568 DECLARE_BITMAP(auto_eoi_bitmap, 256);
569 DECLARE_BITMAP(vec_bitmap, 256);
570 bool active;
571 bool dont_zero_synic_pages;
572};
573
574
575struct kvm_vcpu_hv {
576 struct kvm_vcpu *vcpu;
577 u32 vp_index;
578 u64 hv_vapic;
579 s64 runtime_offset;
580 struct kvm_vcpu_hv_synic synic;
581 struct kvm_hyperv_exit exit;
582 struct kvm_vcpu_hv_stimer stimer[HV_SYNIC_STIMER_COUNT];
583 DECLARE_BITMAP(stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT);
584 cpumask_t tlb_flush;
585 bool enforce_cpuid;
586 struct {
587 u32 features_eax;
588 u32 features_ebx;
589 u32 features_edx;
590 u32 enlightenments_eax;
591 u32 enlightenments_ebx;
592 u32 syndbg_cap_eax;
593 } cpuid_cache;
594};
595
596
597struct kvm_vcpu_xen {
598 u64 hypercall_rip;
599 u32 current_runstate;
600 bool vcpu_info_set;
601 bool vcpu_time_info_set;
602 bool runstate_set;
603 struct gfn_to_hva_cache vcpu_info_cache;
604 struct gfn_to_hva_cache vcpu_time_info_cache;
605 struct gfn_to_hva_cache runstate_cache;
606 u64 last_steal;
607 u64 runstate_entry_time;
608 u64 runstate_times[4];
609};
610
611struct kvm_vcpu_arch {
612
613
614
615
616 unsigned long regs[NR_VCPU_REGS];
617 u32 regs_avail;
618 u32 regs_dirty;
619
620 unsigned long cr0;
621 unsigned long cr0_guest_owned_bits;
622 unsigned long cr2;
623 unsigned long cr3;
624 unsigned long cr4;
625 unsigned long cr4_guest_owned_bits;
626 unsigned long cr4_guest_rsvd_bits;
627 unsigned long cr8;
628 u32 host_pkru;
629 u32 pkru;
630 u32 hflags;
631 u64 efer;
632 u64 apic_base;
633 struct kvm_lapic *apic;
634 bool apicv_active;
635 bool load_eoi_exitmap_pending;
636 DECLARE_BITMAP(ioapic_handled_vectors, 256);
637 unsigned long apic_attention;
638 int32_t apic_arb_prio;
639 int mp_state;
640 u64 ia32_misc_enable_msr;
641 u64 smbase;
642 u64 smi_count;
643 bool tpr_access_reporting;
644 bool xsaves_enabled;
645 u64 ia32_xss;
646 u64 microcode_version;
647 u64 arch_capabilities;
648 u64 perf_capabilities;
649
650
651
652
653
654
655
656
657 struct kvm_mmu *mmu;
658
659
660 struct kvm_mmu root_mmu;
661
662
663 struct kvm_mmu guest_mmu;
664
665
666
667
668
669
670
671
672
673 struct kvm_mmu nested_mmu;
674
675
676
677
678
679 struct kvm_mmu *walk_mmu;
680
681 struct kvm_mmu_memory_cache mmu_pte_list_desc_cache;
682 struct kvm_mmu_memory_cache mmu_shadow_page_cache;
683 struct kvm_mmu_memory_cache mmu_gfn_array_cache;
684 struct kvm_mmu_memory_cache mmu_page_header_cache;
685
686
687
688
689
690
691
692
693
694
695
696
697 struct fpu *user_fpu;
698 struct fpu *guest_fpu;
699
700 u64 xcr0;
701 u64 guest_supported_xcr0;
702
703 struct kvm_pio_request pio;
704 void *pio_data;
705 void *sev_pio_data;
706 unsigned sev_pio_count;
707
708 u8 event_exit_inst_len;
709
710 struct kvm_queued_exception {
711 bool pending;
712 bool injected;
713 bool has_error_code;
714 u8 nr;
715 u32 error_code;
716 unsigned long payload;
717 bool has_payload;
718 u8 nested_apf;
719 } exception;
720
721 struct kvm_queued_interrupt {
722 bool injected;
723 bool soft;
724 u8 nr;
725 } interrupt;
726
727 int halt_request;
728
729 int cpuid_nent;
730 struct kvm_cpuid_entry2 *cpuid_entries;
731
732 u64 reserved_gpa_bits;
733 int maxphyaddr;
734
735
736
737 struct x86_emulate_ctxt *emulate_ctxt;
738 bool emulate_regs_need_sync_to_vcpu;
739 bool emulate_regs_need_sync_from_vcpu;
740 int (*complete_userspace_io)(struct kvm_vcpu *vcpu);
741
742 gpa_t time;
743 struct pvclock_vcpu_time_info hv_clock;
744 unsigned int hw_tsc_khz;
745 struct gfn_to_hva_cache pv_time;
746 bool pv_time_enabled;
747
748 bool pvclock_set_guest_stopped_request;
749
750 struct {
751 u8 preempted;
752 u64 msr_val;
753 u64 last_steal;
754 struct gfn_to_pfn_cache cache;
755 } st;
756
757 u64 l1_tsc_offset;
758 u64 tsc_offset;
759 u64 last_guest_tsc;
760 u64 last_host_tsc;
761 u64 tsc_offset_adjustment;
762 u64 this_tsc_nsec;
763 u64 this_tsc_write;
764 u64 this_tsc_generation;
765 bool tsc_catchup;
766 bool tsc_always_catchup;
767 s8 virtual_tsc_shift;
768 u32 virtual_tsc_mult;
769 u32 virtual_tsc_khz;
770 s64 ia32_tsc_adjust_msr;
771 u64 msr_ia32_power_ctl;
772 u64 l1_tsc_scaling_ratio;
773 u64 tsc_scaling_ratio;
774
775 atomic_t nmi_queued;
776 unsigned nmi_pending;
777 bool nmi_injected;
778 bool smi_pending;
779
780 struct kvm_mtrr mtrr_state;
781 u64 pat;
782
783 unsigned switch_db_regs;
784 unsigned long db[KVM_NR_DB_REGS];
785 unsigned long dr6;
786 unsigned long dr7;
787 unsigned long eff_db[KVM_NR_DB_REGS];
788 unsigned long guest_debug_dr7;
789 u64 msr_platform_info;
790 u64 msr_misc_features_enables;
791
792 u64 mcg_cap;
793 u64 mcg_status;
794 u64 mcg_ctl;
795 u64 mcg_ext_ctl;
796 u64 *mce_banks;
797
798
799 u64 mmio_gva;
800 unsigned mmio_access;
801 gfn_t mmio_gfn;
802 u64 mmio_gen;
803
804 struct kvm_pmu pmu;
805
806
807 unsigned long singlestep_rip;
808
809 bool hyperv_enabled;
810 struct kvm_vcpu_hv *hyperv;
811 struct kvm_vcpu_xen xen;
812
813 cpumask_var_t wbinvd_dirty_mask;
814
815 unsigned long last_retry_eip;
816 unsigned long last_retry_addr;
817
818 struct {
819 bool halted;
820 gfn_t gfns[ASYNC_PF_PER_VCPU];
821 struct gfn_to_hva_cache data;
822 u64 msr_en_val;
823 u64 msr_int_val;
824 u16 vec;
825 u32 id;
826 bool send_user_only;
827 u32 host_apf_flags;
828 unsigned long nested_apf_token;
829 bool delivery_as_pf_vmexit;
830 bool pageready_pending;
831 } apf;
832
833
834 struct {
835 u64 length;
836 u64 status;
837 } osvw;
838
839 struct {
840 u64 msr_val;
841 struct gfn_to_hva_cache data;
842 } pv_eoi;
843
844 u64 msr_kvm_poll_control;
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861 bool write_fault_to_shadow_pgtable;
862
863
864 unsigned long exit_qualification;
865
866
867 struct {
868 bool pv_unhalted;
869 } pv;
870
871 int pending_ioapic_eoi;
872 int pending_external_vector;
873
874
875 bool preempted_in_kernel;
876
877
878 bool l1tf_flush_l1d;
879
880
881 int last_vmentry_cpu;
882
883
884 u64 msr_hwcr;
885
886
887 struct {
888
889
890
891
892 u32 features;
893
894
895
896
897
898 bool enforce;
899 } pv_cpuid;
900
901
902 bool guest_state_protected;
903
904
905
906
907
908 bool pdptrs_from_userspace;
909
910#if IS_ENABLED(CONFIG_HYPERV)
911 hpa_t hv_root_tdp;
912#endif
913};
914
915struct kvm_lpage_info {
916 int disallow_lpage;
917};
918
919struct kvm_arch_memory_slot {
920 struct kvm_rmap_head *rmap[KVM_NR_PAGE_SIZES];
921 struct kvm_lpage_info *lpage_info[KVM_NR_PAGE_SIZES - 1];
922 unsigned short *gfn_track[KVM_PAGE_TRACK_MAX];
923};
924
925
926
927
928
929
930
931
932#define KVM_APIC_MODE_XAPIC_CLUSTER 4
933#define KVM_APIC_MODE_XAPIC_FLAT 8
934#define KVM_APIC_MODE_X2APIC 16
935
936struct kvm_apic_map {
937 struct rcu_head rcu;
938 u8 mode;
939 u32 max_apic_id;
940 union {
941 struct kvm_lapic *xapic_flat_map[8];
942 struct kvm_lapic *xapic_cluster_map[16][4];
943 };
944 struct kvm_lapic *phys_map[];
945};
946
947
948struct kvm_hv_syndbg {
949 struct {
950 u64 control;
951 u64 status;
952 u64 send_page;
953 u64 recv_page;
954 u64 pending_page;
955 } control;
956 u64 options;
957};
958
959
960enum hv_tsc_page_status {
961
962 HV_TSC_PAGE_UNSET = 0,
963
964 HV_TSC_PAGE_GUEST_CHANGED,
965
966 HV_TSC_PAGE_HOST_CHANGED,
967
968 HV_TSC_PAGE_SET,
969
970 HV_TSC_PAGE_UPDATING,
971
972 HV_TSC_PAGE_BROKEN,
973};
974
975
976struct kvm_hv {
977 struct mutex hv_lock;
978 u64 hv_guest_os_id;
979 u64 hv_hypercall;
980 u64 hv_tsc_page;
981 enum hv_tsc_page_status hv_tsc_page_status;
982
983
984 u64 hv_crash_param[HV_X64_MSR_CRASH_PARAMS];
985 u64 hv_crash_ctl;
986
987 struct ms_hyperv_tsc_page tsc_ref;
988
989 struct idr conn_to_evt;
990
991 u64 hv_reenlightenment_control;
992 u64 hv_tsc_emulation_control;
993 u64 hv_tsc_emulation_status;
994
995
996 atomic_t num_mismatched_vp_indexes;
997
998
999
1000
1001
1002 unsigned int synic_auto_eoi_used;
1003
1004 struct hv_partition_assist_pg *hv_pa_pg;
1005 struct kvm_hv_syndbg hv_syndbg;
1006};
1007
1008struct msr_bitmap_range {
1009 u32 flags;
1010 u32 nmsrs;
1011 u32 base;
1012 unsigned long *bitmap;
1013};
1014
1015
1016struct kvm_xen {
1017 bool long_mode;
1018 u8 upcall_vector;
1019 gfn_t shinfo_gfn;
1020};
1021
1022enum kvm_irqchip_mode {
1023 KVM_IRQCHIP_NONE,
1024 KVM_IRQCHIP_KERNEL,
1025 KVM_IRQCHIP_SPLIT,
1026};
1027
1028struct kvm_x86_msr_filter {
1029 u8 count;
1030 bool default_allow:1;
1031 struct msr_bitmap_range ranges[16];
1032};
1033
1034#define APICV_INHIBIT_REASON_DISABLE 0
1035#define APICV_INHIBIT_REASON_HYPERV 1
1036#define APICV_INHIBIT_REASON_NESTED 2
1037#define APICV_INHIBIT_REASON_IRQWIN 3
1038#define APICV_INHIBIT_REASON_PIT_REINJ 4
1039#define APICV_INHIBIT_REASON_X2APIC 5
1040
1041struct kvm_arch {
1042 unsigned long n_used_mmu_pages;
1043 unsigned long n_requested_mmu_pages;
1044 unsigned long n_max_mmu_pages;
1045 unsigned int indirect_shadow_pages;
1046 u8 mmu_valid_gen;
1047 struct hlist_head mmu_page_hash[KVM_NUM_MMU_PAGES];
1048 struct list_head active_mmu_pages;
1049 struct list_head zapped_obsolete_pages;
1050 struct list_head lpage_disallowed_mmu_pages;
1051 struct kvm_page_track_notifier_node mmu_sp_tracker;
1052 struct kvm_page_track_notifier_head track_notifier_head;
1053
1054
1055
1056
1057
1058
1059 spinlock_t mmu_unsync_pages_lock;
1060
1061 struct list_head assigned_dev_head;
1062 struct iommu_domain *iommu_domain;
1063 bool iommu_noncoherent;
1064#define __KVM_HAVE_ARCH_NONCOHERENT_DMA
1065 atomic_t noncoherent_dma_count;
1066#define __KVM_HAVE_ARCH_ASSIGNED_DEVICE
1067 atomic_t assigned_device_count;
1068 struct kvm_pic *vpic;
1069 struct kvm_ioapic *vioapic;
1070 struct kvm_pit *vpit;
1071 atomic_t vapics_in_nmi_mode;
1072 struct mutex apic_map_lock;
1073 struct kvm_apic_map __rcu *apic_map;
1074 atomic_t apic_map_dirty;
1075
1076
1077 struct mutex apicv_update_lock;
1078
1079 bool apic_access_memslot_enabled;
1080 unsigned long apicv_inhibit_reasons;
1081
1082 gpa_t wall_clock;
1083
1084 bool mwait_in_guest;
1085 bool hlt_in_guest;
1086 bool pause_in_guest;
1087 bool cstate_in_guest;
1088
1089 unsigned long irq_sources_bitmap;
1090 s64 kvmclock_offset;
1091 raw_spinlock_t tsc_write_lock;
1092 u64 last_tsc_nsec;
1093 u64 last_tsc_write;
1094 u32 last_tsc_khz;
1095 u64 cur_tsc_nsec;
1096 u64 cur_tsc_write;
1097 u64 cur_tsc_offset;
1098 u64 cur_tsc_generation;
1099 int nr_vcpus_matched_tsc;
1100
1101 raw_spinlock_t pvclock_gtod_sync_lock;
1102 bool use_master_clock;
1103 u64 master_kernel_ns;
1104 u64 master_cycle_now;
1105 struct delayed_work kvmclock_update_work;
1106 struct delayed_work kvmclock_sync_work;
1107
1108 struct kvm_xen_hvm_config xen_hvm_config;
1109
1110
1111 struct hlist_head mask_notifier_list;
1112
1113 struct kvm_hv hyperv;
1114 struct kvm_xen xen;
1115
1116 #ifdef CONFIG_KVM_MMU_AUDIT
1117 int audit_point;
1118 #endif
1119
1120 bool backwards_tsc_observed;
1121 bool boot_vcpu_runs_old_kvmclock;
1122 u32 bsp_vcpu_id;
1123
1124 u64 disabled_quirks;
1125 int cpu_dirty_logging_count;
1126
1127 enum kvm_irqchip_mode irqchip_mode;
1128 u8 nr_reserved_ioapic_pins;
1129
1130 bool disabled_lapic_found;
1131
1132 bool x2apic_format;
1133 bool x2apic_broadcast_quirk_disabled;
1134
1135 bool guest_can_read_msr_platform_info;
1136 bool exception_payload_enabled;
1137
1138 bool bus_lock_detection_enabled;
1139
1140
1141
1142
1143
1144 bool exit_on_emulation_error;
1145
1146
1147 u32 user_space_msr_mask;
1148 struct kvm_x86_msr_filter __rcu *msr_filter;
1149
1150 u32 hypercall_exit_enabled;
1151
1152
1153 bool sgx_provisioning_allowed;
1154
1155 struct kvm_pmu_event_filter __rcu *pmu_event_filter;
1156 struct task_struct *nx_lpage_recovery_thread;
1157
1158#ifdef CONFIG_X86_64
1159
1160
1161
1162
1163
1164
1165
1166 bool tdp_mmu_enabled;
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186 struct list_head tdp_mmu_roots;
1187
1188
1189
1190
1191
1192
1193 struct list_head tdp_mmu_pages;
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207 spinlock_t tdp_mmu_pages_lock;
1208#endif
1209
1210
1211
1212
1213
1214 bool memslots_have_rmaps;
1215
1216#if IS_ENABLED(CONFIG_HYPERV)
1217 hpa_t hv_root_tdp;
1218 spinlock_t hv_root_tdp_lock;
1219#endif
1220};
1221
1222struct kvm_vm_stat {
1223 struct kvm_vm_stat_generic generic;
1224 u64 mmu_shadow_zapped;
1225 u64 mmu_pte_write;
1226 u64 mmu_pde_zapped;
1227 u64 mmu_flooded;
1228 u64 mmu_recycled;
1229 u64 mmu_cache_miss;
1230 u64 mmu_unsync;
1231 union {
1232 struct {
1233 atomic64_t pages_4k;
1234 atomic64_t pages_2m;
1235 atomic64_t pages_1g;
1236 };
1237 atomic64_t pages[KVM_NR_PAGE_SIZES];
1238 };
1239 u64 nx_lpage_splits;
1240 u64 max_mmu_page_hash_collisions;
1241 u64 max_mmu_rmap_size;
1242};
1243
1244struct kvm_vcpu_stat {
1245 struct kvm_vcpu_stat_generic generic;
1246 u64 pf_fixed;
1247 u64 pf_guest;
1248 u64 tlb_flush;
1249 u64 invlpg;
1250
1251 u64 exits;
1252 u64 io_exits;
1253 u64 mmio_exits;
1254 u64 signal_exits;
1255 u64 irq_window_exits;
1256 u64 nmi_window_exits;
1257 u64 l1d_flush;
1258 u64 halt_exits;
1259 u64 request_irq_exits;
1260 u64 irq_exits;
1261 u64 host_state_reload;
1262 u64 fpu_reload;
1263 u64 insn_emulation;
1264 u64 insn_emulation_fail;
1265 u64 hypercalls;
1266 u64 irq_injections;
1267 u64 nmi_injections;
1268 u64 req_event;
1269 u64 nested_run;
1270 u64 directed_yield_attempted;
1271 u64 directed_yield_successful;
1272 u64 guest_mode;
1273};
1274
1275struct x86_instruction_info;
1276
1277struct msr_data {
1278 bool host_initiated;
1279 u32 index;
1280 u64 data;
1281};
1282
1283struct kvm_lapic_irq {
1284 u32 vector;
1285 u16 delivery_mode;
1286 u16 dest_mode;
1287 bool level;
1288 u16 trig_mode;
1289 u32 shorthand;
1290 u32 dest_id;
1291 bool msi_redir_hint;
1292};
1293
1294static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical)
1295{
1296 return dest_mode_logical ? APIC_DEST_LOGICAL : APIC_DEST_PHYSICAL;
1297}
1298
1299struct kvm_x86_ops {
1300 int (*hardware_enable)(void);
1301 void (*hardware_disable)(void);
1302 void (*hardware_unsetup)(void);
1303 bool (*cpu_has_accelerated_tpr)(void);
1304 bool (*has_emulated_msr)(struct kvm *kvm, u32 index);
1305 void (*vcpu_after_set_cpuid)(struct kvm_vcpu *vcpu);
1306
1307 unsigned int vm_size;
1308 int (*vm_init)(struct kvm *kvm);
1309 void (*vm_destroy)(struct kvm *kvm);
1310
1311
1312 int (*vcpu_create)(struct kvm_vcpu *vcpu);
1313 void (*vcpu_free)(struct kvm_vcpu *vcpu);
1314 void (*vcpu_reset)(struct kvm_vcpu *vcpu, bool init_event);
1315
1316 void (*prepare_guest_switch)(struct kvm_vcpu *vcpu);
1317 void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu);
1318 void (*vcpu_put)(struct kvm_vcpu *vcpu);
1319
1320 void (*update_exception_bitmap)(struct kvm_vcpu *vcpu);
1321 int (*get_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr);
1322 int (*set_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr);
1323 u64 (*get_segment_base)(struct kvm_vcpu *vcpu, int seg);
1324 void (*get_segment)(struct kvm_vcpu *vcpu,
1325 struct kvm_segment *var, int seg);
1326 int (*get_cpl)(struct kvm_vcpu *vcpu);
1327 void (*set_segment)(struct kvm_vcpu *vcpu,
1328 struct kvm_segment *var, int seg);
1329 void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l);
1330 void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0);
1331 bool (*is_valid_cr4)(struct kvm_vcpu *vcpu, unsigned long cr0);
1332 void (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4);
1333 int (*set_efer)(struct kvm_vcpu *vcpu, u64 efer);
1334 void (*get_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
1335 void (*set_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
1336 void (*get_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
1337 void (*set_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
1338 void (*sync_dirty_debug_regs)(struct kvm_vcpu *vcpu);
1339 void (*set_dr7)(struct kvm_vcpu *vcpu, unsigned long value);
1340 void (*cache_reg)(struct kvm_vcpu *vcpu, enum kvm_reg reg);
1341 unsigned long (*get_rflags)(struct kvm_vcpu *vcpu);
1342 void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags);
1343
1344 void (*tlb_flush_all)(struct kvm_vcpu *vcpu);
1345 void (*tlb_flush_current)(struct kvm_vcpu *vcpu);
1346 int (*tlb_remote_flush)(struct kvm *kvm);
1347 int (*tlb_remote_flush_with_range)(struct kvm *kvm,
1348 struct kvm_tlb_range *range);
1349
1350
1351
1352
1353
1354
1355
1356 void (*tlb_flush_gva)(struct kvm_vcpu *vcpu, gva_t addr);
1357
1358
1359
1360
1361
1362 void (*tlb_flush_guest)(struct kvm_vcpu *vcpu);
1363
1364 enum exit_fastpath_completion (*run)(struct kvm_vcpu *vcpu);
1365 int (*handle_exit)(struct kvm_vcpu *vcpu,
1366 enum exit_fastpath_completion exit_fastpath);
1367 int (*skip_emulated_instruction)(struct kvm_vcpu *vcpu);
1368 void (*update_emulated_instruction)(struct kvm_vcpu *vcpu);
1369 void (*set_interrupt_shadow)(struct kvm_vcpu *vcpu, int mask);
1370 u32 (*get_interrupt_shadow)(struct kvm_vcpu *vcpu);
1371 void (*patch_hypercall)(struct kvm_vcpu *vcpu,
1372 unsigned char *hypercall_addr);
1373 void (*set_irq)(struct kvm_vcpu *vcpu);
1374 void (*set_nmi)(struct kvm_vcpu *vcpu);
1375 void (*queue_exception)(struct kvm_vcpu *vcpu);
1376 void (*cancel_injection)(struct kvm_vcpu *vcpu);
1377 int (*interrupt_allowed)(struct kvm_vcpu *vcpu, bool for_injection);
1378 int (*nmi_allowed)(struct kvm_vcpu *vcpu, bool for_injection);
1379 bool (*get_nmi_mask)(struct kvm_vcpu *vcpu);
1380 void (*set_nmi_mask)(struct kvm_vcpu *vcpu, bool masked);
1381 void (*enable_nmi_window)(struct kvm_vcpu *vcpu);
1382 void (*enable_irq_window)(struct kvm_vcpu *vcpu);
1383 void (*update_cr8_intercept)(struct kvm_vcpu *vcpu, int tpr, int irr);
1384 bool (*check_apicv_inhibit_reasons)(ulong bit);
1385 void (*refresh_apicv_exec_ctrl)(struct kvm_vcpu *vcpu);
1386 void (*hwapic_irr_update)(struct kvm_vcpu *vcpu, int max_irr);
1387 void (*hwapic_isr_update)(struct kvm_vcpu *vcpu, int isr);
1388 bool (*guest_apic_has_interrupt)(struct kvm_vcpu *vcpu);
1389 void (*load_eoi_exitmap)(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap);
1390 void (*set_virtual_apic_mode)(struct kvm_vcpu *vcpu);
1391 void (*set_apic_access_page_addr)(struct kvm_vcpu *vcpu);
1392 int (*deliver_posted_interrupt)(struct kvm_vcpu *vcpu, int vector);
1393 int (*sync_pir_to_irr)(struct kvm_vcpu *vcpu);
1394 int (*set_tss_addr)(struct kvm *kvm, unsigned int addr);
1395 int (*set_identity_map_addr)(struct kvm *kvm, u64 ident_addr);
1396 u64 (*get_mt_mask)(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio);
1397
1398 void (*load_mmu_pgd)(struct kvm_vcpu *vcpu, hpa_t root_hpa,
1399 int root_level);
1400
1401 bool (*has_wbinvd_exit)(void);
1402
1403 u64 (*get_l2_tsc_offset)(struct kvm_vcpu *vcpu);
1404 u64 (*get_l2_tsc_multiplier)(struct kvm_vcpu *vcpu);
1405 void (*write_tsc_offset)(struct kvm_vcpu *vcpu, u64 offset);
1406 void (*write_tsc_multiplier)(struct kvm_vcpu *vcpu, u64 multiplier);
1407
1408
1409
1410
1411
1412 void (*get_exit_info)(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2,
1413 u32 *exit_int_info, u32 *exit_int_info_err_code);
1414
1415 int (*check_intercept)(struct kvm_vcpu *vcpu,
1416 struct x86_instruction_info *info,
1417 enum x86_intercept_stage stage,
1418 struct x86_exception *exception);
1419 void (*handle_exit_irqoff)(struct kvm_vcpu *vcpu);
1420
1421 void (*request_immediate_exit)(struct kvm_vcpu *vcpu);
1422
1423 void (*sched_in)(struct kvm_vcpu *kvm, int cpu);
1424
1425
1426
1427
1428
1429 int cpu_dirty_log_size;
1430 void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu);
1431
1432
1433 const struct kvm_pmu_ops *pmu_ops;
1434 const struct kvm_x86_nested_ops *nested_ops;
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445 int (*pre_block)(struct kvm_vcpu *vcpu);
1446 void (*post_block)(struct kvm_vcpu *vcpu);
1447
1448 void (*vcpu_blocking)(struct kvm_vcpu *vcpu);
1449 void (*vcpu_unblocking)(struct kvm_vcpu *vcpu);
1450
1451 int (*update_pi_irte)(struct kvm *kvm, unsigned int host_irq,
1452 uint32_t guest_irq, bool set);
1453 void (*start_assignment)(struct kvm *kvm);
1454 void (*apicv_post_state_restore)(struct kvm_vcpu *vcpu);
1455 bool (*dy_apicv_has_pending_interrupt)(struct kvm_vcpu *vcpu);
1456
1457 int (*set_hv_timer)(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc,
1458 bool *expired);
1459 void (*cancel_hv_timer)(struct kvm_vcpu *vcpu);
1460
1461 void (*setup_mce)(struct kvm_vcpu *vcpu);
1462
1463 int (*smi_allowed)(struct kvm_vcpu *vcpu, bool for_injection);
1464 int (*enter_smm)(struct kvm_vcpu *vcpu, char *smstate);
1465 int (*leave_smm)(struct kvm_vcpu *vcpu, const char *smstate);
1466 void (*enable_smi_window)(struct kvm_vcpu *vcpu);
1467
1468 int (*mem_enc_op)(struct kvm *kvm, void __user *argp);
1469 int (*mem_enc_reg_region)(struct kvm *kvm, struct kvm_enc_region *argp);
1470 int (*mem_enc_unreg_region)(struct kvm *kvm, struct kvm_enc_region *argp);
1471 int (*vm_copy_enc_context_from)(struct kvm *kvm, unsigned int source_fd);
1472
1473 int (*get_msr_feature)(struct kvm_msr_entry *entry);
1474
1475 bool (*can_emulate_instruction)(struct kvm_vcpu *vcpu, void *insn, int insn_len);
1476
1477 bool (*apic_init_signal_blocked)(struct kvm_vcpu *vcpu);
1478 int (*enable_direct_tlbflush)(struct kvm_vcpu *vcpu);
1479
1480 void (*migrate_timers)(struct kvm_vcpu *vcpu);
1481 void (*msr_filter_changed)(struct kvm_vcpu *vcpu);
1482 int (*complete_emulated_msr)(struct kvm_vcpu *vcpu, int err);
1483
1484 void (*vcpu_deliver_sipi_vector)(struct kvm_vcpu *vcpu, u8 vector);
1485};
1486
1487struct kvm_x86_nested_ops {
1488 int (*check_events)(struct kvm_vcpu *vcpu);
1489 bool (*hv_timer_pending)(struct kvm_vcpu *vcpu);
1490 void (*triple_fault)(struct kvm_vcpu *vcpu);
1491 int (*get_state)(struct kvm_vcpu *vcpu,
1492 struct kvm_nested_state __user *user_kvm_nested_state,
1493 unsigned user_data_size);
1494 int (*set_state)(struct kvm_vcpu *vcpu,
1495 struct kvm_nested_state __user *user_kvm_nested_state,
1496 struct kvm_nested_state *kvm_state);
1497 bool (*get_nested_state_pages)(struct kvm_vcpu *vcpu);
1498 int (*write_log_dirty)(struct kvm_vcpu *vcpu, gpa_t l2_gpa);
1499
1500 int (*enable_evmcs)(struct kvm_vcpu *vcpu,
1501 uint16_t *vmcs_version);
1502 uint16_t (*get_evmcs_version)(struct kvm_vcpu *vcpu);
1503};
1504
1505struct kvm_x86_init_ops {
1506 int (*cpu_has_kvm_support)(void);
1507 int (*disabled_by_bios)(void);
1508 int (*check_processor_compatibility)(void);
1509 int (*hardware_setup)(void);
1510
1511 struct kvm_x86_ops *runtime_ops;
1512};
1513
1514struct kvm_arch_async_pf {
1515 u32 token;
1516 gfn_t gfn;
1517 unsigned long cr3;
1518 bool direct_map;
1519};
1520
1521extern u32 __read_mostly kvm_nr_uret_msrs;
1522extern u64 __read_mostly host_efer;
1523extern bool __read_mostly allow_smaller_maxphyaddr;
1524extern bool __read_mostly enable_apicv;
1525extern struct kvm_x86_ops kvm_x86_ops;
1526
1527#define KVM_X86_OP(func) \
1528 DECLARE_STATIC_CALL(kvm_x86_##func, *(((struct kvm_x86_ops *)0)->func));
1529#define KVM_X86_OP_NULL KVM_X86_OP
1530#include <asm/kvm-x86-ops.h>
1531
1532static inline void kvm_ops_static_call_update(void)
1533{
1534#define KVM_X86_OP(func) \
1535 static_call_update(kvm_x86_##func, kvm_x86_ops.func);
1536#define KVM_X86_OP_NULL KVM_X86_OP
1537#include <asm/kvm-x86-ops.h>
1538}
1539
1540#define __KVM_HAVE_ARCH_VM_ALLOC
1541static inline struct kvm *kvm_arch_alloc_vm(void)
1542{
1543 return __vmalloc(kvm_x86_ops.vm_size, GFP_KERNEL_ACCOUNT | __GFP_ZERO);
1544}
1545void kvm_arch_free_vm(struct kvm *kvm);
1546
1547#define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLB
1548static inline int kvm_arch_flush_remote_tlb(struct kvm *kvm)
1549{
1550 if (kvm_x86_ops.tlb_remote_flush &&
1551 !static_call(kvm_x86_tlb_remote_flush)(kvm))
1552 return 0;
1553 else
1554 return -ENOTSUPP;
1555}
1556
1557int kvm_mmu_module_init(void);
1558void kvm_mmu_module_exit(void);
1559
1560void kvm_mmu_destroy(struct kvm_vcpu *vcpu);
1561int kvm_mmu_create(struct kvm_vcpu *vcpu);
1562void kvm_mmu_init_vm(struct kvm *kvm);
1563void kvm_mmu_uninit_vm(struct kvm *kvm);
1564
1565void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu);
1566void kvm_mmu_reset_context(struct kvm_vcpu *vcpu);
1567void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
1568 const struct kvm_memory_slot *memslot,
1569 int start_level);
1570void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm,
1571 const struct kvm_memory_slot *memslot);
1572void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
1573 const struct kvm_memory_slot *memslot);
1574void kvm_mmu_zap_all(struct kvm *kvm);
1575void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen);
1576unsigned long kvm_mmu_calculate_default_mmu_pages(struct kvm *kvm);
1577void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long kvm_nr_mmu_pages);
1578
1579int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3);
1580
1581int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
1582 const void *val, int bytes);
1583
1584struct kvm_irq_mask_notifier {
1585 void (*func)(struct kvm_irq_mask_notifier *kimn, bool masked);
1586 int irq;
1587 struct hlist_node link;
1588};
1589
1590void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq,
1591 struct kvm_irq_mask_notifier *kimn);
1592void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq,
1593 struct kvm_irq_mask_notifier *kimn);
1594void kvm_fire_mask_notifiers(struct kvm *kvm, unsigned irqchip, unsigned pin,
1595 bool mask);
1596
1597extern bool tdp_enabled;
1598
1599u64 vcpu_tsc_khz(struct kvm_vcpu *vcpu);
1600
1601
1602extern bool kvm_has_tsc_control;
1603
1604extern u32 kvm_max_guest_tsc_khz;
1605
1606extern u8 kvm_tsc_scaling_ratio_frac_bits;
1607
1608extern u64 kvm_max_tsc_scaling_ratio;
1609
1610extern u64 kvm_default_tsc_scaling_ratio;
1611
1612extern bool kvm_has_bus_lock_exit;
1613
1614extern u64 kvm_mce_cap_supported;
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650#define EMULTYPE_NO_DECODE (1 << 0)
1651#define EMULTYPE_TRAP_UD (1 << 1)
1652#define EMULTYPE_SKIP (1 << 2)
1653#define EMULTYPE_ALLOW_RETRY_PF (1 << 3)
1654#define EMULTYPE_TRAP_UD_FORCED (1 << 4)
1655#define EMULTYPE_VMWARE_GP (1 << 5)
1656#define EMULTYPE_PF (1 << 6)
1657
1658int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type);
1659int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,
1660 void *insn, int insn_len);
1661
1662void kvm_enable_efer_bits(u64);
1663bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer);
1664int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data, bool host_initiated);
1665int kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data);
1666int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data);
1667int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu);
1668int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu);
1669int kvm_emulate_as_nop(struct kvm_vcpu *vcpu);
1670int kvm_emulate_invd(struct kvm_vcpu *vcpu);
1671int kvm_emulate_mwait(struct kvm_vcpu *vcpu);
1672int kvm_handle_invalid_op(struct kvm_vcpu *vcpu);
1673int kvm_emulate_monitor(struct kvm_vcpu *vcpu);
1674
1675int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in);
1676int kvm_emulate_cpuid(struct kvm_vcpu *vcpu);
1677int kvm_emulate_halt(struct kvm_vcpu *vcpu);
1678int kvm_vcpu_halt(struct kvm_vcpu *vcpu);
1679int kvm_emulate_ap_reset_hold(struct kvm_vcpu *vcpu);
1680int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu);
1681
1682void kvm_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg);
1683int kvm_load_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector, int seg);
1684void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector);
1685
1686int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
1687 int reason, bool has_error_code, u32 error_code);
1688
1689void kvm_free_guest_fpu(struct kvm_vcpu *vcpu);
1690
1691void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0);
1692void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4);
1693int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
1694int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3);
1695int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
1696int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8);
1697int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val);
1698void kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val);
1699unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu);
1700void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw);
1701void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l);
1702int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu);
1703
1704int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr);
1705int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr);
1706
1707unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu);
1708void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
1709int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu);
1710
1711void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr);
1712void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);
1713void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr, unsigned long payload);
1714void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr);
1715void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);
1716void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault);
1717bool kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
1718 struct x86_exception *fault);
1719int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
1720 gfn_t gfn, void *data, int offset, int len,
1721 u32 access);
1722bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl);
1723bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr);
1724
1725static inline int __kvm_irq_line_state(unsigned long *irq_state,
1726 int irq_source_id, int level)
1727{
1728
1729 if (level)
1730 __set_bit(irq_source_id, irq_state);
1731 else
1732 __clear_bit(irq_source_id, irq_state);
1733
1734 return !!(*irq_state);
1735}
1736
1737#define KVM_MMU_ROOT_CURRENT BIT(0)
1738#define KVM_MMU_ROOT_PREVIOUS(i) BIT(1+i)
1739#define KVM_MMU_ROOTS_ALL (~0UL)
1740
1741int kvm_pic_set_irq(struct kvm_pic *pic, int irq, int irq_source_id, int level);
1742void kvm_pic_clear_all(struct kvm_pic *pic, int irq_source_id);
1743
1744void kvm_inject_nmi(struct kvm_vcpu *vcpu);
1745
1746void kvm_update_dr7(struct kvm_vcpu *vcpu);
1747
1748int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn);
1749void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu);
1750void kvm_mmu_free_roots(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
1751 ulong roots_to_free);
1752void kvm_mmu_free_guest_mode_roots(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu);
1753gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access,
1754 struct x86_exception *exception);
1755gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
1756 struct x86_exception *exception);
1757gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
1758 struct x86_exception *exception);
1759gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
1760 struct x86_exception *exception);
1761gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
1762 struct x86_exception *exception);
1763
1764bool kvm_apicv_activated(struct kvm *kvm);
1765void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu);
1766void kvm_request_apicv_update(struct kvm *kvm, bool activate,
1767 unsigned long bit);
1768
1769void __kvm_request_apicv_update(struct kvm *kvm, bool activate,
1770 unsigned long bit);
1771
1772int kvm_emulate_hypercall(struct kvm_vcpu *vcpu);
1773
1774int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code,
1775 void *insn, int insn_len);
1776void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva);
1777void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
1778 gva_t gva, hpa_t root_hpa);
1779void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid);
1780void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd);
1781
1782void kvm_configure_mmu(bool enable_tdp, int tdp_forced_root_level,
1783 int tdp_max_root_level, int tdp_huge_page_level);
1784
1785static inline u16 kvm_read_ldt(void)
1786{
1787 u16 ldt;
1788 asm("sldt %0" : "=g"(ldt));
1789 return ldt;
1790}
1791
1792static inline void kvm_load_ldt(u16 sel)
1793{
1794 asm("lldt %0" : : "rm"(sel));
1795}
1796
1797#ifdef CONFIG_X86_64
1798static inline unsigned long read_msr(unsigned long msr)
1799{
1800 u64 value;
1801
1802 rdmsrl(msr, value);
1803 return value;
1804}
1805#endif
1806
1807static inline void kvm_inject_gp(struct kvm_vcpu *vcpu, u32 error_code)
1808{
1809 kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
1810}
1811
1812#define TSS_IOPB_BASE_OFFSET 0x66
1813#define TSS_BASE_SIZE 0x68
1814#define TSS_IOPB_SIZE (65536 / 8)
1815#define TSS_REDIRECTION_SIZE (256 / 8)
1816#define RMODE_TSS_SIZE \
1817 (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1)
1818
1819enum {
1820 TASK_SWITCH_CALL = 0,
1821 TASK_SWITCH_IRET = 1,
1822 TASK_SWITCH_JMP = 2,
1823 TASK_SWITCH_GATE = 3,
1824};
1825
1826#define HF_GIF_MASK (1 << 0)
1827#define HF_NMI_MASK (1 << 3)
1828#define HF_IRET_MASK (1 << 4)
1829#define HF_GUEST_MASK (1 << 5)
1830#define HF_SMM_MASK (1 << 6)
1831#define HF_SMM_INSIDE_NMI_MASK (1 << 7)
1832
1833#define __KVM_VCPU_MULTIPLE_ADDRESS_SPACE
1834#define KVM_ADDRESS_SPACE_NUM 2
1835
1836#define kvm_arch_vcpu_memslots_id(vcpu) ((vcpu)->arch.hflags & HF_SMM_MASK ? 1 : 0)
1837#define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, (role).smm)
1838
1839#define KVM_ARCH_WANT_MMU_NOTIFIER
1840
1841int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v);
1842int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu);
1843int kvm_cpu_has_extint(struct kvm_vcpu *v);
1844int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu);
1845int kvm_cpu_get_interrupt(struct kvm_vcpu *v);
1846void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event);
1847void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu);
1848
1849int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low,
1850 unsigned long ipi_bitmap_high, u32 min,
1851 unsigned long icr, int op_64_bit);
1852
1853int kvm_add_user_return_msr(u32 msr);
1854int kvm_find_user_return_msr(u32 msr);
1855int kvm_set_user_return_msr(unsigned index, u64 val, u64 mask);
1856
1857static inline bool kvm_is_supported_user_return_msr(u32 msr)
1858{
1859 return kvm_find_user_return_msr(msr) >= 0;
1860}
1861
1862u64 kvm_scale_tsc(struct kvm_vcpu *vcpu, u64 tsc, u64 ratio);
1863u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc);
1864u64 kvm_calc_nested_tsc_offset(u64 l1_offset, u64 l2_offset, u64 l2_multiplier);
1865u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier);
1866
1867unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu);
1868bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip);
1869
1870void kvm_make_mclock_inprogress_request(struct kvm *kvm);
1871void kvm_make_scan_ioapic_request(struct kvm *kvm);
1872void kvm_make_scan_ioapic_request_mask(struct kvm *kvm,
1873 unsigned long *vcpu_bitmap);
1874
1875bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1876 struct kvm_async_pf *work);
1877void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1878 struct kvm_async_pf *work);
1879void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1880 struct kvm_async_pf *work);
1881void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu);
1882bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu);
1883extern bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn);
1884
1885int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu);
1886int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err);
1887void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu);
1888
1889int kvm_is_in_guest(void);
1890
1891void __user *__x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa,
1892 u32 size);
1893bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu);
1894bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu);
1895
1896bool kvm_intr_is_single_vcpu(struct kvm *kvm, struct kvm_lapic_irq *irq,
1897 struct kvm_vcpu **dest_vcpu);
1898
1899void kvm_set_msi_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e,
1900 struct kvm_lapic_irq *irq);
1901
1902static inline bool kvm_irq_is_postable(struct kvm_lapic_irq *irq)
1903{
1904
1905 return (irq->delivery_mode == APIC_DM_FIXED ||
1906 irq->delivery_mode == APIC_DM_LOWEST);
1907}
1908
1909static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
1910{
1911 static_call_cond(kvm_x86_vcpu_blocking)(vcpu);
1912}
1913
1914static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
1915{
1916 static_call_cond(kvm_x86_vcpu_unblocking)(vcpu);
1917}
1918
1919static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {}
1920
1921static inline int kvm_cpu_get_apicid(int mps_cpu)
1922{
1923#ifdef CONFIG_X86_LOCAL_APIC
1924 return default_cpu_present_to_apicid(mps_cpu);
1925#else
1926 WARN_ON_ONCE(1);
1927 return BAD_APICID;
1928#endif
1929}
1930
1931#define put_smstate(type, buf, offset, val) \
1932 *(type *)((buf) + (offset) - 0x7e00) = val
1933
1934#define GET_SMSTATE(type, buf, offset) \
1935 (*(type *)((buf) + (offset) - 0x7e00))
1936
1937int kvm_cpu_dirty_log_size(void);
1938
1939int alloc_all_memslots_rmaps(struct kvm *kvm);
1940
1941#endif
1942