1#ifndef __KVM_HOST_H
2#define __KVM_HOST_H
3
4
5
6
7
8
9#include <linux/types.h>
10#include <linux/hardirq.h>
11#include <linux/list.h>
12#include <linux/mutex.h>
13#include <linux/spinlock.h>
14#include <linux/signal.h>
15#include <linux/sched.h>
16#include <linux/bug.h>
17#include <linux/mm.h>
18#include <linux/mmu_notifier.h>
19#include <linux/preempt.h>
20#include <linux/msi.h>
21#include <linux/slab.h>
22#include <linux/rcupdate.h>
23#include <linux/ratelimit.h>
24#include <linux/err.h>
25#include <linux/irqflags.h>
26#include <linux/context_tracking.h>
27#include <linux/irqbypass.h>
28#include <asm/signal.h>
29
30#include <linux/kvm.h>
31#include <linux/kvm_para.h>
32
33#include <linux/kvm_types.h>
34
35#include <asm/kvm_host.h>
36
37
38
39
40
41
42#define KVM_MEMSLOT_INVALID (1UL << 16)
43#define KVM_MEMSLOT_INCOHERENT (1UL << 17)
44
45
46#define KVM_MAX_MMIO_FRAGMENTS 2
47
48#ifndef KVM_ADDRESS_SPACE_NUM
49#define KVM_ADDRESS_SPACE_NUM 1
50#endif
51
52
53
54
55
56
57#define KVM_PFN_ERR_MASK (0x7ffULL << 52)
58#define KVM_PFN_ERR_NOSLOT_MASK (0xfffULL << 52)
59#define KVM_PFN_NOSLOT (0x1ULL << 63)
60
61#define KVM_PFN_ERR_FAULT (KVM_PFN_ERR_MASK)
62#define KVM_PFN_ERR_HWPOISON (KVM_PFN_ERR_MASK + 1)
63#define KVM_PFN_ERR_RO_FAULT (KVM_PFN_ERR_MASK + 2)
64
65
66
67
68
69static inline bool is_error_pfn(kvm_pfn_t pfn)
70{
71 return !!(pfn & KVM_PFN_ERR_MASK);
72}
73
74
75
76
77
78
79static inline bool is_error_noslot_pfn(kvm_pfn_t pfn)
80{
81 return !!(pfn & KVM_PFN_ERR_NOSLOT_MASK);
82}
83
84
85static inline bool is_noslot_pfn(kvm_pfn_t pfn)
86{
87 return pfn == KVM_PFN_NOSLOT;
88}
89
90
91
92
93
94#ifndef KVM_HVA_ERR_BAD
95
96#define KVM_HVA_ERR_BAD (PAGE_OFFSET)
97#define KVM_HVA_ERR_RO_BAD (PAGE_OFFSET + PAGE_SIZE)
98
99static inline bool kvm_is_error_hva(unsigned long addr)
100{
101 return addr >= PAGE_OFFSET;
102}
103
104#endif
105
106#define KVM_ERR_PTR_BAD_PAGE (ERR_PTR(-ENOENT))
107
108static inline bool is_error_page(struct page *page)
109{
110 return IS_ERR(page);
111}
112
113
114
115
116
117#define KVM_REQ_TLB_FLUSH 0
118#define KVM_REQ_MMU_RELOAD 1
119#define KVM_REQ_PENDING_TIMER 2
120#define KVM_REQ_UNHALT 3
121
122#define KVM_USERSPACE_IRQ_SOURCE_ID 0
123#define KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID 1
124
125extern struct kmem_cache *kvm_vcpu_cache;
126
127extern spinlock_t kvm_lock;
128extern struct list_head vm_list;
129
130struct kvm_io_range {
131 gpa_t addr;
132 int len;
133 struct kvm_io_device *dev;
134};
135
136#define NR_IOBUS_DEVS 1000
137
138struct kvm_io_bus {
139 int dev_count;
140 int ioeventfd_count;
141 struct kvm_io_range range[];
142};
143
144enum kvm_bus {
145 KVM_MMIO_BUS,
146 KVM_PIO_BUS,
147 KVM_VIRTIO_CCW_NOTIFY_BUS,
148 KVM_FAST_MMIO_BUS,
149 KVM_NR_BUSES
150};
151
152int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
153 int len, const void *val);
154int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
155 gpa_t addr, int len, const void *val, long cookie);
156int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
157 int len, void *val);
158int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
159 int len, struct kvm_io_device *dev);
160int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
161 struct kvm_io_device *dev);
162
163#ifdef CONFIG_KVM_ASYNC_PF
164struct kvm_async_pf {
165 struct work_struct work;
166 struct list_head link;
167 struct list_head queue;
168 struct kvm_vcpu *vcpu;
169 struct mm_struct *mm;
170 gva_t gva;
171 unsigned long addr;
172 struct kvm_arch_async_pf arch;
173 bool wakeup_all;
174};
175
176void kvm_clear_async_pf_completion_queue(struct kvm_vcpu *vcpu);
177void kvm_check_async_pf_completion(struct kvm_vcpu *vcpu);
178int kvm_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, unsigned long hva,
179 struct kvm_arch_async_pf *arch);
180int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu);
181#endif
182
183enum {
184 OUTSIDE_GUEST_MODE,
185 IN_GUEST_MODE,
186 EXITING_GUEST_MODE,
187 READING_SHADOW_PAGE_TABLES,
188};
189
190
191
192
193
194struct kvm_mmio_fragment {
195 gpa_t gpa;
196 void *data;
197 unsigned len;
198};
199
200struct kvm_vcpu {
201 struct kvm *kvm;
202#ifdef CONFIG_PREEMPT_NOTIFIERS
203 struct preempt_notifier preempt_notifier;
204#endif
205 int cpu;
206 int vcpu_id;
207 int srcu_idx;
208 int mode;
209 unsigned long requests;
210 unsigned long guest_debug;
211
212 int pre_pcpu;
213 struct list_head blocked_vcpu_list;
214
215 struct mutex mutex;
216 struct kvm_run *run;
217
218 int fpu_active;
219 int guest_fpu_loaded, guest_xcr0_loaded;
220 unsigned char fpu_counter;
221 wait_queue_head_t wq;
222 struct pid *pid;
223 int sigset_active;
224 sigset_t sigset;
225 struct kvm_vcpu_stat stat;
226 unsigned int halt_poll_ns;
227
228#ifdef CONFIG_HAS_IOMEM
229 int mmio_needed;
230 int mmio_read_completed;
231 int mmio_is_write;
232 int mmio_cur_fragment;
233 int mmio_nr_fragments;
234 struct kvm_mmio_fragment mmio_fragments[KVM_MAX_MMIO_FRAGMENTS];
235#endif
236
237#ifdef CONFIG_KVM_ASYNC_PF
238 struct {
239 u32 queued;
240 struct list_head queue;
241 struct list_head done;
242 spinlock_t lock;
243 } async_pf;
244#endif
245
246#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
247
248
249
250
251
252
253 struct {
254 bool in_spin_loop;
255 bool dy_eligible;
256 } spin_loop;
257#endif
258 bool preempted;
259 struct kvm_vcpu_arch arch;
260};
261
262static inline int kvm_vcpu_exiting_guest_mode(struct kvm_vcpu *vcpu)
263{
264 return cmpxchg(&vcpu->mode, IN_GUEST_MODE, EXITING_GUEST_MODE);
265}
266
267
268
269
270
271#define KVM_MEM_MAX_NR_PAGES ((1UL << 31) - 1)
272
273struct kvm_memory_slot {
274 gfn_t base_gfn;
275 unsigned long npages;
276 unsigned long *dirty_bitmap;
277 struct kvm_arch_memory_slot arch;
278 unsigned long userspace_addr;
279 u32 flags;
280 short id;
281};
282
283static inline unsigned long kvm_dirty_bitmap_bytes(struct kvm_memory_slot *memslot)
284{
285 return ALIGN(memslot->npages, BITS_PER_LONG) / 8;
286}
287
288struct kvm_s390_adapter_int {
289 u64 ind_addr;
290 u64 summary_addr;
291 u64 ind_offset;
292 u32 summary_offset;
293 u32 adapter_id;
294};
295
296struct kvm_hv_sint {
297 u32 vcpu;
298 u32 sint;
299};
300
301struct kvm_kernel_irq_routing_entry {
302 u32 gsi;
303 u32 type;
304 int (*set)(struct kvm_kernel_irq_routing_entry *e,
305 struct kvm *kvm, int irq_source_id, int level,
306 bool line_status);
307 union {
308 struct {
309 unsigned irqchip;
310 unsigned pin;
311 } irqchip;
312 struct msi_msg msi;
313 struct kvm_s390_adapter_int adapter;
314 struct kvm_hv_sint hv_sint;
315 };
316 struct hlist_node link;
317};
318
319#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
320struct kvm_irq_routing_table {
321 int chip[KVM_NR_IRQCHIPS][KVM_IRQCHIP_NUM_PINS];
322 u32 nr_rt_entries;
323
324
325
326
327 struct hlist_head map[0];
328};
329#endif
330
331#ifndef KVM_PRIVATE_MEM_SLOTS
332#define KVM_PRIVATE_MEM_SLOTS 0
333#endif
334
335#ifndef KVM_MEM_SLOTS_NUM
336#define KVM_MEM_SLOTS_NUM (KVM_USER_MEM_SLOTS + KVM_PRIVATE_MEM_SLOTS)
337#endif
338
339#ifndef __KVM_VCPU_MULTIPLE_ADDRESS_SPACE
340static inline int kvm_arch_vcpu_memslots_id(struct kvm_vcpu *vcpu)
341{
342 return 0;
343}
344#endif
345
346
347
348
349
350
351struct kvm_memslots {
352 u64 generation;
353 struct kvm_memory_slot memslots[KVM_MEM_SLOTS_NUM];
354
355 short id_to_index[KVM_MEM_SLOTS_NUM];
356 atomic_t lru_slot;
357 int used_slots;
358};
359
360struct kvm {
361 spinlock_t mmu_lock;
362 struct mutex slots_lock;
363 struct mm_struct *mm;
364 struct kvm_memslots *memslots[KVM_ADDRESS_SPACE_NUM];
365 struct srcu_struct srcu;
366 struct srcu_struct irq_srcu;
367 struct kvm_vcpu *vcpus[KVM_MAX_VCPUS];
368 atomic_t online_vcpus;
369 int last_boosted_vcpu;
370 struct list_head vm_list;
371 struct mutex lock;
372 struct kvm_io_bus *buses[KVM_NR_BUSES];
373#ifdef CONFIG_HAVE_KVM_EVENTFD
374 struct {
375 spinlock_t lock;
376 struct list_head items;
377 struct list_head resampler_list;
378 struct mutex resampler_lock;
379 } irqfds;
380 struct list_head ioeventfds;
381#endif
382 struct kvm_vm_stat stat;
383 struct kvm_arch arch;
384 atomic_t users_count;
385#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
386 struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
387 spinlock_t ring_lock;
388 struct list_head coalesced_zones;
389#endif
390
391 struct mutex irq_lock;
392#ifdef CONFIG_HAVE_KVM_IRQCHIP
393
394
395
396 struct kvm_irq_routing_table __rcu *irq_routing;
397#endif
398#ifdef CONFIG_HAVE_KVM_IRQFD
399 struct hlist_head irq_ack_notifier_list;
400#endif
401
402#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
403 struct mmu_notifier mmu_notifier;
404 unsigned long mmu_notifier_seq;
405 long mmu_notifier_count;
406#endif
407 long tlbs_dirty;
408 struct list_head devices;
409};
410
411#define kvm_err(fmt, ...) \
412 pr_err("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__)
413#define kvm_info(fmt, ...) \
414 pr_info("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__)
415#define kvm_debug(fmt, ...) \
416 pr_debug("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__)
417#define kvm_pr_unimpl(fmt, ...) \
418 pr_err_ratelimited("kvm [%i]: " fmt, \
419 task_tgid_nr(current), ## __VA_ARGS__)
420
421
422#define vcpu_unimpl(vcpu, fmt, ...) \
423 kvm_pr_unimpl("vcpu%i, guest rIP: 0x%lx " fmt, \
424 (vcpu)->vcpu_id, kvm_rip_read(vcpu), ## __VA_ARGS__)
425
426#define vcpu_debug(vcpu, fmt, ...) \
427 kvm_debug("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__)
428#define vcpu_err(vcpu, fmt, ...) \
429 kvm_err("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__)
430
431static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i)
432{
433
434
435
436
437 smp_rmb();
438 return kvm->vcpus[i];
439}
440
441#define kvm_for_each_vcpu(idx, vcpup, kvm) \
442 for (idx = 0; \
443 idx < atomic_read(&kvm->online_vcpus) && \
444 (vcpup = kvm_get_vcpu(kvm, idx)) != NULL; \
445 idx++)
446
447static inline struct kvm_vcpu *kvm_get_vcpu_by_id(struct kvm *kvm, int id)
448{
449 struct kvm_vcpu *vcpu;
450 int i;
451
452 if (id < 0 || id >= KVM_MAX_VCPUS)
453 return NULL;
454 vcpu = kvm_get_vcpu(kvm, id);
455 if (vcpu && vcpu->vcpu_id == id)
456 return vcpu;
457 kvm_for_each_vcpu(i, vcpu, kvm)
458 if (vcpu->vcpu_id == id)
459 return vcpu;
460 return NULL;
461}
462
463#define kvm_for_each_memslot(memslot, slots) \
464 for (memslot = &slots->memslots[0]; \
465 memslot < slots->memslots + KVM_MEM_SLOTS_NUM && memslot->npages;\
466 memslot++)
467
468int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id);
469void kvm_vcpu_uninit(struct kvm_vcpu *vcpu);
470
471int __must_check vcpu_load(struct kvm_vcpu *vcpu);
472void vcpu_put(struct kvm_vcpu *vcpu);
473
474#ifdef __KVM_HAVE_IOAPIC
475void kvm_vcpu_request_scan_ioapic(struct kvm *kvm);
476void kvm_arch_post_irq_routing_update(struct kvm *kvm);
477#else
478static inline void kvm_vcpu_request_scan_ioapic(struct kvm *kvm)
479{
480}
481static inline void kvm_arch_post_irq_routing_update(struct kvm *kvm)
482{
483}
484#endif
485
486#ifdef CONFIG_HAVE_KVM_IRQFD
487int kvm_irqfd_init(void);
488void kvm_irqfd_exit(void);
489#else
490static inline int kvm_irqfd_init(void)
491{
492 return 0;
493}
494
495static inline void kvm_irqfd_exit(void)
496{
497}
498#endif
499int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
500 struct module *module);
501void kvm_exit(void);
502
503void kvm_get_kvm(struct kvm *kvm);
504void kvm_put_kvm(struct kvm *kvm);
505
506static inline struct kvm_memslots *__kvm_memslots(struct kvm *kvm, int as_id)
507{
508 return rcu_dereference_check(kvm->memslots[as_id],
509 srcu_read_lock_held(&kvm->srcu)
510 || lockdep_is_held(&kvm->slots_lock));
511}
512
513static inline struct kvm_memslots *kvm_memslots(struct kvm *kvm)
514{
515 return __kvm_memslots(kvm, 0);
516}
517
518static inline struct kvm_memslots *kvm_vcpu_memslots(struct kvm_vcpu *vcpu)
519{
520 int as_id = kvm_arch_vcpu_memslots_id(vcpu);
521
522 return __kvm_memslots(vcpu->kvm, as_id);
523}
524
525static inline struct kvm_memory_slot *
526id_to_memslot(struct kvm_memslots *slots, int id)
527{
528 int index = slots->id_to_index[id];
529 struct kvm_memory_slot *slot;
530
531 slot = &slots->memslots[index];
532
533 WARN_ON(slot->id != id);
534 return slot;
535}
536
537
538
539
540
541
542
543
544
545
546
547
548enum kvm_mr_change {
549 KVM_MR_CREATE,
550 KVM_MR_DELETE,
551 KVM_MR_MOVE,
552 KVM_MR_FLAGS_ONLY,
553};
554
555int kvm_set_memory_region(struct kvm *kvm,
556 const struct kvm_userspace_memory_region *mem);
557int __kvm_set_memory_region(struct kvm *kvm,
558 const struct kvm_userspace_memory_region *mem);
559void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
560 struct kvm_memory_slot *dont);
561int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
562 unsigned long npages);
563void kvm_arch_memslots_updated(struct kvm *kvm, struct kvm_memslots *slots);
564int kvm_arch_prepare_memory_region(struct kvm *kvm,
565 struct kvm_memory_slot *memslot,
566 const struct kvm_userspace_memory_region *mem,
567 enum kvm_mr_change change);
568void kvm_arch_commit_memory_region(struct kvm *kvm,
569 const struct kvm_userspace_memory_region *mem,
570 const struct kvm_memory_slot *old,
571 const struct kvm_memory_slot *new,
572 enum kvm_mr_change change);
573bool kvm_largepages_enabled(void);
574void kvm_disable_largepages(void);
575
576void kvm_arch_flush_shadow_all(struct kvm *kvm);
577
578void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
579 struct kvm_memory_slot *slot);
580
581int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
582 struct page **pages, int nr_pages);
583
584struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn);
585unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn);
586unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable);
587unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, gfn_t gfn);
588unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot, gfn_t gfn,
589 bool *writable);
590void kvm_release_page_clean(struct page *page);
591void kvm_release_page_dirty(struct page *page);
592void kvm_set_page_accessed(struct page *page);
593
594kvm_pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn);
595kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn);
596kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
597 bool *writable);
598kvm_pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn);
599kvm_pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn);
600kvm_pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn,
601 bool atomic, bool *async, bool write_fault,
602 bool *writable);
603
604void kvm_release_pfn_clean(kvm_pfn_t pfn);
605void kvm_set_pfn_dirty(kvm_pfn_t pfn);
606void kvm_set_pfn_accessed(kvm_pfn_t pfn);
607void kvm_get_pfn(kvm_pfn_t pfn);
608
609int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
610 int len);
611int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
612 unsigned long len);
613int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len);
614int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
615 void *data, unsigned long len);
616int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
617 int offset, int len);
618int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
619 unsigned long len);
620int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
621 void *data, unsigned long len);
622int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
623 gpa_t gpa, unsigned long len);
624int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len);
625int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len);
626struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn);
627bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn);
628unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn);
629void mark_page_dirty(struct kvm *kvm, gfn_t gfn);
630
631struct kvm_memslots *kvm_vcpu_memslots(struct kvm_vcpu *vcpu);
632struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn);
633kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn);
634kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn);
635struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn);
636unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn);
637unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable);
638int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data, int offset,
639 int len);
640int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa, void *data,
641 unsigned long len);
642int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data,
643 unsigned long len);
644int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, const void *data,
645 int offset, int len);
646int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data,
647 unsigned long len);
648void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn);
649
650void kvm_vcpu_block(struct kvm_vcpu *vcpu);
651void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu);
652void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu);
653void kvm_vcpu_kick(struct kvm_vcpu *vcpu);
654int kvm_vcpu_yield_to(struct kvm_vcpu *target);
655void kvm_vcpu_on_spin(struct kvm_vcpu *vcpu);
656void kvm_load_guest_fpu(struct kvm_vcpu *vcpu);
657void kvm_put_guest_fpu(struct kvm_vcpu *vcpu);
658
659void kvm_flush_remote_tlbs(struct kvm *kvm);
660void kvm_reload_remote_mmus(struct kvm *kvm);
661bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req);
662
663long kvm_arch_dev_ioctl(struct file *filp,
664 unsigned int ioctl, unsigned long arg);
665long kvm_arch_vcpu_ioctl(struct file *filp,
666 unsigned int ioctl, unsigned long arg);
667int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf);
668
669int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext);
670
671int kvm_get_dirty_log(struct kvm *kvm,
672 struct kvm_dirty_log *log, int *is_dirty);
673
674int kvm_get_dirty_log_protect(struct kvm *kvm,
675 struct kvm_dirty_log *log, bool *is_dirty);
676
677void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
678 struct kvm_memory_slot *slot,
679 gfn_t gfn_offset,
680 unsigned long mask);
681
682int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
683 struct kvm_dirty_log *log);
684
685int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
686 bool line_status);
687long kvm_arch_vm_ioctl(struct file *filp,
688 unsigned int ioctl, unsigned long arg);
689
690int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu);
691int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu);
692
693int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
694 struct kvm_translation *tr);
695
696int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs);
697int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs);
698int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
699 struct kvm_sregs *sregs);
700int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
701 struct kvm_sregs *sregs);
702int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
703 struct kvm_mp_state *mp_state);
704int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
705 struct kvm_mp_state *mp_state);
706int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
707 struct kvm_guest_debug *dbg);
708int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run);
709
710int kvm_arch_init(void *opaque);
711void kvm_arch_exit(void);
712
713int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu);
714void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu);
715
716void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu);
717
718void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu);
719void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu);
720void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu);
721struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id);
722int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu);
723void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu);
724void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu);
725
726int kvm_arch_hardware_enable(void);
727void kvm_arch_hardware_disable(void);
728int kvm_arch_hardware_setup(void);
729void kvm_arch_hardware_unsetup(void);
730void kvm_arch_check_processor_compat(void *rtn);
731int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu);
732int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu);
733
734void *kvm_kvzalloc(unsigned long size);
735
736#ifndef __KVM_HAVE_ARCH_VM_ALLOC
737static inline struct kvm *kvm_arch_alloc_vm(void)
738{
739 return kzalloc(sizeof(struct kvm), GFP_KERNEL);
740}
741
742static inline void kvm_arch_free_vm(struct kvm *kvm)
743{
744 kfree(kvm);
745}
746#endif
747
748#ifdef __KVM_HAVE_ARCH_NONCOHERENT_DMA
749void kvm_arch_register_noncoherent_dma(struct kvm *kvm);
750void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm);
751bool kvm_arch_has_noncoherent_dma(struct kvm *kvm);
752#else
753static inline void kvm_arch_register_noncoherent_dma(struct kvm *kvm)
754{
755}
756
757static inline void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm)
758{
759}
760
761static inline bool kvm_arch_has_noncoherent_dma(struct kvm *kvm)
762{
763 return false;
764}
765#endif
766#ifdef __KVM_HAVE_ARCH_ASSIGNED_DEVICE
767void kvm_arch_start_assignment(struct kvm *kvm);
768void kvm_arch_end_assignment(struct kvm *kvm);
769bool kvm_arch_has_assigned_device(struct kvm *kvm);
770#else
771static inline void kvm_arch_start_assignment(struct kvm *kvm)
772{
773}
774
775static inline void kvm_arch_end_assignment(struct kvm *kvm)
776{
777}
778
779static inline bool kvm_arch_has_assigned_device(struct kvm *kvm)
780{
781 return false;
782}
783#endif
784
785static inline wait_queue_head_t *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu)
786{
787#ifdef __KVM_HAVE_ARCH_WQP
788 return vcpu->arch.wqp;
789#else
790 return &vcpu->wq;
791#endif
792}
793
794#ifdef __KVM_HAVE_ARCH_INTC_INITIALIZED
795
796
797
798
799
800bool kvm_arch_intc_initialized(struct kvm *kvm);
801#else
802static inline bool kvm_arch_intc_initialized(struct kvm *kvm)
803{
804 return true;
805}
806#endif
807
808int kvm_arch_init_vm(struct kvm *kvm, unsigned long type);
809void kvm_arch_destroy_vm(struct kvm *kvm);
810void kvm_arch_sync_events(struct kvm *kvm);
811
812int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu);
813void kvm_vcpu_kick(struct kvm_vcpu *vcpu);
814
815bool kvm_is_reserved_pfn(kvm_pfn_t pfn);
816
817struct kvm_irq_ack_notifier {
818 struct hlist_node link;
819 unsigned gsi;
820 void (*irq_acked)(struct kvm_irq_ack_notifier *kian);
821};
822
823int kvm_irq_map_gsi(struct kvm *kvm,
824 struct kvm_kernel_irq_routing_entry *entries, int gsi);
825int kvm_irq_map_chip_pin(struct kvm *kvm, unsigned irqchip, unsigned pin);
826
827int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level,
828 bool line_status);
829int kvm_set_msi(struct kvm_kernel_irq_routing_entry *irq_entry, struct kvm *kvm,
830 int irq_source_id, int level, bool line_status);
831int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e,
832 struct kvm *kvm, int irq_source_id,
833 int level, bool line_status);
834bool kvm_irq_has_notifier(struct kvm *kvm, unsigned irqchip, unsigned pin);
835void kvm_notify_acked_gsi(struct kvm *kvm, int gsi);
836void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin);
837void kvm_register_irq_ack_notifier(struct kvm *kvm,
838 struct kvm_irq_ack_notifier *kian);
839void kvm_unregister_irq_ack_notifier(struct kvm *kvm,
840 struct kvm_irq_ack_notifier *kian);
841int kvm_request_irq_source_id(struct kvm *kvm);
842void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id);
843
844#ifdef CONFIG_KVM_DEVICE_ASSIGNMENT
845int kvm_iommu_map_pages(struct kvm *kvm, struct kvm_memory_slot *slot);
846void kvm_iommu_unmap_pages(struct kvm *kvm, struct kvm_memory_slot *slot);
847#else
848static inline int kvm_iommu_map_pages(struct kvm *kvm,
849 struct kvm_memory_slot *slot)
850{
851 return 0;
852}
853
854static inline void kvm_iommu_unmap_pages(struct kvm *kvm,
855 struct kvm_memory_slot *slot)
856{
857}
858#endif
859
860
861static inline void __kvm_guest_enter(void)
862{
863 guest_enter();
864
865
866
867
868
869
870
871 if (!context_tracking_cpu_is_enabled())
872 rcu_virt_note_context_switch(smp_processor_id());
873}
874
875
876static inline void __kvm_guest_exit(void)
877{
878 guest_exit();
879}
880
881static inline void kvm_guest_enter(void)
882{
883 unsigned long flags;
884
885 local_irq_save(flags);
886 __kvm_guest_enter();
887 local_irq_restore(flags);
888}
889
890static inline void kvm_guest_exit(void)
891{
892 unsigned long flags;
893
894 local_irq_save(flags);
895 __kvm_guest_exit();
896 local_irq_restore(flags);
897}
898
899
900
901
902
903
904
905static inline struct kvm_memory_slot *
906search_memslots(struct kvm_memslots *slots, gfn_t gfn)
907{
908 int start = 0, end = slots->used_slots;
909 int slot = atomic_read(&slots->lru_slot);
910 struct kvm_memory_slot *memslots = slots->memslots;
911
912 if (gfn >= memslots[slot].base_gfn &&
913 gfn < memslots[slot].base_gfn + memslots[slot].npages)
914 return &memslots[slot];
915
916 while (start < end) {
917 slot = start + (end - start) / 2;
918
919 if (gfn >= memslots[slot].base_gfn)
920 end = slot;
921 else
922 start = slot + 1;
923 }
924
925 if (gfn >= memslots[start].base_gfn &&
926 gfn < memslots[start].base_gfn + memslots[start].npages) {
927 atomic_set(&slots->lru_slot, start);
928 return &memslots[start];
929 }
930
931 return NULL;
932}
933
934static inline struct kvm_memory_slot *
935__gfn_to_memslot(struct kvm_memslots *slots, gfn_t gfn)
936{
937 return search_memslots(slots, gfn);
938}
939
940static inline unsigned long
941__gfn_to_hva_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
942{
943 return slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE;
944}
945
946static inline int memslot_id(struct kvm *kvm, gfn_t gfn)
947{
948 return gfn_to_memslot(kvm, gfn)->id;
949}
950
951static inline gfn_t
952hva_to_gfn_memslot(unsigned long hva, struct kvm_memory_slot *slot)
953{
954 gfn_t gfn_offset = (hva - slot->userspace_addr) >> PAGE_SHIFT;
955
956 return slot->base_gfn + gfn_offset;
957}
958
959static inline gpa_t gfn_to_gpa(gfn_t gfn)
960{
961 return (gpa_t)gfn << PAGE_SHIFT;
962}
963
964static inline gfn_t gpa_to_gfn(gpa_t gpa)
965{
966 return (gfn_t)(gpa >> PAGE_SHIFT);
967}
968
969static inline hpa_t pfn_to_hpa(kvm_pfn_t pfn)
970{
971 return (hpa_t)pfn << PAGE_SHIFT;
972}
973
974static inline bool kvm_is_error_gpa(struct kvm *kvm, gpa_t gpa)
975{
976 unsigned long hva = gfn_to_hva(kvm, gpa_to_gfn(gpa));
977
978 return kvm_is_error_hva(hva);
979}
980
981enum kvm_stat_kind {
982 KVM_STAT_VM,
983 KVM_STAT_VCPU,
984};
985
986struct kvm_stats_debugfs_item {
987 const char *name;
988 int offset;
989 enum kvm_stat_kind kind;
990};
991extern struct kvm_stats_debugfs_item debugfs_entries[];
992extern struct dentry *kvm_debugfs_dir;
993
994#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
995static inline int mmu_notifier_retry(struct kvm *kvm, unsigned long mmu_seq)
996{
997 if (unlikely(kvm->mmu_notifier_count))
998 return 1;
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009 smp_rmb();
1010 if (kvm->mmu_notifier_seq != mmu_seq)
1011 return 1;
1012 return 0;
1013}
1014#endif
1015
1016#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
1017
1018#ifdef CONFIG_S390
1019#define KVM_MAX_IRQ_ROUTES 4096
1020#else
1021#define KVM_MAX_IRQ_ROUTES 1024
1022#endif
1023
1024int kvm_setup_default_irq_routing(struct kvm *kvm);
1025int kvm_setup_empty_irq_routing(struct kvm *kvm);
1026int kvm_set_irq_routing(struct kvm *kvm,
1027 const struct kvm_irq_routing_entry *entries,
1028 unsigned nr,
1029 unsigned flags);
1030int kvm_set_routing_entry(struct kvm_kernel_irq_routing_entry *e,
1031 const struct kvm_irq_routing_entry *ue);
1032void kvm_free_irq_routing(struct kvm *kvm);
1033
1034#else
1035
1036static inline void kvm_free_irq_routing(struct kvm *kvm) {}
1037
1038#endif
1039
1040int kvm_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi);
1041
1042#ifdef CONFIG_HAVE_KVM_EVENTFD
1043
1044void kvm_eventfd_init(struct kvm *kvm);
1045int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args);
1046
1047#ifdef CONFIG_HAVE_KVM_IRQFD
1048int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args);
1049void kvm_irqfd_release(struct kvm *kvm);
1050void kvm_irq_routing_update(struct kvm *);
1051#else
1052static inline int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args)
1053{
1054 return -EINVAL;
1055}
1056
1057static inline void kvm_irqfd_release(struct kvm *kvm) {}
1058#endif
1059
1060#else
1061
1062static inline void kvm_eventfd_init(struct kvm *kvm) {}
1063
1064static inline int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args)
1065{
1066 return -EINVAL;
1067}
1068
1069static inline void kvm_irqfd_release(struct kvm *kvm) {}
1070
1071#ifdef CONFIG_HAVE_KVM_IRQCHIP
1072static inline void kvm_irq_routing_update(struct kvm *kvm)
1073{
1074}
1075#endif
1076void kvm_arch_irq_routing_update(struct kvm *kvm);
1077
1078static inline int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
1079{
1080 return -ENOSYS;
1081}
1082
1083#endif
1084
1085#ifdef CONFIG_KVM_APIC_ARCHITECTURE
1086bool kvm_vcpu_compatible(struct kvm_vcpu *vcpu);
1087#else
1088static inline bool kvm_vcpu_compatible(struct kvm_vcpu *vcpu) { return true; }
1089#endif
1090
1091static inline void kvm_make_request(int req, struct kvm_vcpu *vcpu)
1092{
1093 set_bit(req, &vcpu->requests);
1094}
1095
1096static inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu)
1097{
1098 if (test_bit(req, &vcpu->requests)) {
1099 clear_bit(req, &vcpu->requests);
1100 return true;
1101 } else {
1102 return false;
1103 }
1104}
1105
1106extern bool kvm_rebooting;
1107
1108struct kvm_device {
1109 struct kvm_device_ops *ops;
1110 struct kvm *kvm;
1111 void *private;
1112 struct list_head vm_node;
1113};
1114
1115
1116struct kvm_device_ops {
1117 const char *name;
1118 int (*create)(struct kvm_device *dev, u32 type);
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128 void (*destroy)(struct kvm_device *dev);
1129
1130 int (*set_attr)(struct kvm_device *dev, struct kvm_device_attr *attr);
1131 int (*get_attr)(struct kvm_device *dev, struct kvm_device_attr *attr);
1132 int (*has_attr)(struct kvm_device *dev, struct kvm_device_attr *attr);
1133 long (*ioctl)(struct kvm_device *dev, unsigned int ioctl,
1134 unsigned long arg);
1135};
1136
1137void kvm_device_get(struct kvm_device *dev);
1138void kvm_device_put(struct kvm_device *dev);
1139struct kvm_device *kvm_device_from_filp(struct file *filp);
1140int kvm_register_device_ops(struct kvm_device_ops *ops, u32 type);
1141void kvm_unregister_device_ops(u32 type);
1142
1143extern struct kvm_device_ops kvm_mpic_ops;
1144extern struct kvm_device_ops kvm_xics_ops;
1145extern struct kvm_device_ops kvm_arm_vgic_v2_ops;
1146extern struct kvm_device_ops kvm_arm_vgic_v3_ops;
1147
1148#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
1149
1150static inline void kvm_vcpu_set_in_spin_loop(struct kvm_vcpu *vcpu, bool val)
1151{
1152 vcpu->spin_loop.in_spin_loop = val;
1153}
1154static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val)
1155{
1156 vcpu->spin_loop.dy_eligible = val;
1157}
1158
1159#else
1160
1161static inline void kvm_vcpu_set_in_spin_loop(struct kvm_vcpu *vcpu, bool val)
1162{
1163}
1164
1165static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val)
1166{
1167}
1168#endif
1169
1170#ifdef CONFIG_HAVE_KVM_IRQ_BYPASS
1171int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *,
1172 struct irq_bypass_producer *);
1173void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *,
1174 struct irq_bypass_producer *);
1175void kvm_arch_irq_bypass_stop(struct irq_bypass_consumer *);
1176void kvm_arch_irq_bypass_start(struct irq_bypass_consumer *);
1177int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq,
1178 uint32_t guest_irq, bool set);
1179#endif
1180
1181#endif
1182