1#ifndef __KVM_HOST_H
2#define __KVM_HOST_H
3
4
5
6
7
8
9#include <linux/types.h>
10#include <linux/hardirq.h>
11#include <linux/list.h>
12#include <linux/mutex.h>
13#include <linux/spinlock.h>
14#include <linux/signal.h>
15#include <linux/sched.h>
16#include <linux/bug.h>
17#include <linux/mm.h>
18#include <linux/mmu_notifier.h>
19#include <linux/preempt.h>
20#include <linux/msi.h>
21#include <linux/slab.h>
22#include <linux/rcupdate.h>
23#include <linux/ratelimit.h>
24#include <linux/err.h>
25#include <linux/irqflags.h>
26#include <linux/context_tracking.h>
27#include <linux/irqbypass.h>
28#include <linux/swait.h>
29#include <asm/signal.h>
30
31#include <linux/kvm.h>
32#include <linux/kvm_para.h>
33
34#include <linux/kvm_types.h>
35
36#include <asm/kvm_host.h>
37
38#ifndef KVM_MAX_VCPU_ID
39#define KVM_MAX_VCPU_ID KVM_MAX_VCPUS
40#endif
41
42
43
44
45
46
47#define KVM_MEMSLOT_INVALID (1UL << 16)
48#define KVM_MEMSLOT_INCOHERENT (1UL << 17)
49
50
51#define KVM_MAX_MMIO_FRAGMENTS 2
52
53#ifndef KVM_ADDRESS_SPACE_NUM
54#define KVM_ADDRESS_SPACE_NUM 1
55#endif
56
57
58
59
60
61
62#define KVM_PFN_ERR_MASK (0x7ffULL << 52)
63#define KVM_PFN_ERR_NOSLOT_MASK (0xfffULL << 52)
64#define KVM_PFN_NOSLOT (0x1ULL << 63)
65
66#define KVM_PFN_ERR_FAULT (KVM_PFN_ERR_MASK)
67#define KVM_PFN_ERR_HWPOISON (KVM_PFN_ERR_MASK + 1)
68#define KVM_PFN_ERR_RO_FAULT (KVM_PFN_ERR_MASK + 2)
69
70
71
72
73
74static inline bool is_error_pfn(kvm_pfn_t pfn)
75{
76 return !!(pfn & KVM_PFN_ERR_MASK);
77}
78
79
80
81
82
83
84static inline bool is_error_noslot_pfn(kvm_pfn_t pfn)
85{
86 return !!(pfn & KVM_PFN_ERR_NOSLOT_MASK);
87}
88
89
90static inline bool is_noslot_pfn(kvm_pfn_t pfn)
91{
92 return pfn == KVM_PFN_NOSLOT;
93}
94
95
96
97
98
99#ifndef KVM_HVA_ERR_BAD
100
101#define KVM_HVA_ERR_BAD (PAGE_OFFSET)
102#define KVM_HVA_ERR_RO_BAD (PAGE_OFFSET + PAGE_SIZE)
103
104static inline bool kvm_is_error_hva(unsigned long addr)
105{
106 return addr >= PAGE_OFFSET;
107}
108
109#endif
110
111#define KVM_ERR_PTR_BAD_PAGE (ERR_PTR(-ENOENT))
112
113static inline bool is_error_page(struct page *page)
114{
115 return IS_ERR(page);
116}
117
118
119
120
121
122#define KVM_REQ_TLB_FLUSH 0
123#define KVM_REQ_MMU_RELOAD 1
124#define KVM_REQ_PENDING_TIMER 2
125#define KVM_REQ_UNHALT 3
126
127#define KVM_USERSPACE_IRQ_SOURCE_ID 0
128#define KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID 1
129
130extern struct kmem_cache *kvm_vcpu_cache;
131
132extern spinlock_t kvm_lock;
133extern struct list_head vm_list;
134
135struct kvm_io_range {
136 gpa_t addr;
137 int len;
138 struct kvm_io_device *dev;
139};
140
141#define NR_IOBUS_DEVS 1000
142
143struct kvm_io_bus {
144 int dev_count;
145 int ioeventfd_count;
146 struct kvm_io_range range[];
147};
148
149enum kvm_bus {
150 KVM_MMIO_BUS,
151 KVM_PIO_BUS,
152 KVM_VIRTIO_CCW_NOTIFY_BUS,
153 KVM_FAST_MMIO_BUS,
154 KVM_NR_BUSES
155};
156
157int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
158 int len, const void *val);
159int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
160 gpa_t addr, int len, const void *val, long cookie);
161int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
162 int len, void *val);
163int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
164 int len, struct kvm_io_device *dev);
165int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
166 struct kvm_io_device *dev);
167struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
168 gpa_t addr);
169
170#ifdef CONFIG_KVM_ASYNC_PF
171struct kvm_async_pf {
172 struct work_struct work;
173 struct list_head link;
174 struct list_head queue;
175 struct kvm_vcpu *vcpu;
176 struct mm_struct *mm;
177 gva_t gva;
178 unsigned long addr;
179 struct kvm_arch_async_pf arch;
180 bool wakeup_all;
181};
182
183void kvm_clear_async_pf_completion_queue(struct kvm_vcpu *vcpu);
184void kvm_check_async_pf_completion(struct kvm_vcpu *vcpu);
185int kvm_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, unsigned long hva,
186 struct kvm_arch_async_pf *arch);
187int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu);
188#endif
189
190enum {
191 OUTSIDE_GUEST_MODE,
192 IN_GUEST_MODE,
193 EXITING_GUEST_MODE,
194 READING_SHADOW_PAGE_TABLES,
195};
196
197
198
199
200
201struct kvm_mmio_fragment {
202 gpa_t gpa;
203 void *data;
204 unsigned len;
205};
206
207struct kvm_vcpu {
208 struct kvm *kvm;
209#ifdef CONFIG_PREEMPT_NOTIFIERS
210 struct preempt_notifier preempt_notifier;
211#endif
212 int cpu;
213 int vcpu_id;
214 int srcu_idx;
215 int mode;
216 unsigned long requests;
217 unsigned long guest_debug;
218
219 int pre_pcpu;
220 struct list_head blocked_vcpu_list;
221
222 struct mutex mutex;
223 struct kvm_run *run;
224
225 int fpu_active;
226 int guest_fpu_loaded, guest_xcr0_loaded;
227 struct swait_queue_head wq;
228 struct pid *pid;
229 int sigset_active;
230 sigset_t sigset;
231 struct kvm_vcpu_stat stat;
232 unsigned int halt_poll_ns;
233 bool valid_wakeup;
234
235#ifdef CONFIG_HAS_IOMEM
236 int mmio_needed;
237 int mmio_read_completed;
238 int mmio_is_write;
239 int mmio_cur_fragment;
240 int mmio_nr_fragments;
241 struct kvm_mmio_fragment mmio_fragments[KVM_MAX_MMIO_FRAGMENTS];
242#endif
243
244#ifdef CONFIG_KVM_ASYNC_PF
245 struct {
246 u32 queued;
247 struct list_head queue;
248 struct list_head done;
249 spinlock_t lock;
250 } async_pf;
251#endif
252
253#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
254
255
256
257
258
259
260 struct {
261 bool in_spin_loop;
262 bool dy_eligible;
263 } spin_loop;
264#endif
265 bool preempted;
266 struct kvm_vcpu_arch arch;
267 struct dentry *debugfs_dentry;
268};
269
270static inline int kvm_vcpu_exiting_guest_mode(struct kvm_vcpu *vcpu)
271{
272 return cmpxchg(&vcpu->mode, IN_GUEST_MODE, EXITING_GUEST_MODE);
273}
274
275
276
277
278
279#define KVM_MEM_MAX_NR_PAGES ((1UL << 31) - 1)
280
281struct kvm_memory_slot {
282 gfn_t base_gfn;
283 unsigned long npages;
284 unsigned long *dirty_bitmap;
285 struct kvm_arch_memory_slot arch;
286 unsigned long userspace_addr;
287 u32 flags;
288 short id;
289};
290
291static inline unsigned long kvm_dirty_bitmap_bytes(struct kvm_memory_slot *memslot)
292{
293 return ALIGN(memslot->npages, BITS_PER_LONG) / 8;
294}
295
296struct kvm_s390_adapter_int {
297 u64 ind_addr;
298 u64 summary_addr;
299 u64 ind_offset;
300 u32 summary_offset;
301 u32 adapter_id;
302};
303
304struct kvm_hv_sint {
305 u32 vcpu;
306 u32 sint;
307};
308
309struct kvm_kernel_irq_routing_entry {
310 u32 gsi;
311 u32 type;
312 int (*set)(struct kvm_kernel_irq_routing_entry *e,
313 struct kvm *kvm, int irq_source_id, int level,
314 bool line_status);
315 union {
316 struct {
317 unsigned irqchip;
318 unsigned pin;
319 } irqchip;
320 struct {
321 u32 address_lo;
322 u32 address_hi;
323 u32 data;
324 u32 flags;
325 u32 devid;
326 } msi;
327 struct kvm_s390_adapter_int adapter;
328 struct kvm_hv_sint hv_sint;
329 };
330 struct hlist_node link;
331};
332
333#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
334struct kvm_irq_routing_table {
335 int chip[KVM_NR_IRQCHIPS][KVM_IRQCHIP_NUM_PINS];
336 u32 nr_rt_entries;
337
338
339
340
341 struct hlist_head map[0];
342};
343#endif
344
345#ifndef KVM_PRIVATE_MEM_SLOTS
346#define KVM_PRIVATE_MEM_SLOTS 0
347#endif
348
349#ifndef KVM_MEM_SLOTS_NUM
350#define KVM_MEM_SLOTS_NUM (KVM_USER_MEM_SLOTS + KVM_PRIVATE_MEM_SLOTS)
351#endif
352
353#ifndef __KVM_VCPU_MULTIPLE_ADDRESS_SPACE
354static inline int kvm_arch_vcpu_memslots_id(struct kvm_vcpu *vcpu)
355{
356 return 0;
357}
358#endif
359
360
361
362
363
364
365struct kvm_memslots {
366 u64 generation;
367 struct kvm_memory_slot memslots[KVM_MEM_SLOTS_NUM];
368
369 short id_to_index[KVM_MEM_SLOTS_NUM];
370 atomic_t lru_slot;
371 int used_slots;
372};
373
374struct kvm {
375 spinlock_t mmu_lock;
376 struct mutex slots_lock;
377 struct mm_struct *mm;
378 struct kvm_memslots *memslots[KVM_ADDRESS_SPACE_NUM];
379 struct srcu_struct srcu;
380 struct srcu_struct irq_srcu;
381 struct kvm_vcpu *vcpus[KVM_MAX_VCPUS];
382
383
384
385
386
387
388
389 atomic_t online_vcpus;
390 int created_vcpus;
391 int last_boosted_vcpu;
392 struct list_head vm_list;
393 struct mutex lock;
394 struct kvm_io_bus *buses[KVM_NR_BUSES];
395#ifdef CONFIG_HAVE_KVM_EVENTFD
396 struct {
397 spinlock_t lock;
398 struct list_head items;
399 struct list_head resampler_list;
400 struct mutex resampler_lock;
401 } irqfds;
402 struct list_head ioeventfds;
403#endif
404 struct kvm_vm_stat stat;
405 struct kvm_arch arch;
406 atomic_t users_count;
407#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
408 struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
409 spinlock_t ring_lock;
410 struct list_head coalesced_zones;
411#endif
412
413 struct mutex irq_lock;
414#ifdef CONFIG_HAVE_KVM_IRQCHIP
415
416
417
418 struct kvm_irq_routing_table __rcu *irq_routing;
419#endif
420#ifdef CONFIG_HAVE_KVM_IRQFD
421 struct hlist_head irq_ack_notifier_list;
422#endif
423
424#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
425 struct mmu_notifier mmu_notifier;
426 unsigned long mmu_notifier_seq;
427 long mmu_notifier_count;
428#endif
429 long tlbs_dirty;
430 struct list_head devices;
431 struct dentry *debugfs_dentry;
432 struct kvm_stat_data **debugfs_stat_data;
433};
434
435#define kvm_err(fmt, ...) \
436 pr_err("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__)
437#define kvm_info(fmt, ...) \
438 pr_info("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__)
439#define kvm_debug(fmt, ...) \
440 pr_debug("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__)
441#define kvm_debug_ratelimited(fmt, ...) \
442 pr_debug_ratelimited("kvm [%i]: " fmt, task_pid_nr(current), \
443 ## __VA_ARGS__)
444#define kvm_pr_unimpl(fmt, ...) \
445 pr_err_ratelimited("kvm [%i]: " fmt, \
446 task_tgid_nr(current), ## __VA_ARGS__)
447
448
449#define vcpu_unimpl(vcpu, fmt, ...) \
450 kvm_pr_unimpl("vcpu%i, guest rIP: 0x%lx " fmt, \
451 (vcpu)->vcpu_id, kvm_rip_read(vcpu), ## __VA_ARGS__)
452
453#define vcpu_debug(vcpu, fmt, ...) \
454 kvm_debug("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__)
455#define vcpu_debug_ratelimited(vcpu, fmt, ...) \
456 kvm_debug_ratelimited("vcpu%i " fmt, (vcpu)->vcpu_id, \
457 ## __VA_ARGS__)
458#define vcpu_err(vcpu, fmt, ...) \
459 kvm_err("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__)
460
461static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i)
462{
463
464
465
466
467 smp_rmb();
468 return kvm->vcpus[i];
469}
470
471#define kvm_for_each_vcpu(idx, vcpup, kvm) \
472 for (idx = 0; \
473 idx < atomic_read(&kvm->online_vcpus) && \
474 (vcpup = kvm_get_vcpu(kvm, idx)) != NULL; \
475 idx++)
476
477static inline struct kvm_vcpu *kvm_get_vcpu_by_id(struct kvm *kvm, int id)
478{
479 struct kvm_vcpu *vcpu = NULL;
480 int i;
481
482 if (id < 0)
483 return NULL;
484 if (id < KVM_MAX_VCPUS)
485 vcpu = kvm_get_vcpu(kvm, id);
486 if (vcpu && vcpu->vcpu_id == id)
487 return vcpu;
488 kvm_for_each_vcpu(i, vcpu, kvm)
489 if (vcpu->vcpu_id == id)
490 return vcpu;
491 return NULL;
492}
493
494#define kvm_for_each_memslot(memslot, slots) \
495 for (memslot = &slots->memslots[0]; \
496 memslot < slots->memslots + KVM_MEM_SLOTS_NUM && memslot->npages;\
497 memslot++)
498
499int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id);
500void kvm_vcpu_uninit(struct kvm_vcpu *vcpu);
501
502int __must_check vcpu_load(struct kvm_vcpu *vcpu);
503void vcpu_put(struct kvm_vcpu *vcpu);
504
505#ifdef __KVM_HAVE_IOAPIC
506void kvm_vcpu_request_scan_ioapic(struct kvm *kvm);
507void kvm_arch_post_irq_routing_update(struct kvm *kvm);
508#else
509static inline void kvm_vcpu_request_scan_ioapic(struct kvm *kvm)
510{
511}
512static inline void kvm_arch_post_irq_routing_update(struct kvm *kvm)
513{
514}
515#endif
516
517#ifdef CONFIG_HAVE_KVM_IRQFD
518int kvm_irqfd_init(void);
519void kvm_irqfd_exit(void);
520#else
521static inline int kvm_irqfd_init(void)
522{
523 return 0;
524}
525
526static inline void kvm_irqfd_exit(void)
527{
528}
529#endif
530int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
531 struct module *module);
532void kvm_exit(void);
533
534void kvm_get_kvm(struct kvm *kvm);
535void kvm_put_kvm(struct kvm *kvm);
536
537static inline struct kvm_memslots *__kvm_memslots(struct kvm *kvm, int as_id)
538{
539 return rcu_dereference_check(kvm->memslots[as_id],
540 srcu_read_lock_held(&kvm->srcu)
541 || lockdep_is_held(&kvm->slots_lock));
542}
543
544static inline struct kvm_memslots *kvm_memslots(struct kvm *kvm)
545{
546 return __kvm_memslots(kvm, 0);
547}
548
549static inline struct kvm_memslots *kvm_vcpu_memslots(struct kvm_vcpu *vcpu)
550{
551 int as_id = kvm_arch_vcpu_memslots_id(vcpu);
552
553 return __kvm_memslots(vcpu->kvm, as_id);
554}
555
556static inline struct kvm_memory_slot *
557id_to_memslot(struct kvm_memslots *slots, int id)
558{
559 int index = slots->id_to_index[id];
560 struct kvm_memory_slot *slot;
561
562 slot = &slots->memslots[index];
563
564 WARN_ON(slot->id != id);
565 return slot;
566}
567
568
569
570
571
572
573
574
575
576
577
578
579enum kvm_mr_change {
580 KVM_MR_CREATE,
581 KVM_MR_DELETE,
582 KVM_MR_MOVE,
583 KVM_MR_FLAGS_ONLY,
584};
585
586int kvm_set_memory_region(struct kvm *kvm,
587 const struct kvm_userspace_memory_region *mem);
588int __kvm_set_memory_region(struct kvm *kvm,
589 const struct kvm_userspace_memory_region *mem);
590void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
591 struct kvm_memory_slot *dont);
592int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
593 unsigned long npages);
594void kvm_arch_memslots_updated(struct kvm *kvm, struct kvm_memslots *slots);
595int kvm_arch_prepare_memory_region(struct kvm *kvm,
596 struct kvm_memory_slot *memslot,
597 const struct kvm_userspace_memory_region *mem,
598 enum kvm_mr_change change);
599void kvm_arch_commit_memory_region(struct kvm *kvm,
600 const struct kvm_userspace_memory_region *mem,
601 const struct kvm_memory_slot *old,
602 const struct kvm_memory_slot *new,
603 enum kvm_mr_change change);
604bool kvm_largepages_enabled(void);
605void kvm_disable_largepages(void);
606
607void kvm_arch_flush_shadow_all(struct kvm *kvm);
608
609void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
610 struct kvm_memory_slot *slot);
611
612int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
613 struct page **pages, int nr_pages);
614
615struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn);
616unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn);
617unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable);
618unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, gfn_t gfn);
619unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot, gfn_t gfn,
620 bool *writable);
621void kvm_release_page_clean(struct page *page);
622void kvm_release_page_dirty(struct page *page);
623void kvm_set_page_accessed(struct page *page);
624
625kvm_pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn);
626kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn);
627kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
628 bool *writable);
629kvm_pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn);
630kvm_pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn);
631kvm_pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn,
632 bool atomic, bool *async, bool write_fault,
633 bool *writable);
634
635void kvm_release_pfn_clean(kvm_pfn_t pfn);
636void kvm_set_pfn_dirty(kvm_pfn_t pfn);
637void kvm_set_pfn_accessed(kvm_pfn_t pfn);
638void kvm_get_pfn(kvm_pfn_t pfn);
639
640int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
641 int len);
642int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
643 unsigned long len);
644int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len);
645int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
646 void *data, unsigned long len);
647int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
648 int offset, int len);
649int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
650 unsigned long len);
651int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
652 void *data, unsigned long len);
653int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
654 void *data, int offset, unsigned long len);
655int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
656 gpa_t gpa, unsigned long len);
657int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len);
658int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len);
659struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn);
660bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn);
661unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn);
662void mark_page_dirty(struct kvm *kvm, gfn_t gfn);
663
664struct kvm_memslots *kvm_vcpu_memslots(struct kvm_vcpu *vcpu);
665struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn);
666kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn);
667kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn);
668struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn);
669unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn);
670unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable);
671int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data, int offset,
672 int len);
673int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa, void *data,
674 unsigned long len);
675int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data,
676 unsigned long len);
677int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, const void *data,
678 int offset, int len);
679int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data,
680 unsigned long len);
681void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn);
682
683void kvm_vcpu_block(struct kvm_vcpu *vcpu);
684void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu);
685void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu);
686void kvm_vcpu_wake_up(struct kvm_vcpu *vcpu);
687void kvm_vcpu_kick(struct kvm_vcpu *vcpu);
688int kvm_vcpu_yield_to(struct kvm_vcpu *target);
689void kvm_vcpu_on_spin(struct kvm_vcpu *vcpu);
690void kvm_load_guest_fpu(struct kvm_vcpu *vcpu);
691void kvm_put_guest_fpu(struct kvm_vcpu *vcpu);
692
693void kvm_flush_remote_tlbs(struct kvm *kvm);
694void kvm_reload_remote_mmus(struct kvm *kvm);
695bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req);
696
697long kvm_arch_dev_ioctl(struct file *filp,
698 unsigned int ioctl, unsigned long arg);
699long kvm_arch_vcpu_ioctl(struct file *filp,
700 unsigned int ioctl, unsigned long arg);
701int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf);
702
703int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext);
704
705int kvm_get_dirty_log(struct kvm *kvm,
706 struct kvm_dirty_log *log, int *is_dirty);
707
708int kvm_get_dirty_log_protect(struct kvm *kvm,
709 struct kvm_dirty_log *log, bool *is_dirty);
710
711void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
712 struct kvm_memory_slot *slot,
713 gfn_t gfn_offset,
714 unsigned long mask);
715
716int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
717 struct kvm_dirty_log *log);
718
719int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
720 bool line_status);
721long kvm_arch_vm_ioctl(struct file *filp,
722 unsigned int ioctl, unsigned long arg);
723
724int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu);
725int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu);
726
727int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
728 struct kvm_translation *tr);
729
730int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs);
731int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs);
732int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
733 struct kvm_sregs *sregs);
734int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
735 struct kvm_sregs *sregs);
736int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
737 struct kvm_mp_state *mp_state);
738int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
739 struct kvm_mp_state *mp_state);
740int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
741 struct kvm_guest_debug *dbg);
742int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run);
743
744int kvm_arch_init(void *opaque);
745void kvm_arch_exit(void);
746
747int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu);
748void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu);
749
750void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu);
751
752void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu);
753void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu);
754void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu);
755struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id);
756int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu);
757void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu);
758void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu);
759
760bool kvm_arch_has_vcpu_debugfs(void);
761int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu);
762
763int kvm_arch_hardware_enable(void);
764void kvm_arch_hardware_disable(void);
765int kvm_arch_hardware_setup(void);
766void kvm_arch_hardware_unsetup(void);
767void kvm_arch_check_processor_compat(void *rtn);
768int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu);
769int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu);
770
771void *kvm_kvzalloc(unsigned long size);
772
773#ifndef __KVM_HAVE_ARCH_VM_ALLOC
774static inline struct kvm *kvm_arch_alloc_vm(void)
775{
776 return kzalloc(sizeof(struct kvm), GFP_KERNEL);
777}
778
779static inline void kvm_arch_free_vm(struct kvm *kvm)
780{
781 kfree(kvm);
782}
783#endif
784
785#ifdef __KVM_HAVE_ARCH_NONCOHERENT_DMA
786void kvm_arch_register_noncoherent_dma(struct kvm *kvm);
787void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm);
788bool kvm_arch_has_noncoherent_dma(struct kvm *kvm);
789#else
790static inline void kvm_arch_register_noncoherent_dma(struct kvm *kvm)
791{
792}
793
794static inline void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm)
795{
796}
797
798static inline bool kvm_arch_has_noncoherent_dma(struct kvm *kvm)
799{
800 return false;
801}
802#endif
803#ifdef __KVM_HAVE_ARCH_ASSIGNED_DEVICE
804void kvm_arch_start_assignment(struct kvm *kvm);
805void kvm_arch_end_assignment(struct kvm *kvm);
806bool kvm_arch_has_assigned_device(struct kvm *kvm);
807#else
808static inline void kvm_arch_start_assignment(struct kvm *kvm)
809{
810}
811
812static inline void kvm_arch_end_assignment(struct kvm *kvm)
813{
814}
815
816static inline bool kvm_arch_has_assigned_device(struct kvm *kvm)
817{
818 return false;
819}
820#endif
821
822static inline struct swait_queue_head *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu)
823{
824#ifdef __KVM_HAVE_ARCH_WQP
825 return vcpu->arch.wqp;
826#else
827 return &vcpu->wq;
828#endif
829}
830
831#ifdef __KVM_HAVE_ARCH_INTC_INITIALIZED
832
833
834
835
836
837bool kvm_arch_intc_initialized(struct kvm *kvm);
838#else
839static inline bool kvm_arch_intc_initialized(struct kvm *kvm)
840{
841 return true;
842}
843#endif
844
845int kvm_arch_init_vm(struct kvm *kvm, unsigned long type);
846void kvm_arch_destroy_vm(struct kvm *kvm);
847void kvm_arch_sync_events(struct kvm *kvm);
848
849int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu);
850void kvm_vcpu_kick(struct kvm_vcpu *vcpu);
851
852bool kvm_is_reserved_pfn(kvm_pfn_t pfn);
853
854struct kvm_irq_ack_notifier {
855 struct hlist_node link;
856 unsigned gsi;
857 void (*irq_acked)(struct kvm_irq_ack_notifier *kian);
858};
859
860int kvm_irq_map_gsi(struct kvm *kvm,
861 struct kvm_kernel_irq_routing_entry *entries, int gsi);
862int kvm_irq_map_chip_pin(struct kvm *kvm, unsigned irqchip, unsigned pin);
863
864int kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level,
865 bool line_status);
866int kvm_set_msi(struct kvm_kernel_irq_routing_entry *irq_entry, struct kvm *kvm,
867 int irq_source_id, int level, bool line_status);
868int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e,
869 struct kvm *kvm, int irq_source_id,
870 int level, bool line_status);
871bool kvm_irq_has_notifier(struct kvm *kvm, unsigned irqchip, unsigned pin);
872void kvm_notify_acked_gsi(struct kvm *kvm, int gsi);
873void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin);
874void kvm_register_irq_ack_notifier(struct kvm *kvm,
875 struct kvm_irq_ack_notifier *kian);
876void kvm_unregister_irq_ack_notifier(struct kvm *kvm,
877 struct kvm_irq_ack_notifier *kian);
878int kvm_request_irq_source_id(struct kvm *kvm);
879void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id);
880
881#ifdef CONFIG_KVM_DEVICE_ASSIGNMENT
882int kvm_iommu_map_pages(struct kvm *kvm, struct kvm_memory_slot *slot);
883void kvm_iommu_unmap_pages(struct kvm *kvm, struct kvm_memory_slot *slot);
884#else
885static inline int kvm_iommu_map_pages(struct kvm *kvm,
886 struct kvm_memory_slot *slot)
887{
888 return 0;
889}
890
891static inline void kvm_iommu_unmap_pages(struct kvm *kvm,
892 struct kvm_memory_slot *slot)
893{
894}
895#endif
896
897
898
899
900
901
902
903static inline struct kvm_memory_slot *
904search_memslots(struct kvm_memslots *slots, gfn_t gfn)
905{
906 int start = 0, end = slots->used_slots;
907 int slot = atomic_read(&slots->lru_slot);
908 struct kvm_memory_slot *memslots = slots->memslots;
909
910 if (gfn >= memslots[slot].base_gfn &&
911 gfn < memslots[slot].base_gfn + memslots[slot].npages)
912 return &memslots[slot];
913
914 while (start < end) {
915 slot = start + (end - start) / 2;
916
917 if (gfn >= memslots[slot].base_gfn)
918 end = slot;
919 else
920 start = slot + 1;
921 }
922
923 if (gfn >= memslots[start].base_gfn &&
924 gfn < memslots[start].base_gfn + memslots[start].npages) {
925 atomic_set(&slots->lru_slot, start);
926 return &memslots[start];
927 }
928
929 return NULL;
930}
931
932static inline struct kvm_memory_slot *
933__gfn_to_memslot(struct kvm_memslots *slots, gfn_t gfn)
934{
935 return search_memslots(slots, gfn);
936}
937
938static inline unsigned long
939__gfn_to_hva_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
940{
941 return slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE;
942}
943
944static inline int memslot_id(struct kvm *kvm, gfn_t gfn)
945{
946 return gfn_to_memslot(kvm, gfn)->id;
947}
948
949static inline gfn_t
950hva_to_gfn_memslot(unsigned long hva, struct kvm_memory_slot *slot)
951{
952 gfn_t gfn_offset = (hva - slot->userspace_addr) >> PAGE_SHIFT;
953
954 return slot->base_gfn + gfn_offset;
955}
956
957static inline gpa_t gfn_to_gpa(gfn_t gfn)
958{
959 return (gpa_t)gfn << PAGE_SHIFT;
960}
961
962static inline gfn_t gpa_to_gfn(gpa_t gpa)
963{
964 return (gfn_t)(gpa >> PAGE_SHIFT);
965}
966
967static inline hpa_t pfn_to_hpa(kvm_pfn_t pfn)
968{
969 return (hpa_t)pfn << PAGE_SHIFT;
970}
971
972static inline bool kvm_is_error_gpa(struct kvm *kvm, gpa_t gpa)
973{
974 unsigned long hva = gfn_to_hva(kvm, gpa_to_gfn(gpa));
975
976 return kvm_is_error_hva(hva);
977}
978
979enum kvm_stat_kind {
980 KVM_STAT_VM,
981 KVM_STAT_VCPU,
982};
983
984struct kvm_stat_data {
985 int offset;
986 struct kvm *kvm;
987};
988
989struct kvm_stats_debugfs_item {
990 const char *name;
991 int offset;
992 enum kvm_stat_kind kind;
993};
994extern struct kvm_stats_debugfs_item debugfs_entries[];
995extern struct dentry *kvm_debugfs_dir;
996
997#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
998static inline int mmu_notifier_retry(struct kvm *kvm, unsigned long mmu_seq)
999{
1000 if (unlikely(kvm->mmu_notifier_count))
1001 return 1;
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012 smp_rmb();
1013 if (kvm->mmu_notifier_seq != mmu_seq)
1014 return 1;
1015 return 0;
1016}
1017#endif
1018
1019#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
1020
1021#ifdef CONFIG_S390
1022#define KVM_MAX_IRQ_ROUTES 4096
1023#elif defined(CONFIG_ARM64)
1024#define KVM_MAX_IRQ_ROUTES 4096
1025#else
1026#define KVM_MAX_IRQ_ROUTES 1024
1027#endif
1028
1029int kvm_set_irq_routing(struct kvm *kvm,
1030 const struct kvm_irq_routing_entry *entries,
1031 unsigned nr,
1032 unsigned flags);
1033int kvm_set_routing_entry(struct kvm *kvm,
1034 struct kvm_kernel_irq_routing_entry *e,
1035 const struct kvm_irq_routing_entry *ue);
1036void kvm_free_irq_routing(struct kvm *kvm);
1037
1038#else
1039
1040static inline void kvm_free_irq_routing(struct kvm *kvm) {}
1041
1042#endif
1043
1044int kvm_send_userspace_msi(struct kvm *kvm, struct kvm_msi *msi);
1045
1046#ifdef CONFIG_HAVE_KVM_EVENTFD
1047
1048void kvm_eventfd_init(struct kvm *kvm);
1049int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args);
1050
1051#ifdef CONFIG_HAVE_KVM_IRQFD
1052int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args);
1053void kvm_irqfd_release(struct kvm *kvm);
1054void kvm_irq_routing_update(struct kvm *);
1055#else
1056static inline int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args)
1057{
1058 return -EINVAL;
1059}
1060
1061static inline void kvm_irqfd_release(struct kvm *kvm) {}
1062#endif
1063
1064#else
1065
1066static inline void kvm_eventfd_init(struct kvm *kvm) {}
1067
1068static inline int kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args)
1069{
1070 return -EINVAL;
1071}
1072
1073static inline void kvm_irqfd_release(struct kvm *kvm) {}
1074
1075#ifdef CONFIG_HAVE_KVM_IRQCHIP
1076static inline void kvm_irq_routing_update(struct kvm *kvm)
1077{
1078}
1079#endif
1080void kvm_arch_irq_routing_update(struct kvm *kvm);
1081
1082static inline int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
1083{
1084 return -ENOSYS;
1085}
1086
1087#endif
1088
1089static inline void kvm_make_request(int req, struct kvm_vcpu *vcpu)
1090{
1091
1092
1093
1094
1095 smp_wmb();
1096 set_bit(req, &vcpu->requests);
1097}
1098
1099static inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu)
1100{
1101 if (test_bit(req, &vcpu->requests)) {
1102 clear_bit(req, &vcpu->requests);
1103
1104
1105
1106
1107
1108 smp_mb__after_atomic();
1109 return true;
1110 } else {
1111 return false;
1112 }
1113}
1114
1115extern bool kvm_rebooting;
1116
1117extern unsigned int halt_poll_ns;
1118extern unsigned int halt_poll_ns_grow;
1119extern unsigned int halt_poll_ns_shrink;
1120
1121struct kvm_device {
1122 struct kvm_device_ops *ops;
1123 struct kvm *kvm;
1124 void *private;
1125 struct list_head vm_node;
1126};
1127
1128
1129struct kvm_device_ops {
1130 const char *name;
1131
1132
1133
1134
1135
1136
1137 int (*create)(struct kvm_device *dev, u32 type);
1138
1139
1140
1141
1142
1143 void (*init)(struct kvm_device *dev);
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153 void (*destroy)(struct kvm_device *dev);
1154
1155 int (*set_attr)(struct kvm_device *dev, struct kvm_device_attr *attr);
1156 int (*get_attr)(struct kvm_device *dev, struct kvm_device_attr *attr);
1157 int (*has_attr)(struct kvm_device *dev, struct kvm_device_attr *attr);
1158 long (*ioctl)(struct kvm_device *dev, unsigned int ioctl,
1159 unsigned long arg);
1160};
1161
1162void kvm_device_get(struct kvm_device *dev);
1163void kvm_device_put(struct kvm_device *dev);
1164struct kvm_device *kvm_device_from_filp(struct file *filp);
1165int kvm_register_device_ops(struct kvm_device_ops *ops, u32 type);
1166void kvm_unregister_device_ops(u32 type);
1167
1168extern struct kvm_device_ops kvm_mpic_ops;
1169extern struct kvm_device_ops kvm_xics_ops;
1170extern struct kvm_device_ops kvm_arm_vgic_v2_ops;
1171extern struct kvm_device_ops kvm_arm_vgic_v3_ops;
1172
1173#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
1174
1175static inline void kvm_vcpu_set_in_spin_loop(struct kvm_vcpu *vcpu, bool val)
1176{
1177 vcpu->spin_loop.in_spin_loop = val;
1178}
1179static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val)
1180{
1181 vcpu->spin_loop.dy_eligible = val;
1182}
1183
1184#else
1185
1186static inline void kvm_vcpu_set_in_spin_loop(struct kvm_vcpu *vcpu, bool val)
1187{
1188}
1189
1190static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val)
1191{
1192}
1193#endif
1194
1195#ifdef CONFIG_HAVE_KVM_IRQ_BYPASS
1196bool kvm_arch_has_irq_bypass(void);
1197int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *,
1198 struct irq_bypass_producer *);
1199void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *,
1200 struct irq_bypass_producer *);
1201void kvm_arch_irq_bypass_stop(struct irq_bypass_consumer *);
1202void kvm_arch_irq_bypass_start(struct irq_bypass_consumer *);
1203int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq,
1204 uint32_t guest_irq, bool set);
1205#endif
1206
1207#ifdef CONFIG_HAVE_KVM_INVALID_WAKEUPS
1208
1209static inline bool vcpu_valid_wakeup(struct kvm_vcpu *vcpu)
1210{
1211 return vcpu->valid_wakeup;
1212}
1213
1214#else
1215static inline bool vcpu_valid_wakeup(struct kvm_vcpu *vcpu)
1216{
1217 return true;
1218}
1219#endif
1220
1221#endif
1222