1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19#include <linux/bug.h>
20#include <linux/cpu_pm.h>
21#include <linux/errno.h>
22#include <linux/err.h>
23#include <linux/kvm_host.h>
24#include <linux/list.h>
25#include <linux/module.h>
26#include <linux/vmalloc.h>
27#include <linux/fs.h>
28#include <linux/mman.h>
29#include <linux/sched.h>
30#include <linux/kvm.h>
31#include <linux/kvm_irqfd.h>
32#include <linux/irqbypass.h>
33#include <linux/sched/stat.h>
34#include <trace/events/kvm.h>
35#include <kvm/arm_pmu.h>
36#include <kvm/arm_psci.h>
37
38#define CREATE_TRACE_POINTS
39#include "trace.h"
40
41#include <linux/uaccess.h>
42#include <asm/ptrace.h>
43#include <asm/mman.h>
44#include <asm/tlbflush.h>
45#include <asm/cacheflush.h>
46#include <asm/cpufeature.h>
47#include <asm/virt.h>
48#include <asm/kvm_arm.h>
49#include <asm/kvm_asm.h>
50#include <asm/kvm_mmu.h>
51#include <asm/kvm_emulate.h>
52#include <asm/kvm_coproc.h>
53#include <asm/sections.h>
54
55#ifdef REQUIRES_VIRT
56__asm__(".arch_extension virt");
57#endif
58
59DEFINE_PER_CPU(kvm_cpu_context_t, kvm_host_cpu_state);
60static DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);
61
62
63static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_arm_running_vcpu);
64
65
66static atomic64_t kvm_vmid_gen = ATOMIC64_INIT(1);
67static u32 kvm_next_vmid;
68static unsigned int kvm_vmid_bits __read_mostly;
69static DEFINE_SPINLOCK(kvm_vmid_lock);
70
71static bool vgic_present;
72
73static DEFINE_PER_CPU(unsigned char, kvm_arm_hardware_enabled);
74
75static void kvm_arm_set_running_vcpu(struct kvm_vcpu *vcpu)
76{
77 __this_cpu_write(kvm_arm_running_vcpu, vcpu);
78}
79
80DEFINE_STATIC_KEY_FALSE(userspace_irqchip_in_use);
81
82
83
84
85
86struct kvm_vcpu *kvm_arm_get_running_vcpu(void)
87{
88 return __this_cpu_read(kvm_arm_running_vcpu);
89}
90
91
92
93
94struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void)
95{
96 return &kvm_arm_running_vcpu;
97}
98
99int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
100{
101 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
102}
103
104int kvm_arch_hardware_setup(void)
105{
106 return 0;
107}
108
109void kvm_arch_check_processor_compat(void *rtn)
110{
111 *(int *)rtn = 0;
112}
113
114
115
116
117
118
119int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
120{
121 int ret, cpu;
122
123 ret = kvm_arm_setup_stage2(kvm, type);
124 if (ret)
125 return ret;
126
127 kvm->arch.last_vcpu_ran = alloc_percpu(typeof(*kvm->arch.last_vcpu_ran));
128 if (!kvm->arch.last_vcpu_ran)
129 return -ENOMEM;
130
131 for_each_possible_cpu(cpu)
132 *per_cpu_ptr(kvm->arch.last_vcpu_ran, cpu) = -1;
133
134 ret = kvm_alloc_stage2_pgd(kvm);
135 if (ret)
136 goto out_fail_alloc;
137
138 ret = create_hyp_mappings(kvm, kvm + 1, PAGE_HYP);
139 if (ret)
140 goto out_free_stage2_pgd;
141
142 kvm_vgic_early_init(kvm);
143
144
145 kvm->arch.vmid_gen = 0;
146
147
148 kvm->arch.max_vcpus = vgic_present ?
149 kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS;
150
151 return ret;
152out_free_stage2_pgd:
153 kvm_free_stage2_pgd(kvm);
154out_fail_alloc:
155 free_percpu(kvm->arch.last_vcpu_ran);
156 kvm->arch.last_vcpu_ran = NULL;
157 return ret;
158}
159
160bool kvm_arch_has_vcpu_debugfs(void)
161{
162 return false;
163}
164
165int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
166{
167 return 0;
168}
169
170vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
171{
172 return VM_FAULT_SIGBUS;
173}
174
175
176
177
178
179
180void kvm_arch_destroy_vm(struct kvm *kvm)
181{
182 int i;
183
184 kvm_vgic_destroy(kvm);
185
186 free_percpu(kvm->arch.last_vcpu_ran);
187 kvm->arch.last_vcpu_ran = NULL;
188
189 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
190 if (kvm->vcpus[i]) {
191 kvm_arch_vcpu_free(kvm->vcpus[i]);
192 kvm->vcpus[i] = NULL;
193 }
194 }
195 atomic_set(&kvm->online_vcpus, 0);
196}
197
198int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
199{
200 int r;
201 switch (ext) {
202 case KVM_CAP_IRQCHIP:
203 r = vgic_present;
204 break;
205 case KVM_CAP_IOEVENTFD:
206 case KVM_CAP_DEVICE_CTRL:
207 case KVM_CAP_USER_MEMORY:
208 case KVM_CAP_SYNC_MMU:
209 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
210 case KVM_CAP_ONE_REG:
211 case KVM_CAP_ARM_PSCI:
212 case KVM_CAP_ARM_PSCI_0_2:
213 case KVM_CAP_READONLY_MEM:
214 case KVM_CAP_MP_STATE:
215 case KVM_CAP_IMMEDIATE_EXIT:
216 case KVM_CAP_VCPU_EVENTS:
217 r = 1;
218 break;
219 case KVM_CAP_ARM_SET_DEVICE_ADDR:
220 r = 1;
221 break;
222 case KVM_CAP_NR_VCPUS:
223 r = num_online_cpus();
224 break;
225 case KVM_CAP_MAX_VCPUS:
226 r = KVM_MAX_VCPUS;
227 break;
228 case KVM_CAP_NR_MEMSLOTS:
229 r = KVM_USER_MEM_SLOTS;
230 break;
231 case KVM_CAP_MSI_DEVID:
232 if (!kvm)
233 r = -EINVAL;
234 else
235 r = kvm->arch.vgic.msis_require_devid;
236 break;
237 case KVM_CAP_ARM_USER_IRQ:
238
239
240
241
242 r = 1;
243 break;
244 default:
245 r = kvm_arch_vm_ioctl_check_extension(kvm, ext);
246 break;
247 }
248 return r;
249}
250
251long kvm_arch_dev_ioctl(struct file *filp,
252 unsigned int ioctl, unsigned long arg)
253{
254 return -EINVAL;
255}
256
257struct kvm *kvm_arch_alloc_vm(void)
258{
259 if (!has_vhe())
260 return kzalloc(sizeof(struct kvm), GFP_KERNEL);
261
262 return vzalloc(sizeof(struct kvm));
263}
264
265void kvm_arch_free_vm(struct kvm *kvm)
266{
267 if (!has_vhe())
268 kfree(kvm);
269 else
270 vfree(kvm);
271}
272
273struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id)
274{
275 int err;
276 struct kvm_vcpu *vcpu;
277
278 if (irqchip_in_kernel(kvm) && vgic_initialized(kvm)) {
279 err = -EBUSY;
280 goto out;
281 }
282
283 if (id >= kvm->arch.max_vcpus) {
284 err = -EINVAL;
285 goto out;
286 }
287
288 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
289 if (!vcpu) {
290 err = -ENOMEM;
291 goto out;
292 }
293
294 err = kvm_vcpu_init(vcpu, kvm, id);
295 if (err)
296 goto free_vcpu;
297
298 err = create_hyp_mappings(vcpu, vcpu + 1, PAGE_HYP);
299 if (err)
300 goto vcpu_uninit;
301
302 return vcpu;
303vcpu_uninit:
304 kvm_vcpu_uninit(vcpu);
305free_vcpu:
306 kmem_cache_free(kvm_vcpu_cache, vcpu);
307out:
308 return ERR_PTR(err);
309}
310
311void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
312{
313}
314
315void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
316{
317 if (vcpu->arch.has_run_once && unlikely(!irqchip_in_kernel(vcpu->kvm)))
318 static_branch_dec(&userspace_irqchip_in_use);
319
320 kvm_mmu_free_memory_caches(vcpu);
321 kvm_timer_vcpu_terminate(vcpu);
322 kvm_pmu_vcpu_destroy(vcpu);
323 kvm_vcpu_uninit(vcpu);
324 kmem_cache_free(kvm_vcpu_cache, vcpu);
325}
326
327void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
328{
329 kvm_arch_vcpu_free(vcpu);
330}
331
332int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
333{
334 return kvm_timer_is_pending(vcpu);
335}
336
337void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
338{
339 kvm_timer_schedule(vcpu);
340 kvm_vgic_v4_enable_doorbell(vcpu);
341}
342
343void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
344{
345 kvm_timer_unschedule(vcpu);
346 kvm_vgic_v4_disable_doorbell(vcpu);
347}
348
349int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
350{
351
352 vcpu->arch.target = -1;
353 bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES);
354
355
356 kvm_timer_vcpu_init(vcpu);
357
358 kvm_arm_reset_debug_ptr(vcpu);
359
360 return kvm_vgic_vcpu_init(vcpu);
361}
362
363void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
364{
365 int *last_ran;
366
367 last_ran = this_cpu_ptr(vcpu->kvm->arch.last_vcpu_ran);
368
369
370
371
372
373 if (*last_ran != vcpu->vcpu_id) {
374 kvm_call_hyp(__kvm_tlb_flush_local_vmid, vcpu);
375 *last_ran = vcpu->vcpu_id;
376 }
377
378 vcpu->cpu = cpu;
379 vcpu->arch.host_cpu_context = this_cpu_ptr(&kvm_host_cpu_state);
380
381 kvm_arm_set_running_vcpu(vcpu);
382 kvm_vgic_load(vcpu);
383 kvm_timer_vcpu_load(vcpu);
384 kvm_vcpu_load_sysregs(vcpu);
385 kvm_arch_vcpu_load_fp(vcpu);
386
387 if (single_task_running())
388 vcpu_clear_wfe_traps(vcpu);
389 else
390 vcpu_set_wfe_traps(vcpu);
391}
392
393void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
394{
395 kvm_arch_vcpu_put_fp(vcpu);
396 kvm_vcpu_put_sysregs(vcpu);
397 kvm_timer_vcpu_put(vcpu);
398 kvm_vgic_put(vcpu);
399
400 vcpu->cpu = -1;
401
402 kvm_arm_set_running_vcpu(NULL);
403}
404
405static void vcpu_power_off(struct kvm_vcpu *vcpu)
406{
407 vcpu->arch.power_off = true;
408 kvm_make_request(KVM_REQ_SLEEP, vcpu);
409 kvm_vcpu_kick(vcpu);
410}
411
412int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
413 struct kvm_mp_state *mp_state)
414{
415 if (vcpu->arch.power_off)
416 mp_state->mp_state = KVM_MP_STATE_STOPPED;
417 else
418 mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
419
420 return 0;
421}
422
423int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
424 struct kvm_mp_state *mp_state)
425{
426 int ret = 0;
427
428 switch (mp_state->mp_state) {
429 case KVM_MP_STATE_RUNNABLE:
430 vcpu->arch.power_off = false;
431 break;
432 case KVM_MP_STATE_STOPPED:
433 vcpu_power_off(vcpu);
434 break;
435 default:
436 ret = -EINVAL;
437 }
438
439 return ret;
440}
441
442
443
444
445
446
447
448
449int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
450{
451 bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF);
452 return ((irq_lines || kvm_vgic_vcpu_pending_irq(v))
453 && !v->arch.power_off && !v->arch.pause);
454}
455
456bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
457{
458 return vcpu_mode_priv(vcpu);
459}
460
461
462static void exit_vm_noop(void *info)
463{
464}
465
466void force_vm_exit(const cpumask_t *mask)
467{
468 preempt_disable();
469 smp_call_function_many(mask, exit_vm_noop, NULL, true);
470 preempt_enable();
471}
472
473
474
475
476
477
478
479
480
481
482
483
484
485static bool need_new_vmid_gen(struct kvm *kvm)
486{
487 u64 current_vmid_gen = atomic64_read(&kvm_vmid_gen);
488 smp_rmb();
489 return unlikely(READ_ONCE(kvm->arch.vmid_gen) != current_vmid_gen);
490}
491
492
493
494
495
496
497
498
499
500static void update_vttbr(struct kvm *kvm)
501{
502 phys_addr_t pgd_phys;
503 u64 vmid, cnp = kvm_cpu_has_cnp() ? VTTBR_CNP_BIT : 0;
504
505 if (!need_new_vmid_gen(kvm))
506 return;
507
508 spin_lock(&kvm_vmid_lock);
509
510
511
512
513
514
515 if (!need_new_vmid_gen(kvm)) {
516 spin_unlock(&kvm_vmid_lock);
517 return;
518 }
519
520
521 if (unlikely(kvm_next_vmid == 0)) {
522 atomic64_inc(&kvm_vmid_gen);
523 kvm_next_vmid = 1;
524
525
526
527
528
529
530 force_vm_exit(cpu_all_mask);
531
532
533
534
535
536 kvm_call_hyp(__kvm_flush_vm_context);
537 }
538
539 kvm->arch.vmid = kvm_next_vmid;
540 kvm_next_vmid++;
541 kvm_next_vmid &= (1 << kvm_vmid_bits) - 1;
542
543
544 pgd_phys = virt_to_phys(kvm->arch.pgd);
545 BUG_ON(pgd_phys & ~kvm_vttbr_baddr_mask(kvm));
546 vmid = ((u64)(kvm->arch.vmid) << VTTBR_VMID_SHIFT) & VTTBR_VMID_MASK(kvm_vmid_bits);
547 kvm->arch.vttbr = kvm_phys_to_vttbr(pgd_phys) | vmid | cnp;
548
549 smp_wmb();
550 WRITE_ONCE(kvm->arch.vmid_gen, atomic64_read(&kvm_vmid_gen));
551
552 spin_unlock(&kvm_vmid_lock);
553}
554
555static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu)
556{
557 struct kvm *kvm = vcpu->kvm;
558 int ret = 0;
559
560 if (likely(vcpu->arch.has_run_once))
561 return 0;
562
563 vcpu->arch.has_run_once = true;
564
565 if (likely(irqchip_in_kernel(kvm))) {
566
567
568
569
570 if (unlikely(!vgic_ready(kvm))) {
571 ret = kvm_vgic_map_resources(kvm);
572 if (ret)
573 return ret;
574 }
575 } else {
576
577
578
579
580 static_branch_inc(&userspace_irqchip_in_use);
581 }
582
583 ret = kvm_timer_enable(vcpu);
584 if (ret)
585 return ret;
586
587 ret = kvm_arm_pmu_v3_enable(vcpu);
588
589 return ret;
590}
591
592bool kvm_arch_intc_initialized(struct kvm *kvm)
593{
594 return vgic_initialized(kvm);
595}
596
597void kvm_arm_halt_guest(struct kvm *kvm)
598{
599 int i;
600 struct kvm_vcpu *vcpu;
601
602 kvm_for_each_vcpu(i, vcpu, kvm)
603 vcpu->arch.pause = true;
604 kvm_make_all_cpus_request(kvm, KVM_REQ_SLEEP);
605}
606
607void kvm_arm_resume_guest(struct kvm *kvm)
608{
609 int i;
610 struct kvm_vcpu *vcpu;
611
612 kvm_for_each_vcpu(i, vcpu, kvm) {
613 vcpu->arch.pause = false;
614 swake_up_one(kvm_arch_vcpu_wq(vcpu));
615 }
616}
617
618static void vcpu_req_sleep(struct kvm_vcpu *vcpu)
619{
620 struct swait_queue_head *wq = kvm_arch_vcpu_wq(vcpu);
621
622 swait_event_interruptible_exclusive(*wq, ((!vcpu->arch.power_off) &&
623 (!vcpu->arch.pause)));
624
625 if (vcpu->arch.power_off || vcpu->arch.pause) {
626
627 kvm_make_request(KVM_REQ_SLEEP, vcpu);
628 }
629
630
631
632
633
634
635 smp_rmb();
636}
637
638static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu)
639{
640 return vcpu->arch.target >= 0;
641}
642
643static void check_vcpu_requests(struct kvm_vcpu *vcpu)
644{
645 if (kvm_request_pending(vcpu)) {
646 if (kvm_check_request(KVM_REQ_SLEEP, vcpu))
647 vcpu_req_sleep(vcpu);
648
649 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
650 kvm_reset_vcpu(vcpu);
651
652
653
654
655
656 kvm_check_request(KVM_REQ_IRQ_PENDING, vcpu);
657 }
658}
659
660
661
662
663
664
665
666
667
668
669
670
671int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
672{
673 int ret;
674
675 if (unlikely(!kvm_vcpu_initialized(vcpu)))
676 return -ENOEXEC;
677
678 ret = kvm_vcpu_first_run_init(vcpu);
679 if (ret)
680 return ret;
681
682 if (run->exit_reason == KVM_EXIT_MMIO) {
683 ret = kvm_handle_mmio_return(vcpu, vcpu->run);
684 if (ret)
685 return ret;
686 }
687
688 if (run->immediate_exit)
689 return -EINTR;
690
691 vcpu_load(vcpu);
692
693 kvm_sigset_activate(vcpu);
694
695 ret = 1;
696 run->exit_reason = KVM_EXIT_UNKNOWN;
697 while (ret > 0) {
698
699
700
701 cond_resched();
702
703 update_vttbr(vcpu->kvm);
704
705 check_vcpu_requests(vcpu);
706
707
708
709
710
711
712 preempt_disable();
713
714 kvm_pmu_flush_hwstate(vcpu);
715
716 local_irq_disable();
717
718 kvm_vgic_flush_hwstate(vcpu);
719
720
721
722
723
724 if (signal_pending(current)) {
725 ret = -EINTR;
726 run->exit_reason = KVM_EXIT_INTR;
727 }
728
729
730
731
732
733
734
735
736 if (static_branch_unlikely(&userspace_irqchip_in_use)) {
737 if (kvm_timer_should_notify_user(vcpu) ||
738 kvm_pmu_should_notify_user(vcpu)) {
739 ret = -EINTR;
740 run->exit_reason = KVM_EXIT_INTR;
741 }
742 }
743
744
745
746
747
748
749
750 smp_store_mb(vcpu->mode, IN_GUEST_MODE);
751
752 if (ret <= 0 || need_new_vmid_gen(vcpu->kvm) ||
753 kvm_request_pending(vcpu)) {
754 vcpu->mode = OUTSIDE_GUEST_MODE;
755 isb();
756 kvm_pmu_sync_hwstate(vcpu);
757 if (static_branch_unlikely(&userspace_irqchip_in_use))
758 kvm_timer_sync_hwstate(vcpu);
759 kvm_vgic_sync_hwstate(vcpu);
760 local_irq_enable();
761 preempt_enable();
762 continue;
763 }
764
765 kvm_arm_setup_debug(vcpu);
766
767
768
769
770 trace_kvm_entry(*vcpu_pc(vcpu));
771 guest_enter_irqoff();
772
773 if (has_vhe()) {
774 kvm_arm_vhe_guest_enter();
775 ret = kvm_vcpu_run_vhe(vcpu);
776 kvm_arm_vhe_guest_exit();
777 } else {
778 ret = kvm_call_hyp(__kvm_vcpu_run_nvhe, vcpu);
779 }
780
781 vcpu->mode = OUTSIDE_GUEST_MODE;
782 vcpu->stat.exits++;
783
784
785
786
787 kvm_arm_clear_debug(vcpu);
788
789
790
791
792
793
794 kvm_pmu_sync_hwstate(vcpu);
795
796
797
798
799
800
801 kvm_vgic_sync_hwstate(vcpu);
802
803
804
805
806
807
808 if (static_branch_unlikely(&userspace_irqchip_in_use))
809 kvm_timer_sync_hwstate(vcpu);
810
811 kvm_arch_vcpu_ctxsync_fp(vcpu);
812
813
814
815
816
817
818
819
820
821
822
823 local_irq_enable();
824
825
826
827
828
829
830
831
832
833 guest_exit();
834 trace_kvm_exit(ret, kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu));
835
836
837 handle_exit_early(vcpu, run, ret);
838
839 preempt_enable();
840
841 ret = handle_exit(vcpu, run, ret);
842 }
843
844
845 if (unlikely(!irqchip_in_kernel(vcpu->kvm))) {
846 kvm_timer_update_run(vcpu);
847 kvm_pmu_update_run(vcpu);
848 }
849
850 kvm_sigset_deactivate(vcpu);
851
852 vcpu_put(vcpu);
853 return ret;
854}
855
856static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level)
857{
858 int bit_index;
859 bool set;
860 unsigned long *hcr;
861
862 if (number == KVM_ARM_IRQ_CPU_IRQ)
863 bit_index = __ffs(HCR_VI);
864 else
865 bit_index = __ffs(HCR_VF);
866
867 hcr = vcpu_hcr(vcpu);
868 if (level)
869 set = test_and_set_bit(bit_index, hcr);
870 else
871 set = test_and_clear_bit(bit_index, hcr);
872
873
874
875
876 if (set == level)
877 return 0;
878
879
880
881
882
883
884 kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
885 kvm_vcpu_kick(vcpu);
886
887 return 0;
888}
889
890int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
891 bool line_status)
892{
893 u32 irq = irq_level->irq;
894 unsigned int irq_type, vcpu_idx, irq_num;
895 int nrcpus = atomic_read(&kvm->online_vcpus);
896 struct kvm_vcpu *vcpu = NULL;
897 bool level = irq_level->level;
898
899 irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK;
900 vcpu_idx = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;
901 irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK;
902
903 trace_kvm_irq_line(irq_type, vcpu_idx, irq_num, irq_level->level);
904
905 switch (irq_type) {
906 case KVM_ARM_IRQ_TYPE_CPU:
907 if (irqchip_in_kernel(kvm))
908 return -ENXIO;
909
910 if (vcpu_idx >= nrcpus)
911 return -EINVAL;
912
913 vcpu = kvm_get_vcpu(kvm, vcpu_idx);
914 if (!vcpu)
915 return -EINVAL;
916
917 if (irq_num > KVM_ARM_IRQ_CPU_FIQ)
918 return -EINVAL;
919
920 return vcpu_interrupt_line(vcpu, irq_num, level);
921 case KVM_ARM_IRQ_TYPE_PPI:
922 if (!irqchip_in_kernel(kvm))
923 return -ENXIO;
924
925 if (vcpu_idx >= nrcpus)
926 return -EINVAL;
927
928 vcpu = kvm_get_vcpu(kvm, vcpu_idx);
929 if (!vcpu)
930 return -EINVAL;
931
932 if (irq_num < VGIC_NR_SGIS || irq_num >= VGIC_NR_PRIVATE_IRQS)
933 return -EINVAL;
934
935 return kvm_vgic_inject_irq(kvm, vcpu->vcpu_id, irq_num, level, NULL);
936 case KVM_ARM_IRQ_TYPE_SPI:
937 if (!irqchip_in_kernel(kvm))
938 return -ENXIO;
939
940 if (irq_num < VGIC_NR_PRIVATE_IRQS)
941 return -EINVAL;
942
943 return kvm_vgic_inject_irq(kvm, 0, irq_num, level, NULL);
944 }
945
946 return -EINVAL;
947}
948
949static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
950 const struct kvm_vcpu_init *init)
951{
952 unsigned int i;
953 int phys_target = kvm_target_cpu();
954
955 if (init->target != phys_target)
956 return -EINVAL;
957
958
959
960
961
962 if (vcpu->arch.target != -1 && vcpu->arch.target != init->target)
963 return -EINVAL;
964
965
966 for (i = 0; i < sizeof(init->features) * 8; i++) {
967 bool set = (init->features[i / 32] & (1 << (i % 32)));
968
969 if (set && i >= KVM_VCPU_MAX_FEATURES)
970 return -ENOENT;
971
972
973
974
975
976 if (vcpu->arch.target != -1 && i < KVM_VCPU_MAX_FEATURES &&
977 test_bit(i, vcpu->arch.features) != set)
978 return -EINVAL;
979
980 if (set)
981 set_bit(i, vcpu->arch.features);
982 }
983
984 vcpu->arch.target = phys_target;
985
986
987 return kvm_reset_vcpu(vcpu);
988}
989
990
991static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu,
992 struct kvm_vcpu_init *init)
993{
994 int ret;
995
996 ret = kvm_vcpu_set_target(vcpu, init);
997 if (ret)
998 return ret;
999
1000
1001
1002
1003
1004 if (vcpu->arch.has_run_once)
1005 stage2_unmap_vm(vcpu->kvm);
1006
1007 vcpu_reset_hcr(vcpu);
1008
1009
1010
1011
1012 if (test_bit(KVM_ARM_VCPU_POWER_OFF, vcpu->arch.features))
1013 vcpu_power_off(vcpu);
1014 else
1015 vcpu->arch.power_off = false;
1016
1017 return 0;
1018}
1019
1020static int kvm_arm_vcpu_set_attr(struct kvm_vcpu *vcpu,
1021 struct kvm_device_attr *attr)
1022{
1023 int ret = -ENXIO;
1024
1025 switch (attr->group) {
1026 default:
1027 ret = kvm_arm_vcpu_arch_set_attr(vcpu, attr);
1028 break;
1029 }
1030
1031 return ret;
1032}
1033
1034static int kvm_arm_vcpu_get_attr(struct kvm_vcpu *vcpu,
1035 struct kvm_device_attr *attr)
1036{
1037 int ret = -ENXIO;
1038
1039 switch (attr->group) {
1040 default:
1041 ret = kvm_arm_vcpu_arch_get_attr(vcpu, attr);
1042 break;
1043 }
1044
1045 return ret;
1046}
1047
1048static int kvm_arm_vcpu_has_attr(struct kvm_vcpu *vcpu,
1049 struct kvm_device_attr *attr)
1050{
1051 int ret = -ENXIO;
1052
1053 switch (attr->group) {
1054 default:
1055 ret = kvm_arm_vcpu_arch_has_attr(vcpu, attr);
1056 break;
1057 }
1058
1059 return ret;
1060}
1061
1062static int kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
1063 struct kvm_vcpu_events *events)
1064{
1065 memset(events, 0, sizeof(*events));
1066
1067 return __kvm_arm_vcpu_get_events(vcpu, events);
1068}
1069
1070static int kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
1071 struct kvm_vcpu_events *events)
1072{
1073 int i;
1074
1075
1076 for (i = 0; i < ARRAY_SIZE(events->reserved); i++)
1077 if (events->reserved[i])
1078 return -EINVAL;
1079
1080
1081 for (i = 0; i < ARRAY_SIZE(events->exception.pad); i++)
1082 if (events->exception.pad[i])
1083 return -EINVAL;
1084
1085 return __kvm_arm_vcpu_set_events(vcpu, events);
1086}
1087
1088long kvm_arch_vcpu_ioctl(struct file *filp,
1089 unsigned int ioctl, unsigned long arg)
1090{
1091 struct kvm_vcpu *vcpu = filp->private_data;
1092 void __user *argp = (void __user *)arg;
1093 struct kvm_device_attr attr;
1094 long r;
1095
1096 switch (ioctl) {
1097 case KVM_ARM_VCPU_INIT: {
1098 struct kvm_vcpu_init init;
1099
1100 r = -EFAULT;
1101 if (copy_from_user(&init, argp, sizeof(init)))
1102 break;
1103
1104 r = kvm_arch_vcpu_ioctl_vcpu_init(vcpu, &init);
1105 break;
1106 }
1107 case KVM_SET_ONE_REG:
1108 case KVM_GET_ONE_REG: {
1109 struct kvm_one_reg reg;
1110
1111 r = -ENOEXEC;
1112 if (unlikely(!kvm_vcpu_initialized(vcpu)))
1113 break;
1114
1115 r = -EFAULT;
1116 if (copy_from_user(®, argp, sizeof(reg)))
1117 break;
1118
1119 if (ioctl == KVM_SET_ONE_REG)
1120 r = kvm_arm_set_reg(vcpu, ®);
1121 else
1122 r = kvm_arm_get_reg(vcpu, ®);
1123 break;
1124 }
1125 case KVM_GET_REG_LIST: {
1126 struct kvm_reg_list __user *user_list = argp;
1127 struct kvm_reg_list reg_list;
1128 unsigned n;
1129
1130 r = -ENOEXEC;
1131 if (unlikely(!kvm_vcpu_initialized(vcpu)))
1132 break;
1133
1134 r = -EFAULT;
1135 if (copy_from_user(®_list, user_list, sizeof(reg_list)))
1136 break;
1137 n = reg_list.n;
1138 reg_list.n = kvm_arm_num_regs(vcpu);
1139 if (copy_to_user(user_list, ®_list, sizeof(reg_list)))
1140 break;
1141 r = -E2BIG;
1142 if (n < reg_list.n)
1143 break;
1144 r = kvm_arm_copy_reg_indices(vcpu, user_list->reg);
1145 break;
1146 }
1147 case KVM_SET_DEVICE_ATTR: {
1148 r = -EFAULT;
1149 if (copy_from_user(&attr, argp, sizeof(attr)))
1150 break;
1151 r = kvm_arm_vcpu_set_attr(vcpu, &attr);
1152 break;
1153 }
1154 case KVM_GET_DEVICE_ATTR: {
1155 r = -EFAULT;
1156 if (copy_from_user(&attr, argp, sizeof(attr)))
1157 break;
1158 r = kvm_arm_vcpu_get_attr(vcpu, &attr);
1159 break;
1160 }
1161 case KVM_HAS_DEVICE_ATTR: {
1162 r = -EFAULT;
1163 if (copy_from_user(&attr, argp, sizeof(attr)))
1164 break;
1165 r = kvm_arm_vcpu_has_attr(vcpu, &attr);
1166 break;
1167 }
1168 case KVM_GET_VCPU_EVENTS: {
1169 struct kvm_vcpu_events events;
1170
1171 if (kvm_arm_vcpu_get_events(vcpu, &events))
1172 return -EINVAL;
1173
1174 if (copy_to_user(argp, &events, sizeof(events)))
1175 return -EFAULT;
1176
1177 return 0;
1178 }
1179 case KVM_SET_VCPU_EVENTS: {
1180 struct kvm_vcpu_events events;
1181
1182 if (copy_from_user(&events, argp, sizeof(events)))
1183 return -EFAULT;
1184
1185 return kvm_arm_vcpu_set_events(vcpu, &events);
1186 }
1187 default:
1188 r = -EINVAL;
1189 }
1190
1191 return r;
1192}
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
1214{
1215 bool flush = false;
1216 int r;
1217
1218 mutex_lock(&kvm->slots_lock);
1219
1220 r = kvm_get_dirty_log_protect(kvm, log, &flush);
1221
1222 if (flush)
1223 kvm_flush_remote_tlbs(kvm);
1224
1225 mutex_unlock(&kvm->slots_lock);
1226 return r;
1227}
1228
1229int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, struct kvm_clear_dirty_log *log)
1230{
1231 bool flush = false;
1232 int r;
1233
1234 mutex_lock(&kvm->slots_lock);
1235
1236 r = kvm_clear_dirty_log_protect(kvm, log, &flush);
1237
1238 if (flush)
1239 kvm_flush_remote_tlbs(kvm);
1240
1241 mutex_unlock(&kvm->slots_lock);
1242 return r;
1243}
1244
1245static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm,
1246 struct kvm_arm_device_addr *dev_addr)
1247{
1248 unsigned long dev_id, type;
1249
1250 dev_id = (dev_addr->id & KVM_ARM_DEVICE_ID_MASK) >>
1251 KVM_ARM_DEVICE_ID_SHIFT;
1252 type = (dev_addr->id & KVM_ARM_DEVICE_TYPE_MASK) >>
1253 KVM_ARM_DEVICE_TYPE_SHIFT;
1254
1255 switch (dev_id) {
1256 case KVM_ARM_DEVICE_VGIC_V2:
1257 if (!vgic_present)
1258 return -ENXIO;
1259 return kvm_vgic_addr(kvm, type, &dev_addr->addr, true);
1260 default:
1261 return -ENODEV;
1262 }
1263}
1264
1265long kvm_arch_vm_ioctl(struct file *filp,
1266 unsigned int ioctl, unsigned long arg)
1267{
1268 struct kvm *kvm = filp->private_data;
1269 void __user *argp = (void __user *)arg;
1270
1271 switch (ioctl) {
1272 case KVM_CREATE_IRQCHIP: {
1273 int ret;
1274 if (!vgic_present)
1275 return -ENXIO;
1276 mutex_lock(&kvm->lock);
1277 ret = kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
1278 mutex_unlock(&kvm->lock);
1279 return ret;
1280 }
1281 case KVM_ARM_SET_DEVICE_ADDR: {
1282 struct kvm_arm_device_addr dev_addr;
1283
1284 if (copy_from_user(&dev_addr, argp, sizeof(dev_addr)))
1285 return -EFAULT;
1286 return kvm_vm_ioctl_set_device_addr(kvm, &dev_addr);
1287 }
1288 case KVM_ARM_PREFERRED_TARGET: {
1289 int err;
1290 struct kvm_vcpu_init init;
1291
1292 err = kvm_vcpu_preferred_target(&init);
1293 if (err)
1294 return err;
1295
1296 if (copy_to_user(argp, &init, sizeof(init)))
1297 return -EFAULT;
1298
1299 return 0;
1300 }
1301 default:
1302 return -EINVAL;
1303 }
1304}
1305
1306static void cpu_init_hyp_mode(void *dummy)
1307{
1308 phys_addr_t pgd_ptr;
1309 unsigned long hyp_stack_ptr;
1310 unsigned long stack_page;
1311 unsigned long vector_ptr;
1312
1313
1314 __hyp_set_vectors(kvm_get_idmap_vector());
1315
1316 pgd_ptr = kvm_mmu_get_httbr();
1317 stack_page = __this_cpu_read(kvm_arm_hyp_stack_page);
1318 hyp_stack_ptr = stack_page + PAGE_SIZE;
1319 vector_ptr = (unsigned long)kvm_get_hyp_vector();
1320
1321 __cpu_init_hyp_mode(pgd_ptr, hyp_stack_ptr, vector_ptr);
1322 __cpu_init_stage2();
1323}
1324
1325static void cpu_hyp_reset(void)
1326{
1327 if (!is_kernel_in_hyp_mode())
1328 __hyp_reset_vectors();
1329}
1330
1331static void cpu_hyp_reinit(void)
1332{
1333 cpu_hyp_reset();
1334
1335 if (is_kernel_in_hyp_mode())
1336 kvm_timer_init_vhe();
1337 else
1338 cpu_init_hyp_mode(NULL);
1339
1340 kvm_arm_init_debug();
1341
1342 if (vgic_present)
1343 kvm_vgic_init_cpu_hardware();
1344}
1345
1346static void _kvm_arch_hardware_enable(void *discard)
1347{
1348 if (!__this_cpu_read(kvm_arm_hardware_enabled)) {
1349 cpu_hyp_reinit();
1350 __this_cpu_write(kvm_arm_hardware_enabled, 1);
1351 }
1352}
1353
1354int kvm_arch_hardware_enable(void)
1355{
1356 _kvm_arch_hardware_enable(NULL);
1357 return 0;
1358}
1359
1360static void _kvm_arch_hardware_disable(void *discard)
1361{
1362 if (__this_cpu_read(kvm_arm_hardware_enabled)) {
1363 cpu_hyp_reset();
1364 __this_cpu_write(kvm_arm_hardware_enabled, 0);
1365 }
1366}
1367
1368void kvm_arch_hardware_disable(void)
1369{
1370 _kvm_arch_hardware_disable(NULL);
1371}
1372
1373#ifdef CONFIG_CPU_PM
1374static int hyp_init_cpu_pm_notifier(struct notifier_block *self,
1375 unsigned long cmd,
1376 void *v)
1377{
1378
1379
1380
1381
1382
1383 switch (cmd) {
1384 case CPU_PM_ENTER:
1385 if (__this_cpu_read(kvm_arm_hardware_enabled))
1386
1387
1388
1389
1390
1391 cpu_hyp_reset();
1392
1393 return NOTIFY_OK;
1394 case CPU_PM_ENTER_FAILED:
1395 case CPU_PM_EXIT:
1396 if (__this_cpu_read(kvm_arm_hardware_enabled))
1397
1398 cpu_hyp_reinit();
1399
1400 return NOTIFY_OK;
1401
1402 default:
1403 return NOTIFY_DONE;
1404 }
1405}
1406
1407static struct notifier_block hyp_init_cpu_pm_nb = {
1408 .notifier_call = hyp_init_cpu_pm_notifier,
1409};
1410
1411static void __init hyp_cpu_pm_init(void)
1412{
1413 cpu_pm_register_notifier(&hyp_init_cpu_pm_nb);
1414}
1415static void __init hyp_cpu_pm_exit(void)
1416{
1417 cpu_pm_unregister_notifier(&hyp_init_cpu_pm_nb);
1418}
1419#else
1420static inline void hyp_cpu_pm_init(void)
1421{
1422}
1423static inline void hyp_cpu_pm_exit(void)
1424{
1425}
1426#endif
1427
1428static int init_common_resources(void)
1429{
1430
1431 kvm_vmid_bits = kvm_get_vmid_bits();
1432 kvm_info("%d-bit VMID\n", kvm_vmid_bits);
1433
1434 kvm_set_ipa_limit();
1435
1436 return 0;
1437}
1438
1439static int init_subsystems(void)
1440{
1441 int err = 0;
1442
1443
1444
1445
1446 on_each_cpu(_kvm_arch_hardware_enable, NULL, 1);
1447
1448
1449
1450
1451 hyp_cpu_pm_init();
1452
1453
1454
1455
1456 err = kvm_vgic_hyp_init();
1457 switch (err) {
1458 case 0:
1459 vgic_present = true;
1460 break;
1461 case -ENODEV:
1462 case -ENXIO:
1463 vgic_present = false;
1464 err = 0;
1465 break;
1466 default:
1467 goto out;
1468 }
1469
1470
1471
1472
1473 err = kvm_timer_hyp_init(vgic_present);
1474 if (err)
1475 goto out;
1476
1477 kvm_perf_init();
1478 kvm_coproc_table_init();
1479
1480out:
1481 on_each_cpu(_kvm_arch_hardware_disable, NULL, 1);
1482
1483 return err;
1484}
1485
1486static void teardown_hyp_mode(void)
1487{
1488 int cpu;
1489
1490 free_hyp_pgds();
1491 for_each_possible_cpu(cpu)
1492 free_page(per_cpu(kvm_arm_hyp_stack_page, cpu));
1493 hyp_cpu_pm_exit();
1494}
1495
1496
1497
1498
1499static int init_hyp_mode(void)
1500{
1501 int cpu;
1502 int err = 0;
1503
1504
1505
1506
1507 err = kvm_mmu_init();
1508 if (err)
1509 goto out_err;
1510
1511
1512
1513
1514 for_each_possible_cpu(cpu) {
1515 unsigned long stack_page;
1516
1517 stack_page = __get_free_page(GFP_KERNEL);
1518 if (!stack_page) {
1519 err = -ENOMEM;
1520 goto out_err;
1521 }
1522
1523 per_cpu(kvm_arm_hyp_stack_page, cpu) = stack_page;
1524 }
1525
1526
1527
1528
1529 err = create_hyp_mappings(kvm_ksym_ref(__hyp_text_start),
1530 kvm_ksym_ref(__hyp_text_end), PAGE_HYP_EXEC);
1531 if (err) {
1532 kvm_err("Cannot map world-switch code\n");
1533 goto out_err;
1534 }
1535
1536 err = create_hyp_mappings(kvm_ksym_ref(__start_rodata),
1537 kvm_ksym_ref(__end_rodata), PAGE_HYP_RO);
1538 if (err) {
1539 kvm_err("Cannot map rodata section\n");
1540 goto out_err;
1541 }
1542
1543 err = create_hyp_mappings(kvm_ksym_ref(__bss_start),
1544 kvm_ksym_ref(__bss_stop), PAGE_HYP_RO);
1545 if (err) {
1546 kvm_err("Cannot map bss section\n");
1547 goto out_err;
1548 }
1549
1550 err = kvm_map_vectors();
1551 if (err) {
1552 kvm_err("Cannot map vectors\n");
1553 goto out_err;
1554 }
1555
1556
1557
1558
1559 for_each_possible_cpu(cpu) {
1560 char *stack_page = (char *)per_cpu(kvm_arm_hyp_stack_page, cpu);
1561 err = create_hyp_mappings(stack_page, stack_page + PAGE_SIZE,
1562 PAGE_HYP);
1563
1564 if (err) {
1565 kvm_err("Cannot map hyp stack\n");
1566 goto out_err;
1567 }
1568 }
1569
1570 for_each_possible_cpu(cpu) {
1571 kvm_cpu_context_t *cpu_ctxt;
1572
1573 cpu_ctxt = per_cpu_ptr(&kvm_host_cpu_state, cpu);
1574 err = create_hyp_mappings(cpu_ctxt, cpu_ctxt + 1, PAGE_HYP);
1575
1576 if (err) {
1577 kvm_err("Cannot map host CPU state: %d\n", err);
1578 goto out_err;
1579 }
1580 }
1581
1582 err = hyp_map_aux_data();
1583 if (err)
1584 kvm_err("Cannot map host auxilary data: %d\n", err);
1585
1586 return 0;
1587
1588out_err:
1589 teardown_hyp_mode();
1590 kvm_err("error initializing Hyp mode: %d\n", err);
1591 return err;
1592}
1593
1594static void check_kvm_target_cpu(void *ret)
1595{
1596 *(int *)ret = kvm_target_cpu();
1597}
1598
1599struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr)
1600{
1601 struct kvm_vcpu *vcpu;
1602 int i;
1603
1604 mpidr &= MPIDR_HWID_BITMASK;
1605 kvm_for_each_vcpu(i, vcpu, kvm) {
1606 if (mpidr == kvm_vcpu_get_mpidr_aff(vcpu))
1607 return vcpu;
1608 }
1609 return NULL;
1610}
1611
1612bool kvm_arch_has_irq_bypass(void)
1613{
1614 return true;
1615}
1616
1617int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
1618 struct irq_bypass_producer *prod)
1619{
1620 struct kvm_kernel_irqfd *irqfd =
1621 container_of(cons, struct kvm_kernel_irqfd, consumer);
1622
1623 return kvm_vgic_v4_set_forwarding(irqfd->kvm, prod->irq,
1624 &irqfd->irq_entry);
1625}
1626void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
1627 struct irq_bypass_producer *prod)
1628{
1629 struct kvm_kernel_irqfd *irqfd =
1630 container_of(cons, struct kvm_kernel_irqfd, consumer);
1631
1632 kvm_vgic_v4_unset_forwarding(irqfd->kvm, prod->irq,
1633 &irqfd->irq_entry);
1634}
1635
1636void kvm_arch_irq_bypass_stop(struct irq_bypass_consumer *cons)
1637{
1638 struct kvm_kernel_irqfd *irqfd =
1639 container_of(cons, struct kvm_kernel_irqfd, consumer);
1640
1641 kvm_arm_halt_guest(irqfd->kvm);
1642}
1643
1644void kvm_arch_irq_bypass_start(struct irq_bypass_consumer *cons)
1645{
1646 struct kvm_kernel_irqfd *irqfd =
1647 container_of(cons, struct kvm_kernel_irqfd, consumer);
1648
1649 kvm_arm_resume_guest(irqfd->kvm);
1650}
1651
1652
1653
1654
1655int kvm_arch_init(void *opaque)
1656{
1657 int err;
1658 int ret, cpu;
1659 bool in_hyp_mode;
1660
1661 if (!is_hyp_mode_available()) {
1662 kvm_info("HYP mode not available\n");
1663 return -ENODEV;
1664 }
1665
1666 in_hyp_mode = is_kernel_in_hyp_mode();
1667
1668 if (!in_hyp_mode && kvm_arch_requires_vhe()) {
1669 kvm_pr_unimpl("CPU unsupported in non-VHE mode, not initializing\n");
1670 return -ENODEV;
1671 }
1672
1673 for_each_online_cpu(cpu) {
1674 smp_call_function_single(cpu, check_kvm_target_cpu, &ret, 1);
1675 if (ret < 0) {
1676 kvm_err("Error, CPU %d not supported!\n", cpu);
1677 return -ENODEV;
1678 }
1679 }
1680
1681 err = init_common_resources();
1682 if (err)
1683 return err;
1684
1685 if (!in_hyp_mode) {
1686 err = init_hyp_mode();
1687 if (err)
1688 goto out_err;
1689 }
1690
1691 err = init_subsystems();
1692 if (err)
1693 goto out_hyp;
1694
1695 if (in_hyp_mode)
1696 kvm_info("VHE mode initialized successfully\n");
1697 else
1698 kvm_info("Hyp mode initialized successfully\n");
1699
1700 return 0;
1701
1702out_hyp:
1703 if (!in_hyp_mode)
1704 teardown_hyp_mode();
1705out_err:
1706 return err;
1707}
1708
1709
1710void kvm_arch_exit(void)
1711{
1712 kvm_perf_teardown();
1713}
1714
1715static int arm_init(void)
1716{
1717 int rc = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
1718 return rc;
1719}
1720
1721module_init(arm_init);
1722