1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19#include <linux/cpu_pm.h>
20#include <linux/errno.h>
21#include <linux/err.h>
22#include <linux/kvm_host.h>
23#include <linux/list.h>
24#include <linux/module.h>
25#include <linux/vmalloc.h>
26#include <linux/fs.h>
27#include <linux/mman.h>
28#include <linux/sched.h>
29#include <linux/kvm.h>
30#include <trace/events/kvm.h>
31#include <kvm/arm_pmu.h>
32
33#define CREATE_TRACE_POINTS
34#include "trace.h"
35
36#include <asm/uaccess.h>
37#include <asm/ptrace.h>
38#include <asm/mman.h>
39#include <asm/tlbflush.h>
40#include <asm/cacheflush.h>
41#include <asm/virt.h>
42#include <asm/kvm_arm.h>
43#include <asm/kvm_asm.h>
44#include <asm/kvm_mmu.h>
45#include <asm/kvm_emulate.h>
46#include <asm/kvm_coproc.h>
47#include <asm/kvm_psci.h>
48#include <asm/sections.h>
49
50#ifdef REQUIRES_VIRT
51__asm__(".arch_extension virt");
52#endif
53
54static DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);
55static kvm_cpu_context_t __percpu *kvm_host_cpu_state;
56static unsigned long hyp_default_vectors;
57
58
59static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_arm_running_vcpu);
60
61
62static atomic64_t kvm_vmid_gen = ATOMIC64_INIT(1);
63static u32 kvm_next_vmid;
64static unsigned int kvm_vmid_bits __read_mostly;
65static DEFINE_SPINLOCK(kvm_vmid_lock);
66
67static bool vgic_present;
68
69static DEFINE_PER_CPU(unsigned char, kvm_arm_hardware_enabled);
70
71static void kvm_arm_set_running_vcpu(struct kvm_vcpu *vcpu)
72{
73 BUG_ON(preemptible());
74 __this_cpu_write(kvm_arm_running_vcpu, vcpu);
75}
76
77
78
79
80
81struct kvm_vcpu *kvm_arm_get_running_vcpu(void)
82{
83 BUG_ON(preemptible());
84 return __this_cpu_read(kvm_arm_running_vcpu);
85}
86
87
88
89
90struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void)
91{
92 return &kvm_arm_running_vcpu;
93}
94
95int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
96{
97 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
98}
99
100int kvm_arch_hardware_setup(void)
101{
102 return 0;
103}
104
105void kvm_arch_check_processor_compat(void *rtn)
106{
107 *(int *)rtn = 0;
108}
109
110
111
112
113
114
115int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
116{
117 int ret = 0;
118
119 if (type)
120 return -EINVAL;
121
122 ret = kvm_alloc_stage2_pgd(kvm);
123 if (ret)
124 goto out_fail_alloc;
125
126 ret = create_hyp_mappings(kvm, kvm + 1, PAGE_HYP);
127 if (ret)
128 goto out_free_stage2_pgd;
129
130 kvm_vgic_early_init(kvm);
131 kvm_timer_init(kvm);
132
133
134 kvm->arch.vmid_gen = 0;
135
136
137 kvm->arch.max_vcpus = vgic_present ?
138 kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS;
139
140 return ret;
141out_free_stage2_pgd:
142 kvm_free_stage2_pgd(kvm);
143out_fail_alloc:
144 return ret;
145}
146
147int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
148{
149 return VM_FAULT_SIGBUS;
150}
151
152
153
154
155
156
157void kvm_arch_destroy_vm(struct kvm *kvm)
158{
159 int i;
160
161 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
162 if (kvm->vcpus[i]) {
163 kvm_arch_vcpu_free(kvm->vcpus[i]);
164 kvm->vcpus[i] = NULL;
165 }
166 }
167
168 kvm_vgic_destroy(kvm);
169}
170
171int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
172{
173 int r;
174 switch (ext) {
175 case KVM_CAP_IRQCHIP:
176 r = vgic_present;
177 break;
178 case KVM_CAP_IOEVENTFD:
179 case KVM_CAP_DEVICE_CTRL:
180 case KVM_CAP_USER_MEMORY:
181 case KVM_CAP_SYNC_MMU:
182 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
183 case KVM_CAP_ONE_REG:
184 case KVM_CAP_ARM_PSCI:
185 case KVM_CAP_ARM_PSCI_0_2:
186 case KVM_CAP_READONLY_MEM:
187 case KVM_CAP_MP_STATE:
188 r = 1;
189 break;
190 case KVM_CAP_COALESCED_MMIO:
191 r = KVM_COALESCED_MMIO_PAGE_OFFSET;
192 break;
193 case KVM_CAP_ARM_SET_DEVICE_ADDR:
194 r = 1;
195 break;
196 case KVM_CAP_NR_VCPUS:
197 r = num_online_cpus();
198 break;
199 case KVM_CAP_MAX_VCPUS:
200 r = KVM_MAX_VCPUS;
201 break;
202 default:
203 r = kvm_arch_dev_ioctl_check_extension(kvm, ext);
204 break;
205 }
206 return r;
207}
208
209long kvm_arch_dev_ioctl(struct file *filp,
210 unsigned int ioctl, unsigned long arg)
211{
212 return -EINVAL;
213}
214
215
216struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id)
217{
218 int err;
219 struct kvm_vcpu *vcpu;
220
221 if (irqchip_in_kernel(kvm) && vgic_initialized(kvm)) {
222 err = -EBUSY;
223 goto out;
224 }
225
226 if (id >= kvm->arch.max_vcpus) {
227 err = -EINVAL;
228 goto out;
229 }
230
231 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
232 if (!vcpu) {
233 err = -ENOMEM;
234 goto out;
235 }
236
237 err = kvm_vcpu_init(vcpu, kvm, id);
238 if (err)
239 goto free_vcpu;
240
241 err = create_hyp_mappings(vcpu, vcpu + 1, PAGE_HYP);
242 if (err)
243 goto vcpu_uninit;
244
245 return vcpu;
246vcpu_uninit:
247 kvm_vcpu_uninit(vcpu);
248free_vcpu:
249 kmem_cache_free(kvm_vcpu_cache, vcpu);
250out:
251 return ERR_PTR(err);
252}
253
254void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
255{
256 kvm_vgic_vcpu_early_init(vcpu);
257}
258
259void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
260{
261 kvm_mmu_free_memory_caches(vcpu);
262 kvm_timer_vcpu_terminate(vcpu);
263 kvm_vgic_vcpu_destroy(vcpu);
264 kvm_pmu_vcpu_destroy(vcpu);
265 kvm_vcpu_uninit(vcpu);
266 kmem_cache_free(kvm_vcpu_cache, vcpu);
267}
268
269void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
270{
271 kvm_arch_vcpu_free(vcpu);
272}
273
274int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
275{
276 return kvm_timer_should_fire(vcpu);
277}
278
279void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
280{
281 kvm_timer_schedule(vcpu);
282}
283
284void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
285{
286 kvm_timer_unschedule(vcpu);
287}
288
289int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
290{
291
292 vcpu->arch.target = -1;
293 bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES);
294
295
296 kvm_timer_vcpu_init(vcpu);
297
298 kvm_arm_reset_debug_ptr(vcpu);
299
300 return 0;
301}
302
303void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
304{
305 vcpu->cpu = cpu;
306 vcpu->arch.host_cpu_context = this_cpu_ptr(kvm_host_cpu_state);
307
308 kvm_arm_set_running_vcpu(vcpu);
309}
310
311void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
312{
313
314
315
316
317
318 vcpu->cpu = -1;
319
320 kvm_arm_set_running_vcpu(NULL);
321 kvm_timer_vcpu_put(vcpu);
322}
323
324int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
325 struct kvm_mp_state *mp_state)
326{
327 if (vcpu->arch.power_off)
328 mp_state->mp_state = KVM_MP_STATE_STOPPED;
329 else
330 mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
331
332 return 0;
333}
334
335int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
336 struct kvm_mp_state *mp_state)
337{
338 switch (mp_state->mp_state) {
339 case KVM_MP_STATE_RUNNABLE:
340 vcpu->arch.power_off = false;
341 break;
342 case KVM_MP_STATE_STOPPED:
343 vcpu->arch.power_off = true;
344 break;
345 default:
346 return -EINVAL;
347 }
348
349 return 0;
350}
351
352
353
354
355
356
357
358
359int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
360{
361 return ((!!v->arch.irq_lines || kvm_vgic_vcpu_pending_irq(v))
362 && !v->arch.power_off && !v->arch.pause);
363}
364
365
366static void exit_vm_noop(void *info)
367{
368}
369
370void force_vm_exit(const cpumask_t *mask)
371{
372 preempt_disable();
373 smp_call_function_many(mask, exit_vm_noop, NULL, true);
374 preempt_enable();
375}
376
377
378
379
380
381
382
383
384
385
386
387
388
389static bool need_new_vmid_gen(struct kvm *kvm)
390{
391 return unlikely(kvm->arch.vmid_gen != atomic64_read(&kvm_vmid_gen));
392}
393
394
395
396
397
398
399
400
401
402static void update_vttbr(struct kvm *kvm)
403{
404 phys_addr_t pgd_phys;
405 u64 vmid;
406
407 if (!need_new_vmid_gen(kvm))
408 return;
409
410 spin_lock(&kvm_vmid_lock);
411
412
413
414
415
416
417 if (!need_new_vmid_gen(kvm)) {
418 spin_unlock(&kvm_vmid_lock);
419 return;
420 }
421
422
423 if (unlikely(kvm_next_vmid == 0)) {
424 atomic64_inc(&kvm_vmid_gen);
425 kvm_next_vmid = 1;
426
427
428
429
430
431
432 force_vm_exit(cpu_all_mask);
433
434
435
436
437
438 kvm_call_hyp(__kvm_flush_vm_context);
439 }
440
441 kvm->arch.vmid_gen = atomic64_read(&kvm_vmid_gen);
442 kvm->arch.vmid = kvm_next_vmid;
443 kvm_next_vmid++;
444 kvm_next_vmid &= (1 << kvm_vmid_bits) - 1;
445
446
447 pgd_phys = virt_to_phys(kvm->arch.pgd);
448 BUG_ON(pgd_phys & ~VTTBR_BADDR_MASK);
449 vmid = ((u64)(kvm->arch.vmid) << VTTBR_VMID_SHIFT) & VTTBR_VMID_MASK(kvm_vmid_bits);
450 kvm->arch.vttbr = pgd_phys | vmid;
451
452 spin_unlock(&kvm_vmid_lock);
453}
454
455static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu)
456{
457 struct kvm *kvm = vcpu->kvm;
458 int ret = 0;
459
460 if (likely(vcpu->arch.has_run_once))
461 return 0;
462
463 vcpu->arch.has_run_once = true;
464
465
466
467
468
469 if (unlikely(irqchip_in_kernel(kvm) && !vgic_ready(kvm))) {
470 ret = kvm_vgic_map_resources(kvm);
471 if (ret)
472 return ret;
473 }
474
475
476
477
478
479
480 if (irqchip_in_kernel(kvm) && vgic_initialized(kvm))
481 ret = kvm_timer_enable(vcpu);
482
483 return ret;
484}
485
486bool kvm_arch_intc_initialized(struct kvm *kvm)
487{
488 return vgic_initialized(kvm);
489}
490
491void kvm_arm_halt_guest(struct kvm *kvm)
492{
493 int i;
494 struct kvm_vcpu *vcpu;
495
496 kvm_for_each_vcpu(i, vcpu, kvm)
497 vcpu->arch.pause = true;
498 kvm_make_all_cpus_request(kvm, KVM_REQ_VCPU_EXIT);
499}
500
501void kvm_arm_halt_vcpu(struct kvm_vcpu *vcpu)
502{
503 vcpu->arch.pause = true;
504 kvm_vcpu_kick(vcpu);
505}
506
507void kvm_arm_resume_vcpu(struct kvm_vcpu *vcpu)
508{
509 struct swait_queue_head *wq = kvm_arch_vcpu_wq(vcpu);
510
511 vcpu->arch.pause = false;
512 swake_up(wq);
513}
514
515void kvm_arm_resume_guest(struct kvm *kvm)
516{
517 int i;
518 struct kvm_vcpu *vcpu;
519
520 kvm_for_each_vcpu(i, vcpu, kvm)
521 kvm_arm_resume_vcpu(vcpu);
522}
523
524static void vcpu_sleep(struct kvm_vcpu *vcpu)
525{
526 struct swait_queue_head *wq = kvm_arch_vcpu_wq(vcpu);
527
528 swait_event_interruptible(*wq, ((!vcpu->arch.power_off) &&
529 (!vcpu->arch.pause)));
530}
531
532static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu)
533{
534 return vcpu->arch.target >= 0;
535}
536
537
538
539
540
541
542
543
544
545
546
547
548int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
549{
550 int ret;
551 sigset_t sigsaved;
552
553 if (unlikely(!kvm_vcpu_initialized(vcpu)))
554 return -ENOEXEC;
555
556 ret = kvm_vcpu_first_run_init(vcpu);
557 if (ret)
558 return ret;
559
560 if (run->exit_reason == KVM_EXIT_MMIO) {
561 ret = kvm_handle_mmio_return(vcpu, vcpu->run);
562 if (ret)
563 return ret;
564 }
565
566 if (vcpu->sigset_active)
567 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
568
569 ret = 1;
570 run->exit_reason = KVM_EXIT_UNKNOWN;
571 while (ret > 0) {
572
573
574
575 cond_resched();
576
577 update_vttbr(vcpu->kvm);
578
579 if (vcpu->arch.power_off || vcpu->arch.pause)
580 vcpu_sleep(vcpu);
581
582
583
584
585
586
587 preempt_disable();
588 kvm_pmu_flush_hwstate(vcpu);
589 kvm_timer_flush_hwstate(vcpu);
590 kvm_vgic_flush_hwstate(vcpu);
591
592 local_irq_disable();
593
594
595
596
597 if (signal_pending(current)) {
598 ret = -EINTR;
599 run->exit_reason = KVM_EXIT_INTR;
600 }
601
602 if (ret <= 0 || need_new_vmid_gen(vcpu->kvm) ||
603 vcpu->arch.power_off || vcpu->arch.pause) {
604 local_irq_enable();
605 kvm_pmu_sync_hwstate(vcpu);
606 kvm_timer_sync_hwstate(vcpu);
607 kvm_vgic_sync_hwstate(vcpu);
608 preempt_enable();
609 continue;
610 }
611
612 kvm_arm_setup_debug(vcpu);
613
614
615
616
617 trace_kvm_entry(*vcpu_pc(vcpu));
618 guest_enter_irqoff();
619 vcpu->mode = IN_GUEST_MODE;
620
621 ret = kvm_call_hyp(__kvm_vcpu_run, vcpu);
622
623 vcpu->mode = OUTSIDE_GUEST_MODE;
624 vcpu->stat.exits++;
625
626
627
628
629 kvm_arm_clear_debug(vcpu);
630
631
632
633
634
635
636
637
638
639
640
641 local_irq_enable();
642
643
644
645
646
647
648
649
650
651 guest_exit();
652 trace_kvm_exit(ret, kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu));
653
654
655
656
657
658
659 kvm_pmu_sync_hwstate(vcpu);
660 kvm_timer_sync_hwstate(vcpu);
661
662 kvm_vgic_sync_hwstate(vcpu);
663
664 preempt_enable();
665
666 ret = handle_exit(vcpu, run, ret);
667 }
668
669 if (vcpu->sigset_active)
670 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
671 return ret;
672}
673
674static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level)
675{
676 int bit_index;
677 bool set;
678 unsigned long *ptr;
679
680 if (number == KVM_ARM_IRQ_CPU_IRQ)
681 bit_index = __ffs(HCR_VI);
682 else
683 bit_index = __ffs(HCR_VF);
684
685 ptr = (unsigned long *)&vcpu->arch.irq_lines;
686 if (level)
687 set = test_and_set_bit(bit_index, ptr);
688 else
689 set = test_and_clear_bit(bit_index, ptr);
690
691
692
693
694 if (set == level)
695 return 0;
696
697
698
699
700
701
702 kvm_vcpu_kick(vcpu);
703
704 return 0;
705}
706
707int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
708 bool line_status)
709{
710 u32 irq = irq_level->irq;
711 unsigned int irq_type, vcpu_idx, irq_num;
712 int nrcpus = atomic_read(&kvm->online_vcpus);
713 struct kvm_vcpu *vcpu = NULL;
714 bool level = irq_level->level;
715
716 irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK;
717 vcpu_idx = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;
718 irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK;
719
720 trace_kvm_irq_line(irq_type, vcpu_idx, irq_num, irq_level->level);
721
722 switch (irq_type) {
723 case KVM_ARM_IRQ_TYPE_CPU:
724 if (irqchip_in_kernel(kvm))
725 return -ENXIO;
726
727 if (vcpu_idx >= nrcpus)
728 return -EINVAL;
729
730 vcpu = kvm_get_vcpu(kvm, vcpu_idx);
731 if (!vcpu)
732 return -EINVAL;
733
734 if (irq_num > KVM_ARM_IRQ_CPU_FIQ)
735 return -EINVAL;
736
737 return vcpu_interrupt_line(vcpu, irq_num, level);
738 case KVM_ARM_IRQ_TYPE_PPI:
739 if (!irqchip_in_kernel(kvm))
740 return -ENXIO;
741
742 if (vcpu_idx >= nrcpus)
743 return -EINVAL;
744
745 vcpu = kvm_get_vcpu(kvm, vcpu_idx);
746 if (!vcpu)
747 return -EINVAL;
748
749 if (irq_num < VGIC_NR_SGIS || irq_num >= VGIC_NR_PRIVATE_IRQS)
750 return -EINVAL;
751
752 return kvm_vgic_inject_irq(kvm, vcpu->vcpu_id, irq_num, level);
753 case KVM_ARM_IRQ_TYPE_SPI:
754 if (!irqchip_in_kernel(kvm))
755 return -ENXIO;
756
757 if (irq_num < VGIC_NR_PRIVATE_IRQS)
758 return -EINVAL;
759
760 return kvm_vgic_inject_irq(kvm, 0, irq_num, level);
761 }
762
763 return -EINVAL;
764}
765
766static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
767 const struct kvm_vcpu_init *init)
768{
769 unsigned int i;
770 int phys_target = kvm_target_cpu();
771
772 if (init->target != phys_target)
773 return -EINVAL;
774
775
776
777
778
779 if (vcpu->arch.target != -1 && vcpu->arch.target != init->target)
780 return -EINVAL;
781
782
783 for (i = 0; i < sizeof(init->features) * 8; i++) {
784 bool set = (init->features[i / 32] & (1 << (i % 32)));
785
786 if (set && i >= KVM_VCPU_MAX_FEATURES)
787 return -ENOENT;
788
789
790
791
792
793 if (vcpu->arch.target != -1 && i < KVM_VCPU_MAX_FEATURES &&
794 test_bit(i, vcpu->arch.features) != set)
795 return -EINVAL;
796
797 if (set)
798 set_bit(i, vcpu->arch.features);
799 }
800
801 vcpu->arch.target = phys_target;
802
803
804 return kvm_reset_vcpu(vcpu);
805}
806
807
808static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu,
809 struct kvm_vcpu_init *init)
810{
811 int ret;
812
813 ret = kvm_vcpu_set_target(vcpu, init);
814 if (ret)
815 return ret;
816
817
818
819
820
821 if (vcpu->arch.has_run_once)
822 stage2_unmap_vm(vcpu->kvm);
823
824 vcpu_reset_hcr(vcpu);
825
826
827
828
829 if (test_bit(KVM_ARM_VCPU_POWER_OFF, vcpu->arch.features))
830 vcpu->arch.power_off = true;
831 else
832 vcpu->arch.power_off = false;
833
834 return 0;
835}
836
837static int kvm_arm_vcpu_set_attr(struct kvm_vcpu *vcpu,
838 struct kvm_device_attr *attr)
839{
840 int ret = -ENXIO;
841
842 switch (attr->group) {
843 default:
844 ret = kvm_arm_vcpu_arch_set_attr(vcpu, attr);
845 break;
846 }
847
848 return ret;
849}
850
851static int kvm_arm_vcpu_get_attr(struct kvm_vcpu *vcpu,
852 struct kvm_device_attr *attr)
853{
854 int ret = -ENXIO;
855
856 switch (attr->group) {
857 default:
858 ret = kvm_arm_vcpu_arch_get_attr(vcpu, attr);
859 break;
860 }
861
862 return ret;
863}
864
865static int kvm_arm_vcpu_has_attr(struct kvm_vcpu *vcpu,
866 struct kvm_device_attr *attr)
867{
868 int ret = -ENXIO;
869
870 switch (attr->group) {
871 default:
872 ret = kvm_arm_vcpu_arch_has_attr(vcpu, attr);
873 break;
874 }
875
876 return ret;
877}
878
879long kvm_arch_vcpu_ioctl(struct file *filp,
880 unsigned int ioctl, unsigned long arg)
881{
882 struct kvm_vcpu *vcpu = filp->private_data;
883 void __user *argp = (void __user *)arg;
884 struct kvm_device_attr attr;
885
886 switch (ioctl) {
887 case KVM_ARM_VCPU_INIT: {
888 struct kvm_vcpu_init init;
889
890 if (copy_from_user(&init, argp, sizeof(init)))
891 return -EFAULT;
892
893 return kvm_arch_vcpu_ioctl_vcpu_init(vcpu, &init);
894 }
895 case KVM_SET_ONE_REG:
896 case KVM_GET_ONE_REG: {
897 struct kvm_one_reg reg;
898
899 if (unlikely(!kvm_vcpu_initialized(vcpu)))
900 return -ENOEXEC;
901
902 if (copy_from_user(®, argp, sizeof(reg)))
903 return -EFAULT;
904 if (ioctl == KVM_SET_ONE_REG)
905 return kvm_arm_set_reg(vcpu, ®);
906 else
907 return kvm_arm_get_reg(vcpu, ®);
908 }
909 case KVM_GET_REG_LIST: {
910 struct kvm_reg_list __user *user_list = argp;
911 struct kvm_reg_list reg_list;
912 unsigned n;
913
914 if (unlikely(!kvm_vcpu_initialized(vcpu)))
915 return -ENOEXEC;
916
917 if (copy_from_user(®_list, user_list, sizeof(reg_list)))
918 return -EFAULT;
919 n = reg_list.n;
920 reg_list.n = kvm_arm_num_regs(vcpu);
921 if (copy_to_user(user_list, ®_list, sizeof(reg_list)))
922 return -EFAULT;
923 if (n < reg_list.n)
924 return -E2BIG;
925 return kvm_arm_copy_reg_indices(vcpu, user_list->reg);
926 }
927 case KVM_SET_DEVICE_ATTR: {
928 if (copy_from_user(&attr, argp, sizeof(attr)))
929 return -EFAULT;
930 return kvm_arm_vcpu_set_attr(vcpu, &attr);
931 }
932 case KVM_GET_DEVICE_ATTR: {
933 if (copy_from_user(&attr, argp, sizeof(attr)))
934 return -EFAULT;
935 return kvm_arm_vcpu_get_attr(vcpu, &attr);
936 }
937 case KVM_HAS_DEVICE_ATTR: {
938 if (copy_from_user(&attr, argp, sizeof(attr)))
939 return -EFAULT;
940 return kvm_arm_vcpu_has_attr(vcpu, &attr);
941 }
942 default:
943 return -EINVAL;
944 }
945}
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
967{
968 bool is_dirty = false;
969 int r;
970
971 mutex_lock(&kvm->slots_lock);
972
973 r = kvm_get_dirty_log_protect(kvm, log, &is_dirty);
974
975 if (is_dirty)
976 kvm_flush_remote_tlbs(kvm);
977
978 mutex_unlock(&kvm->slots_lock);
979 return r;
980}
981
982static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm,
983 struct kvm_arm_device_addr *dev_addr)
984{
985 unsigned long dev_id, type;
986
987 dev_id = (dev_addr->id & KVM_ARM_DEVICE_ID_MASK) >>
988 KVM_ARM_DEVICE_ID_SHIFT;
989 type = (dev_addr->id & KVM_ARM_DEVICE_TYPE_MASK) >>
990 KVM_ARM_DEVICE_TYPE_SHIFT;
991
992 switch (dev_id) {
993 case KVM_ARM_DEVICE_VGIC_V2:
994 if (!vgic_present)
995 return -ENXIO;
996 return kvm_vgic_addr(kvm, type, &dev_addr->addr, true);
997 default:
998 return -ENODEV;
999 }
1000}
1001
1002long kvm_arch_vm_ioctl(struct file *filp,
1003 unsigned int ioctl, unsigned long arg)
1004{
1005 struct kvm *kvm = filp->private_data;
1006 void __user *argp = (void __user *)arg;
1007
1008 switch (ioctl) {
1009 case KVM_CREATE_IRQCHIP: {
1010 int ret;
1011 if (!vgic_present)
1012 return -ENXIO;
1013 mutex_lock(&kvm->lock);
1014 ret = kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
1015 mutex_unlock(&kvm->lock);
1016 return ret;
1017 }
1018 case KVM_ARM_SET_DEVICE_ADDR: {
1019 struct kvm_arm_device_addr dev_addr;
1020
1021 if (copy_from_user(&dev_addr, argp, sizeof(dev_addr)))
1022 return -EFAULT;
1023 return kvm_vm_ioctl_set_device_addr(kvm, &dev_addr);
1024 }
1025 case KVM_ARM_PREFERRED_TARGET: {
1026 int err;
1027 struct kvm_vcpu_init init;
1028
1029 err = kvm_vcpu_preferred_target(&init);
1030 if (err)
1031 return err;
1032
1033 if (copy_to_user(argp, &init, sizeof(init)))
1034 return -EFAULT;
1035
1036 return 0;
1037 }
1038 default:
1039 return -EINVAL;
1040 }
1041}
1042
1043static void cpu_init_hyp_mode(void *dummy)
1044{
1045 phys_addr_t pgd_ptr;
1046 unsigned long hyp_stack_ptr;
1047 unsigned long stack_page;
1048 unsigned long vector_ptr;
1049
1050
1051 __hyp_set_vectors(kvm_get_idmap_vector());
1052
1053 pgd_ptr = kvm_mmu_get_httbr();
1054 stack_page = __this_cpu_read(kvm_arm_hyp_stack_page);
1055 hyp_stack_ptr = stack_page + PAGE_SIZE;
1056 vector_ptr = (unsigned long)kvm_ksym_ref(__kvm_hyp_vector);
1057
1058 __cpu_init_hyp_mode(pgd_ptr, hyp_stack_ptr, vector_ptr);
1059 __cpu_init_stage2();
1060
1061 kvm_arm_init_debug();
1062}
1063
1064static void cpu_hyp_reinit(void)
1065{
1066 if (is_kernel_in_hyp_mode()) {
1067
1068
1069
1070
1071 __cpu_init_stage2();
1072 } else {
1073 if (__hyp_get_vectors() == hyp_default_vectors)
1074 cpu_init_hyp_mode(NULL);
1075 }
1076}
1077
1078static void cpu_hyp_reset(void)
1079{
1080 if (!is_kernel_in_hyp_mode())
1081 __cpu_reset_hyp_mode(hyp_default_vectors,
1082 kvm_get_idmap_start());
1083}
1084
1085static void _kvm_arch_hardware_enable(void *discard)
1086{
1087 if (!__this_cpu_read(kvm_arm_hardware_enabled)) {
1088 cpu_hyp_reinit();
1089 __this_cpu_write(kvm_arm_hardware_enabled, 1);
1090 }
1091}
1092
1093int kvm_arch_hardware_enable(void)
1094{
1095 _kvm_arch_hardware_enable(NULL);
1096 return 0;
1097}
1098
1099static void _kvm_arch_hardware_disable(void *discard)
1100{
1101 if (__this_cpu_read(kvm_arm_hardware_enabled)) {
1102 cpu_hyp_reset();
1103 __this_cpu_write(kvm_arm_hardware_enabled, 0);
1104 }
1105}
1106
1107void kvm_arch_hardware_disable(void)
1108{
1109 _kvm_arch_hardware_disable(NULL);
1110}
1111
1112#ifdef CONFIG_CPU_PM
1113static int hyp_init_cpu_pm_notifier(struct notifier_block *self,
1114 unsigned long cmd,
1115 void *v)
1116{
1117
1118
1119
1120
1121
1122 switch (cmd) {
1123 case CPU_PM_ENTER:
1124 if (__this_cpu_read(kvm_arm_hardware_enabled))
1125
1126
1127
1128
1129
1130 cpu_hyp_reset();
1131
1132 return NOTIFY_OK;
1133 case CPU_PM_EXIT:
1134 if (__this_cpu_read(kvm_arm_hardware_enabled))
1135
1136 cpu_hyp_reinit();
1137
1138 return NOTIFY_OK;
1139
1140 default:
1141 return NOTIFY_DONE;
1142 }
1143}
1144
1145static struct notifier_block hyp_init_cpu_pm_nb = {
1146 .notifier_call = hyp_init_cpu_pm_notifier,
1147};
1148
1149static void __init hyp_cpu_pm_init(void)
1150{
1151 cpu_pm_register_notifier(&hyp_init_cpu_pm_nb);
1152}
1153static void __init hyp_cpu_pm_exit(void)
1154{
1155 cpu_pm_unregister_notifier(&hyp_init_cpu_pm_nb);
1156}
1157#else
1158static inline void hyp_cpu_pm_init(void)
1159{
1160}
1161static inline void hyp_cpu_pm_exit(void)
1162{
1163}
1164#endif
1165
1166static void teardown_common_resources(void)
1167{
1168 free_percpu(kvm_host_cpu_state);
1169}
1170
1171static int init_common_resources(void)
1172{
1173 kvm_host_cpu_state = alloc_percpu(kvm_cpu_context_t);
1174 if (!kvm_host_cpu_state) {
1175 kvm_err("Cannot allocate host CPU state\n");
1176 return -ENOMEM;
1177 }
1178
1179 return 0;
1180}
1181
1182static int init_subsystems(void)
1183{
1184 int err = 0;
1185
1186
1187
1188
1189 on_each_cpu(_kvm_arch_hardware_enable, NULL, 1);
1190
1191
1192
1193
1194 hyp_cpu_pm_init();
1195
1196
1197
1198
1199 err = kvm_vgic_hyp_init();
1200 switch (err) {
1201 case 0:
1202 vgic_present = true;
1203 break;
1204 case -ENODEV:
1205 case -ENXIO:
1206 vgic_present = false;
1207 err = 0;
1208 break;
1209 default:
1210 goto out;
1211 }
1212
1213
1214
1215
1216 err = kvm_timer_hyp_init();
1217 if (err)
1218 goto out;
1219
1220 kvm_perf_init();
1221 kvm_coproc_table_init();
1222
1223out:
1224 on_each_cpu(_kvm_arch_hardware_disable, NULL, 1);
1225
1226 return err;
1227}
1228
1229static void teardown_hyp_mode(void)
1230{
1231 int cpu;
1232
1233 if (is_kernel_in_hyp_mode())
1234 return;
1235
1236 free_hyp_pgds();
1237 for_each_possible_cpu(cpu)
1238 free_page(per_cpu(kvm_arm_hyp_stack_page, cpu));
1239 hyp_cpu_pm_exit();
1240}
1241
1242static int init_vhe_mode(void)
1243{
1244
1245 kvm_vmid_bits = kvm_get_vmid_bits();
1246 kvm_info("%d-bit VMID\n", kvm_vmid_bits);
1247
1248 kvm_info("VHE mode initialized successfully\n");
1249 return 0;
1250}
1251
1252
1253
1254
1255static int init_hyp_mode(void)
1256{
1257 int cpu;
1258 int err = 0;
1259
1260
1261
1262
1263 err = kvm_mmu_init();
1264 if (err)
1265 goto out_err;
1266
1267
1268
1269
1270
1271 hyp_default_vectors = __hyp_get_vectors();
1272
1273
1274
1275
1276 for_each_possible_cpu(cpu) {
1277 unsigned long stack_page;
1278
1279 stack_page = __get_free_page(GFP_KERNEL);
1280 if (!stack_page) {
1281 err = -ENOMEM;
1282 goto out_err;
1283 }
1284
1285 per_cpu(kvm_arm_hyp_stack_page, cpu) = stack_page;
1286 }
1287
1288
1289
1290
1291 err = create_hyp_mappings(kvm_ksym_ref(__hyp_text_start),
1292 kvm_ksym_ref(__hyp_text_end), PAGE_HYP_EXEC);
1293 if (err) {
1294 kvm_err("Cannot map world-switch code\n");
1295 goto out_err;
1296 }
1297
1298 err = create_hyp_mappings(kvm_ksym_ref(__start_rodata),
1299 kvm_ksym_ref(__end_rodata), PAGE_HYP_RO);
1300 if (err) {
1301 kvm_err("Cannot map rodata section\n");
1302 goto out_err;
1303 }
1304
1305
1306
1307
1308 for_each_possible_cpu(cpu) {
1309 char *stack_page = (char *)per_cpu(kvm_arm_hyp_stack_page, cpu);
1310 err = create_hyp_mappings(stack_page, stack_page + PAGE_SIZE,
1311 PAGE_HYP);
1312
1313 if (err) {
1314 kvm_err("Cannot map hyp stack\n");
1315 goto out_err;
1316 }
1317 }
1318
1319 for_each_possible_cpu(cpu) {
1320 kvm_cpu_context_t *cpu_ctxt;
1321
1322 cpu_ctxt = per_cpu_ptr(kvm_host_cpu_state, cpu);
1323 err = create_hyp_mappings(cpu_ctxt, cpu_ctxt + 1, PAGE_HYP);
1324
1325 if (err) {
1326 kvm_err("Cannot map host CPU state: %d\n", err);
1327 goto out_err;
1328 }
1329 }
1330
1331
1332 kvm_vmid_bits = kvm_get_vmid_bits();
1333 kvm_info("%d-bit VMID\n", kvm_vmid_bits);
1334
1335 kvm_info("Hyp mode initialized successfully\n");
1336
1337 return 0;
1338
1339out_err:
1340 teardown_hyp_mode();
1341 kvm_err("error initializing Hyp mode: %d\n", err);
1342 return err;
1343}
1344
1345static void check_kvm_target_cpu(void *ret)
1346{
1347 *(int *)ret = kvm_target_cpu();
1348}
1349
1350struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr)
1351{
1352 struct kvm_vcpu *vcpu;
1353 int i;
1354
1355 mpidr &= MPIDR_HWID_BITMASK;
1356 kvm_for_each_vcpu(i, vcpu, kvm) {
1357 if (mpidr == kvm_vcpu_get_mpidr_aff(vcpu))
1358 return vcpu;
1359 }
1360 return NULL;
1361}
1362
1363
1364
1365
1366int kvm_arch_init(void *opaque)
1367{
1368 int err;
1369 int ret, cpu;
1370
1371 if (!is_hyp_mode_available()) {
1372 kvm_err("HYP mode not available\n");
1373 return -ENODEV;
1374 }
1375
1376 for_each_online_cpu(cpu) {
1377 smp_call_function_single(cpu, check_kvm_target_cpu, &ret, 1);
1378 if (ret < 0) {
1379 kvm_err("Error, CPU %d not supported!\n", cpu);
1380 return -ENODEV;
1381 }
1382 }
1383
1384 err = init_common_resources();
1385 if (err)
1386 return err;
1387
1388 if (is_kernel_in_hyp_mode())
1389 err = init_vhe_mode();
1390 else
1391 err = init_hyp_mode();
1392 if (err)
1393 goto out_err;
1394
1395 err = init_subsystems();
1396 if (err)
1397 goto out_hyp;
1398
1399 return 0;
1400
1401out_hyp:
1402 teardown_hyp_mode();
1403out_err:
1404 teardown_common_resources();
1405 return err;
1406}
1407
1408
1409void kvm_arch_exit(void)
1410{
1411 kvm_perf_teardown();
1412}
1413
1414static int arm_init(void)
1415{
1416 int rc = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
1417 return rc;
1418}
1419
1420module_init(arm_init);
1421