1
2
3
4
5
6
7
8
9
10
11#include "qemu/osdep.h"
12#include <sys/ioctl.h>
13
14#include <linux/kvm.h>
15
16#include "qemu-common.h"
17#include "qemu/timer.h"
18#include "qemu/error-report.h"
19#include "sysemu/sysemu.h"
20#include "sysemu/kvm.h"
21#include "sysemu/kvm_int.h"
22#include "kvm_arm.h"
23#include "cpu.h"
24#include "trace.h"
25#include "internals.h"
26#include "hw/arm/arm.h"
27#include "hw/pci/pci.h"
28#include "exec/memattrs.h"
29#include "exec/address-spaces.h"
30#include "hw/boards.h"
31#include "qemu/log.h"
32
33const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
34 KVM_CAP_LAST_INFO
35};
36
37static bool cap_has_mp_state;
38static bool cap_has_inject_serror_esr;
39
40static ARMHostCPUFeatures arm_host_cpu_features;
41
42int kvm_arm_vcpu_init(CPUState *cs)
43{
44 ARMCPU *cpu = ARM_CPU(cs);
45 struct kvm_vcpu_init init;
46
47 init.target = cpu->kvm_target;
48 memcpy(init.features, cpu->kvm_init_features, sizeof(init.features));
49
50 return kvm_vcpu_ioctl(cs, KVM_ARM_VCPU_INIT, &init);
51}
52
53void kvm_arm_init_serror_injection(CPUState *cs)
54{
55 cap_has_inject_serror_esr = kvm_check_extension(cs->kvm_state,
56 KVM_CAP_ARM_INJECT_SERROR_ESR);
57}
58
59bool kvm_arm_create_scratch_host_vcpu(const uint32_t *cpus_to_try,
60 int *fdarray,
61 struct kvm_vcpu_init *init)
62{
63 int ret, kvmfd = -1, vmfd = -1, cpufd = -1;
64
65 kvmfd = qemu_open("/dev/kvm", O_RDWR);
66 if (kvmfd < 0) {
67 goto err;
68 }
69 vmfd = ioctl(kvmfd, KVM_CREATE_VM, 0);
70 if (vmfd < 0) {
71 goto err;
72 }
73 cpufd = ioctl(vmfd, KVM_CREATE_VCPU, 0);
74 if (cpufd < 0) {
75 goto err;
76 }
77
78 if (!init) {
79
80 goto finish;
81 }
82
83 ret = ioctl(vmfd, KVM_ARM_PREFERRED_TARGET, init);
84 if (ret >= 0) {
85 ret = ioctl(cpufd, KVM_ARM_VCPU_INIT, init);
86 if (ret < 0) {
87 goto err;
88 }
89 } else if (cpus_to_try) {
90
91
92
93
94
95 while (*cpus_to_try != QEMU_KVM_ARM_TARGET_NONE) {
96 init->target = *cpus_to_try++;
97 memset(init->features, 0, sizeof(init->features));
98 ret = ioctl(cpufd, KVM_ARM_VCPU_INIT, init);
99 if (ret >= 0) {
100 break;
101 }
102 }
103 if (ret < 0) {
104 goto err;
105 }
106 } else {
107
108
109
110
111 goto err;
112 }
113
114finish:
115 fdarray[0] = kvmfd;
116 fdarray[1] = vmfd;
117 fdarray[2] = cpufd;
118
119 return true;
120
121err:
122 if (cpufd >= 0) {
123 close(cpufd);
124 }
125 if (vmfd >= 0) {
126 close(vmfd);
127 }
128 if (kvmfd >= 0) {
129 close(kvmfd);
130 }
131
132 return false;
133}
134
135void kvm_arm_destroy_scratch_host_vcpu(int *fdarray)
136{
137 int i;
138
139 for (i = 2; i >= 0; i--) {
140 close(fdarray[i]);
141 }
142}
143
144void kvm_arm_set_cpu_features_from_host(ARMCPU *cpu)
145{
146 CPUARMState *env = &cpu->env;
147
148 if (!arm_host_cpu_features.dtb_compatible) {
149 if (!kvm_enabled() ||
150 !kvm_arm_get_host_cpu_features(&arm_host_cpu_features)) {
151
152
153
154 cpu->kvm_target = QEMU_KVM_ARM_TARGET_NONE;
155 cpu->host_cpu_probe_failed = true;
156 return;
157 }
158 }
159
160 cpu->kvm_target = arm_host_cpu_features.target;
161 cpu->dtb_compatible = arm_host_cpu_features.dtb_compatible;
162 cpu->isar = arm_host_cpu_features.isar;
163 env->features = arm_host_cpu_features.features;
164}
165
166int kvm_arm_get_max_vm_ipa_size(MachineState *ms)
167{
168 KVMState *s = KVM_STATE(ms->accelerator);
169 int ret;
170
171 ret = kvm_check_extension(s, KVM_CAP_ARM_VM_IPA_SIZE);
172 return ret > 0 ? ret : 40;
173}
174
175int kvm_arch_init(MachineState *ms, KVMState *s)
176{
177
178
179
180 kvm_async_interrupts_allowed = true;
181
182
183
184
185
186 kvm_halt_in_kernel_allowed = true;
187
188 cap_has_mp_state = kvm_check_extension(s, KVM_CAP_MP_STATE);
189
190 return 0;
191}
192
193unsigned long kvm_arch_vcpu_id(CPUState *cpu)
194{
195 return cpu->cpu_index;
196}
197
198
199
200
201
202
203
204
205
206
207
208
209
210typedef struct KVMDevice {
211 struct kvm_arm_device_addr kda;
212 struct kvm_device_attr kdattr;
213 uint64_t kda_addr_ormask;
214 MemoryRegion *mr;
215 QSLIST_ENTRY(KVMDevice) entries;
216 int dev_fd;
217} KVMDevice;
218
219static QSLIST_HEAD(, KVMDevice) kvm_devices_head;
220
221static void kvm_arm_devlistener_add(MemoryListener *listener,
222 MemoryRegionSection *section)
223{
224 KVMDevice *kd;
225
226 QSLIST_FOREACH(kd, &kvm_devices_head, entries) {
227 if (section->mr == kd->mr) {
228 kd->kda.addr = section->offset_within_address_space;
229 }
230 }
231}
232
233static void kvm_arm_devlistener_del(MemoryListener *listener,
234 MemoryRegionSection *section)
235{
236 KVMDevice *kd;
237
238 QSLIST_FOREACH(kd, &kvm_devices_head, entries) {
239 if (section->mr == kd->mr) {
240 kd->kda.addr = -1;
241 }
242 }
243}
244
245static MemoryListener devlistener = {
246 .region_add = kvm_arm_devlistener_add,
247 .region_del = kvm_arm_devlistener_del,
248};
249
250static void kvm_arm_set_device_addr(KVMDevice *kd)
251{
252 struct kvm_device_attr *attr = &kd->kdattr;
253 int ret;
254
255
256
257
258 if (kd->dev_fd >= 0) {
259 uint64_t addr = kd->kda.addr;
260
261 addr |= kd->kda_addr_ormask;
262 attr->addr = (uintptr_t)&addr;
263 ret = kvm_device_ioctl(kd->dev_fd, KVM_SET_DEVICE_ATTR, attr);
264 } else {
265 ret = kvm_vm_ioctl(kvm_state, KVM_ARM_SET_DEVICE_ADDR, &kd->kda);
266 }
267
268 if (ret < 0) {
269 fprintf(stderr, "Failed to set device address: %s\n",
270 strerror(-ret));
271 abort();
272 }
273}
274
275static void kvm_arm_machine_init_done(Notifier *notifier, void *data)
276{
277 KVMDevice *kd, *tkd;
278
279 QSLIST_FOREACH_SAFE(kd, &kvm_devices_head, entries, tkd) {
280 if (kd->kda.addr != -1) {
281 kvm_arm_set_device_addr(kd);
282 }
283 memory_region_unref(kd->mr);
284 QSLIST_REMOVE_HEAD(&kvm_devices_head, entries);
285 g_free(kd);
286 }
287 memory_listener_unregister(&devlistener);
288}
289
290static Notifier notify = {
291 .notify = kvm_arm_machine_init_done,
292};
293
294void kvm_arm_register_device(MemoryRegion *mr, uint64_t devid, uint64_t group,
295 uint64_t attr, int dev_fd, uint64_t addr_ormask)
296{
297 KVMDevice *kd;
298
299 if (!kvm_irqchip_in_kernel()) {
300 return;
301 }
302
303 if (QSLIST_EMPTY(&kvm_devices_head)) {
304 memory_listener_register(&devlistener, &address_space_memory);
305 qemu_add_machine_init_done_notifier(¬ify);
306 }
307 kd = g_new0(KVMDevice, 1);
308 kd->mr = mr;
309 kd->kda.id = devid;
310 kd->kda.addr = -1;
311 kd->kdattr.flags = 0;
312 kd->kdattr.group = group;
313 kd->kdattr.attr = attr;
314 kd->dev_fd = dev_fd;
315 kd->kda_addr_ormask = addr_ormask;
316 QSLIST_INSERT_HEAD(&kvm_devices_head, kd, entries);
317 memory_region_ref(kd->mr);
318}
319
320static int compare_u64(const void *a, const void *b)
321{
322 if (*(uint64_t *)a > *(uint64_t *)b) {
323 return 1;
324 }
325 if (*(uint64_t *)a < *(uint64_t *)b) {
326 return -1;
327 }
328 return 0;
329}
330
331
332
333
334
335int kvm_arm_init_cpreg_list(ARMCPU *cpu)
336{
337 struct kvm_reg_list rl;
338 struct kvm_reg_list *rlp;
339 int i, ret, arraylen;
340 CPUState *cs = CPU(cpu);
341
342 rl.n = 0;
343 ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, &rl);
344 if (ret != -E2BIG) {
345 return ret;
346 }
347 rlp = g_malloc(sizeof(struct kvm_reg_list) + rl.n * sizeof(uint64_t));
348 rlp->n = rl.n;
349 ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, rlp);
350 if (ret) {
351 goto out;
352 }
353
354
355
356 qsort(&rlp->reg, rlp->n, sizeof(rlp->reg[0]), compare_u64);
357
358 for (i = 0, arraylen = 0; i < rlp->n; i++) {
359 if (!kvm_arm_reg_syncs_via_cpreg_list(rlp->reg[i])) {
360 continue;
361 }
362 switch (rlp->reg[i] & KVM_REG_SIZE_MASK) {
363 case KVM_REG_SIZE_U32:
364 case KVM_REG_SIZE_U64:
365 break;
366 default:
367 fprintf(stderr, "Can't handle size of register in kernel list\n");
368 ret = -EINVAL;
369 goto out;
370 }
371
372 arraylen++;
373 }
374
375 cpu->cpreg_indexes = g_renew(uint64_t, cpu->cpreg_indexes, arraylen);
376 cpu->cpreg_values = g_renew(uint64_t, cpu->cpreg_values, arraylen);
377 cpu->cpreg_vmstate_indexes = g_renew(uint64_t, cpu->cpreg_vmstate_indexes,
378 arraylen);
379 cpu->cpreg_vmstate_values = g_renew(uint64_t, cpu->cpreg_vmstate_values,
380 arraylen);
381 cpu->cpreg_array_len = arraylen;
382 cpu->cpreg_vmstate_array_len = arraylen;
383
384 for (i = 0, arraylen = 0; i < rlp->n; i++) {
385 uint64_t regidx = rlp->reg[i];
386 if (!kvm_arm_reg_syncs_via_cpreg_list(regidx)) {
387 continue;
388 }
389 cpu->cpreg_indexes[arraylen] = regidx;
390 arraylen++;
391 }
392 assert(cpu->cpreg_array_len == arraylen);
393
394 if (!write_kvmstate_to_list(cpu)) {
395
396
397
398 fprintf(stderr, "Initial read of kernel register state failed\n");
399 ret = -EINVAL;
400 goto out;
401 }
402
403out:
404 g_free(rlp);
405 return ret;
406}
407
408bool write_kvmstate_to_list(ARMCPU *cpu)
409{
410 CPUState *cs = CPU(cpu);
411 int i;
412 bool ok = true;
413
414 for (i = 0; i < cpu->cpreg_array_len; i++) {
415 struct kvm_one_reg r;
416 uint64_t regidx = cpu->cpreg_indexes[i];
417 uint32_t v32;
418 int ret;
419
420 r.id = regidx;
421
422 switch (regidx & KVM_REG_SIZE_MASK) {
423 case KVM_REG_SIZE_U32:
424 r.addr = (uintptr_t)&v32;
425 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &r);
426 if (!ret) {
427 cpu->cpreg_values[i] = v32;
428 }
429 break;
430 case KVM_REG_SIZE_U64:
431 r.addr = (uintptr_t)(cpu->cpreg_values + i);
432 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &r);
433 break;
434 default:
435 abort();
436 }
437 if (ret) {
438 ok = false;
439 }
440 }
441 return ok;
442}
443
444bool write_list_to_kvmstate(ARMCPU *cpu, int level)
445{
446 CPUState *cs = CPU(cpu);
447 int i;
448 bool ok = true;
449
450 for (i = 0; i < cpu->cpreg_array_len; i++) {
451 struct kvm_one_reg r;
452 uint64_t regidx = cpu->cpreg_indexes[i];
453 uint32_t v32;
454 int ret;
455
456 if (kvm_arm_cpreg_level(regidx) > level) {
457 continue;
458 }
459
460 r.id = regidx;
461 switch (regidx & KVM_REG_SIZE_MASK) {
462 case KVM_REG_SIZE_U32:
463 v32 = cpu->cpreg_values[i];
464 r.addr = (uintptr_t)&v32;
465 break;
466 case KVM_REG_SIZE_U64:
467 r.addr = (uintptr_t)(cpu->cpreg_values + i);
468 break;
469 default:
470 abort();
471 }
472 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &r);
473 if (ret) {
474
475
476
477
478 ok = false;
479 }
480 }
481 return ok;
482}
483
484void kvm_arm_reset_vcpu(ARMCPU *cpu)
485{
486 int ret;
487
488
489
490
491 ret = kvm_arm_vcpu_init(CPU(cpu));
492 if (ret < 0) {
493 fprintf(stderr, "kvm_arm_vcpu_init failed: %s\n", strerror(-ret));
494 abort();
495 }
496 if (!write_kvmstate_to_list(cpu)) {
497 fprintf(stderr, "write_kvmstate_to_list failed\n");
498 abort();
499 }
500}
501
502
503
504
505int kvm_arm_sync_mpstate_to_kvm(ARMCPU *cpu)
506{
507 if (cap_has_mp_state) {
508 struct kvm_mp_state mp_state = {
509 .mp_state = (cpu->power_state == PSCI_OFF) ?
510 KVM_MP_STATE_STOPPED : KVM_MP_STATE_RUNNABLE
511 };
512 int ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MP_STATE, &mp_state);
513 if (ret) {
514 fprintf(stderr, "%s: failed to set MP_STATE %d/%s\n",
515 __func__, ret, strerror(-ret));
516 return -1;
517 }
518 }
519
520 return 0;
521}
522
523
524
525
526int kvm_arm_sync_mpstate_to_qemu(ARMCPU *cpu)
527{
528 if (cap_has_mp_state) {
529 struct kvm_mp_state mp_state;
530 int ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MP_STATE, &mp_state);
531 if (ret) {
532 fprintf(stderr, "%s: failed to get MP_STATE %d/%s\n",
533 __func__, ret, strerror(-ret));
534 abort();
535 }
536 cpu->power_state = (mp_state.mp_state == KVM_MP_STATE_STOPPED) ?
537 PSCI_OFF : PSCI_ON;
538 }
539
540 return 0;
541}
542
543int kvm_put_vcpu_events(ARMCPU *cpu)
544{
545 CPUARMState *env = &cpu->env;
546 struct kvm_vcpu_events events;
547 int ret;
548
549 if (!kvm_has_vcpu_events()) {
550 return 0;
551 }
552
553 memset(&events, 0, sizeof(events));
554 events.exception.serror_pending = env->serror.pending;
555
556
557
558
559 if (cap_has_inject_serror_esr) {
560 events.exception.serror_has_esr = env->serror.has_esr;
561 events.exception.serror_esr = env->serror.esr;
562 }
563
564 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_VCPU_EVENTS, &events);
565 if (ret) {
566 error_report("failed to put vcpu events");
567 }
568
569 return ret;
570}
571
572int kvm_get_vcpu_events(ARMCPU *cpu)
573{
574 CPUARMState *env = &cpu->env;
575 struct kvm_vcpu_events events;
576 int ret;
577
578 if (!kvm_has_vcpu_events()) {
579 return 0;
580 }
581
582 memset(&events, 0, sizeof(events));
583 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_VCPU_EVENTS, &events);
584 if (ret) {
585 error_report("failed to get vcpu events");
586 return ret;
587 }
588
589 env->serror.pending = events.exception.serror_pending;
590 env->serror.has_esr = events.exception.serror_has_esr;
591 env->serror.esr = events.exception.serror_esr;
592
593 return 0;
594}
595
596void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run)
597{
598}
599
600MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run *run)
601{
602 ARMCPU *cpu;
603 uint32_t switched_level;
604
605 if (kvm_irqchip_in_kernel()) {
606
607
608
609
610 return MEMTXATTRS_UNSPECIFIED;
611 }
612
613 cpu = ARM_CPU(cs);
614
615
616 if (run->s.regs.device_irq_level != cpu->device_irq_level) {
617 switched_level = cpu->device_irq_level ^ run->s.regs.device_irq_level;
618
619 qemu_mutex_lock_iothread();
620
621 if (switched_level & KVM_ARM_DEV_EL1_VTIMER) {
622 qemu_set_irq(cpu->gt_timer_outputs[GTIMER_VIRT],
623 !!(run->s.regs.device_irq_level &
624 KVM_ARM_DEV_EL1_VTIMER));
625 switched_level &= ~KVM_ARM_DEV_EL1_VTIMER;
626 }
627
628 if (switched_level & KVM_ARM_DEV_EL1_PTIMER) {
629 qemu_set_irq(cpu->gt_timer_outputs[GTIMER_PHYS],
630 !!(run->s.regs.device_irq_level &
631 KVM_ARM_DEV_EL1_PTIMER));
632 switched_level &= ~KVM_ARM_DEV_EL1_PTIMER;
633 }
634
635 if (switched_level & KVM_ARM_DEV_PMU) {
636 qemu_set_irq(cpu->pmu_interrupt,
637 !!(run->s.regs.device_irq_level & KVM_ARM_DEV_PMU));
638 switched_level &= ~KVM_ARM_DEV_PMU;
639 }
640
641 if (switched_level) {
642 qemu_log_mask(LOG_UNIMP, "%s: unhandled in-kernel device IRQ %x\n",
643 __func__, switched_level);
644 }
645
646
647 cpu->device_irq_level = run->s.regs.device_irq_level;
648 qemu_mutex_unlock_iothread();
649 }
650
651 return MEMTXATTRS_UNSPECIFIED;
652}
653
654
655int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
656{
657 int ret = 0;
658
659 switch (run->exit_reason) {
660 case KVM_EXIT_DEBUG:
661 if (kvm_arm_handle_debug(cs, &run->debug.arch)) {
662 ret = EXCP_DEBUG;
663 }
664 break;
665 default:
666 qemu_log_mask(LOG_UNIMP, "%s: un-handled exit reason %d\n",
667 __func__, run->exit_reason);
668 break;
669 }
670 return ret;
671}
672
673bool kvm_arch_stop_on_emulation_error(CPUState *cs)
674{
675 return true;
676}
677
678int kvm_arch_process_async_events(CPUState *cs)
679{
680 return 0;
681}
682
683
684
685
686void kvm_arch_update_guest_debug(CPUState *cs, struct kvm_guest_debug *dbg)
687{
688#ifdef KVM_GUESTDBG_USE_SW_BP
689 if (kvm_sw_breakpoints_active(cs)) {
690 dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP;
691 }
692#endif
693#ifdef KVM_GUESTDBG_USE_HW
694 if (kvm_arm_hw_debug_active(cs)) {
695 dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW;
696 kvm_arm_copy_hw_debug_data(&dbg->arch);
697 }
698#endif
699}
700
701void kvm_arch_init_irq_routing(KVMState *s)
702{
703}
704
705int kvm_arch_irqchip_create(MachineState *ms, KVMState *s)
706{
707 if (machine_kernel_irqchip_split(ms)) {
708 perror("-machine kernel_irqchip=split is not supported on ARM.");
709 exit(1);
710 }
711
712
713
714
715 return kvm_check_extension(s, KVM_CAP_DEVICE_CTRL);
716}
717
718int kvm_arm_vgic_probe(void)
719{
720 if (kvm_create_device(kvm_state,
721 KVM_DEV_TYPE_ARM_VGIC_V3, true) == 0) {
722 return 3;
723 } else if (kvm_create_device(kvm_state,
724 KVM_DEV_TYPE_ARM_VGIC_V2, true) == 0) {
725 return 2;
726 } else {
727 return 0;
728 }
729}
730
731int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
732 uint64_t address, uint32_t data, PCIDevice *dev)
733{
734 AddressSpace *as = pci_device_iommu_address_space(dev);
735 hwaddr xlat, len, doorbell_gpa;
736 MemoryRegionSection mrs;
737 MemoryRegion *mr;
738 int ret = 1;
739
740 if (as == &address_space_memory) {
741 return 0;
742 }
743
744
745
746 rcu_read_lock();
747 mr = address_space_translate(as, address, &xlat, &len, true,
748 MEMTXATTRS_UNSPECIFIED);
749 if (!mr) {
750 goto unlock;
751 }
752 mrs = memory_region_find(mr, xlat, 1);
753 if (!mrs.mr) {
754 goto unlock;
755 }
756
757 doorbell_gpa = mrs.offset_within_address_space;
758 memory_region_unref(mrs.mr);
759
760 route->u.msi.address_lo = doorbell_gpa;
761 route->u.msi.address_hi = doorbell_gpa >> 32;
762
763 trace_kvm_arm_fixup_msi_route(address, doorbell_gpa);
764
765 ret = 0;
766
767unlock:
768 rcu_read_unlock();
769 return ret;
770}
771
772int kvm_arch_add_msi_route_post(struct kvm_irq_routing_entry *route,
773 int vector, PCIDevice *dev)
774{
775 return 0;
776}
777
778int kvm_arch_release_virq_post(int virq)
779{
780 return 0;
781}
782
783int kvm_arch_msi_data_to_gsi(uint32_t data)
784{
785 return (data - 32) & 0xffff;
786}
787