1
2
3
4
5
6
7
8
9
10
11#include "qemu/osdep.h"
12#include <sys/ioctl.h>
13
14#include <linux/kvm.h>
15
16#include "qemu-common.h"
17#include "qemu/timer.h"
18#include "qemu/error-report.h"
19#include "sysemu/sysemu.h"
20#include "sysemu/kvm.h"
21#include "sysemu/kvm_int.h"
22#include "kvm_arm.h"
23#include "cpu.h"
24#include "trace.h"
25#include "internals.h"
26#include "hw/pci/pci.h"
27#include "exec/memattrs.h"
28#include "exec/address-spaces.h"
29#include "hw/boards.h"
30#include "qemu/log.h"
31
32const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
33 KVM_CAP_LAST_INFO
34};
35
36static bool cap_has_mp_state;
37static bool cap_has_inject_serror_esr;
38
39static ARMHostCPUFeatures arm_host_cpu_features;
40
41int kvm_arm_vcpu_init(CPUState *cs)
42{
43 ARMCPU *cpu = ARM_CPU(cs);
44 struct kvm_vcpu_init init;
45
46 init.target = cpu->kvm_target;
47 memcpy(init.features, cpu->kvm_init_features, sizeof(init.features));
48
49 return kvm_vcpu_ioctl(cs, KVM_ARM_VCPU_INIT, &init);
50}
51
52void kvm_arm_init_serror_injection(CPUState *cs)
53{
54 cap_has_inject_serror_esr = kvm_check_extension(cs->kvm_state,
55 KVM_CAP_ARM_INJECT_SERROR_ESR);
56}
57
58bool kvm_arm_create_scratch_host_vcpu(const uint32_t *cpus_to_try,
59 int *fdarray,
60 struct kvm_vcpu_init *init)
61{
62 int ret, kvmfd = -1, vmfd = -1, cpufd = -1;
63
64 kvmfd = qemu_open("/dev/kvm", O_RDWR);
65 if (kvmfd < 0) {
66 goto err;
67 }
68 vmfd = ioctl(kvmfd, KVM_CREATE_VM, 0);
69 if (vmfd < 0) {
70 goto err;
71 }
72 cpufd = ioctl(vmfd, KVM_CREATE_VCPU, 0);
73 if (cpufd < 0) {
74 goto err;
75 }
76
77 if (!init) {
78
79 goto finish;
80 }
81
82 ret = ioctl(vmfd, KVM_ARM_PREFERRED_TARGET, init);
83 if (ret >= 0) {
84 ret = ioctl(cpufd, KVM_ARM_VCPU_INIT, init);
85 if (ret < 0) {
86 goto err;
87 }
88 } else if (cpus_to_try) {
89
90
91
92
93
94 while (*cpus_to_try != QEMU_KVM_ARM_TARGET_NONE) {
95 init->target = *cpus_to_try++;
96 memset(init->features, 0, sizeof(init->features));
97 ret = ioctl(cpufd, KVM_ARM_VCPU_INIT, init);
98 if (ret >= 0) {
99 break;
100 }
101 }
102 if (ret < 0) {
103 goto err;
104 }
105 } else {
106
107
108
109
110 goto err;
111 }
112
113finish:
114 fdarray[0] = kvmfd;
115 fdarray[1] = vmfd;
116 fdarray[2] = cpufd;
117
118 return true;
119
120err:
121 if (cpufd >= 0) {
122 close(cpufd);
123 }
124 if (vmfd >= 0) {
125 close(vmfd);
126 }
127 if (kvmfd >= 0) {
128 close(kvmfd);
129 }
130
131 return false;
132}
133
134void kvm_arm_destroy_scratch_host_vcpu(int *fdarray)
135{
136 int i;
137
138 for (i = 2; i >= 0; i--) {
139 close(fdarray[i]);
140 }
141}
142
143void kvm_arm_set_cpu_features_from_host(ARMCPU *cpu)
144{
145 CPUARMState *env = &cpu->env;
146
147 if (!arm_host_cpu_features.dtb_compatible) {
148 if (!kvm_enabled() ||
149 !kvm_arm_get_host_cpu_features(&arm_host_cpu_features)) {
150
151
152
153 cpu->kvm_target = QEMU_KVM_ARM_TARGET_NONE;
154 cpu->host_cpu_probe_failed = true;
155 return;
156 }
157 }
158
159 cpu->kvm_target = arm_host_cpu_features.target;
160 cpu->dtb_compatible = arm_host_cpu_features.dtb_compatible;
161 cpu->isar = arm_host_cpu_features.isar;
162 env->features = arm_host_cpu_features.features;
163}
164
165int kvm_arm_get_max_vm_ipa_size(MachineState *ms)
166{
167 KVMState *s = KVM_STATE(ms->accelerator);
168 int ret;
169
170 ret = kvm_check_extension(s, KVM_CAP_ARM_VM_IPA_SIZE);
171 return ret > 0 ? ret : 40;
172}
173
174int kvm_arch_init(MachineState *ms, KVMState *s)
175{
176
177
178
179 kvm_async_interrupts_allowed = true;
180
181
182
183
184
185 kvm_halt_in_kernel_allowed = true;
186
187 cap_has_mp_state = kvm_check_extension(s, KVM_CAP_MP_STATE);
188
189 return 0;
190}
191
192unsigned long kvm_arch_vcpu_id(CPUState *cpu)
193{
194 return cpu->cpu_index;
195}
196
197
198
199
200
201
202
203
204
205
206
207
208
209typedef struct KVMDevice {
210 struct kvm_arm_device_addr kda;
211 struct kvm_device_attr kdattr;
212 uint64_t kda_addr_ormask;
213 MemoryRegion *mr;
214 QSLIST_ENTRY(KVMDevice) entries;
215 int dev_fd;
216} KVMDevice;
217
218static QSLIST_HEAD(, KVMDevice) kvm_devices_head;
219
220static void kvm_arm_devlistener_add(MemoryListener *listener,
221 MemoryRegionSection *section)
222{
223 KVMDevice *kd;
224
225 QSLIST_FOREACH(kd, &kvm_devices_head, entries) {
226 if (section->mr == kd->mr) {
227 kd->kda.addr = section->offset_within_address_space;
228 }
229 }
230}
231
232static void kvm_arm_devlistener_del(MemoryListener *listener,
233 MemoryRegionSection *section)
234{
235 KVMDevice *kd;
236
237 QSLIST_FOREACH(kd, &kvm_devices_head, entries) {
238 if (section->mr == kd->mr) {
239 kd->kda.addr = -1;
240 }
241 }
242}
243
244static MemoryListener devlistener = {
245 .region_add = kvm_arm_devlistener_add,
246 .region_del = kvm_arm_devlistener_del,
247};
248
249static void kvm_arm_set_device_addr(KVMDevice *kd)
250{
251 struct kvm_device_attr *attr = &kd->kdattr;
252 int ret;
253
254
255
256
257 if (kd->dev_fd >= 0) {
258 uint64_t addr = kd->kda.addr;
259
260 addr |= kd->kda_addr_ormask;
261 attr->addr = (uintptr_t)&addr;
262 ret = kvm_device_ioctl(kd->dev_fd, KVM_SET_DEVICE_ATTR, attr);
263 } else {
264 ret = kvm_vm_ioctl(kvm_state, KVM_ARM_SET_DEVICE_ADDR, &kd->kda);
265 }
266
267 if (ret < 0) {
268 fprintf(stderr, "Failed to set device address: %s\n",
269 strerror(-ret));
270 abort();
271 }
272}
273
274static void kvm_arm_machine_init_done(Notifier *notifier, void *data)
275{
276 KVMDevice *kd, *tkd;
277
278 QSLIST_FOREACH_SAFE(kd, &kvm_devices_head, entries, tkd) {
279 if (kd->kda.addr != -1) {
280 kvm_arm_set_device_addr(kd);
281 }
282 memory_region_unref(kd->mr);
283 QSLIST_REMOVE_HEAD(&kvm_devices_head, entries);
284 g_free(kd);
285 }
286 memory_listener_unregister(&devlistener);
287}
288
289static Notifier notify = {
290 .notify = kvm_arm_machine_init_done,
291};
292
293void kvm_arm_register_device(MemoryRegion *mr, uint64_t devid, uint64_t group,
294 uint64_t attr, int dev_fd, uint64_t addr_ormask)
295{
296 KVMDevice *kd;
297
298 if (!kvm_irqchip_in_kernel()) {
299 return;
300 }
301
302 if (QSLIST_EMPTY(&kvm_devices_head)) {
303 memory_listener_register(&devlistener, &address_space_memory);
304 qemu_add_machine_init_done_notifier(¬ify);
305 }
306 kd = g_new0(KVMDevice, 1);
307 kd->mr = mr;
308 kd->kda.id = devid;
309 kd->kda.addr = -1;
310 kd->kdattr.flags = 0;
311 kd->kdattr.group = group;
312 kd->kdattr.attr = attr;
313 kd->dev_fd = dev_fd;
314 kd->kda_addr_ormask = addr_ormask;
315 QSLIST_INSERT_HEAD(&kvm_devices_head, kd, entries);
316 memory_region_ref(kd->mr);
317}
318
319static int compare_u64(const void *a, const void *b)
320{
321 if (*(uint64_t *)a > *(uint64_t *)b) {
322 return 1;
323 }
324 if (*(uint64_t *)a < *(uint64_t *)b) {
325 return -1;
326 }
327 return 0;
328}
329
330
331
332
333
334int kvm_arm_init_cpreg_list(ARMCPU *cpu)
335{
336 struct kvm_reg_list rl;
337 struct kvm_reg_list *rlp;
338 int i, ret, arraylen;
339 CPUState *cs = CPU(cpu);
340
341 rl.n = 0;
342 ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, &rl);
343 if (ret != -E2BIG) {
344 return ret;
345 }
346 rlp = g_malloc(sizeof(struct kvm_reg_list) + rl.n * sizeof(uint64_t));
347 rlp->n = rl.n;
348 ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, rlp);
349 if (ret) {
350 goto out;
351 }
352
353
354
355 qsort(&rlp->reg, rlp->n, sizeof(rlp->reg[0]), compare_u64);
356
357 for (i = 0, arraylen = 0; i < rlp->n; i++) {
358 if (!kvm_arm_reg_syncs_via_cpreg_list(rlp->reg[i])) {
359 continue;
360 }
361 switch (rlp->reg[i] & KVM_REG_SIZE_MASK) {
362 case KVM_REG_SIZE_U32:
363 case KVM_REG_SIZE_U64:
364 break;
365 default:
366 fprintf(stderr, "Can't handle size of register in kernel list\n");
367 ret = -EINVAL;
368 goto out;
369 }
370
371 arraylen++;
372 }
373
374 cpu->cpreg_indexes = g_renew(uint64_t, cpu->cpreg_indexes, arraylen);
375 cpu->cpreg_values = g_renew(uint64_t, cpu->cpreg_values, arraylen);
376 cpu->cpreg_vmstate_indexes = g_renew(uint64_t, cpu->cpreg_vmstate_indexes,
377 arraylen);
378 cpu->cpreg_vmstate_values = g_renew(uint64_t, cpu->cpreg_vmstate_values,
379 arraylen);
380 cpu->cpreg_array_len = arraylen;
381 cpu->cpreg_vmstate_array_len = arraylen;
382
383 for (i = 0, arraylen = 0; i < rlp->n; i++) {
384 uint64_t regidx = rlp->reg[i];
385 if (!kvm_arm_reg_syncs_via_cpreg_list(regidx)) {
386 continue;
387 }
388 cpu->cpreg_indexes[arraylen] = regidx;
389 arraylen++;
390 }
391 assert(cpu->cpreg_array_len == arraylen);
392
393 if (!write_kvmstate_to_list(cpu)) {
394
395
396
397 fprintf(stderr, "Initial read of kernel register state failed\n");
398 ret = -EINVAL;
399 goto out;
400 }
401
402out:
403 g_free(rlp);
404 return ret;
405}
406
407bool write_kvmstate_to_list(ARMCPU *cpu)
408{
409 CPUState *cs = CPU(cpu);
410 int i;
411 bool ok = true;
412
413 for (i = 0; i < cpu->cpreg_array_len; i++) {
414 struct kvm_one_reg r;
415 uint64_t regidx = cpu->cpreg_indexes[i];
416 uint32_t v32;
417 int ret;
418
419 r.id = regidx;
420
421 switch (regidx & KVM_REG_SIZE_MASK) {
422 case KVM_REG_SIZE_U32:
423 r.addr = (uintptr_t)&v32;
424 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &r);
425 if (!ret) {
426 cpu->cpreg_values[i] = v32;
427 }
428 break;
429 case KVM_REG_SIZE_U64:
430 r.addr = (uintptr_t)(cpu->cpreg_values + i);
431 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &r);
432 break;
433 default:
434 abort();
435 }
436 if (ret) {
437 ok = false;
438 }
439 }
440 return ok;
441}
442
443bool write_list_to_kvmstate(ARMCPU *cpu, int level)
444{
445 CPUState *cs = CPU(cpu);
446 int i;
447 bool ok = true;
448
449 for (i = 0; i < cpu->cpreg_array_len; i++) {
450 struct kvm_one_reg r;
451 uint64_t regidx = cpu->cpreg_indexes[i];
452 uint32_t v32;
453 int ret;
454
455 if (kvm_arm_cpreg_level(regidx) > level) {
456 continue;
457 }
458
459 r.id = regidx;
460 switch (regidx & KVM_REG_SIZE_MASK) {
461 case KVM_REG_SIZE_U32:
462 v32 = cpu->cpreg_values[i];
463 r.addr = (uintptr_t)&v32;
464 break;
465 case KVM_REG_SIZE_U64:
466 r.addr = (uintptr_t)(cpu->cpreg_values + i);
467 break;
468 default:
469 abort();
470 }
471 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &r);
472 if (ret) {
473
474
475
476
477 ok = false;
478 }
479 }
480 return ok;
481}
482
483void kvm_arm_reset_vcpu(ARMCPU *cpu)
484{
485 int ret;
486
487
488
489
490 ret = kvm_arm_vcpu_init(CPU(cpu));
491 if (ret < 0) {
492 fprintf(stderr, "kvm_arm_vcpu_init failed: %s\n", strerror(-ret));
493 abort();
494 }
495 if (!write_kvmstate_to_list(cpu)) {
496 fprintf(stderr, "write_kvmstate_to_list failed\n");
497 abort();
498 }
499
500
501
502
503
504
505
506 write_list_to_cpustate(cpu);
507}
508
509
510
511
512int kvm_arm_sync_mpstate_to_kvm(ARMCPU *cpu)
513{
514 if (cap_has_mp_state) {
515 struct kvm_mp_state mp_state = {
516 .mp_state = (cpu->power_state == PSCI_OFF) ?
517 KVM_MP_STATE_STOPPED : KVM_MP_STATE_RUNNABLE
518 };
519 int ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MP_STATE, &mp_state);
520 if (ret) {
521 fprintf(stderr, "%s: failed to set MP_STATE %d/%s\n",
522 __func__, ret, strerror(-ret));
523 return -1;
524 }
525 }
526
527 return 0;
528}
529
530
531
532
533int kvm_arm_sync_mpstate_to_qemu(ARMCPU *cpu)
534{
535 if (cap_has_mp_state) {
536 struct kvm_mp_state mp_state;
537 int ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MP_STATE, &mp_state);
538 if (ret) {
539 fprintf(stderr, "%s: failed to get MP_STATE %d/%s\n",
540 __func__, ret, strerror(-ret));
541 abort();
542 }
543 cpu->power_state = (mp_state.mp_state == KVM_MP_STATE_STOPPED) ?
544 PSCI_OFF : PSCI_ON;
545 }
546
547 return 0;
548}
549
550int kvm_put_vcpu_events(ARMCPU *cpu)
551{
552 CPUARMState *env = &cpu->env;
553 struct kvm_vcpu_events events;
554 int ret;
555
556 if (!kvm_has_vcpu_events()) {
557 return 0;
558 }
559
560 memset(&events, 0, sizeof(events));
561 events.exception.serror_pending = env->serror.pending;
562
563
564
565
566 if (cap_has_inject_serror_esr) {
567 events.exception.serror_has_esr = env->serror.has_esr;
568 events.exception.serror_esr = env->serror.esr;
569 }
570
571 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_VCPU_EVENTS, &events);
572 if (ret) {
573 error_report("failed to put vcpu events");
574 }
575
576 return ret;
577}
578
579int kvm_get_vcpu_events(ARMCPU *cpu)
580{
581 CPUARMState *env = &cpu->env;
582 struct kvm_vcpu_events events;
583 int ret;
584
585 if (!kvm_has_vcpu_events()) {
586 return 0;
587 }
588
589 memset(&events, 0, sizeof(events));
590 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_VCPU_EVENTS, &events);
591 if (ret) {
592 error_report("failed to get vcpu events");
593 return ret;
594 }
595
596 env->serror.pending = events.exception.serror_pending;
597 env->serror.has_esr = events.exception.serror_has_esr;
598 env->serror.esr = events.exception.serror_esr;
599
600 return 0;
601}
602
603void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run)
604{
605}
606
607MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run *run)
608{
609 ARMCPU *cpu;
610 uint32_t switched_level;
611
612 if (kvm_irqchip_in_kernel()) {
613
614
615
616
617 return MEMTXATTRS_UNSPECIFIED;
618 }
619
620 cpu = ARM_CPU(cs);
621
622
623 if (run->s.regs.device_irq_level != cpu->device_irq_level) {
624 switched_level = cpu->device_irq_level ^ run->s.regs.device_irq_level;
625
626 qemu_mutex_lock_iothread();
627
628 if (switched_level & KVM_ARM_DEV_EL1_VTIMER) {
629 qemu_set_irq(cpu->gt_timer_outputs[GTIMER_VIRT],
630 !!(run->s.regs.device_irq_level &
631 KVM_ARM_DEV_EL1_VTIMER));
632 switched_level &= ~KVM_ARM_DEV_EL1_VTIMER;
633 }
634
635 if (switched_level & KVM_ARM_DEV_EL1_PTIMER) {
636 qemu_set_irq(cpu->gt_timer_outputs[GTIMER_PHYS],
637 !!(run->s.regs.device_irq_level &
638 KVM_ARM_DEV_EL1_PTIMER));
639 switched_level &= ~KVM_ARM_DEV_EL1_PTIMER;
640 }
641
642 if (switched_level & KVM_ARM_DEV_PMU) {
643 qemu_set_irq(cpu->pmu_interrupt,
644 !!(run->s.regs.device_irq_level & KVM_ARM_DEV_PMU));
645 switched_level &= ~KVM_ARM_DEV_PMU;
646 }
647
648 if (switched_level) {
649 qemu_log_mask(LOG_UNIMP, "%s: unhandled in-kernel device IRQ %x\n",
650 __func__, switched_level);
651 }
652
653
654 cpu->device_irq_level = run->s.regs.device_irq_level;
655 qemu_mutex_unlock_iothread();
656 }
657
658 return MEMTXATTRS_UNSPECIFIED;
659}
660
661
662int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
663{
664 int ret = 0;
665
666 switch (run->exit_reason) {
667 case KVM_EXIT_DEBUG:
668 if (kvm_arm_handle_debug(cs, &run->debug.arch)) {
669 ret = EXCP_DEBUG;
670 }
671 break;
672 default:
673 qemu_log_mask(LOG_UNIMP, "%s: un-handled exit reason %d\n",
674 __func__, run->exit_reason);
675 break;
676 }
677 return ret;
678}
679
680bool kvm_arch_stop_on_emulation_error(CPUState *cs)
681{
682 return true;
683}
684
685int kvm_arch_process_async_events(CPUState *cs)
686{
687 return 0;
688}
689
690
691
692
693void kvm_arch_update_guest_debug(CPUState *cs, struct kvm_guest_debug *dbg)
694{
695#ifdef KVM_GUESTDBG_USE_SW_BP
696 if (kvm_sw_breakpoints_active(cs)) {
697 dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP;
698 }
699#endif
700#ifdef KVM_GUESTDBG_USE_HW
701 if (kvm_arm_hw_debug_active(cs)) {
702 dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW;
703 kvm_arm_copy_hw_debug_data(&dbg->arch);
704 }
705#endif
706}
707
708void kvm_arch_init_irq_routing(KVMState *s)
709{
710}
711
712int kvm_arch_irqchip_create(MachineState *ms, KVMState *s)
713{
714 if (machine_kernel_irqchip_split(ms)) {
715 perror("-machine kernel_irqchip=split is not supported on ARM.");
716 exit(1);
717 }
718
719
720
721
722 return kvm_check_extension(s, KVM_CAP_DEVICE_CTRL);
723}
724
725int kvm_arm_vgic_probe(void)
726{
727 if (kvm_create_device(kvm_state,
728 KVM_DEV_TYPE_ARM_VGIC_V3, true) == 0) {
729 return 3;
730 } else if (kvm_create_device(kvm_state,
731 KVM_DEV_TYPE_ARM_VGIC_V2, true) == 0) {
732 return 2;
733 } else {
734 return 0;
735 }
736}
737
738int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
739 uint64_t address, uint32_t data, PCIDevice *dev)
740{
741 AddressSpace *as = pci_device_iommu_address_space(dev);
742 hwaddr xlat, len, doorbell_gpa;
743 MemoryRegionSection mrs;
744 MemoryRegion *mr;
745 int ret = 1;
746
747 if (as == &address_space_memory) {
748 return 0;
749 }
750
751
752
753 rcu_read_lock();
754 mr = address_space_translate(as, address, &xlat, &len, true,
755 MEMTXATTRS_UNSPECIFIED);
756 if (!mr) {
757 goto unlock;
758 }
759 mrs = memory_region_find(mr, xlat, 1);
760 if (!mrs.mr) {
761 goto unlock;
762 }
763
764 doorbell_gpa = mrs.offset_within_address_space;
765 memory_region_unref(mrs.mr);
766
767 route->u.msi.address_lo = doorbell_gpa;
768 route->u.msi.address_hi = doorbell_gpa >> 32;
769
770 trace_kvm_arm_fixup_msi_route(address, doorbell_gpa);
771
772 ret = 0;
773
774unlock:
775 rcu_read_unlock();
776 return ret;
777}
778
779int kvm_arch_add_msi_route_post(struct kvm_irq_routing_entry *route,
780 int vector, PCIDevice *dev)
781{
782 return 0;
783}
784
785int kvm_arch_release_virq_post(int virq)
786{
787 return 0;
788}
789
790int kvm_arch_msi_data_to_gsi(uint32_t data)
791{
792 return (data - 32) & 0xffff;
793}
794