1
2
3
4
5
6
7
8
9
10
11#include "qemu/osdep.h"
12#include <sys/ioctl.h>
13
14#include <linux/kvm.h>
15
16#include "qemu-common.h"
17#include "qemu/timer.h"
18#include "qemu/error-report.h"
19#include "qemu/main-loop.h"
20#include "qom/object.h"
21#include "qapi/error.h"
22#include "sysemu/sysemu.h"
23#include "sysemu/kvm.h"
24#include "sysemu/kvm_int.h"
25#include "kvm_arm.h"
26#include "cpu.h"
27#include "trace.h"
28#include "internals.h"
29#include "hw/pci/pci.h"
30#include "exec/memattrs.h"
31#include "exec/address-spaces.h"
32#include "hw/boards.h"
33#include "hw/irq.h"
34#include "qemu/log.h"
35
36const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
37 KVM_CAP_LAST_INFO
38};
39
40static bool cap_has_mp_state;
41static bool cap_has_inject_serror_esr;
42static bool cap_has_inject_ext_dabt;
43
44static ARMHostCPUFeatures arm_host_cpu_features;
45
46int kvm_arm_vcpu_init(CPUState *cs)
47{
48 ARMCPU *cpu = ARM_CPU(cs);
49 struct kvm_vcpu_init init;
50
51 init.target = cpu->kvm_target;
52 memcpy(init.features, cpu->kvm_init_features, sizeof(init.features));
53
54 return kvm_vcpu_ioctl(cs, KVM_ARM_VCPU_INIT, &init);
55}
56
57int kvm_arm_vcpu_finalize(CPUState *cs, int feature)
58{
59 return kvm_vcpu_ioctl(cs, KVM_ARM_VCPU_FINALIZE, &feature);
60}
61
62void kvm_arm_init_serror_injection(CPUState *cs)
63{
64 cap_has_inject_serror_esr = kvm_check_extension(cs->kvm_state,
65 KVM_CAP_ARM_INJECT_SERROR_ESR);
66}
67
68bool kvm_arm_create_scratch_host_vcpu(const uint32_t *cpus_to_try,
69 int *fdarray,
70 struct kvm_vcpu_init *init)
71{
72 int ret = 0, kvmfd = -1, vmfd = -1, cpufd = -1;
73
74 kvmfd = qemu_open_old("/dev/kvm", O_RDWR);
75 if (kvmfd < 0) {
76 goto err;
77 }
78 vmfd = ioctl(kvmfd, KVM_CREATE_VM, 0);
79 if (vmfd < 0) {
80 goto err;
81 }
82 cpufd = ioctl(vmfd, KVM_CREATE_VCPU, 0);
83 if (cpufd < 0) {
84 goto err;
85 }
86
87 if (!init) {
88
89 goto finish;
90 }
91
92 if (init->target == -1) {
93 struct kvm_vcpu_init preferred;
94
95 ret = ioctl(vmfd, KVM_ARM_PREFERRED_TARGET, &preferred);
96 if (!ret) {
97 init->target = preferred.target;
98 }
99 }
100 if (ret >= 0) {
101 ret = ioctl(cpufd, KVM_ARM_VCPU_INIT, init);
102 if (ret < 0) {
103 goto err;
104 }
105 } else if (cpus_to_try) {
106
107
108
109
110
111 struct kvm_vcpu_init try;
112
113 while (*cpus_to_try != QEMU_KVM_ARM_TARGET_NONE) {
114 try.target = *cpus_to_try++;
115 memcpy(try.features, init->features, sizeof(init->features));
116 ret = ioctl(cpufd, KVM_ARM_VCPU_INIT, &try);
117 if (ret >= 0) {
118 break;
119 }
120 }
121 if (ret < 0) {
122 goto err;
123 }
124 init->target = try.target;
125 } else {
126
127
128
129
130 goto err;
131 }
132
133finish:
134 fdarray[0] = kvmfd;
135 fdarray[1] = vmfd;
136 fdarray[2] = cpufd;
137
138 return true;
139
140err:
141 if (cpufd >= 0) {
142 close(cpufd);
143 }
144 if (vmfd >= 0) {
145 close(vmfd);
146 }
147 if (kvmfd >= 0) {
148 close(kvmfd);
149 }
150
151 return false;
152}
153
154void kvm_arm_destroy_scratch_host_vcpu(int *fdarray)
155{
156 int i;
157
158 for (i = 2; i >= 0; i--) {
159 close(fdarray[i]);
160 }
161}
162
163void kvm_arm_set_cpu_features_from_host(ARMCPU *cpu)
164{
165 CPUARMState *env = &cpu->env;
166
167 if (!arm_host_cpu_features.dtb_compatible) {
168 if (!kvm_enabled() ||
169 !kvm_arm_get_host_cpu_features(&arm_host_cpu_features)) {
170
171
172
173 cpu->kvm_target = QEMU_KVM_ARM_TARGET_NONE;
174 cpu->host_cpu_probe_failed = true;
175 return;
176 }
177 }
178
179 cpu->kvm_target = arm_host_cpu_features.target;
180 cpu->dtb_compatible = arm_host_cpu_features.dtb_compatible;
181 cpu->isar = arm_host_cpu_features.isar;
182 env->features = arm_host_cpu_features.features;
183}
184
185static bool kvm_no_adjvtime_get(Object *obj, Error **errp)
186{
187 return !ARM_CPU(obj)->kvm_adjvtime;
188}
189
190static void kvm_no_adjvtime_set(Object *obj, bool value, Error **errp)
191{
192 ARM_CPU(obj)->kvm_adjvtime = !value;
193}
194
195static bool kvm_steal_time_get(Object *obj, Error **errp)
196{
197 return ARM_CPU(obj)->kvm_steal_time != ON_OFF_AUTO_OFF;
198}
199
200static void kvm_steal_time_set(Object *obj, bool value, Error **errp)
201{
202 ARM_CPU(obj)->kvm_steal_time = value ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
203}
204
205
206void kvm_arm_add_vcpu_properties(Object *obj)
207{
208 ARMCPU *cpu = ARM_CPU(obj);
209 CPUARMState *env = &cpu->env;
210
211 if (arm_feature(env, ARM_FEATURE_GENERIC_TIMER)) {
212 cpu->kvm_adjvtime = true;
213 object_property_add_bool(obj, "kvm-no-adjvtime", kvm_no_adjvtime_get,
214 kvm_no_adjvtime_set);
215 object_property_set_description(obj, "kvm-no-adjvtime",
216 "Set on to disable the adjustment of "
217 "the virtual counter. VM stopped time "
218 "will be counted.");
219 }
220
221 cpu->kvm_steal_time = ON_OFF_AUTO_AUTO;
222 object_property_add_bool(obj, "kvm-steal-time", kvm_steal_time_get,
223 kvm_steal_time_set);
224 object_property_set_description(obj, "kvm-steal-time",
225 "Set off to disable KVM steal time.");
226}
227
228bool kvm_arm_pmu_supported(void)
229{
230 return kvm_check_extension(kvm_state, KVM_CAP_ARM_PMU_V3);
231}
232
233int kvm_arm_get_max_vm_ipa_size(MachineState *ms, bool *fixed_ipa)
234{
235 KVMState *s = KVM_STATE(ms->accelerator);
236 int ret;
237
238 ret = kvm_check_extension(s, KVM_CAP_ARM_VM_IPA_SIZE);
239 *fixed_ipa = ret <= 0;
240
241 return ret > 0 ? ret : 40;
242}
243
244int kvm_arch_init(MachineState *ms, KVMState *s)
245{
246 int ret = 0;
247
248
249
250 kvm_async_interrupts_allowed = true;
251
252
253
254
255
256 kvm_halt_in_kernel_allowed = true;
257
258 cap_has_mp_state = kvm_check_extension(s, KVM_CAP_MP_STATE);
259
260 if (ms->smp.cpus > 256 &&
261 !kvm_check_extension(s, KVM_CAP_ARM_IRQ_LINE_LAYOUT_2)) {
262 error_report("Using more than 256 vcpus requires a host kernel "
263 "with KVM_CAP_ARM_IRQ_LINE_LAYOUT_2");
264 ret = -EINVAL;
265 }
266
267 if (kvm_check_extension(s, KVM_CAP_ARM_NISV_TO_USER)) {
268 if (kvm_vm_enable_cap(s, KVM_CAP_ARM_NISV_TO_USER, 0)) {
269 error_report("Failed to enable KVM_CAP_ARM_NISV_TO_USER cap");
270 } else {
271
272 cap_has_inject_ext_dabt = kvm_check_extension(s,
273 KVM_CAP_ARM_INJECT_EXT_DABT);
274 }
275 }
276
277 return ret;
278}
279
280unsigned long kvm_arch_vcpu_id(CPUState *cpu)
281{
282 return cpu->cpu_index;
283}
284
285
286
287
288
289
290
291
292
293
294
295
296
297typedef struct KVMDevice {
298 struct kvm_arm_device_addr kda;
299 struct kvm_device_attr kdattr;
300 uint64_t kda_addr_ormask;
301 MemoryRegion *mr;
302 QSLIST_ENTRY(KVMDevice) entries;
303 int dev_fd;
304} KVMDevice;
305
306static QSLIST_HEAD(, KVMDevice) kvm_devices_head;
307
308static void kvm_arm_devlistener_add(MemoryListener *listener,
309 MemoryRegionSection *section)
310{
311 KVMDevice *kd;
312
313 QSLIST_FOREACH(kd, &kvm_devices_head, entries) {
314 if (section->mr == kd->mr) {
315 kd->kda.addr = section->offset_within_address_space;
316 }
317 }
318}
319
320static void kvm_arm_devlistener_del(MemoryListener *listener,
321 MemoryRegionSection *section)
322{
323 KVMDevice *kd;
324
325 QSLIST_FOREACH(kd, &kvm_devices_head, entries) {
326 if (section->mr == kd->mr) {
327 kd->kda.addr = -1;
328 }
329 }
330}
331
332static MemoryListener devlistener = {
333 .region_add = kvm_arm_devlistener_add,
334 .region_del = kvm_arm_devlistener_del,
335};
336
337static void kvm_arm_set_device_addr(KVMDevice *kd)
338{
339 struct kvm_device_attr *attr = &kd->kdattr;
340 int ret;
341
342
343
344
345 if (kd->dev_fd >= 0) {
346 uint64_t addr = kd->kda.addr;
347
348 addr |= kd->kda_addr_ormask;
349 attr->addr = (uintptr_t)&addr;
350 ret = kvm_device_ioctl(kd->dev_fd, KVM_SET_DEVICE_ATTR, attr);
351 } else {
352 ret = kvm_vm_ioctl(kvm_state, KVM_ARM_SET_DEVICE_ADDR, &kd->kda);
353 }
354
355 if (ret < 0) {
356 fprintf(stderr, "Failed to set device address: %s\n",
357 strerror(-ret));
358 abort();
359 }
360}
361
362static void kvm_arm_machine_init_done(Notifier *notifier, void *data)
363{
364 KVMDevice *kd, *tkd;
365
366 QSLIST_FOREACH_SAFE(kd, &kvm_devices_head, entries, tkd) {
367 if (kd->kda.addr != -1) {
368 kvm_arm_set_device_addr(kd);
369 }
370 memory_region_unref(kd->mr);
371 QSLIST_REMOVE_HEAD(&kvm_devices_head, entries);
372 g_free(kd);
373 }
374 memory_listener_unregister(&devlistener);
375}
376
377static Notifier notify = {
378 .notify = kvm_arm_machine_init_done,
379};
380
381void kvm_arm_register_device(MemoryRegion *mr, uint64_t devid, uint64_t group,
382 uint64_t attr, int dev_fd, uint64_t addr_ormask)
383{
384 KVMDevice *kd;
385
386 if (!kvm_irqchip_in_kernel()) {
387 return;
388 }
389
390 if (QSLIST_EMPTY(&kvm_devices_head)) {
391 memory_listener_register(&devlistener, &address_space_memory);
392 qemu_add_machine_init_done_notifier(¬ify);
393 }
394 kd = g_new0(KVMDevice, 1);
395 kd->mr = mr;
396 kd->kda.id = devid;
397 kd->kda.addr = -1;
398 kd->kdattr.flags = 0;
399 kd->kdattr.group = group;
400 kd->kdattr.attr = attr;
401 kd->dev_fd = dev_fd;
402 kd->kda_addr_ormask = addr_ormask;
403 QSLIST_INSERT_HEAD(&kvm_devices_head, kd, entries);
404 memory_region_ref(kd->mr);
405}
406
407static int compare_u64(const void *a, const void *b)
408{
409 if (*(uint64_t *)a > *(uint64_t *)b) {
410 return 1;
411 }
412 if (*(uint64_t *)a < *(uint64_t *)b) {
413 return -1;
414 }
415 return 0;
416}
417
418
419
420
421
422
423static uint64_t *kvm_arm_get_cpreg_ptr(ARMCPU *cpu, uint64_t regidx)
424{
425 uint64_t *res;
426
427 res = bsearch(®idx, cpu->cpreg_indexes, cpu->cpreg_array_len,
428 sizeof(uint64_t), compare_u64);
429 assert(res);
430
431 return &cpu->cpreg_values[res - cpu->cpreg_indexes];
432}
433
434
435
436
437
438int kvm_arm_init_cpreg_list(ARMCPU *cpu)
439{
440 struct kvm_reg_list rl;
441 struct kvm_reg_list *rlp;
442 int i, ret, arraylen;
443 CPUState *cs = CPU(cpu);
444
445 rl.n = 0;
446 ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, &rl);
447 if (ret != -E2BIG) {
448 return ret;
449 }
450 rlp = g_malloc(sizeof(struct kvm_reg_list) + rl.n * sizeof(uint64_t));
451 rlp->n = rl.n;
452 ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, rlp);
453 if (ret) {
454 goto out;
455 }
456
457
458
459 qsort(&rlp->reg, rlp->n, sizeof(rlp->reg[0]), compare_u64);
460
461 for (i = 0, arraylen = 0; i < rlp->n; i++) {
462 if (!kvm_arm_reg_syncs_via_cpreg_list(rlp->reg[i])) {
463 continue;
464 }
465 switch (rlp->reg[i] & KVM_REG_SIZE_MASK) {
466 case KVM_REG_SIZE_U32:
467 case KVM_REG_SIZE_U64:
468 break;
469 default:
470 fprintf(stderr, "Can't handle size of register in kernel list\n");
471 ret = -EINVAL;
472 goto out;
473 }
474
475 arraylen++;
476 }
477
478 cpu->cpreg_indexes = g_renew(uint64_t, cpu->cpreg_indexes, arraylen);
479 cpu->cpreg_values = g_renew(uint64_t, cpu->cpreg_values, arraylen);
480 cpu->cpreg_vmstate_indexes = g_renew(uint64_t, cpu->cpreg_vmstate_indexes,
481 arraylen);
482 cpu->cpreg_vmstate_values = g_renew(uint64_t, cpu->cpreg_vmstate_values,
483 arraylen);
484 cpu->cpreg_array_len = arraylen;
485 cpu->cpreg_vmstate_array_len = arraylen;
486
487 for (i = 0, arraylen = 0; i < rlp->n; i++) {
488 uint64_t regidx = rlp->reg[i];
489 if (!kvm_arm_reg_syncs_via_cpreg_list(regidx)) {
490 continue;
491 }
492 cpu->cpreg_indexes[arraylen] = regidx;
493 arraylen++;
494 }
495 assert(cpu->cpreg_array_len == arraylen);
496
497 if (!write_kvmstate_to_list(cpu)) {
498
499
500
501 fprintf(stderr, "Initial read of kernel register state failed\n");
502 ret = -EINVAL;
503 goto out;
504 }
505
506out:
507 g_free(rlp);
508 return ret;
509}
510
511bool write_kvmstate_to_list(ARMCPU *cpu)
512{
513 CPUState *cs = CPU(cpu);
514 int i;
515 bool ok = true;
516
517 for (i = 0; i < cpu->cpreg_array_len; i++) {
518 struct kvm_one_reg r;
519 uint64_t regidx = cpu->cpreg_indexes[i];
520 uint32_t v32;
521 int ret;
522
523 r.id = regidx;
524
525 switch (regidx & KVM_REG_SIZE_MASK) {
526 case KVM_REG_SIZE_U32:
527 r.addr = (uintptr_t)&v32;
528 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &r);
529 if (!ret) {
530 cpu->cpreg_values[i] = v32;
531 }
532 break;
533 case KVM_REG_SIZE_U64:
534 r.addr = (uintptr_t)(cpu->cpreg_values + i);
535 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &r);
536 break;
537 default:
538 abort();
539 }
540 if (ret) {
541 ok = false;
542 }
543 }
544 return ok;
545}
546
547bool write_list_to_kvmstate(ARMCPU *cpu, int level)
548{
549 CPUState *cs = CPU(cpu);
550 int i;
551 bool ok = true;
552
553 for (i = 0; i < cpu->cpreg_array_len; i++) {
554 struct kvm_one_reg r;
555 uint64_t regidx = cpu->cpreg_indexes[i];
556 uint32_t v32;
557 int ret;
558
559 if (kvm_arm_cpreg_level(regidx) > level) {
560 continue;
561 }
562
563 r.id = regidx;
564 switch (regidx & KVM_REG_SIZE_MASK) {
565 case KVM_REG_SIZE_U32:
566 v32 = cpu->cpreg_values[i];
567 r.addr = (uintptr_t)&v32;
568 break;
569 case KVM_REG_SIZE_U64:
570 r.addr = (uintptr_t)(cpu->cpreg_values + i);
571 break;
572 default:
573 abort();
574 }
575 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &r);
576 if (ret) {
577
578
579
580
581 ok = false;
582 }
583 }
584 return ok;
585}
586
587void kvm_arm_cpu_pre_save(ARMCPU *cpu)
588{
589
590 if (cpu->kvm_vtime_dirty) {
591 *kvm_arm_get_cpreg_ptr(cpu, KVM_REG_ARM_TIMER_CNT) = cpu->kvm_vtime;
592 }
593}
594
595void kvm_arm_cpu_post_load(ARMCPU *cpu)
596{
597
598 if (cpu->kvm_adjvtime) {
599 cpu->kvm_vtime = *kvm_arm_get_cpreg_ptr(cpu, KVM_REG_ARM_TIMER_CNT);
600 cpu->kvm_vtime_dirty = true;
601 }
602}
603
604void kvm_arm_reset_vcpu(ARMCPU *cpu)
605{
606 int ret;
607
608
609
610
611 ret = kvm_arm_vcpu_init(CPU(cpu));
612 if (ret < 0) {
613 fprintf(stderr, "kvm_arm_vcpu_init failed: %s\n", strerror(-ret));
614 abort();
615 }
616 if (!write_kvmstate_to_list(cpu)) {
617 fprintf(stderr, "write_kvmstate_to_list failed\n");
618 abort();
619 }
620
621
622
623
624
625
626
627 write_list_to_cpustate(cpu);
628}
629
630
631
632
633int kvm_arm_sync_mpstate_to_kvm(ARMCPU *cpu)
634{
635 if (cap_has_mp_state) {
636 struct kvm_mp_state mp_state = {
637 .mp_state = (cpu->power_state == PSCI_OFF) ?
638 KVM_MP_STATE_STOPPED : KVM_MP_STATE_RUNNABLE
639 };
640 int ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MP_STATE, &mp_state);
641 if (ret) {
642 fprintf(stderr, "%s: failed to set MP_STATE %d/%s\n",
643 __func__, ret, strerror(-ret));
644 return -1;
645 }
646 }
647
648 return 0;
649}
650
651
652
653
654int kvm_arm_sync_mpstate_to_qemu(ARMCPU *cpu)
655{
656 if (cap_has_mp_state) {
657 struct kvm_mp_state mp_state;
658 int ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MP_STATE, &mp_state);
659 if (ret) {
660 fprintf(stderr, "%s: failed to get MP_STATE %d/%s\n",
661 __func__, ret, strerror(-ret));
662 abort();
663 }
664 cpu->power_state = (mp_state.mp_state == KVM_MP_STATE_STOPPED) ?
665 PSCI_OFF : PSCI_ON;
666 }
667
668 return 0;
669}
670
671void kvm_arm_get_virtual_time(CPUState *cs)
672{
673 ARMCPU *cpu = ARM_CPU(cs);
674 struct kvm_one_reg reg = {
675 .id = KVM_REG_ARM_TIMER_CNT,
676 .addr = (uintptr_t)&cpu->kvm_vtime,
677 };
678 int ret;
679
680 if (cpu->kvm_vtime_dirty) {
681 return;
682 }
683
684 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®);
685 if (ret) {
686 error_report("Failed to get KVM_REG_ARM_TIMER_CNT");
687 abort();
688 }
689
690 cpu->kvm_vtime_dirty = true;
691}
692
693void kvm_arm_put_virtual_time(CPUState *cs)
694{
695 ARMCPU *cpu = ARM_CPU(cs);
696 struct kvm_one_reg reg = {
697 .id = KVM_REG_ARM_TIMER_CNT,
698 .addr = (uintptr_t)&cpu->kvm_vtime,
699 };
700 int ret;
701
702 if (!cpu->kvm_vtime_dirty) {
703 return;
704 }
705
706 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®);
707 if (ret) {
708 error_report("Failed to set KVM_REG_ARM_TIMER_CNT");
709 abort();
710 }
711
712 cpu->kvm_vtime_dirty = false;
713}
714
715int kvm_put_vcpu_events(ARMCPU *cpu)
716{
717 CPUARMState *env = &cpu->env;
718 struct kvm_vcpu_events events;
719 int ret;
720
721 if (!kvm_has_vcpu_events()) {
722 return 0;
723 }
724
725 memset(&events, 0, sizeof(events));
726 events.exception.serror_pending = env->serror.pending;
727
728
729
730
731 if (cap_has_inject_serror_esr) {
732 events.exception.serror_has_esr = env->serror.has_esr;
733 events.exception.serror_esr = env->serror.esr;
734 }
735
736 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_VCPU_EVENTS, &events);
737 if (ret) {
738 error_report("failed to put vcpu events");
739 }
740
741 return ret;
742}
743
744int kvm_get_vcpu_events(ARMCPU *cpu)
745{
746 CPUARMState *env = &cpu->env;
747 struct kvm_vcpu_events events;
748 int ret;
749
750 if (!kvm_has_vcpu_events()) {
751 return 0;
752 }
753
754 memset(&events, 0, sizeof(events));
755 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_VCPU_EVENTS, &events);
756 if (ret) {
757 error_report("failed to get vcpu events");
758 return ret;
759 }
760
761 env->serror.pending = events.exception.serror_pending;
762 env->serror.has_esr = events.exception.serror_has_esr;
763 env->serror.esr = events.exception.serror_esr;
764
765 return 0;
766}
767
768void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run)
769{
770 ARMCPU *cpu = ARM_CPU(cs);
771 CPUARMState *env = &cpu->env;
772
773 if (unlikely(env->ext_dabt_raised)) {
774
775
776
777
778
779
780
781 if (!arm_feature(env, ARM_FEATURE_AARCH64) &&
782 unlikely(!kvm_arm_verify_ext_dabt_pending(cs))) {
783
784 error_report("Data abort exception with no valid ISS generated by "
785 "guest memory access. KVM unable to emulate faulting "
786 "instruction. Failed to inject an external data abort "
787 "into the guest.");
788 abort();
789 }
790
791 env->ext_dabt_raised = 0;
792 }
793}
794
795MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run *run)
796{
797 ARMCPU *cpu;
798 uint32_t switched_level;
799
800 if (kvm_irqchip_in_kernel()) {
801
802
803
804
805 return MEMTXATTRS_UNSPECIFIED;
806 }
807
808 cpu = ARM_CPU(cs);
809
810
811 if (run->s.regs.device_irq_level != cpu->device_irq_level) {
812 switched_level = cpu->device_irq_level ^ run->s.regs.device_irq_level;
813
814 qemu_mutex_lock_iothread();
815
816 if (switched_level & KVM_ARM_DEV_EL1_VTIMER) {
817 qemu_set_irq(cpu->gt_timer_outputs[GTIMER_VIRT],
818 !!(run->s.regs.device_irq_level &
819 KVM_ARM_DEV_EL1_VTIMER));
820 switched_level &= ~KVM_ARM_DEV_EL1_VTIMER;
821 }
822
823 if (switched_level & KVM_ARM_DEV_EL1_PTIMER) {
824 qemu_set_irq(cpu->gt_timer_outputs[GTIMER_PHYS],
825 !!(run->s.regs.device_irq_level &
826 KVM_ARM_DEV_EL1_PTIMER));
827 switched_level &= ~KVM_ARM_DEV_EL1_PTIMER;
828 }
829
830 if (switched_level & KVM_ARM_DEV_PMU) {
831 qemu_set_irq(cpu->pmu_interrupt,
832 !!(run->s.regs.device_irq_level & KVM_ARM_DEV_PMU));
833 switched_level &= ~KVM_ARM_DEV_PMU;
834 }
835
836 if (switched_level) {
837 qemu_log_mask(LOG_UNIMP, "%s: unhandled in-kernel device IRQ %x\n",
838 __func__, switched_level);
839 }
840
841
842 cpu->device_irq_level = run->s.regs.device_irq_level;
843 qemu_mutex_unlock_iothread();
844 }
845
846 return MEMTXATTRS_UNSPECIFIED;
847}
848
849void kvm_arm_vm_state_change(void *opaque, bool running, RunState state)
850{
851 CPUState *cs = opaque;
852 ARMCPU *cpu = ARM_CPU(cs);
853
854 if (running) {
855 if (cpu->kvm_adjvtime) {
856 kvm_arm_put_virtual_time(cs);
857 }
858 } else {
859 if (cpu->kvm_adjvtime) {
860 kvm_arm_get_virtual_time(cs);
861 }
862 }
863}
864
865
866
867
868
869
870
871
872
873
874static int kvm_arm_handle_dabt_nisv(CPUState *cs, uint64_t esr_iss,
875 uint64_t fault_ipa)
876{
877 ARMCPU *cpu = ARM_CPU(cs);
878 CPUARMState *env = &cpu->env;
879
880
881
882 if (cap_has_inject_ext_dabt) {
883 struct kvm_vcpu_events events = { };
884
885
886
887
888
889
890
891 events.exception.ext_dabt_pending = 1;
892
893 if (!kvm_vcpu_ioctl(cs, KVM_SET_VCPU_EVENTS, &events)) {
894 env->ext_dabt_raised = 1;
895 return 0;
896 }
897 } else {
898 error_report("Data abort exception triggered by guest memory access "
899 "at physical address: 0x" TARGET_FMT_lx,
900 (target_ulong)fault_ipa);
901 error_printf("KVM unable to emulate faulting instruction.\n");
902 }
903 return -1;
904}
905
906int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
907{
908 int ret = 0;
909
910 switch (run->exit_reason) {
911 case KVM_EXIT_DEBUG:
912 if (kvm_arm_handle_debug(cs, &run->debug.arch)) {
913 ret = EXCP_DEBUG;
914 }
915 break;
916 case KVM_EXIT_ARM_NISV:
917
918 ret = kvm_arm_handle_dabt_nisv(cs, run->arm_nisv.esr_iss,
919 run->arm_nisv.fault_ipa);
920 break;
921 default:
922 qemu_log_mask(LOG_UNIMP, "%s: un-handled exit reason %d\n",
923 __func__, run->exit_reason);
924 break;
925 }
926 return ret;
927}
928
929bool kvm_arch_stop_on_emulation_error(CPUState *cs)
930{
931 return true;
932}
933
934int kvm_arch_process_async_events(CPUState *cs)
935{
936 return 0;
937}
938
939void kvm_arch_update_guest_debug(CPUState *cs, struct kvm_guest_debug *dbg)
940{
941 if (kvm_sw_breakpoints_active(cs)) {
942 dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP;
943 }
944 if (kvm_arm_hw_debug_active(cs)) {
945 dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW;
946 kvm_arm_copy_hw_debug_data(&dbg->arch);
947 }
948}
949
950void kvm_arch_init_irq_routing(KVMState *s)
951{
952}
953
954int kvm_arch_irqchip_create(KVMState *s)
955{
956 if (kvm_kernel_irqchip_split()) {
957 perror("-machine kernel_irqchip=split is not supported on ARM.");
958 exit(1);
959 }
960
961
962
963
964 return kvm_check_extension(s, KVM_CAP_DEVICE_CTRL);
965}
966
967int kvm_arm_vgic_probe(void)
968{
969 int val = 0;
970
971 if (kvm_create_device(kvm_state,
972 KVM_DEV_TYPE_ARM_VGIC_V3, true) == 0) {
973 val |= KVM_ARM_VGIC_V3;
974 }
975 if (kvm_create_device(kvm_state,
976 KVM_DEV_TYPE_ARM_VGIC_V2, true) == 0) {
977 val |= KVM_ARM_VGIC_V2;
978 }
979 return val;
980}
981
982int kvm_arm_set_irq(int cpu, int irqtype, int irq, int level)
983{
984 int kvm_irq = (irqtype << KVM_ARM_IRQ_TYPE_SHIFT) | irq;
985 int cpu_idx1 = cpu % 256;
986 int cpu_idx2 = cpu / 256;
987
988 kvm_irq |= (cpu_idx1 << KVM_ARM_IRQ_VCPU_SHIFT) |
989 (cpu_idx2 << KVM_ARM_IRQ_VCPU2_SHIFT);
990
991 return kvm_set_irq(kvm_state, kvm_irq, !!level);
992}
993
994int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
995 uint64_t address, uint32_t data, PCIDevice *dev)
996{
997 AddressSpace *as = pci_device_iommu_address_space(dev);
998 hwaddr xlat, len, doorbell_gpa;
999 MemoryRegionSection mrs;
1000 MemoryRegion *mr;
1001 int ret = 1;
1002
1003 if (as == &address_space_memory) {
1004 return 0;
1005 }
1006
1007
1008
1009 rcu_read_lock();
1010 mr = address_space_translate(as, address, &xlat, &len, true,
1011 MEMTXATTRS_UNSPECIFIED);
1012 if (!mr) {
1013 goto unlock;
1014 }
1015 mrs = memory_region_find(mr, xlat, 1);
1016 if (!mrs.mr) {
1017 goto unlock;
1018 }
1019
1020 doorbell_gpa = mrs.offset_within_address_space;
1021 memory_region_unref(mrs.mr);
1022
1023 route->u.msi.address_lo = doorbell_gpa;
1024 route->u.msi.address_hi = doorbell_gpa >> 32;
1025
1026 trace_kvm_arm_fixup_msi_route(address, doorbell_gpa);
1027
1028 ret = 0;
1029
1030unlock:
1031 rcu_read_unlock();
1032 return ret;
1033}
1034
1035int kvm_arch_add_msi_route_post(struct kvm_irq_routing_entry *route,
1036 int vector, PCIDevice *dev)
1037{
1038 return 0;
1039}
1040
1041int kvm_arch_release_virq_post(int virq)
1042{
1043 return 0;
1044}
1045
1046int kvm_arch_msi_data_to_gsi(uint32_t data)
1047{
1048 return (data - 32) & 0xffff;
1049}
1050
1051bool kvm_arch_cpu_check_are_resettable(void)
1052{
1053 return true;
1054}
1055