1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16#include "qemu/osdep.h"
17#include <sys/ioctl.h>
18
19#include <linux/kvm.h>
20
21#include "qemu-common.h"
22#include "qemu/atomic.h"
23#include "qemu/option.h"
24#include "qemu/config-file.h"
25#include "qemu/error-report.h"
26#include "qapi/error.h"
27#include "hw/hw.h"
28#include "hw/pci/msi.h"
29#include "hw/pci/msix.h"
30#include "hw/s390x/adapter.h"
31#include "exec/gdbstub.h"
32#include "sysemu/kvm_int.h"
33#include "sysemu/cpus.h"
34#include "qemu/bswap.h"
35#include "exec/memory.h"
36#include "exec/ram_addr.h"
37#include "exec/address-spaces.h"
38#include "qemu/event_notifier.h"
39#include "trace.h"
40#include "hw/irq.h"
41
42#include "hw/boards.h"
43
44
45#ifdef CONFIG_EVENTFD
46#include <sys/eventfd.h>
47#endif
48
49
50
51
52#define PAGE_SIZE getpagesize()
53
54
55
56#ifdef DEBUG_KVM
57#define DPRINTF(fmt, ...) \
58 do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
59#else
60#define DPRINTF(fmt, ...) \
61 do { } while (0)
62#endif
63
64#define KVM_MSI_HASHTAB_SIZE 256
65
66struct KVMParkedVcpu {
67 unsigned long vcpu_id;
68 int kvm_fd;
69 QLIST_ENTRY(KVMParkedVcpu) node;
70};
71
72struct KVMState
73{
74 AccelState parent_obj;
75
76 int nr_slots;
77 int fd;
78 int vmfd;
79 int coalesced_mmio;
80 struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
81 bool coalesced_flush_in_progress;
82 int broken_set_mem_region;
83 int vcpu_events;
84 int robust_singlestep;
85 int debugregs;
86#ifdef KVM_CAP_SET_GUEST_DEBUG
87 struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
88#endif
89 int many_ioeventfds;
90 int intx_set_mask;
91
92
93
94 unsigned irq_set_ioctl;
95 unsigned int sigmask_len;
96 GHashTable *gsimap;
97#ifdef KVM_CAP_IRQ_ROUTING
98 struct kvm_irq_routing *irq_routes;
99 int nr_allocated_irq_routes;
100 unsigned long *used_gsi_bitmap;
101 unsigned int gsi_count;
102 QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
103#endif
104 KVMMemoryListener memory_listener;
105 QLIST_HEAD(, KVMParkedVcpu) kvm_parked_vcpus;
106};
107
108KVMState *kvm_state;
109bool kvm_kernel_irqchip;
110bool kvm_split_irqchip;
111bool kvm_async_interrupts_allowed;
112bool kvm_halt_in_kernel_allowed;
113bool kvm_eventfds_allowed;
114bool kvm_irqfds_allowed;
115bool kvm_resamplefds_allowed;
116bool kvm_msi_via_irqfd_allowed;
117bool kvm_gsi_routing_allowed;
118bool kvm_gsi_direct_mapping;
119bool kvm_allowed;
120bool kvm_readonly_mem_allowed;
121bool kvm_vm_attributes_allowed;
122bool kvm_direct_msi_allowed;
123bool kvm_ioeventfd_any_length_allowed;
124bool kvm_msi_use_devid;
125static bool kvm_immediate_exit;
126
127static const KVMCapabilityInfo kvm_required_capabilites[] = {
128 KVM_CAP_INFO(USER_MEMORY),
129 KVM_CAP_INFO(DESTROY_MEMORY_REGION_WORKS),
130 KVM_CAP_LAST_INFO
131};
132
133int kvm_get_max_memslots(void)
134{
135 KVMState *s = KVM_STATE(current_machine->accelerator);
136
137 return s->nr_slots;
138}
139
140static KVMSlot *kvm_get_free_slot(KVMMemoryListener *kml)
141{
142 KVMState *s = kvm_state;
143 int i;
144
145 for (i = 0; i < s->nr_slots; i++) {
146 if (kml->slots[i].memory_size == 0) {
147 return &kml->slots[i];
148 }
149 }
150
151 return NULL;
152}
153
154bool kvm_has_free_slot(MachineState *ms)
155{
156 KVMState *s = KVM_STATE(ms->accelerator);
157
158 return kvm_get_free_slot(&s->memory_listener);
159}
160
161static KVMSlot *kvm_alloc_slot(KVMMemoryListener *kml)
162{
163 KVMSlot *slot = kvm_get_free_slot(kml);
164
165 if (slot) {
166 return slot;
167 }
168
169 fprintf(stderr, "%s: no free slot available\n", __func__);
170 abort();
171}
172
173static KVMSlot *kvm_lookup_matching_slot(KVMMemoryListener *kml,
174 hwaddr start_addr,
175 hwaddr end_addr)
176{
177 KVMState *s = kvm_state;
178 int i;
179
180 for (i = 0; i < s->nr_slots; i++) {
181 KVMSlot *mem = &kml->slots[i];
182
183 if (start_addr == mem->start_addr &&
184 end_addr == mem->start_addr + mem->memory_size) {
185 return mem;
186 }
187 }
188
189 return NULL;
190}
191
192
193
194
195static KVMSlot *kvm_lookup_overlapping_slot(KVMMemoryListener *kml,
196 hwaddr start_addr,
197 hwaddr end_addr)
198{
199 KVMState *s = kvm_state;
200 KVMSlot *found = NULL;
201 int i;
202
203 for (i = 0; i < s->nr_slots; i++) {
204 KVMSlot *mem = &kml->slots[i];
205
206 if (mem->memory_size == 0 ||
207 (found && found->start_addr < mem->start_addr)) {
208 continue;
209 }
210
211 if (end_addr > mem->start_addr &&
212 start_addr < mem->start_addr + mem->memory_size) {
213 found = mem;
214 }
215 }
216
217 return found;
218}
219
220int kvm_physical_memory_addr_from_host(KVMState *s, void *ram,
221 hwaddr *phys_addr)
222{
223 KVMMemoryListener *kml = &s->memory_listener;
224 int i;
225
226 for (i = 0; i < s->nr_slots; i++) {
227 KVMSlot *mem = &kml->slots[i];
228
229 if (ram >= mem->ram && ram < mem->ram + mem->memory_size) {
230 *phys_addr = mem->start_addr + (ram - mem->ram);
231 return 1;
232 }
233 }
234
235 return 0;
236}
237
238static int kvm_set_user_memory_region(KVMMemoryListener *kml, KVMSlot *slot)
239{
240 KVMState *s = kvm_state;
241 struct kvm_userspace_memory_region mem;
242
243 mem.slot = slot->slot | (kml->as_id << 16);
244 mem.guest_phys_addr = slot->start_addr;
245 mem.userspace_addr = (unsigned long)slot->ram;
246 mem.flags = slot->flags;
247
248 if (slot->memory_size && mem.flags & KVM_MEM_READONLY) {
249
250
251 mem.memory_size = 0;
252 kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
253 }
254 mem.memory_size = slot->memory_size;
255 return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
256}
257
258int kvm_destroy_vcpu(CPUState *cpu)
259{
260 KVMState *s = kvm_state;
261 long mmap_size;
262 struct KVMParkedVcpu *vcpu = NULL;
263 int ret = 0;
264
265 DPRINTF("kvm_destroy_vcpu\n");
266
267 mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
268 if (mmap_size < 0) {
269 ret = mmap_size;
270 DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
271 goto err;
272 }
273
274 ret = munmap(cpu->kvm_run, mmap_size);
275 if (ret < 0) {
276 goto err;
277 }
278
279 vcpu = g_malloc0(sizeof(*vcpu));
280 vcpu->vcpu_id = kvm_arch_vcpu_id(cpu);
281 vcpu->kvm_fd = cpu->kvm_fd;
282 QLIST_INSERT_HEAD(&kvm_state->kvm_parked_vcpus, vcpu, node);
283err:
284 return ret;
285}
286
287static int kvm_get_vcpu(KVMState *s, unsigned long vcpu_id)
288{
289 struct KVMParkedVcpu *cpu;
290
291 QLIST_FOREACH(cpu, &s->kvm_parked_vcpus, node) {
292 if (cpu->vcpu_id == vcpu_id) {
293 int kvm_fd;
294
295 QLIST_REMOVE(cpu, node);
296 kvm_fd = cpu->kvm_fd;
297 g_free(cpu);
298 return kvm_fd;
299 }
300 }
301
302 return kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)vcpu_id);
303}
304
305int kvm_init_vcpu(CPUState *cpu)
306{
307 KVMState *s = kvm_state;
308 long mmap_size;
309 int ret;
310
311 DPRINTF("kvm_init_vcpu\n");
312
313 ret = kvm_get_vcpu(s, kvm_arch_vcpu_id(cpu));
314 if (ret < 0) {
315 DPRINTF("kvm_create_vcpu failed\n");
316 goto err;
317 }
318
319 cpu->kvm_fd = ret;
320 cpu->kvm_state = s;
321 cpu->vcpu_dirty = true;
322
323 mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
324 if (mmap_size < 0) {
325 ret = mmap_size;
326 DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
327 goto err;
328 }
329
330 cpu->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
331 cpu->kvm_fd, 0);
332 if (cpu->kvm_run == MAP_FAILED) {
333 ret = -errno;
334 DPRINTF("mmap'ing vcpu state failed\n");
335 goto err;
336 }
337
338 if (s->coalesced_mmio && !s->coalesced_mmio_ring) {
339 s->coalesced_mmio_ring =
340 (void *)cpu->kvm_run + s->coalesced_mmio * PAGE_SIZE;
341 }
342
343 ret = kvm_arch_init_vcpu(cpu);
344err:
345 return ret;
346}
347
348
349
350
351
352static int kvm_mem_flags(MemoryRegion *mr)
353{
354 bool readonly = mr->readonly || memory_region_is_romd(mr);
355 int flags = 0;
356
357 if (memory_region_get_dirty_log_mask(mr) != 0) {
358 flags |= KVM_MEM_LOG_DIRTY_PAGES;
359 }
360 if (readonly && kvm_readonly_mem_allowed) {
361 flags |= KVM_MEM_READONLY;
362 }
363 return flags;
364}
365
366static int kvm_slot_update_flags(KVMMemoryListener *kml, KVMSlot *mem,
367 MemoryRegion *mr)
368{
369 int old_flags;
370
371 old_flags = mem->flags;
372 mem->flags = kvm_mem_flags(mr);
373
374
375 if (mem->flags == old_flags) {
376 return 0;
377 }
378
379 return kvm_set_user_memory_region(kml, mem);
380}
381
382static int kvm_section_update_flags(KVMMemoryListener *kml,
383 MemoryRegionSection *section)
384{
385 hwaddr phys_addr = section->offset_within_address_space;
386 ram_addr_t size = int128_get64(section->size);
387 KVMSlot *mem = kvm_lookup_matching_slot(kml, phys_addr, phys_addr + size);
388
389 if (mem == NULL) {
390 return 0;
391 } else {
392 return kvm_slot_update_flags(kml, mem, section->mr);
393 }
394}
395
396static void kvm_log_start(MemoryListener *listener,
397 MemoryRegionSection *section,
398 int old, int new)
399{
400 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
401 int r;
402
403 if (old != 0) {
404 return;
405 }
406
407 r = kvm_section_update_flags(kml, section);
408 if (r < 0) {
409 abort();
410 }
411}
412
413static void kvm_log_stop(MemoryListener *listener,
414 MemoryRegionSection *section,
415 int old, int new)
416{
417 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
418 int r;
419
420 if (new != 0) {
421 return;
422 }
423
424 r = kvm_section_update_flags(kml, section);
425 if (r < 0) {
426 abort();
427 }
428}
429
430
431static int kvm_get_dirty_pages_log_range(MemoryRegionSection *section,
432 unsigned long *bitmap)
433{
434 ram_addr_t start = section->offset_within_region +
435 memory_region_get_ram_addr(section->mr);
436 ram_addr_t pages = int128_get64(section->size) / getpagesize();
437
438 cpu_physical_memory_set_dirty_lebitmap(bitmap, start, pages);
439 return 0;
440}
441
442#define ALIGN(x, y) (((x)+(y)-1) & ~((y)-1))
443
444
445
446
447
448
449
450
451
452
453static int kvm_physical_sync_dirty_bitmap(KVMMemoryListener *kml,
454 MemoryRegionSection *section)
455{
456 KVMState *s = kvm_state;
457 unsigned long size, allocated_size = 0;
458 struct kvm_dirty_log d = {};
459 KVMSlot *mem;
460 int ret = 0;
461 hwaddr start_addr = section->offset_within_address_space;
462 hwaddr end_addr = start_addr + int128_get64(section->size);
463
464 d.dirty_bitmap = NULL;
465 while (start_addr < end_addr) {
466 mem = kvm_lookup_overlapping_slot(kml, start_addr, end_addr);
467 if (mem == NULL) {
468 break;
469 }
470
471
472
473
474
475
476
477
478
479
480
481
482
483 size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS),
484 64) / 8;
485 if (!d.dirty_bitmap) {
486 d.dirty_bitmap = g_malloc(size);
487 } else if (size > allocated_size) {
488 d.dirty_bitmap = g_realloc(d.dirty_bitmap, size);
489 }
490 allocated_size = size;
491 memset(d.dirty_bitmap, 0, allocated_size);
492
493 d.slot = mem->slot | (kml->as_id << 16);
494 if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
495 DPRINTF("ioctl failed %d\n", errno);
496 ret = -1;
497 break;
498 }
499
500 kvm_get_dirty_pages_log_range(section, d.dirty_bitmap);
501 start_addr = mem->start_addr + mem->memory_size;
502 }
503 g_free(d.dirty_bitmap);
504
505 return ret;
506}
507
508static void kvm_coalesce_mmio_region(MemoryListener *listener,
509 MemoryRegionSection *secion,
510 hwaddr start, hwaddr size)
511{
512 KVMState *s = kvm_state;
513
514 if (s->coalesced_mmio) {
515 struct kvm_coalesced_mmio_zone zone;
516
517 zone.addr = start;
518 zone.size = size;
519 zone.pad = 0;
520
521 (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
522 }
523}
524
525static void kvm_uncoalesce_mmio_region(MemoryListener *listener,
526 MemoryRegionSection *secion,
527 hwaddr start, hwaddr size)
528{
529 KVMState *s = kvm_state;
530
531 if (s->coalesced_mmio) {
532 struct kvm_coalesced_mmio_zone zone;
533
534 zone.addr = start;
535 zone.size = size;
536 zone.pad = 0;
537
538 (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
539 }
540}
541
542int kvm_check_extension(KVMState *s, unsigned int extension)
543{
544 int ret;
545
546 ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
547 if (ret < 0) {
548 ret = 0;
549 }
550
551 return ret;
552}
553
554int kvm_vm_check_extension(KVMState *s, unsigned int extension)
555{
556 int ret;
557
558 ret = kvm_vm_ioctl(s, KVM_CHECK_EXTENSION, extension);
559 if (ret < 0) {
560
561 ret = kvm_check_extension(s, extension);
562 }
563
564 return ret;
565}
566
567static uint32_t adjust_ioeventfd_endianness(uint32_t val, uint32_t size)
568{
569#if defined(HOST_WORDS_BIGENDIAN) != defined(TARGET_WORDS_BIGENDIAN)
570
571
572
573
574
575 switch (size) {
576 case 2:
577 val = bswap16(val);
578 break;
579 case 4:
580 val = bswap32(val);
581 break;
582 }
583#endif
584 return val;
585}
586
587static int kvm_set_ioeventfd_mmio(int fd, hwaddr addr, uint32_t val,
588 bool assign, uint32_t size, bool datamatch)
589{
590 int ret;
591 struct kvm_ioeventfd iofd = {
592 .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0,
593 .addr = addr,
594 .len = size,
595 .flags = 0,
596 .fd = fd,
597 };
598
599 if (!kvm_enabled()) {
600 return -ENOSYS;
601 }
602
603 if (datamatch) {
604 iofd.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH;
605 }
606 if (!assign) {
607 iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
608 }
609
610 ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd);
611
612 if (ret < 0) {
613 return -errno;
614 }
615
616 return 0;
617}
618
619static int kvm_set_ioeventfd_pio(int fd, uint16_t addr, uint16_t val,
620 bool assign, uint32_t size, bool datamatch)
621{
622 struct kvm_ioeventfd kick = {
623 .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0,
624 .addr = addr,
625 .flags = KVM_IOEVENTFD_FLAG_PIO,
626 .len = size,
627 .fd = fd,
628 };
629 int r;
630 if (!kvm_enabled()) {
631 return -ENOSYS;
632 }
633 if (datamatch) {
634 kick.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH;
635 }
636 if (!assign) {
637 kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
638 }
639 r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
640 if (r < 0) {
641 return r;
642 }
643 return 0;
644}
645
646
647static int kvm_check_many_ioeventfds(void)
648{
649
650
651
652
653
654
655
656#if defined(CONFIG_EVENTFD)
657 int ioeventfds[7];
658 int i, ret = 0;
659 for (i = 0; i < ARRAY_SIZE(ioeventfds); i++) {
660 ioeventfds[i] = eventfd(0, EFD_CLOEXEC);
661 if (ioeventfds[i] < 0) {
662 break;
663 }
664 ret = kvm_set_ioeventfd_pio(ioeventfds[i], 0, i, true, 2, true);
665 if (ret < 0) {
666 close(ioeventfds[i]);
667 break;
668 }
669 }
670
671
672 ret = i == ARRAY_SIZE(ioeventfds);
673
674 while (i-- > 0) {
675 kvm_set_ioeventfd_pio(ioeventfds[i], 0, i, false, 2, true);
676 close(ioeventfds[i]);
677 }
678 return ret;
679#else
680 return 0;
681#endif
682}
683
684static const KVMCapabilityInfo *
685kvm_check_extension_list(KVMState *s, const KVMCapabilityInfo *list)
686{
687 while (list->name) {
688 if (!kvm_check_extension(s, list->value)) {
689 return list;
690 }
691 list++;
692 }
693 return NULL;
694}
695
696static void kvm_set_phys_mem(KVMMemoryListener *kml,
697 MemoryRegionSection *section, bool add)
698{
699 KVMState *s = kvm_state;
700 KVMSlot *mem, old;
701 int err;
702 MemoryRegion *mr = section->mr;
703 bool writeable = !mr->readonly && !mr->rom_device;
704 hwaddr start_addr = section->offset_within_address_space;
705 ram_addr_t size = int128_get64(section->size);
706 void *ram = NULL;
707 unsigned delta;
708
709
710
711
712 delta = qemu_real_host_page_size - (start_addr & ~qemu_real_host_page_mask);
713 delta &= ~qemu_real_host_page_mask;
714 if (delta > size) {
715 return;
716 }
717 start_addr += delta;
718 size -= delta;
719 size &= qemu_real_host_page_mask;
720 if (!size || (start_addr & ~qemu_real_host_page_mask)) {
721 return;
722 }
723
724 if (!memory_region_is_ram(mr)) {
725 if (writeable || !kvm_readonly_mem_allowed) {
726 return;
727 } else if (!mr->romd_mode) {
728
729
730 add = false;
731 }
732 }
733
734 ram = memory_region_get_ram_ptr(mr) + section->offset_within_region + delta;
735
736 while (1) {
737 mem = kvm_lookup_overlapping_slot(kml, start_addr, start_addr + size);
738 if (!mem) {
739 break;
740 }
741
742 if (add && start_addr >= mem->start_addr &&
743 (start_addr + size <= mem->start_addr + mem->memory_size) &&
744 (ram - start_addr == mem->ram - mem->start_addr)) {
745
746
747 kvm_slot_update_flags(kml, mem, mr);
748 return;
749 }
750
751 old = *mem;
752
753 if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) {
754 kvm_physical_sync_dirty_bitmap(kml, section);
755 }
756
757
758 mem->memory_size = 0;
759 err = kvm_set_user_memory_region(kml, mem);
760 if (err) {
761 fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
762 __func__, strerror(-err));
763 abort();
764 }
765
766
767
768
769
770
771
772
773
774 if (s->broken_set_mem_region &&
775 old.start_addr == start_addr && old.memory_size < size && add) {
776 mem = kvm_alloc_slot(kml);
777 mem->memory_size = old.memory_size;
778 mem->start_addr = old.start_addr;
779 mem->ram = old.ram;
780 mem->flags = kvm_mem_flags(mr);
781
782 err = kvm_set_user_memory_region(kml, mem);
783 if (err) {
784 fprintf(stderr, "%s: error updating slot: %s\n", __func__,
785 strerror(-err));
786 abort();
787 }
788
789 start_addr += old.memory_size;
790 ram += old.memory_size;
791 size -= old.memory_size;
792 continue;
793 }
794
795
796 if (old.start_addr < start_addr) {
797 mem = kvm_alloc_slot(kml);
798 mem->memory_size = start_addr - old.start_addr;
799 mem->start_addr = old.start_addr;
800 mem->ram = old.ram;
801 mem->flags = kvm_mem_flags(mr);
802
803 err = kvm_set_user_memory_region(kml, mem);
804 if (err) {
805 fprintf(stderr, "%s: error registering prefix slot: %s\n",
806 __func__, strerror(-err));
807#ifdef TARGET_PPC
808 fprintf(stderr, "%s: This is probably because your kernel's " \
809 "PAGE_SIZE is too big. Please try to use 4k " \
810 "PAGE_SIZE!\n", __func__);
811#endif
812 abort();
813 }
814 }
815
816
817 if (old.start_addr + old.memory_size > start_addr + size) {
818 ram_addr_t size_delta;
819
820 mem = kvm_alloc_slot(kml);
821 mem->start_addr = start_addr + size;
822 size_delta = mem->start_addr - old.start_addr;
823 mem->memory_size = old.memory_size - size_delta;
824 mem->ram = old.ram + size_delta;
825 mem->flags = kvm_mem_flags(mr);
826
827 err = kvm_set_user_memory_region(kml, mem);
828 if (err) {
829 fprintf(stderr, "%s: error registering suffix slot: %s\n",
830 __func__, strerror(-err));
831 abort();
832 }
833 }
834 }
835
836
837 if (!size) {
838 return;
839 }
840 if (!add) {
841 return;
842 }
843 mem = kvm_alloc_slot(kml);
844 mem->memory_size = size;
845 mem->start_addr = start_addr;
846 mem->ram = ram;
847 mem->flags = kvm_mem_flags(mr);
848
849 err = kvm_set_user_memory_region(kml, mem);
850 if (err) {
851 fprintf(stderr, "%s: error registering slot: %s\n", __func__,
852 strerror(-err));
853 abort();
854 }
855}
856
857static void kvm_region_add(MemoryListener *listener,
858 MemoryRegionSection *section)
859{
860 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
861
862 memory_region_ref(section->mr);
863 kvm_set_phys_mem(kml, section, true);
864}
865
866static void kvm_region_del(MemoryListener *listener,
867 MemoryRegionSection *section)
868{
869 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
870
871 kvm_set_phys_mem(kml, section, false);
872 memory_region_unref(section->mr);
873}
874
875static void kvm_log_sync(MemoryListener *listener,
876 MemoryRegionSection *section)
877{
878 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
879 int r;
880
881 r = kvm_physical_sync_dirty_bitmap(kml, section);
882 if (r < 0) {
883 abort();
884 }
885}
886
887static void kvm_mem_ioeventfd_add(MemoryListener *listener,
888 MemoryRegionSection *section,
889 bool match_data, uint64_t data,
890 EventNotifier *e)
891{
892 int fd = event_notifier_get_fd(e);
893 int r;
894
895 r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
896 data, true, int128_get64(section->size),
897 match_data);
898 if (r < 0) {
899 fprintf(stderr, "%s: error adding ioeventfd: %s\n",
900 __func__, strerror(-r));
901 abort();
902 }
903}
904
905static void kvm_mem_ioeventfd_del(MemoryListener *listener,
906 MemoryRegionSection *section,
907 bool match_data, uint64_t data,
908 EventNotifier *e)
909{
910 int fd = event_notifier_get_fd(e);
911 int r;
912
913 r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
914 data, false, int128_get64(section->size),
915 match_data);
916 if (r < 0) {
917 abort();
918 }
919}
920
921static void kvm_io_ioeventfd_add(MemoryListener *listener,
922 MemoryRegionSection *section,
923 bool match_data, uint64_t data,
924 EventNotifier *e)
925{
926 int fd = event_notifier_get_fd(e);
927 int r;
928
929 r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
930 data, true, int128_get64(section->size),
931 match_data);
932 if (r < 0) {
933 fprintf(stderr, "%s: error adding ioeventfd: %s\n",
934 __func__, strerror(-r));
935 abort();
936 }
937}
938
939static void kvm_io_ioeventfd_del(MemoryListener *listener,
940 MemoryRegionSection *section,
941 bool match_data, uint64_t data,
942 EventNotifier *e)
943
944{
945 int fd = event_notifier_get_fd(e);
946 int r;
947
948 r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
949 data, false, int128_get64(section->size),
950 match_data);
951 if (r < 0) {
952 abort();
953 }
954}
955
956void kvm_memory_listener_register(KVMState *s, KVMMemoryListener *kml,
957 AddressSpace *as, int as_id)
958{
959 int i;
960
961 kml->slots = g_malloc0(s->nr_slots * sizeof(KVMSlot));
962 kml->as_id = as_id;
963
964 for (i = 0; i < s->nr_slots; i++) {
965 kml->slots[i].slot = i;
966 }
967
968 kml->listener.region_add = kvm_region_add;
969 kml->listener.region_del = kvm_region_del;
970 kml->listener.log_start = kvm_log_start;
971 kml->listener.log_stop = kvm_log_stop;
972 kml->listener.log_sync = kvm_log_sync;
973 kml->listener.priority = 10;
974
975 memory_listener_register(&kml->listener, as);
976}
977
978static MemoryListener kvm_io_listener = {
979 .eventfd_add = kvm_io_ioeventfd_add,
980 .eventfd_del = kvm_io_ioeventfd_del,
981 .priority = 10,
982};
983
984int kvm_set_irq(KVMState *s, int irq, int level)
985{
986 struct kvm_irq_level event;
987 int ret;
988
989 assert(kvm_async_interrupts_enabled());
990
991 event.level = level;
992 event.irq = irq;
993 ret = kvm_vm_ioctl(s, s->irq_set_ioctl, &event);
994 if (ret < 0) {
995 perror("kvm_set_irq");
996 abort();
997 }
998
999 return (s->irq_set_ioctl == KVM_IRQ_LINE) ? 1 : event.status;
1000}
1001
1002#ifdef KVM_CAP_IRQ_ROUTING
1003typedef struct KVMMSIRoute {
1004 struct kvm_irq_routing_entry kroute;
1005 QTAILQ_ENTRY(KVMMSIRoute) entry;
1006} KVMMSIRoute;
1007
1008static void set_gsi(KVMState *s, unsigned int gsi)
1009{
1010 set_bit(gsi, s->used_gsi_bitmap);
1011}
1012
1013static void clear_gsi(KVMState *s, unsigned int gsi)
1014{
1015 clear_bit(gsi, s->used_gsi_bitmap);
1016}
1017
1018void kvm_init_irq_routing(KVMState *s)
1019{
1020 int gsi_count, i;
1021
1022 gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING) - 1;
1023 if (gsi_count > 0) {
1024
1025 s->used_gsi_bitmap = bitmap_new(gsi_count);
1026 s->gsi_count = gsi_count;
1027 }
1028
1029 s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
1030 s->nr_allocated_irq_routes = 0;
1031
1032 if (!kvm_direct_msi_allowed) {
1033 for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) {
1034 QTAILQ_INIT(&s->msi_hashtab[i]);
1035 }
1036 }
1037
1038 kvm_arch_init_irq_routing(s);
1039}
1040
1041void kvm_irqchip_commit_routes(KVMState *s)
1042{
1043 int ret;
1044
1045 if (kvm_gsi_direct_mapping()) {
1046 return;
1047 }
1048
1049 if (!kvm_gsi_routing_enabled()) {
1050 return;
1051 }
1052
1053 s->irq_routes->flags = 0;
1054 trace_kvm_irqchip_commit_routes();
1055 ret = kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
1056 assert(ret == 0);
1057}
1058
1059static void kvm_add_routing_entry(KVMState *s,
1060 struct kvm_irq_routing_entry *entry)
1061{
1062 struct kvm_irq_routing_entry *new;
1063 int n, size;
1064
1065 if (s->irq_routes->nr == s->nr_allocated_irq_routes) {
1066 n = s->nr_allocated_irq_routes * 2;
1067 if (n < 64) {
1068 n = 64;
1069 }
1070 size = sizeof(struct kvm_irq_routing);
1071 size += n * sizeof(*new);
1072 s->irq_routes = g_realloc(s->irq_routes, size);
1073 s->nr_allocated_irq_routes = n;
1074 }
1075 n = s->irq_routes->nr++;
1076 new = &s->irq_routes->entries[n];
1077
1078 *new = *entry;
1079
1080 set_gsi(s, entry->gsi);
1081}
1082
1083static int kvm_update_routing_entry(KVMState *s,
1084 struct kvm_irq_routing_entry *new_entry)
1085{
1086 struct kvm_irq_routing_entry *entry;
1087 int n;
1088
1089 for (n = 0; n < s->irq_routes->nr; n++) {
1090 entry = &s->irq_routes->entries[n];
1091 if (entry->gsi != new_entry->gsi) {
1092 continue;
1093 }
1094
1095 if(!memcmp(entry, new_entry, sizeof *entry)) {
1096 return 0;
1097 }
1098
1099 *entry = *new_entry;
1100
1101 return 0;
1102 }
1103
1104 return -ESRCH;
1105}
1106
1107void kvm_irqchip_add_irq_route(KVMState *s, int irq, int irqchip, int pin)
1108{
1109 struct kvm_irq_routing_entry e = {};
1110
1111 assert(pin < s->gsi_count);
1112
1113 e.gsi = irq;
1114 e.type = KVM_IRQ_ROUTING_IRQCHIP;
1115 e.flags = 0;
1116 e.u.irqchip.irqchip = irqchip;
1117 e.u.irqchip.pin = pin;
1118 kvm_add_routing_entry(s, &e);
1119}
1120
1121void kvm_irqchip_release_virq(KVMState *s, int virq)
1122{
1123 struct kvm_irq_routing_entry *e;
1124 int i;
1125
1126 if (kvm_gsi_direct_mapping()) {
1127 return;
1128 }
1129
1130 for (i = 0; i < s->irq_routes->nr; i++) {
1131 e = &s->irq_routes->entries[i];
1132 if (e->gsi == virq) {
1133 s->irq_routes->nr--;
1134 *e = s->irq_routes->entries[s->irq_routes->nr];
1135 }
1136 }
1137 clear_gsi(s, virq);
1138 kvm_arch_release_virq_post(virq);
1139 trace_kvm_irqchip_release_virq(virq);
1140}
1141
1142static unsigned int kvm_hash_msi(uint32_t data)
1143{
1144
1145
1146 return data & 0xff;
1147}
1148
1149static void kvm_flush_dynamic_msi_routes(KVMState *s)
1150{
1151 KVMMSIRoute *route, *next;
1152 unsigned int hash;
1153
1154 for (hash = 0; hash < KVM_MSI_HASHTAB_SIZE; hash++) {
1155 QTAILQ_FOREACH_SAFE(route, &s->msi_hashtab[hash], entry, next) {
1156 kvm_irqchip_release_virq(s, route->kroute.gsi);
1157 QTAILQ_REMOVE(&s->msi_hashtab[hash], route, entry);
1158 g_free(route);
1159 }
1160 }
1161}
1162
1163static int kvm_irqchip_get_virq(KVMState *s)
1164{
1165 int next_virq;
1166
1167
1168
1169
1170
1171
1172
1173 if (!kvm_direct_msi_allowed && s->irq_routes->nr == s->gsi_count) {
1174 kvm_flush_dynamic_msi_routes(s);
1175 }
1176
1177
1178 next_virq = find_first_zero_bit(s->used_gsi_bitmap, s->gsi_count);
1179 if (next_virq >= s->gsi_count) {
1180 return -ENOSPC;
1181 } else {
1182 return next_virq;
1183 }
1184}
1185
1186static KVMMSIRoute *kvm_lookup_msi_route(KVMState *s, MSIMessage msg)
1187{
1188 unsigned int hash = kvm_hash_msi(msg.data);
1189 KVMMSIRoute *route;
1190
1191 QTAILQ_FOREACH(route, &s->msi_hashtab[hash], entry) {
1192 if (route->kroute.u.msi.address_lo == (uint32_t)msg.address &&
1193 route->kroute.u.msi.address_hi == (msg.address >> 32) &&
1194 route->kroute.u.msi.data == le32_to_cpu(msg.data)) {
1195 return route;
1196 }
1197 }
1198 return NULL;
1199}
1200
1201int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
1202{
1203 struct kvm_msi msi;
1204 KVMMSIRoute *route;
1205
1206 if (kvm_direct_msi_allowed) {
1207 msi.address_lo = (uint32_t)msg.address;
1208 msi.address_hi = msg.address >> 32;
1209 msi.data = le32_to_cpu(msg.data);
1210 msi.flags = 0;
1211 memset(msi.pad, 0, sizeof(msi.pad));
1212
1213 return kvm_vm_ioctl(s, KVM_SIGNAL_MSI, &msi);
1214 }
1215
1216 route = kvm_lookup_msi_route(s, msg);
1217 if (!route) {
1218 int virq;
1219
1220 virq = kvm_irqchip_get_virq(s);
1221 if (virq < 0) {
1222 return virq;
1223 }
1224
1225 route = g_malloc0(sizeof(KVMMSIRoute));
1226 route->kroute.gsi = virq;
1227 route->kroute.type = KVM_IRQ_ROUTING_MSI;
1228 route->kroute.flags = 0;
1229 route->kroute.u.msi.address_lo = (uint32_t)msg.address;
1230 route->kroute.u.msi.address_hi = msg.address >> 32;
1231 route->kroute.u.msi.data = le32_to_cpu(msg.data);
1232
1233 kvm_add_routing_entry(s, &route->kroute);
1234 kvm_irqchip_commit_routes(s);
1235
1236 QTAILQ_INSERT_TAIL(&s->msi_hashtab[kvm_hash_msi(msg.data)], route,
1237 entry);
1238 }
1239
1240 assert(route->kroute.type == KVM_IRQ_ROUTING_MSI);
1241
1242 return kvm_set_irq(s, route->kroute.gsi, 1);
1243}
1244
1245int kvm_irqchip_add_msi_route(KVMState *s, int vector, PCIDevice *dev)
1246{
1247 struct kvm_irq_routing_entry kroute = {};
1248 int virq;
1249 MSIMessage msg = {0, 0};
1250
1251 if (dev) {
1252 msg = pci_get_msi_message(dev, vector);
1253 }
1254
1255 if (kvm_gsi_direct_mapping()) {
1256 return kvm_arch_msi_data_to_gsi(msg.data);
1257 }
1258
1259 if (!kvm_gsi_routing_enabled()) {
1260 return -ENOSYS;
1261 }
1262
1263 virq = kvm_irqchip_get_virq(s);
1264 if (virq < 0) {
1265 return virq;
1266 }
1267
1268 kroute.gsi = virq;
1269 kroute.type = KVM_IRQ_ROUTING_MSI;
1270 kroute.flags = 0;
1271 kroute.u.msi.address_lo = (uint32_t)msg.address;
1272 kroute.u.msi.address_hi = msg.address >> 32;
1273 kroute.u.msi.data = le32_to_cpu(msg.data);
1274 if (kvm_msi_devid_required()) {
1275 kroute.flags = KVM_MSI_VALID_DEVID;
1276 kroute.u.msi.devid = pci_requester_id(dev);
1277 }
1278 if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) {
1279 kvm_irqchip_release_virq(s, virq);
1280 return -EINVAL;
1281 }
1282
1283 trace_kvm_irqchip_add_msi_route(dev ? dev->name : (char *)"N/A",
1284 vector, virq);
1285
1286 kvm_add_routing_entry(s, &kroute);
1287 kvm_arch_add_msi_route_post(&kroute, vector, dev);
1288 kvm_irqchip_commit_routes(s);
1289
1290 return virq;
1291}
1292
1293int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg,
1294 PCIDevice *dev)
1295{
1296 struct kvm_irq_routing_entry kroute = {};
1297
1298 if (kvm_gsi_direct_mapping()) {
1299 return 0;
1300 }
1301
1302 if (!kvm_irqchip_in_kernel()) {
1303 return -ENOSYS;
1304 }
1305
1306 kroute.gsi = virq;
1307 kroute.type = KVM_IRQ_ROUTING_MSI;
1308 kroute.flags = 0;
1309 kroute.u.msi.address_lo = (uint32_t)msg.address;
1310 kroute.u.msi.address_hi = msg.address >> 32;
1311 kroute.u.msi.data = le32_to_cpu(msg.data);
1312 if (kvm_msi_devid_required()) {
1313 kroute.flags = KVM_MSI_VALID_DEVID;
1314 kroute.u.msi.devid = pci_requester_id(dev);
1315 }
1316 if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) {
1317 return -EINVAL;
1318 }
1319
1320 trace_kvm_irqchip_update_msi_route(virq);
1321
1322 return kvm_update_routing_entry(s, &kroute);
1323}
1324
1325static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int rfd, int virq,
1326 bool assign)
1327{
1328 struct kvm_irqfd irqfd = {
1329 .fd = fd,
1330 .gsi = virq,
1331 .flags = assign ? 0 : KVM_IRQFD_FLAG_DEASSIGN,
1332 };
1333
1334 if (rfd != -1) {
1335 irqfd.flags |= KVM_IRQFD_FLAG_RESAMPLE;
1336 irqfd.resamplefd = rfd;
1337 }
1338
1339 if (!kvm_irqfds_enabled()) {
1340 return -ENOSYS;
1341 }
1342
1343 return kvm_vm_ioctl(s, KVM_IRQFD, &irqfd);
1344}
1345
1346int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
1347{
1348 struct kvm_irq_routing_entry kroute = {};
1349 int virq;
1350
1351 if (!kvm_gsi_routing_enabled()) {
1352 return -ENOSYS;
1353 }
1354
1355 virq = kvm_irqchip_get_virq(s);
1356 if (virq < 0) {
1357 return virq;
1358 }
1359
1360 kroute.gsi = virq;
1361 kroute.type = KVM_IRQ_ROUTING_S390_ADAPTER;
1362 kroute.flags = 0;
1363 kroute.u.adapter.summary_addr = adapter->summary_addr;
1364 kroute.u.adapter.ind_addr = adapter->ind_addr;
1365 kroute.u.adapter.summary_offset = adapter->summary_offset;
1366 kroute.u.adapter.ind_offset = adapter->ind_offset;
1367 kroute.u.adapter.adapter_id = adapter->adapter_id;
1368
1369 kvm_add_routing_entry(s, &kroute);
1370
1371 return virq;
1372}
1373
1374int kvm_irqchip_add_hv_sint_route(KVMState *s, uint32_t vcpu, uint32_t sint)
1375{
1376 struct kvm_irq_routing_entry kroute = {};
1377 int virq;
1378
1379 if (!kvm_gsi_routing_enabled()) {
1380 return -ENOSYS;
1381 }
1382 if (!kvm_check_extension(s, KVM_CAP_HYPERV_SYNIC)) {
1383 return -ENOSYS;
1384 }
1385 virq = kvm_irqchip_get_virq(s);
1386 if (virq < 0) {
1387 return virq;
1388 }
1389
1390 kroute.gsi = virq;
1391 kroute.type = KVM_IRQ_ROUTING_HV_SINT;
1392 kroute.flags = 0;
1393 kroute.u.hv_sint.vcpu = vcpu;
1394 kroute.u.hv_sint.sint = sint;
1395
1396 kvm_add_routing_entry(s, &kroute);
1397 kvm_irqchip_commit_routes(s);
1398
1399 return virq;
1400}
1401
1402#else
1403
1404void kvm_init_irq_routing(KVMState *s)
1405{
1406}
1407
1408void kvm_irqchip_release_virq(KVMState *s, int virq)
1409{
1410}
1411
1412int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
1413{
1414 abort();
1415}
1416
1417int kvm_irqchip_add_msi_route(KVMState *s, int vector, PCIDevice *dev)
1418{
1419 return -ENOSYS;
1420}
1421
1422int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
1423{
1424 return -ENOSYS;
1425}
1426
1427int kvm_irqchip_add_hv_sint_route(KVMState *s, uint32_t vcpu, uint32_t sint)
1428{
1429 return -ENOSYS;
1430}
1431
1432static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
1433{
1434 abort();
1435}
1436
1437int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
1438{
1439 return -ENOSYS;
1440}
1441#endif
1442
1443int kvm_irqchip_add_irqfd_notifier_gsi(KVMState *s, EventNotifier *n,
1444 EventNotifier *rn, int virq)
1445{
1446 return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n),
1447 rn ? event_notifier_get_fd(rn) : -1, virq, true);
1448}
1449
1450int kvm_irqchip_remove_irqfd_notifier_gsi(KVMState *s, EventNotifier *n,
1451 int virq)
1452{
1453 return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n), -1, virq,
1454 false);
1455}
1456
1457int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n,
1458 EventNotifier *rn, qemu_irq irq)
1459{
1460 gpointer key, gsi;
1461 gboolean found = g_hash_table_lookup_extended(s->gsimap, irq, &key, &gsi);
1462
1463 if (!found) {
1464 return -ENXIO;
1465 }
1466 return kvm_irqchip_add_irqfd_notifier_gsi(s, n, rn, GPOINTER_TO_INT(gsi));
1467}
1468
1469int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n,
1470 qemu_irq irq)
1471{
1472 gpointer key, gsi;
1473 gboolean found = g_hash_table_lookup_extended(s->gsimap, irq, &key, &gsi);
1474
1475 if (!found) {
1476 return -ENXIO;
1477 }
1478 return kvm_irqchip_remove_irqfd_notifier_gsi(s, n, GPOINTER_TO_INT(gsi));
1479}
1480
1481void kvm_irqchip_set_qemuirq_gsi(KVMState *s, qemu_irq irq, int gsi)
1482{
1483 g_hash_table_insert(s->gsimap, irq, GINT_TO_POINTER(gsi));
1484}
1485
1486static void kvm_irqchip_create(MachineState *machine, KVMState *s)
1487{
1488 int ret;
1489
1490 if (kvm_check_extension(s, KVM_CAP_IRQCHIP)) {
1491 ;
1492 } else if (kvm_check_extension(s, KVM_CAP_S390_IRQCHIP)) {
1493 ret = kvm_vm_enable_cap(s, KVM_CAP_S390_IRQCHIP, 0);
1494 if (ret < 0) {
1495 fprintf(stderr, "Enable kernel irqchip failed: %s\n", strerror(-ret));
1496 exit(1);
1497 }
1498 } else {
1499 return;
1500 }
1501
1502
1503
1504 ret = kvm_arch_irqchip_create(machine, s);
1505 if (ret == 0) {
1506 if (machine_kernel_irqchip_split(machine)) {
1507 perror("Split IRQ chip mode not supported.");
1508 exit(1);
1509 } else {
1510 ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP);
1511 }
1512 }
1513 if (ret < 0) {
1514 fprintf(stderr, "Create kernel irqchip failed: %s\n", strerror(-ret));
1515 exit(1);
1516 }
1517
1518 kvm_kernel_irqchip = true;
1519
1520
1521
1522 kvm_async_interrupts_allowed = true;
1523 kvm_halt_in_kernel_allowed = true;
1524
1525 kvm_init_irq_routing(s);
1526
1527 s->gsimap = g_hash_table_new(g_direct_hash, g_direct_equal);
1528}
1529
1530
1531
1532
1533
1534static int kvm_recommended_vcpus(KVMState *s)
1535{
1536 int ret = kvm_check_extension(s, KVM_CAP_NR_VCPUS);
1537 return (ret) ? ret : 4;
1538}
1539
1540static int kvm_max_vcpus(KVMState *s)
1541{
1542 int ret = kvm_check_extension(s, KVM_CAP_MAX_VCPUS);
1543 return (ret) ? ret : kvm_recommended_vcpus(s);
1544}
1545
1546static int kvm_max_vcpu_id(KVMState *s)
1547{
1548 int ret = kvm_check_extension(s, KVM_CAP_MAX_VCPU_ID);
1549 return (ret) ? ret : kvm_max_vcpus(s);
1550}
1551
1552bool kvm_vcpu_id_is_valid(int vcpu_id)
1553{
1554 KVMState *s = KVM_STATE(current_machine->accelerator);
1555 return vcpu_id >= 0 && vcpu_id < kvm_max_vcpu_id(s);
1556}
1557
1558static int kvm_init(MachineState *ms)
1559{
1560 MachineClass *mc = MACHINE_GET_CLASS(ms);
1561 static const char upgrade_note[] =
1562 "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
1563 "(see http://sourceforge.net/projects/kvm).\n";
1564 struct {
1565 const char *name;
1566 int num;
1567 } num_cpus[] = {
1568 { "SMP", smp_cpus },
1569 { "hotpluggable", max_cpus },
1570 { NULL, }
1571 }, *nc = num_cpus;
1572 int soft_vcpus_limit, hard_vcpus_limit;
1573 KVMState *s;
1574 const KVMCapabilityInfo *missing_cap;
1575 int ret;
1576 int type = 0;
1577 const char *kvm_type;
1578
1579 s = KVM_STATE(ms->accelerator);
1580
1581
1582
1583
1584
1585
1586
1587 assert(TARGET_PAGE_SIZE <= getpagesize());
1588
1589 s->sigmask_len = 8;
1590
1591#ifdef KVM_CAP_SET_GUEST_DEBUG
1592 QTAILQ_INIT(&s->kvm_sw_breakpoints);
1593#endif
1594 QLIST_INIT(&s->kvm_parked_vcpus);
1595 s->vmfd = -1;
1596 s->fd = qemu_open("/dev/kvm", O_RDWR);
1597 if (s->fd == -1) {
1598 fprintf(stderr, "Could not access KVM kernel module: %m\n");
1599 ret = -errno;
1600 goto err;
1601 }
1602
1603 ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
1604 if (ret < KVM_API_VERSION) {
1605 if (ret >= 0) {
1606 ret = -EINVAL;
1607 }
1608 fprintf(stderr, "kvm version too old\n");
1609 goto err;
1610 }
1611
1612 if (ret > KVM_API_VERSION) {
1613 ret = -EINVAL;
1614 fprintf(stderr, "kvm version not supported\n");
1615 goto err;
1616 }
1617
1618 kvm_immediate_exit = kvm_check_extension(s, KVM_CAP_IMMEDIATE_EXIT);
1619 s->nr_slots = kvm_check_extension(s, KVM_CAP_NR_MEMSLOTS);
1620
1621
1622 if (!s->nr_slots) {
1623 s->nr_slots = 32;
1624 }
1625
1626
1627 soft_vcpus_limit = kvm_recommended_vcpus(s);
1628 hard_vcpus_limit = kvm_max_vcpus(s);
1629
1630 while (nc->name) {
1631 if (nc->num > soft_vcpus_limit) {
1632 fprintf(stderr,
1633 "Warning: Number of %s cpus requested (%d) exceeds "
1634 "the recommended cpus supported by KVM (%d)\n",
1635 nc->name, nc->num, soft_vcpus_limit);
1636
1637 if (nc->num > hard_vcpus_limit) {
1638 fprintf(stderr, "Number of %s cpus requested (%d) exceeds "
1639 "the maximum cpus supported by KVM (%d)\n",
1640 nc->name, nc->num, hard_vcpus_limit);
1641 exit(1);
1642 }
1643 }
1644 nc++;
1645 }
1646
1647 kvm_type = qemu_opt_get(qemu_get_machine_opts(), "kvm-type");
1648 if (mc->kvm_type) {
1649 type = mc->kvm_type(kvm_type);
1650 } else if (kvm_type) {
1651 ret = -EINVAL;
1652 fprintf(stderr, "Invalid argument kvm-type=%s\n", kvm_type);
1653 goto err;
1654 }
1655
1656 do {
1657 ret = kvm_ioctl(s, KVM_CREATE_VM, type);
1658 } while (ret == -EINTR);
1659
1660 if (ret < 0) {
1661 fprintf(stderr, "ioctl(KVM_CREATE_VM) failed: %d %s\n", -ret,
1662 strerror(-ret));
1663
1664#ifdef TARGET_S390X
1665 if (ret == -EINVAL) {
1666 fprintf(stderr,
1667 "Host kernel setup problem detected. Please verify:\n");
1668 fprintf(stderr, "- for kernels supporting the switch_amode or"
1669 " user_mode parameters, whether\n");
1670 fprintf(stderr,
1671 " user space is running in primary address space\n");
1672 fprintf(stderr,
1673 "- for kernels supporting the vm.allocate_pgste sysctl, "
1674 "whether it is enabled\n");
1675 }
1676#endif
1677 goto err;
1678 }
1679
1680 s->vmfd = ret;
1681 missing_cap = kvm_check_extension_list(s, kvm_required_capabilites);
1682 if (!missing_cap) {
1683 missing_cap =
1684 kvm_check_extension_list(s, kvm_arch_required_capabilities);
1685 }
1686 if (missing_cap) {
1687 ret = -EINVAL;
1688 fprintf(stderr, "kvm does not support %s\n%s",
1689 missing_cap->name, upgrade_note);
1690 goto err;
1691 }
1692
1693 s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
1694
1695 s->broken_set_mem_region = 1;
1696 ret = kvm_check_extension(s, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
1697 if (ret > 0) {
1698 s->broken_set_mem_region = 0;
1699 }
1700
1701#ifdef KVM_CAP_VCPU_EVENTS
1702 s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
1703#endif
1704
1705 s->robust_singlestep =
1706 kvm_check_extension(s, KVM_CAP_X86_ROBUST_SINGLESTEP);
1707
1708#ifdef KVM_CAP_DEBUGREGS
1709 s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
1710#endif
1711
1712#ifdef KVM_CAP_IRQ_ROUTING
1713 kvm_direct_msi_allowed = (kvm_check_extension(s, KVM_CAP_SIGNAL_MSI) > 0);
1714#endif
1715
1716 s->intx_set_mask = kvm_check_extension(s, KVM_CAP_PCI_2_3);
1717
1718 s->irq_set_ioctl = KVM_IRQ_LINE;
1719 if (kvm_check_extension(s, KVM_CAP_IRQ_INJECT_STATUS)) {
1720 s->irq_set_ioctl = KVM_IRQ_LINE_STATUS;
1721 }
1722
1723#ifdef KVM_CAP_READONLY_MEM
1724 kvm_readonly_mem_allowed =
1725 (kvm_check_extension(s, KVM_CAP_READONLY_MEM) > 0);
1726#endif
1727
1728 kvm_eventfds_allowed =
1729 (kvm_check_extension(s, KVM_CAP_IOEVENTFD) > 0);
1730
1731 kvm_irqfds_allowed =
1732 (kvm_check_extension(s, KVM_CAP_IRQFD) > 0);
1733
1734 kvm_resamplefds_allowed =
1735 (kvm_check_extension(s, KVM_CAP_IRQFD_RESAMPLE) > 0);
1736
1737 kvm_vm_attributes_allowed =
1738 (kvm_check_extension(s, KVM_CAP_VM_ATTRIBUTES) > 0);
1739
1740 kvm_ioeventfd_any_length_allowed =
1741 (kvm_check_extension(s, KVM_CAP_IOEVENTFD_ANY_LENGTH) > 0);
1742
1743 kvm_state = s;
1744
1745 ret = kvm_arch_init(ms, s);
1746 if (ret < 0) {
1747 goto err;
1748 }
1749
1750 if (machine_kernel_irqchip_allowed(ms)) {
1751 kvm_irqchip_create(ms, s);
1752 }
1753
1754 if (kvm_eventfds_allowed) {
1755 s->memory_listener.listener.eventfd_add = kvm_mem_ioeventfd_add;
1756 s->memory_listener.listener.eventfd_del = kvm_mem_ioeventfd_del;
1757 }
1758 s->memory_listener.listener.coalesced_mmio_add = kvm_coalesce_mmio_region;
1759 s->memory_listener.listener.coalesced_mmio_del = kvm_uncoalesce_mmio_region;
1760
1761 kvm_memory_listener_register(s, &s->memory_listener,
1762 &address_space_memory, 0);
1763 memory_listener_register(&kvm_io_listener,
1764 &address_space_io);
1765
1766 s->many_ioeventfds = kvm_check_many_ioeventfds();
1767
1768 return 0;
1769
1770err:
1771 assert(ret < 0);
1772 if (s->vmfd >= 0) {
1773 close(s->vmfd);
1774 }
1775 if (s->fd != -1) {
1776 close(s->fd);
1777 }
1778 g_free(s->memory_listener.slots);
1779
1780 return ret;
1781}
1782
1783void kvm_set_sigmask_len(KVMState *s, unsigned int sigmask_len)
1784{
1785 s->sigmask_len = sigmask_len;
1786}
1787
1788static void kvm_handle_io(uint16_t port, MemTxAttrs attrs, void *data, int direction,
1789 int size, uint32_t count)
1790{
1791 int i;
1792 uint8_t *ptr = data;
1793
1794 for (i = 0; i < count; i++) {
1795 address_space_rw(&address_space_io, port, attrs,
1796 ptr, size,
1797 direction == KVM_EXIT_IO_OUT);
1798 ptr += size;
1799 }
1800}
1801
1802static int kvm_handle_internal_error(CPUState *cpu, struct kvm_run *run)
1803{
1804 fprintf(stderr, "KVM internal error. Suberror: %d\n",
1805 run->internal.suberror);
1806
1807 if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
1808 int i;
1809
1810 for (i = 0; i < run->internal.ndata; ++i) {
1811 fprintf(stderr, "extra data[%d]: %"PRIx64"\n",
1812 i, (uint64_t)run->internal.data[i]);
1813 }
1814 }
1815 if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
1816 fprintf(stderr, "emulation failure\n");
1817 if (!kvm_arch_stop_on_emulation_error(cpu)) {
1818 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_CODE);
1819 return EXCP_INTERRUPT;
1820 }
1821 }
1822
1823
1824
1825 return -1;
1826}
1827
1828void kvm_flush_coalesced_mmio_buffer(void)
1829{
1830 KVMState *s = kvm_state;
1831
1832 if (s->coalesced_flush_in_progress) {
1833 return;
1834 }
1835
1836 s->coalesced_flush_in_progress = true;
1837
1838 if (s->coalesced_mmio_ring) {
1839 struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
1840 while (ring->first != ring->last) {
1841 struct kvm_coalesced_mmio *ent;
1842
1843 ent = &ring->coalesced_mmio[ring->first];
1844
1845 cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
1846 smp_wmb();
1847 ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
1848 }
1849 }
1850
1851 s->coalesced_flush_in_progress = false;
1852}
1853
1854static void do_kvm_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg)
1855{
1856 if (!cpu->vcpu_dirty) {
1857 kvm_arch_get_registers(cpu);
1858 cpu->vcpu_dirty = true;
1859 }
1860}
1861
1862void kvm_cpu_synchronize_state(CPUState *cpu)
1863{
1864 if (!cpu->vcpu_dirty) {
1865 run_on_cpu(cpu, do_kvm_cpu_synchronize_state, RUN_ON_CPU_NULL);
1866 }
1867}
1868
1869static void do_kvm_cpu_synchronize_post_reset(CPUState *cpu, run_on_cpu_data arg)
1870{
1871 kvm_arch_put_registers(cpu, KVM_PUT_RESET_STATE);
1872 cpu->vcpu_dirty = false;
1873}
1874
1875void kvm_cpu_synchronize_post_reset(CPUState *cpu)
1876{
1877 run_on_cpu(cpu, do_kvm_cpu_synchronize_post_reset, RUN_ON_CPU_NULL);
1878}
1879
1880static void do_kvm_cpu_synchronize_post_init(CPUState *cpu, run_on_cpu_data arg)
1881{
1882 kvm_arch_put_registers(cpu, KVM_PUT_FULL_STATE);
1883 cpu->vcpu_dirty = false;
1884}
1885
1886void kvm_cpu_synchronize_post_init(CPUState *cpu)
1887{
1888 run_on_cpu(cpu, do_kvm_cpu_synchronize_post_init, RUN_ON_CPU_NULL);
1889}
1890
1891static void do_kvm_cpu_synchronize_pre_loadvm(CPUState *cpu, run_on_cpu_data arg)
1892{
1893 cpu->vcpu_dirty = true;
1894}
1895
1896void kvm_cpu_synchronize_pre_loadvm(CPUState *cpu)
1897{
1898 run_on_cpu(cpu, do_kvm_cpu_synchronize_pre_loadvm, RUN_ON_CPU_NULL);
1899}
1900
1901#ifdef KVM_HAVE_MCE_INJECTION
1902static __thread void *pending_sigbus_addr;
1903static __thread int pending_sigbus_code;
1904static __thread bool have_sigbus_pending;
1905#endif
1906
1907static void kvm_cpu_kick(CPUState *cpu)
1908{
1909 atomic_set(&cpu->kvm_run->immediate_exit, 1);
1910}
1911
1912static void kvm_cpu_kick_self(void)
1913{
1914 if (kvm_immediate_exit) {
1915 kvm_cpu_kick(current_cpu);
1916 } else {
1917 qemu_cpu_kick_self();
1918 }
1919}
1920
1921static void kvm_eat_signals(CPUState *cpu)
1922{
1923 struct timespec ts = { 0, 0 };
1924 siginfo_t siginfo;
1925 sigset_t waitset;
1926 sigset_t chkset;
1927 int r;
1928
1929 if (kvm_immediate_exit) {
1930 atomic_set(&cpu->kvm_run->immediate_exit, 0);
1931
1932
1933
1934 smp_wmb();
1935 return;
1936 }
1937
1938 sigemptyset(&waitset);
1939 sigaddset(&waitset, SIG_IPI);
1940
1941 do {
1942 r = sigtimedwait(&waitset, &siginfo, &ts);
1943 if (r == -1 && !(errno == EAGAIN || errno == EINTR)) {
1944 perror("sigtimedwait");
1945 exit(1);
1946 }
1947
1948 r = sigpending(&chkset);
1949 if (r == -1) {
1950 perror("sigpending");
1951 exit(1);
1952 }
1953 } while (sigismember(&chkset, SIG_IPI));
1954}
1955
1956int kvm_cpu_exec(CPUState *cpu)
1957{
1958 struct kvm_run *run = cpu->kvm_run;
1959 int ret, run_ret;
1960
1961 DPRINTF("kvm_cpu_exec()\n");
1962
1963 if (kvm_arch_process_async_events(cpu)) {
1964 atomic_set(&cpu->exit_request, 0);
1965 return EXCP_HLT;
1966 }
1967
1968 qemu_mutex_unlock_iothread();
1969 cpu_exec_start(cpu);
1970
1971 do {
1972 MemTxAttrs attrs;
1973
1974 if (cpu->vcpu_dirty) {
1975 kvm_arch_put_registers(cpu, KVM_PUT_RUNTIME_STATE);
1976 cpu->vcpu_dirty = false;
1977 }
1978
1979 kvm_arch_pre_run(cpu, run);
1980 if (atomic_read(&cpu->exit_request)) {
1981 DPRINTF("interrupt exit requested\n");
1982
1983
1984
1985
1986
1987 kvm_cpu_kick_self();
1988 }
1989
1990
1991
1992
1993 smp_rmb();
1994
1995 run_ret = kvm_vcpu_ioctl(cpu, KVM_RUN, 0);
1996
1997 attrs = kvm_arch_post_run(cpu, run);
1998
1999#ifdef KVM_HAVE_MCE_INJECTION
2000 if (unlikely(have_sigbus_pending)) {
2001 qemu_mutex_lock_iothread();
2002 kvm_arch_on_sigbus_vcpu(cpu, pending_sigbus_code,
2003 pending_sigbus_addr);
2004 have_sigbus_pending = false;
2005 qemu_mutex_unlock_iothread();
2006 }
2007#endif
2008
2009 if (run_ret < 0) {
2010 if (run_ret == -EINTR || run_ret == -EAGAIN) {
2011 DPRINTF("io window exit\n");
2012 kvm_eat_signals(cpu);
2013 ret = EXCP_INTERRUPT;
2014 break;
2015 }
2016 fprintf(stderr, "error: kvm run failed %s\n",
2017 strerror(-run_ret));
2018#ifdef TARGET_PPC
2019 if (run_ret == -EBUSY) {
2020 fprintf(stderr,
2021 "This is probably because your SMT is enabled.\n"
2022 "VCPU can only run on primary threads with all "
2023 "secondary threads offline.\n");
2024 }
2025#endif
2026 ret = -1;
2027 break;
2028 }
2029
2030 trace_kvm_run_exit(cpu->cpu_index, run->exit_reason);
2031 switch (run->exit_reason) {
2032 case KVM_EXIT_IO:
2033 DPRINTF("handle_io\n");
2034
2035 kvm_handle_io(run->io.port, attrs,
2036 (uint8_t *)run + run->io.data_offset,
2037 run->io.direction,
2038 run->io.size,
2039 run->io.count);
2040 ret = 0;
2041 break;
2042 case KVM_EXIT_MMIO:
2043 DPRINTF("handle_mmio\n");
2044
2045 address_space_rw(&address_space_memory,
2046 run->mmio.phys_addr, attrs,
2047 run->mmio.data,
2048 run->mmio.len,
2049 run->mmio.is_write);
2050 ret = 0;
2051 break;
2052 case KVM_EXIT_IRQ_WINDOW_OPEN:
2053 DPRINTF("irq_window_open\n");
2054 ret = EXCP_INTERRUPT;
2055 break;
2056 case KVM_EXIT_SHUTDOWN:
2057 DPRINTF("shutdown\n");
2058 qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET);
2059 ret = EXCP_INTERRUPT;
2060 break;
2061 case KVM_EXIT_UNKNOWN:
2062 fprintf(stderr, "KVM: unknown exit, hardware reason %" PRIx64 "\n",
2063 (uint64_t)run->hw.hardware_exit_reason);
2064 ret = -1;
2065 break;
2066 case KVM_EXIT_INTERNAL_ERROR:
2067 ret = kvm_handle_internal_error(cpu, run);
2068 break;
2069 case KVM_EXIT_SYSTEM_EVENT:
2070 switch (run->system_event.type) {
2071 case KVM_SYSTEM_EVENT_SHUTDOWN:
2072 qemu_system_shutdown_request(SHUTDOWN_CAUSE_GUEST_SHUTDOWN);
2073 ret = EXCP_INTERRUPT;
2074 break;
2075 case KVM_SYSTEM_EVENT_RESET:
2076 qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET);
2077 ret = EXCP_INTERRUPT;
2078 break;
2079 case KVM_SYSTEM_EVENT_CRASH:
2080 kvm_cpu_synchronize_state(cpu);
2081 qemu_mutex_lock_iothread();
2082 qemu_system_guest_panicked(cpu_get_crash_info(cpu));
2083 qemu_mutex_unlock_iothread();
2084 ret = 0;
2085 break;
2086 default:
2087 DPRINTF("kvm_arch_handle_exit\n");
2088 ret = kvm_arch_handle_exit(cpu, run);
2089 break;
2090 }
2091 break;
2092 default:
2093 DPRINTF("kvm_arch_handle_exit\n");
2094 ret = kvm_arch_handle_exit(cpu, run);
2095 break;
2096 }
2097 } while (ret == 0);
2098
2099 cpu_exec_end(cpu);
2100 qemu_mutex_lock_iothread();
2101
2102 if (ret < 0) {
2103 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_CODE);
2104 vm_stop(RUN_STATE_INTERNAL_ERROR);
2105 }
2106
2107 atomic_set(&cpu->exit_request, 0);
2108 return ret;
2109}
2110
2111int kvm_ioctl(KVMState *s, int type, ...)
2112{
2113 int ret;
2114 void *arg;
2115 va_list ap;
2116
2117 va_start(ap, type);
2118 arg = va_arg(ap, void *);
2119 va_end(ap);
2120
2121 trace_kvm_ioctl(type, arg);
2122 ret = ioctl(s->fd, type, arg);
2123 if (ret == -1) {
2124 ret = -errno;
2125 }
2126 return ret;
2127}
2128
2129int kvm_vm_ioctl(KVMState *s, int type, ...)
2130{
2131 int ret;
2132 void *arg;
2133 va_list ap;
2134
2135 va_start(ap, type);
2136 arg = va_arg(ap, void *);
2137 va_end(ap);
2138
2139 trace_kvm_vm_ioctl(type, arg);
2140 ret = ioctl(s->vmfd, type, arg);
2141 if (ret == -1) {
2142 ret = -errno;
2143 }
2144 return ret;
2145}
2146
2147int kvm_vcpu_ioctl(CPUState *cpu, int type, ...)
2148{
2149 int ret;
2150 void *arg;
2151 va_list ap;
2152
2153 va_start(ap, type);
2154 arg = va_arg(ap, void *);
2155 va_end(ap);
2156
2157 trace_kvm_vcpu_ioctl(cpu->cpu_index, type, arg);
2158 ret = ioctl(cpu->kvm_fd, type, arg);
2159 if (ret == -1) {
2160 ret = -errno;
2161 }
2162 return ret;
2163}
2164
2165int kvm_device_ioctl(int fd, int type, ...)
2166{
2167 int ret;
2168 void *arg;
2169 va_list ap;
2170
2171 va_start(ap, type);
2172 arg = va_arg(ap, void *);
2173 va_end(ap);
2174
2175 trace_kvm_device_ioctl(fd, type, arg);
2176 ret = ioctl(fd, type, arg);
2177 if (ret == -1) {
2178 ret = -errno;
2179 }
2180 return ret;
2181}
2182
2183int kvm_vm_check_attr(KVMState *s, uint32_t group, uint64_t attr)
2184{
2185 int ret;
2186 struct kvm_device_attr attribute = {
2187 .group = group,
2188 .attr = attr,
2189 };
2190
2191 if (!kvm_vm_attributes_allowed) {
2192 return 0;
2193 }
2194
2195 ret = kvm_vm_ioctl(s, KVM_HAS_DEVICE_ATTR, &attribute);
2196
2197 return ret ? 0 : 1;
2198}
2199
2200int kvm_device_check_attr(int dev_fd, uint32_t group, uint64_t attr)
2201{
2202 struct kvm_device_attr attribute = {
2203 .group = group,
2204 .attr = attr,
2205 .flags = 0,
2206 };
2207
2208 return kvm_device_ioctl(dev_fd, KVM_HAS_DEVICE_ATTR, &attribute) ? 0 : 1;
2209}
2210
2211int kvm_device_access(int fd, int group, uint64_t attr,
2212 void *val, bool write, Error **errp)
2213{
2214 struct kvm_device_attr kvmattr;
2215 int err;
2216
2217 kvmattr.flags = 0;
2218 kvmattr.group = group;
2219 kvmattr.attr = attr;
2220 kvmattr.addr = (uintptr_t)val;
2221
2222 err = kvm_device_ioctl(fd,
2223 write ? KVM_SET_DEVICE_ATTR : KVM_GET_DEVICE_ATTR,
2224 &kvmattr);
2225 if (err < 0) {
2226 error_setg_errno(errp, -err,
2227 "KVM_%s_DEVICE_ATTR failed: Group %d "
2228 "attr 0x%016" PRIx64,
2229 write ? "SET" : "GET", group, attr);
2230 }
2231 return err;
2232}
2233
2234
2235int kvm_has_sync_mmu(void)
2236{
2237 return kvm_check_extension(kvm_state, KVM_CAP_SYNC_MMU);
2238}
2239
2240int kvm_has_vcpu_events(void)
2241{
2242 return kvm_state->vcpu_events;
2243}
2244
2245int kvm_has_robust_singlestep(void)
2246{
2247 return kvm_state->robust_singlestep;
2248}
2249
2250int kvm_has_debugregs(void)
2251{
2252 return kvm_state->debugregs;
2253}
2254
2255int kvm_has_many_ioeventfds(void)
2256{
2257 if (!kvm_enabled()) {
2258 return 0;
2259 }
2260 return kvm_state->many_ioeventfds;
2261}
2262
2263int kvm_has_gsi_routing(void)
2264{
2265#ifdef KVM_CAP_IRQ_ROUTING
2266 return kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING);
2267#else
2268 return false;
2269#endif
2270}
2271
2272int kvm_has_intx_set_mask(void)
2273{
2274 return kvm_state->intx_set_mask;
2275}
2276
2277bool kvm_arm_supports_user_irq(void)
2278{
2279 return kvm_check_extension(kvm_state, KVM_CAP_ARM_USER_IRQ);
2280}
2281
2282#ifdef KVM_CAP_SET_GUEST_DEBUG
2283struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *cpu,
2284 target_ulong pc)
2285{
2286 struct kvm_sw_breakpoint *bp;
2287
2288 QTAILQ_FOREACH(bp, &cpu->kvm_state->kvm_sw_breakpoints, entry) {
2289 if (bp->pc == pc) {
2290 return bp;
2291 }
2292 }
2293 return NULL;
2294}
2295
2296int kvm_sw_breakpoints_active(CPUState *cpu)
2297{
2298 return !QTAILQ_EMPTY(&cpu->kvm_state->kvm_sw_breakpoints);
2299}
2300
2301struct kvm_set_guest_debug_data {
2302 struct kvm_guest_debug dbg;
2303 int err;
2304};
2305
2306static void kvm_invoke_set_guest_debug(CPUState *cpu, run_on_cpu_data data)
2307{
2308 struct kvm_set_guest_debug_data *dbg_data =
2309 (struct kvm_set_guest_debug_data *) data.host_ptr;
2310
2311 dbg_data->err = kvm_vcpu_ioctl(cpu, KVM_SET_GUEST_DEBUG,
2312 &dbg_data->dbg);
2313}
2314
2315int kvm_update_guest_debug(CPUState *cpu, unsigned long reinject_trap)
2316{
2317 struct kvm_set_guest_debug_data data;
2318
2319 data.dbg.control = reinject_trap;
2320
2321 if (cpu->singlestep_enabled) {
2322 data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
2323 }
2324 kvm_arch_update_guest_debug(cpu, &data.dbg);
2325
2326 run_on_cpu(cpu, kvm_invoke_set_guest_debug,
2327 RUN_ON_CPU_HOST_PTR(&data));
2328 return data.err;
2329}
2330
2331int kvm_insert_breakpoint(CPUState *cpu, target_ulong addr,
2332 target_ulong len, int type)
2333{
2334 struct kvm_sw_breakpoint *bp;
2335 int err;
2336
2337 if (type == GDB_BREAKPOINT_SW) {
2338 bp = kvm_find_sw_breakpoint(cpu, addr);
2339 if (bp) {
2340 bp->use_count++;
2341 return 0;
2342 }
2343
2344 bp = g_malloc(sizeof(struct kvm_sw_breakpoint));
2345 bp->pc = addr;
2346 bp->use_count = 1;
2347 err = kvm_arch_insert_sw_breakpoint(cpu, bp);
2348 if (err) {
2349 g_free(bp);
2350 return err;
2351 }
2352
2353 QTAILQ_INSERT_HEAD(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry);
2354 } else {
2355 err = kvm_arch_insert_hw_breakpoint(addr, len, type);
2356 if (err) {
2357 return err;
2358 }
2359 }
2360
2361 CPU_FOREACH(cpu) {
2362 err = kvm_update_guest_debug(cpu, 0);
2363 if (err) {
2364 return err;
2365 }
2366 }
2367 return 0;
2368}
2369
2370int kvm_remove_breakpoint(CPUState *cpu, target_ulong addr,
2371 target_ulong len, int type)
2372{
2373 struct kvm_sw_breakpoint *bp;
2374 int err;
2375
2376 if (type == GDB_BREAKPOINT_SW) {
2377 bp = kvm_find_sw_breakpoint(cpu, addr);
2378 if (!bp) {
2379 return -ENOENT;
2380 }
2381
2382 if (bp->use_count > 1) {
2383 bp->use_count--;
2384 return 0;
2385 }
2386
2387 err = kvm_arch_remove_sw_breakpoint(cpu, bp);
2388 if (err) {
2389 return err;
2390 }
2391
2392 QTAILQ_REMOVE(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry);
2393 g_free(bp);
2394 } else {
2395 err = kvm_arch_remove_hw_breakpoint(addr, len, type);
2396 if (err) {
2397 return err;
2398 }
2399 }
2400
2401 CPU_FOREACH(cpu) {
2402 err = kvm_update_guest_debug(cpu, 0);
2403 if (err) {
2404 return err;
2405 }
2406 }
2407 return 0;
2408}
2409
2410void kvm_remove_all_breakpoints(CPUState *cpu)
2411{
2412 struct kvm_sw_breakpoint *bp, *next;
2413 KVMState *s = cpu->kvm_state;
2414 CPUState *tmpcpu;
2415
2416 QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
2417 if (kvm_arch_remove_sw_breakpoint(cpu, bp) != 0) {
2418
2419 CPU_FOREACH(tmpcpu) {
2420 if (kvm_arch_remove_sw_breakpoint(tmpcpu, bp) == 0) {
2421 break;
2422 }
2423 }
2424 }
2425 QTAILQ_REMOVE(&s->kvm_sw_breakpoints, bp, entry);
2426 g_free(bp);
2427 }
2428 kvm_arch_remove_all_hw_breakpoints();
2429
2430 CPU_FOREACH(cpu) {
2431 kvm_update_guest_debug(cpu, 0);
2432 }
2433}
2434
2435#else
2436
2437int kvm_update_guest_debug(CPUState *cpu, unsigned long reinject_trap)
2438{
2439 return -EINVAL;
2440}
2441
2442int kvm_insert_breakpoint(CPUState *cpu, target_ulong addr,
2443 target_ulong len, int type)
2444{
2445 return -EINVAL;
2446}
2447
2448int kvm_remove_breakpoint(CPUState *cpu, target_ulong addr,
2449 target_ulong len, int type)
2450{
2451 return -EINVAL;
2452}
2453
2454void kvm_remove_all_breakpoints(CPUState *cpu)
2455{
2456}
2457#endif
2458
2459static int kvm_set_signal_mask(CPUState *cpu, const sigset_t *sigset)
2460{
2461 KVMState *s = kvm_state;
2462 struct kvm_signal_mask *sigmask;
2463 int r;
2464
2465 sigmask = g_malloc(sizeof(*sigmask) + sizeof(*sigset));
2466
2467 sigmask->len = s->sigmask_len;
2468 memcpy(sigmask->sigset, sigset, sizeof(*sigset));
2469 r = kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, sigmask);
2470 g_free(sigmask);
2471
2472 return r;
2473}
2474
2475static void kvm_ipi_signal(int sig)
2476{
2477 if (current_cpu) {
2478 assert(kvm_immediate_exit);
2479 kvm_cpu_kick(current_cpu);
2480 }
2481}
2482
2483void kvm_init_cpu_signals(CPUState *cpu)
2484{
2485 int r;
2486 sigset_t set;
2487 struct sigaction sigact;
2488
2489 memset(&sigact, 0, sizeof(sigact));
2490 sigact.sa_handler = kvm_ipi_signal;
2491 sigaction(SIG_IPI, &sigact, NULL);
2492
2493 pthread_sigmask(SIG_BLOCK, NULL, &set);
2494#if defined KVM_HAVE_MCE_INJECTION
2495 sigdelset(&set, SIGBUS);
2496 pthread_sigmask(SIG_SETMASK, &set, NULL);
2497#endif
2498 sigdelset(&set, SIG_IPI);
2499 if (kvm_immediate_exit) {
2500 r = pthread_sigmask(SIG_SETMASK, &set, NULL);
2501 } else {
2502 r = kvm_set_signal_mask(cpu, &set);
2503 }
2504 if (r) {
2505 fprintf(stderr, "kvm_set_signal_mask: %s\n", strerror(-r));
2506 exit(1);
2507 }
2508}
2509
2510
2511int kvm_on_sigbus_vcpu(CPUState *cpu, int code, void *addr)
2512{
2513#ifdef KVM_HAVE_MCE_INJECTION
2514 if (have_sigbus_pending) {
2515 return 1;
2516 }
2517 have_sigbus_pending = true;
2518 pending_sigbus_addr = addr;
2519 pending_sigbus_code = code;
2520 atomic_set(&cpu->exit_request, 1);
2521 return 0;
2522#else
2523 return 1;
2524#endif
2525}
2526
2527
2528int kvm_on_sigbus(int code, void *addr)
2529{
2530#ifdef KVM_HAVE_MCE_INJECTION
2531
2532
2533
2534
2535 assert(code != BUS_MCEERR_AR);
2536 kvm_arch_on_sigbus_vcpu(first_cpu, code, addr);
2537 return 0;
2538#else
2539 return 1;
2540#endif
2541}
2542
2543int kvm_create_device(KVMState *s, uint64_t type, bool test)
2544{
2545 int ret;
2546 struct kvm_create_device create_dev;
2547
2548 create_dev.type = type;
2549 create_dev.fd = -1;
2550 create_dev.flags = test ? KVM_CREATE_DEVICE_TEST : 0;
2551
2552 if (!kvm_check_extension(s, KVM_CAP_DEVICE_CTRL)) {
2553 return -ENOTSUP;
2554 }
2555
2556 ret = kvm_vm_ioctl(s, KVM_CREATE_DEVICE, &create_dev);
2557 if (ret) {
2558 return ret;
2559 }
2560
2561 return test ? 0 : create_dev.fd;
2562}
2563
2564bool kvm_device_supported(int vmfd, uint64_t type)
2565{
2566 struct kvm_create_device create_dev = {
2567 .type = type,
2568 .fd = -1,
2569 .flags = KVM_CREATE_DEVICE_TEST,
2570 };
2571
2572 if (ioctl(vmfd, KVM_CHECK_EXTENSION, KVM_CAP_DEVICE_CTRL) <= 0) {
2573 return false;
2574 }
2575
2576 return (ioctl(vmfd, KVM_CREATE_DEVICE, &create_dev) >= 0);
2577}
2578
2579int kvm_set_one_reg(CPUState *cs, uint64_t id, void *source)
2580{
2581 struct kvm_one_reg reg;
2582 int r;
2583
2584 reg.id = id;
2585 reg.addr = (uintptr_t) source;
2586 r = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®);
2587 if (r) {
2588 trace_kvm_failed_reg_set(id, strerror(-r));
2589 }
2590 return r;
2591}
2592
2593int kvm_get_one_reg(CPUState *cs, uint64_t id, void *target)
2594{
2595 struct kvm_one_reg reg;
2596 int r;
2597
2598 reg.id = id;
2599 reg.addr = (uintptr_t) target;
2600 r = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®);
2601 if (r) {
2602 trace_kvm_failed_reg_get(id, strerror(-r));
2603 }
2604 return r;
2605}
2606
2607static void kvm_accel_class_init(ObjectClass *oc, void *data)
2608{
2609 AccelClass *ac = ACCEL_CLASS(oc);
2610 ac->name = "KVM";
2611 ac->init_machine = kvm_init;
2612 ac->allowed = &kvm_allowed;
2613}
2614
2615static const TypeInfo kvm_accel_type = {
2616 .name = TYPE_KVM_ACCEL,
2617 .parent = TYPE_ACCEL,
2618 .class_init = kvm_accel_class_init,
2619 .instance_size = sizeof(KVMState),
2620};
2621
2622static void kvm_type_init(void)
2623{
2624 type_register_static(&kvm_accel_type);
2625}
2626
2627type_init(kvm_type_init);
2628