1
2
3
4
5
6
7
8#include "test_util.h"
9#include "kvm_util.h"
10#include "kvm_util_internal.h"
11
12#include <assert.h>
13#include <sys/mman.h>
14#include <sys/types.h>
15#include <sys/stat.h>
16#include <linux/kernel.h>
17
18#define KVM_UTIL_PGS_PER_HUGEPG 512
19#define KVM_UTIL_MIN_PFN 2
20
21
22static void *align(void *x, size_t size)
23{
24 size_t mask = size - 1;
25 TEST_ASSERT(size != 0 && !(size & (size - 1)),
26 "size not a power of 2: %lu", size);
27 return (void *) (((size_t) x + mask) & ~mask);
28}
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46int kvm_check_cap(long cap)
47{
48 int ret;
49 int kvm_fd;
50
51 kvm_fd = open(KVM_DEV_PATH, O_RDONLY);
52 if (kvm_fd < 0)
53 exit(KSFT_SKIP);
54
55 ret = ioctl(kvm_fd, KVM_CHECK_EXTENSION, cap);
56 TEST_ASSERT(ret != -1, "KVM_CHECK_EXTENSION IOCTL failed,\n"
57 " rc: %i errno: %i", ret, errno);
58
59 close(kvm_fd);
60
61 return ret;
62}
63
64
65
66
67
68
69
70
71
72
73
74
75
76int vm_enable_cap(struct kvm_vm *vm, struct kvm_enable_cap *cap)
77{
78 int ret;
79
80 ret = ioctl(vm->fd, KVM_ENABLE_CAP, cap);
81 TEST_ASSERT(ret == 0, "KVM_ENABLE_CAP IOCTL failed,\n"
82 " rc: %i errno: %i", ret, errno);
83
84 return ret;
85}
86
87static void vm_open(struct kvm_vm *vm, int perm, unsigned long type)
88{
89 vm->kvm_fd = open(KVM_DEV_PATH, perm);
90 if (vm->kvm_fd < 0)
91 exit(KSFT_SKIP);
92
93 if (!kvm_check_cap(KVM_CAP_IMMEDIATE_EXIT)) {
94 fprintf(stderr, "immediate_exit not available, skipping test\n");
95 exit(KSFT_SKIP);
96 }
97
98 vm->fd = ioctl(vm->kvm_fd, KVM_CREATE_VM, type);
99 TEST_ASSERT(vm->fd >= 0, "KVM_CREATE_VM ioctl failed, "
100 "rc: %i errno: %i", vm->fd, errno);
101}
102
103const char * const vm_guest_mode_string[] = {
104 "PA-bits:52, VA-bits:48, 4K pages",
105 "PA-bits:52, VA-bits:48, 64K pages",
106 "PA-bits:48, VA-bits:48, 4K pages",
107 "PA-bits:48, VA-bits:48, 64K pages",
108 "PA-bits:40, VA-bits:48, 4K pages",
109 "PA-bits:40, VA-bits:48, 64K pages",
110};
111_Static_assert(sizeof(vm_guest_mode_string)/sizeof(char *) == NUM_VM_MODES,
112 "Missing new mode strings?");
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133struct kvm_vm *_vm_create(enum vm_guest_mode mode, uint64_t phy_pages,
134 int perm, unsigned long type)
135{
136 struct kvm_vm *vm;
137
138 vm = calloc(1, sizeof(*vm));
139 TEST_ASSERT(vm != NULL, "Insufficient Memory");
140
141 vm->mode = mode;
142 vm->type = type;
143 vm_open(vm, perm, type);
144
145
146 switch (vm->mode) {
147 case VM_MODE_P52V48_4K:
148 vm->pgtable_levels = 4;
149 vm->pa_bits = 52;
150 vm->va_bits = 48;
151 vm->page_size = 0x1000;
152 vm->page_shift = 12;
153 break;
154 case VM_MODE_P52V48_64K:
155 vm->pgtable_levels = 3;
156 vm->pa_bits = 52;
157 vm->va_bits = 48;
158 vm->page_size = 0x10000;
159 vm->page_shift = 16;
160 break;
161 case VM_MODE_P48V48_4K:
162 vm->pgtable_levels = 4;
163 vm->pa_bits = 48;
164 vm->va_bits = 48;
165 vm->page_size = 0x1000;
166 vm->page_shift = 12;
167 break;
168 case VM_MODE_P48V48_64K:
169 vm->pgtable_levels = 3;
170 vm->pa_bits = 48;
171 vm->va_bits = 48;
172 vm->page_size = 0x10000;
173 vm->page_shift = 16;
174 break;
175 case VM_MODE_P40V48_4K:
176 vm->pgtable_levels = 4;
177 vm->pa_bits = 40;
178 vm->va_bits = 48;
179 vm->page_size = 0x1000;
180 vm->page_shift = 12;
181 break;
182 case VM_MODE_P40V48_64K:
183 vm->pgtable_levels = 3;
184 vm->pa_bits = 40;
185 vm->va_bits = 48;
186 vm->page_size = 0x10000;
187 vm->page_shift = 16;
188 break;
189 default:
190 TEST_ASSERT(false, "Unknown guest mode, mode: 0x%x", mode);
191 }
192
193
194 vm->vpages_valid = sparsebit_alloc();
195 sparsebit_set_num(vm->vpages_valid,
196 0, (1ULL << (vm->va_bits - 1)) >> vm->page_shift);
197 sparsebit_set_num(vm->vpages_valid,
198 (~((1ULL << (vm->va_bits - 1)) - 1)) >> vm->page_shift,
199 (1ULL << (vm->va_bits - 1)) >> vm->page_shift);
200
201
202 vm->max_gfn = ((1ULL << vm->pa_bits) >> vm->page_shift) - 1;
203
204
205 vm->vpages_mapped = sparsebit_alloc();
206 if (phy_pages != 0)
207 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS,
208 0, 0, phy_pages, 0);
209
210 return vm;
211}
212
213struct kvm_vm *vm_create(enum vm_guest_mode mode, uint64_t phy_pages, int perm)
214{
215 return _vm_create(mode, phy_pages, perm, 0);
216}
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231void kvm_vm_restart(struct kvm_vm *vmp, int perm)
232{
233 struct userspace_mem_region *region;
234
235 vm_open(vmp, perm, vmp->type);
236 if (vmp->has_irqchip)
237 vm_create_irqchip(vmp);
238
239 for (region = vmp->userspace_mem_region_head; region;
240 region = region->next) {
241 int ret = ioctl(vmp->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region);
242 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
243 " rc: %i errno: %i\n"
244 " slot: %u flags: 0x%x\n"
245 " guest_phys_addr: 0x%lx size: 0x%lx",
246 ret, errno, region->region.slot,
247 region->region.flags,
248 region->region.guest_phys_addr,
249 region->region.memory_size);
250 }
251}
252
253void kvm_vm_get_dirty_log(struct kvm_vm *vm, int slot, void *log)
254{
255 struct kvm_dirty_log args = { .dirty_bitmap = log, .slot = slot };
256 int ret;
257
258 ret = ioctl(vm->fd, KVM_GET_DIRTY_LOG, &args);
259 TEST_ASSERT(ret == 0, "%s: KVM_GET_DIRTY_LOG failed: %s",
260 strerror(-ret));
261}
262
263void kvm_vm_clear_dirty_log(struct kvm_vm *vm, int slot, void *log,
264 uint64_t first_page, uint32_t num_pages)
265{
266 struct kvm_clear_dirty_log args = { .dirty_bitmap = log, .slot = slot,
267 .first_page = first_page,
268 .num_pages = num_pages };
269 int ret;
270
271 ret = ioctl(vm->fd, KVM_CLEAR_DIRTY_LOG, &args);
272 TEST_ASSERT(ret == 0, "%s: KVM_CLEAR_DIRTY_LOG failed: %s",
273 strerror(-ret));
274}
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295static struct userspace_mem_region *
296userspace_mem_region_find(struct kvm_vm *vm, uint64_t start, uint64_t end)
297{
298 struct userspace_mem_region *region;
299
300 for (region = vm->userspace_mem_region_head; region;
301 region = region->next) {
302 uint64_t existing_start = region->region.guest_phys_addr;
303 uint64_t existing_end = region->region.guest_phys_addr
304 + region->region.memory_size - 1;
305 if (start <= existing_end && end >= existing_start)
306 return region;
307 }
308
309 return NULL;
310}
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328struct kvm_userspace_memory_region *
329kvm_userspace_memory_region_find(struct kvm_vm *vm, uint64_t start,
330 uint64_t end)
331{
332 struct userspace_mem_region *region;
333
334 region = userspace_mem_region_find(vm, start, end);
335 if (!region)
336 return NULL;
337
338 return ®ion->region;
339}
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357struct vcpu *vcpu_find(struct kvm_vm *vm, uint32_t vcpuid)
358{
359 struct vcpu *vcpup;
360
361 for (vcpup = vm->vcpu_head; vcpup; vcpup = vcpup->next) {
362 if (vcpup->id == vcpuid)
363 return vcpup;
364 }
365
366 return NULL;
367}
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382static void vm_vcpu_rm(struct kvm_vm *vm, uint32_t vcpuid)
383{
384 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
385 int ret;
386
387 ret = munmap(vcpu->state, sizeof(*vcpu->state));
388 TEST_ASSERT(ret == 0, "munmap of VCPU fd failed, rc: %i "
389 "errno: %i", ret, errno);
390 close(vcpu->fd);
391 TEST_ASSERT(ret == 0, "Close of VCPU fd failed, rc: %i "
392 "errno: %i", ret, errno);
393
394 if (vcpu->next)
395 vcpu->next->prev = vcpu->prev;
396 if (vcpu->prev)
397 vcpu->prev->next = vcpu->next;
398 else
399 vm->vcpu_head = vcpu->next;
400 free(vcpu);
401}
402
403void kvm_vm_release(struct kvm_vm *vmp)
404{
405 int ret;
406
407 while (vmp->vcpu_head)
408 vm_vcpu_rm(vmp, vmp->vcpu_head->id);
409
410 ret = close(vmp->fd);
411 TEST_ASSERT(ret == 0, "Close of vm fd failed,\n"
412 " vmp->fd: %i rc: %i errno: %i", vmp->fd, ret, errno);
413
414 close(vmp->kvm_fd);
415 TEST_ASSERT(ret == 0, "Close of /dev/kvm fd failed,\n"
416 " vmp->kvm_fd: %i rc: %i errno: %i", vmp->kvm_fd, ret, errno);
417}
418
419
420
421
422void kvm_vm_free(struct kvm_vm *vmp)
423{
424 int ret;
425
426 if (vmp == NULL)
427 return;
428
429
430 while (vmp->userspace_mem_region_head) {
431 struct userspace_mem_region *region
432 = vmp->userspace_mem_region_head;
433
434 region->region.memory_size = 0;
435 ret = ioctl(vmp->fd, KVM_SET_USER_MEMORY_REGION,
436 ®ion->region);
437 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed, "
438 "rc: %i errno: %i", ret, errno);
439
440 vmp->userspace_mem_region_head = region->next;
441 sparsebit_free(®ion->unused_phy_pages);
442 ret = munmap(region->mmap_start, region->mmap_size);
443 TEST_ASSERT(ret == 0, "munmap failed, rc: %i errno: %i",
444 ret, errno);
445
446 free(region);
447 }
448
449
450 sparsebit_free(&vmp->vpages_valid);
451 sparsebit_free(&vmp->vpages_mapped);
452
453 kvm_vm_release(vmp);
454
455
456 free(vmp);
457}
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482int kvm_memcmp_hva_gva(void *hva, struct kvm_vm *vm, vm_vaddr_t gva, size_t len)
483{
484 size_t amt;
485
486
487
488
489
490 for (uintptr_t offset = 0; offset < len; offset += amt) {
491 uintptr_t ptr1 = (uintptr_t)hva + offset;
492
493
494
495
496
497 uintptr_t ptr2 = (uintptr_t)addr_gva2hva(vm, gva + offset);
498
499
500
501
502
503 amt = len - offset;
504 if ((ptr1 >> vm->page_shift) != ((ptr1 + amt) >> vm->page_shift))
505 amt = vm->page_size - (ptr1 % vm->page_size);
506 if ((ptr2 >> vm->page_shift) != ((ptr2 + amt) >> vm->page_shift))
507 amt = vm->page_size - (ptr2 % vm->page_size);
508
509 assert((ptr1 >> vm->page_shift) == ((ptr1 + amt - 1) >> vm->page_shift));
510 assert((ptr2 >> vm->page_shift) == ((ptr2 + amt - 1) >> vm->page_shift));
511
512
513
514
515
516
517 int ret = memcmp((void *)ptr1, (void *)ptr2, amt);
518 if (ret != 0)
519 return ret;
520 }
521
522
523
524
525
526 return 0;
527}
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551void vm_userspace_mem_region_add(struct kvm_vm *vm,
552 enum vm_mem_backing_src_type src_type,
553 uint64_t guest_paddr, uint32_t slot, uint64_t npages,
554 uint32_t flags)
555{
556 int ret;
557 struct userspace_mem_region *region;
558 size_t huge_page_size = KVM_UTIL_PGS_PER_HUGEPG * vm->page_size;
559
560 TEST_ASSERT((guest_paddr % vm->page_size) == 0, "Guest physical "
561 "address not on a page boundary.\n"
562 " guest_paddr: 0x%lx vm->page_size: 0x%x",
563 guest_paddr, vm->page_size);
564 TEST_ASSERT((((guest_paddr >> vm->page_shift) + npages) - 1)
565 <= vm->max_gfn, "Physical range beyond maximum "
566 "supported physical address,\n"
567 " guest_paddr: 0x%lx npages: 0x%lx\n"
568 " vm->max_gfn: 0x%lx vm->page_size: 0x%x",
569 guest_paddr, npages, vm->max_gfn, vm->page_size);
570
571
572
573
574
575 region = (struct userspace_mem_region *) userspace_mem_region_find(
576 vm, guest_paddr, (guest_paddr + npages * vm->page_size) - 1);
577 if (region != NULL)
578 TEST_ASSERT(false, "overlapping userspace_mem_region already "
579 "exists\n"
580 " requested guest_paddr: 0x%lx npages: 0x%lx "
581 "page_size: 0x%x\n"
582 " existing guest_paddr: 0x%lx size: 0x%lx",
583 guest_paddr, npages, vm->page_size,
584 (uint64_t) region->region.guest_phys_addr,
585 (uint64_t) region->region.memory_size);
586
587
588 for (region = vm->userspace_mem_region_head; region;
589 region = region->next) {
590 if (region->region.slot == slot)
591 break;
592 }
593 if (region != NULL)
594 TEST_ASSERT(false, "A mem region with the requested slot "
595 "already exists.\n"
596 " requested slot: %u paddr: 0x%lx npages: 0x%lx\n"
597 " existing slot: %u paddr: 0x%lx size: 0x%lx",
598 slot, guest_paddr, npages,
599 region->region.slot,
600 (uint64_t) region->region.guest_phys_addr,
601 (uint64_t) region->region.memory_size);
602
603
604 region = calloc(1, sizeof(*region));
605 TEST_ASSERT(region != NULL, "Insufficient Memory");
606 region->mmap_size = npages * vm->page_size;
607
608
609 if (src_type == VM_MEM_SRC_ANONYMOUS_THP)
610 region->mmap_size += huge_page_size;
611 region->mmap_start = mmap(NULL, region->mmap_size,
612 PROT_READ | PROT_WRITE,
613 MAP_PRIVATE | MAP_ANONYMOUS
614 | (src_type == VM_MEM_SRC_ANONYMOUS_HUGETLB ? MAP_HUGETLB : 0),
615 -1, 0);
616 TEST_ASSERT(region->mmap_start != MAP_FAILED,
617 "test_malloc failed, mmap_start: %p errno: %i",
618 region->mmap_start, errno);
619
620
621 region->host_mem = align(region->mmap_start,
622 src_type == VM_MEM_SRC_ANONYMOUS_THP ? huge_page_size : 1);
623
624
625 if (src_type == VM_MEM_SRC_ANONYMOUS || src_type == VM_MEM_SRC_ANONYMOUS_THP) {
626 ret = madvise(region->host_mem, npages * vm->page_size,
627 src_type == VM_MEM_SRC_ANONYMOUS ? MADV_NOHUGEPAGE : MADV_HUGEPAGE);
628 TEST_ASSERT(ret == 0, "madvise failed,\n"
629 " addr: %p\n"
630 " length: 0x%lx\n"
631 " src_type: %x",
632 region->host_mem, npages * vm->page_size, src_type);
633 }
634
635 region->unused_phy_pages = sparsebit_alloc();
636 sparsebit_set_num(region->unused_phy_pages,
637 guest_paddr >> vm->page_shift, npages);
638 region->region.slot = slot;
639 region->region.flags = flags;
640 region->region.guest_phys_addr = guest_paddr;
641 region->region.memory_size = npages * vm->page_size;
642 region->region.userspace_addr = (uintptr_t) region->host_mem;
643 ret = ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region);
644 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
645 " rc: %i errno: %i\n"
646 " slot: %u flags: 0x%x\n"
647 " guest_phys_addr: 0x%lx size: 0x%lx",
648 ret, errno, slot, flags,
649 guest_paddr, (uint64_t) region->region.memory_size);
650
651
652 if (vm->userspace_mem_region_head)
653 vm->userspace_mem_region_head->prev = region;
654 region->next = vm->userspace_mem_region_head;
655 vm->userspace_mem_region_head = region;
656}
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673static struct userspace_mem_region *
674memslot2region(struct kvm_vm *vm, uint32_t memslot)
675{
676 struct userspace_mem_region *region;
677
678 for (region = vm->userspace_mem_region_head; region;
679 region = region->next) {
680 if (region->region.slot == memslot)
681 break;
682 }
683 if (region == NULL) {
684 fprintf(stderr, "No mem region with the requested slot found,\n"
685 " requested slot: %u\n", memslot);
686 fputs("---- vm dump ----\n", stderr);
687 vm_dump(stderr, vm, 2);
688 TEST_ASSERT(false, "Mem region not found");
689 }
690
691 return region;
692}
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags)
709{
710 int ret;
711 struct userspace_mem_region *region;
712
713 region = memslot2region(vm, slot);
714
715 region->region.flags = flags;
716
717 ret = ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region);
718
719 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
720 " rc: %i errno: %i slot: %u flags: 0x%x",
721 ret, errno, slot, flags);
722}
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737static int vcpu_mmap_sz(void)
738{
739 int dev_fd, ret;
740
741 dev_fd = open(KVM_DEV_PATH, O_RDONLY);
742 if (dev_fd < 0)
743 exit(KSFT_SKIP);
744
745 ret = ioctl(dev_fd, KVM_GET_VCPU_MMAP_SIZE, NULL);
746 TEST_ASSERT(ret >= sizeof(struct kvm_run),
747 "%s KVM_GET_VCPU_MMAP_SIZE ioctl failed, rc: %i errno: %i",
748 __func__, ret, errno);
749
750 close(dev_fd);
751
752 return ret;
753}
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769void vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpuid, int pgd_memslot,
770 int gdt_memslot)
771{
772 struct vcpu *vcpu;
773
774
775 vcpu = vcpu_find(vm, vcpuid);
776 if (vcpu != NULL)
777 TEST_ASSERT(false, "vcpu with the specified id "
778 "already exists,\n"
779 " requested vcpuid: %u\n"
780 " existing vcpuid: %u state: %p",
781 vcpuid, vcpu->id, vcpu->state);
782
783
784 vcpu = calloc(1, sizeof(*vcpu));
785 TEST_ASSERT(vcpu != NULL, "Insufficient Memory");
786 vcpu->id = vcpuid;
787 vcpu->fd = ioctl(vm->fd, KVM_CREATE_VCPU, vcpuid);
788 TEST_ASSERT(vcpu->fd >= 0, "KVM_CREATE_VCPU failed, rc: %i errno: %i",
789 vcpu->fd, errno);
790
791 TEST_ASSERT(vcpu_mmap_sz() >= sizeof(*vcpu->state), "vcpu mmap size "
792 "smaller than expected, vcpu_mmap_sz: %i expected_min: %zi",
793 vcpu_mmap_sz(), sizeof(*vcpu->state));
794 vcpu->state = (struct kvm_run *) mmap(NULL, sizeof(*vcpu->state),
795 PROT_READ | PROT_WRITE, MAP_SHARED, vcpu->fd, 0);
796 TEST_ASSERT(vcpu->state != MAP_FAILED, "mmap vcpu_state failed, "
797 "vcpu id: %u errno: %i", vcpuid, errno);
798
799
800 if (vm->vcpu_head)
801 vm->vcpu_head->prev = vcpu;
802 vcpu->next = vm->vcpu_head;
803 vm->vcpu_head = vcpu;
804
805 vcpu_setup(vm, vcpuid, pgd_memslot, gdt_memslot);
806}
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828static vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz,
829 vm_vaddr_t vaddr_min)
830{
831 uint64_t pages = (sz + vm->page_size - 1) >> vm->page_shift;
832
833
834 uint64_t pgidx_start = (vaddr_min + vm->page_size - 1) >> vm->page_shift;
835 if ((pgidx_start * vm->page_size) < vaddr_min)
836 goto no_va_found;
837
838
839 if (!sparsebit_is_set_num(vm->vpages_valid,
840 pgidx_start, pages))
841 pgidx_start = sparsebit_next_set_num(vm->vpages_valid,
842 pgidx_start, pages);
843 do {
844
845
846
847
848
849
850 if (sparsebit_is_clear_num(vm->vpages_mapped,
851 pgidx_start, pages))
852 goto va_found;
853 pgidx_start = sparsebit_next_clear_num(vm->vpages_mapped,
854 pgidx_start, pages);
855 if (pgidx_start == 0)
856 goto no_va_found;
857
858
859
860
861
862 if (!sparsebit_is_set_num(vm->vpages_valid,
863 pgidx_start, pages)) {
864 pgidx_start = sparsebit_next_set_num(
865 vm->vpages_valid, pgidx_start, pages);
866 if (pgidx_start == 0)
867 goto no_va_found;
868 }
869 } while (pgidx_start != 0);
870
871no_va_found:
872 TEST_ASSERT(false, "No vaddr of specified pages available, "
873 "pages: 0x%lx", pages);
874
875
876 return -1;
877
878va_found:
879 TEST_ASSERT(sparsebit_is_set_num(vm->vpages_valid,
880 pgidx_start, pages),
881 "Unexpected, invalid virtual page index range,\n"
882 " pgidx_start: 0x%lx\n"
883 " pages: 0x%lx",
884 pgidx_start, pages);
885 TEST_ASSERT(sparsebit_is_clear_num(vm->vpages_mapped,
886 pgidx_start, pages),
887 "Unexpected, pages already mapped,\n"
888 " pgidx_start: 0x%lx\n"
889 " pages: 0x%lx",
890 pgidx_start, pages);
891
892 return pgidx_start * vm->page_size;
893}
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min,
917 uint32_t data_memslot, uint32_t pgd_memslot)
918{
919 uint64_t pages = (sz >> vm->page_shift) + ((sz % vm->page_size) != 0);
920
921 virt_pgd_alloc(vm, pgd_memslot);
922
923
924
925
926
927 vm_vaddr_t vaddr_start = vm_vaddr_unused_gap(vm, sz, vaddr_min);
928
929
930 for (vm_vaddr_t vaddr = vaddr_start; pages > 0;
931 pages--, vaddr += vm->page_size) {
932 vm_paddr_t paddr;
933
934 paddr = vm_phy_page_alloc(vm,
935 KVM_UTIL_MIN_PFN * vm->page_size, data_memslot);
936
937 virt_pg_map(vm, vaddr, paddr, pgd_memslot);
938
939 sparsebit_set(vm->vpages_mapped,
940 vaddr >> vm->page_shift);
941 }
942
943 return vaddr_start;
944}
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
964 size_t size, uint32_t pgd_memslot)
965{
966 size_t page_size = vm->page_size;
967 size_t npages = size / page_size;
968
969 TEST_ASSERT(vaddr + size > vaddr, "Vaddr overflow");
970 TEST_ASSERT(paddr + size > paddr, "Paddr overflow");
971
972 while (npages--) {
973 virt_pg_map(vm, vaddr, paddr, pgd_memslot);
974 vaddr += page_size;
975 paddr += page_size;
976 }
977}
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa)
997{
998 struct userspace_mem_region *region;
999 for (region = vm->userspace_mem_region_head; region;
1000 region = region->next) {
1001 if ((gpa >= region->region.guest_phys_addr)
1002 && (gpa <= (region->region.guest_phys_addr
1003 + region->region.memory_size - 1)))
1004 return (void *) ((uintptr_t) region->host_mem
1005 + (gpa - region->region.guest_phys_addr));
1006 }
1007
1008 TEST_ASSERT(false, "No vm physical memory at 0x%lx", gpa);
1009 return NULL;
1010}
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva)
1030{
1031 struct userspace_mem_region *region;
1032 for (region = vm->userspace_mem_region_head; region;
1033 region = region->next) {
1034 if ((hva >= region->host_mem)
1035 && (hva <= (region->host_mem
1036 + region->region.memory_size - 1)))
1037 return (vm_paddr_t) ((uintptr_t)
1038 region->region.guest_phys_addr
1039 + (hva - (uintptr_t) region->host_mem));
1040 }
1041
1042 TEST_ASSERT(false, "No mapping to a guest physical address, "
1043 "hva: %p", hva);
1044 return -1;
1045}
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059void vm_create_irqchip(struct kvm_vm *vm)
1060{
1061 int ret;
1062
1063 ret = ioctl(vm->fd, KVM_CREATE_IRQCHIP, 0);
1064 TEST_ASSERT(ret == 0, "KVM_CREATE_IRQCHIP IOCTL failed, "
1065 "rc: %i errno: %i", ret, errno);
1066
1067 vm->has_irqchip = true;
1068}
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085struct kvm_run *vcpu_state(struct kvm_vm *vm, uint32_t vcpuid)
1086{
1087 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1088 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1089
1090 return vcpu->state;
1091}
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107void vcpu_run(struct kvm_vm *vm, uint32_t vcpuid)
1108{
1109 int ret = _vcpu_run(vm, vcpuid);
1110 TEST_ASSERT(ret == 0, "KVM_RUN IOCTL failed, "
1111 "rc: %i errno: %i", ret, errno);
1112}
1113
1114int _vcpu_run(struct kvm_vm *vm, uint32_t vcpuid)
1115{
1116 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1117 int rc;
1118
1119 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1120 do {
1121 rc = ioctl(vcpu->fd, KVM_RUN, NULL);
1122 } while (rc == -1 && errno == EINTR);
1123 return rc;
1124}
1125
1126void vcpu_run_complete_io(struct kvm_vm *vm, uint32_t vcpuid)
1127{
1128 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1129 int ret;
1130
1131 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1132
1133 vcpu->state->immediate_exit = 1;
1134 ret = ioctl(vcpu->fd, KVM_RUN, NULL);
1135 vcpu->state->immediate_exit = 0;
1136
1137 TEST_ASSERT(ret == -1 && errno == EINTR,
1138 "KVM_RUN IOCTL didn't exit immediately, rc: %i, errno: %i",
1139 ret, errno);
1140}
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157void vcpu_set_mp_state(struct kvm_vm *vm, uint32_t vcpuid,
1158 struct kvm_mp_state *mp_state)
1159{
1160 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1161 int ret;
1162
1163 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1164
1165 ret = ioctl(vcpu->fd, KVM_SET_MP_STATE, mp_state);
1166 TEST_ASSERT(ret == 0, "KVM_SET_MP_STATE IOCTL failed, "
1167 "rc: %i errno: %i", ret, errno);
1168}
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185void vcpu_regs_get(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_regs *regs)
1186{
1187 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1188 int ret;
1189
1190 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1191
1192 ret = ioctl(vcpu->fd, KVM_GET_REGS, regs);
1193 TEST_ASSERT(ret == 0, "KVM_GET_REGS failed, rc: %i errno: %i",
1194 ret, errno);
1195}
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212void vcpu_regs_set(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_regs *regs)
1213{
1214 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1215 int ret;
1216
1217 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1218
1219 ret = ioctl(vcpu->fd, KVM_SET_REGS, regs);
1220 TEST_ASSERT(ret == 0, "KVM_SET_REGS failed, rc: %i errno: %i",
1221 ret, errno);
1222}
1223
1224void vcpu_events_get(struct kvm_vm *vm, uint32_t vcpuid,
1225 struct kvm_vcpu_events *events)
1226{
1227 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1228 int ret;
1229
1230 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1231
1232 ret = ioctl(vcpu->fd, KVM_GET_VCPU_EVENTS, events);
1233 TEST_ASSERT(ret == 0, "KVM_GET_VCPU_EVENTS, failed, rc: %i errno: %i",
1234 ret, errno);
1235}
1236
1237void vcpu_events_set(struct kvm_vm *vm, uint32_t vcpuid,
1238 struct kvm_vcpu_events *events)
1239{
1240 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1241 int ret;
1242
1243 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1244
1245 ret = ioctl(vcpu->fd, KVM_SET_VCPU_EVENTS, events);
1246 TEST_ASSERT(ret == 0, "KVM_SET_VCPU_EVENTS, failed, rc: %i errno: %i",
1247 ret, errno);
1248}
1249
1250#ifdef __x86_64__
1251void vcpu_nested_state_get(struct kvm_vm *vm, uint32_t vcpuid,
1252 struct kvm_nested_state *state)
1253{
1254 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1255 int ret;
1256
1257 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1258
1259 ret = ioctl(vcpu->fd, KVM_GET_NESTED_STATE, state);
1260 TEST_ASSERT(ret == 0,
1261 "KVM_SET_NESTED_STATE failed, ret: %i errno: %i",
1262 ret, errno);
1263}
1264
1265int vcpu_nested_state_set(struct kvm_vm *vm, uint32_t vcpuid,
1266 struct kvm_nested_state *state, bool ignore_error)
1267{
1268 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1269 int ret;
1270
1271 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1272
1273 ret = ioctl(vcpu->fd, KVM_SET_NESTED_STATE, state);
1274 if (!ignore_error) {
1275 TEST_ASSERT(ret == 0,
1276 "KVM_SET_NESTED_STATE failed, ret: %i errno: %i",
1277 ret, errno);
1278 }
1279
1280 return ret;
1281}
1282#endif
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299void vcpu_sregs_get(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_sregs *sregs)
1300{
1301 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1302 int ret;
1303
1304 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1305
1306 ret = ioctl(vcpu->fd, KVM_GET_SREGS, sregs);
1307 TEST_ASSERT(ret == 0, "KVM_GET_SREGS failed, rc: %i errno: %i",
1308 ret, errno);
1309}
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326void vcpu_sregs_set(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_sregs *sregs)
1327{
1328 int ret = _vcpu_sregs_set(vm, vcpuid, sregs);
1329 TEST_ASSERT(ret == 0, "KVM_RUN IOCTL failed, "
1330 "rc: %i errno: %i", ret, errno);
1331}
1332
1333int _vcpu_sregs_set(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_sregs *sregs)
1334{
1335 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1336
1337 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1338
1339 return ioctl(vcpu->fd, KVM_SET_SREGS, sregs);
1340}
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355void vcpu_ioctl(struct kvm_vm *vm, uint32_t vcpuid,
1356 unsigned long cmd, void *arg)
1357{
1358 int ret;
1359
1360 ret = _vcpu_ioctl(vm, vcpuid, cmd, arg);
1361 TEST_ASSERT(ret == 0, "vcpu ioctl %lu failed, rc: %i errno: %i (%s)",
1362 cmd, ret, errno, strerror(errno));
1363}
1364
1365int _vcpu_ioctl(struct kvm_vm *vm, uint32_t vcpuid,
1366 unsigned long cmd, void *arg)
1367{
1368 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1369 int ret;
1370
1371 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1372
1373 ret = ioctl(vcpu->fd, cmd, arg);
1374
1375 return ret;
1376}
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390void vm_ioctl(struct kvm_vm *vm, unsigned long cmd, void *arg)
1391{
1392 int ret;
1393
1394 ret = ioctl(vm->fd, cmd, arg);
1395 TEST_ASSERT(ret == 0, "vm ioctl %lu failed, rc: %i errno: %i (%s)",
1396 cmd, ret, errno, strerror(errno));
1397}
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
1415{
1416 struct userspace_mem_region *region;
1417 struct vcpu *vcpu;
1418
1419 fprintf(stream, "%*smode: 0x%x\n", indent, "", vm->mode);
1420 fprintf(stream, "%*sfd: %i\n", indent, "", vm->fd);
1421 fprintf(stream, "%*spage_size: 0x%x\n", indent, "", vm->page_size);
1422 fprintf(stream, "%*sMem Regions:\n", indent, "");
1423 for (region = vm->userspace_mem_region_head; region;
1424 region = region->next) {
1425 fprintf(stream, "%*sguest_phys: 0x%lx size: 0x%lx "
1426 "host_virt: %p\n", indent + 2, "",
1427 (uint64_t) region->region.guest_phys_addr,
1428 (uint64_t) region->region.memory_size,
1429 region->host_mem);
1430 fprintf(stream, "%*sunused_phy_pages: ", indent + 2, "");
1431 sparsebit_dump(stream, region->unused_phy_pages, 0);
1432 }
1433 fprintf(stream, "%*sMapped Virtual Pages:\n", indent, "");
1434 sparsebit_dump(stream, vm->vpages_mapped, indent + 2);
1435 fprintf(stream, "%*spgd_created: %u\n", indent, "",
1436 vm->pgd_created);
1437 if (vm->pgd_created) {
1438 fprintf(stream, "%*sVirtual Translation Tables:\n",
1439 indent + 2, "");
1440 virt_dump(stream, vm, indent + 4);
1441 }
1442 fprintf(stream, "%*sVCPUs:\n", indent, "");
1443 for (vcpu = vm->vcpu_head; vcpu; vcpu = vcpu->next)
1444 vcpu_dump(stream, vm, vcpu->id, indent + 2);
1445}
1446
1447
1448static struct exit_reason {
1449 unsigned int reason;
1450 const char *name;
1451} exit_reasons_known[] = {
1452 {KVM_EXIT_UNKNOWN, "UNKNOWN"},
1453 {KVM_EXIT_EXCEPTION, "EXCEPTION"},
1454 {KVM_EXIT_IO, "IO"},
1455 {KVM_EXIT_HYPERCALL, "HYPERCALL"},
1456 {KVM_EXIT_DEBUG, "DEBUG"},
1457 {KVM_EXIT_HLT, "HLT"},
1458 {KVM_EXIT_MMIO, "MMIO"},
1459 {KVM_EXIT_IRQ_WINDOW_OPEN, "IRQ_WINDOW_OPEN"},
1460 {KVM_EXIT_SHUTDOWN, "SHUTDOWN"},
1461 {KVM_EXIT_FAIL_ENTRY, "FAIL_ENTRY"},
1462 {KVM_EXIT_INTR, "INTR"},
1463 {KVM_EXIT_SET_TPR, "SET_TPR"},
1464 {KVM_EXIT_TPR_ACCESS, "TPR_ACCESS"},
1465 {KVM_EXIT_S390_SIEIC, "S390_SIEIC"},
1466 {KVM_EXIT_S390_RESET, "S390_RESET"},
1467 {KVM_EXIT_DCR, "DCR"},
1468 {KVM_EXIT_NMI, "NMI"},
1469 {KVM_EXIT_INTERNAL_ERROR, "INTERNAL_ERROR"},
1470 {KVM_EXIT_OSI, "OSI"},
1471 {KVM_EXIT_PAPR_HCALL, "PAPR_HCALL"},
1472#ifdef KVM_EXIT_MEMORY_NOT_PRESENT
1473 {KVM_EXIT_MEMORY_NOT_PRESENT, "MEMORY_NOT_PRESENT"},
1474#endif
1475};
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492const char *exit_reason_str(unsigned int exit_reason)
1493{
1494 unsigned int n1;
1495
1496 for (n1 = 0; n1 < ARRAY_SIZE(exit_reasons_known); n1++) {
1497 if (exit_reason == exit_reasons_known[n1].reason)
1498 return exit_reasons_known[n1].name;
1499 }
1500
1501 return "Unknown";
1502}
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523vm_paddr_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
1524 vm_paddr_t paddr_min, uint32_t memslot)
1525{
1526 struct userspace_mem_region *region;
1527 sparsebit_idx_t pg, base;
1528
1529 TEST_ASSERT(num > 0, "Must allocate at least one page");
1530
1531 TEST_ASSERT((paddr_min % vm->page_size) == 0, "Min physical address "
1532 "not divisible by page size.\n"
1533 " paddr_min: 0x%lx page_size: 0x%x",
1534 paddr_min, vm->page_size);
1535
1536 region = memslot2region(vm, memslot);
1537 base = pg = paddr_min >> vm->page_shift;
1538
1539 do {
1540 for (; pg < base + num; ++pg) {
1541 if (!sparsebit_is_set(region->unused_phy_pages, pg)) {
1542 base = pg = sparsebit_next_set(region->unused_phy_pages, pg);
1543 break;
1544 }
1545 }
1546 } while (pg && pg != base + num);
1547
1548 if (pg == 0) {
1549 fprintf(stderr, "No guest physical page available, "
1550 "paddr_min: 0x%lx page_size: 0x%x memslot: %u\n",
1551 paddr_min, vm->page_size, memslot);
1552 fputs("---- vm dump ----\n", stderr);
1553 vm_dump(stderr, vm, 2);
1554 abort();
1555 }
1556
1557 for (pg = base; pg < base + num; ++pg)
1558 sparsebit_clear(region->unused_phy_pages, pg);
1559
1560 return base * vm->page_size;
1561}
1562
1563vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min,
1564 uint32_t memslot)
1565{
1566 return vm_phy_pages_alloc(vm, 1, paddr_min, memslot);
1567}
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581void *addr_gva2hva(struct kvm_vm *vm, vm_vaddr_t gva)
1582{
1583 return addr_gpa2hva(vm, addr_gva2gpa(vm, gva));
1584}
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598bool vm_is_unrestricted_guest(struct kvm_vm *vm)
1599{
1600 char val = 'N';
1601 size_t count;
1602 FILE *f;
1603
1604 if (vm == NULL) {
1605
1606 f = fopen(KVM_DEV_PATH, "r");
1607 TEST_ASSERT(f != NULL, "Error in opening KVM dev file: %d",
1608 errno);
1609 fclose(f);
1610 }
1611
1612 f = fopen("/sys/module/kvm_intel/parameters/unrestricted_guest", "r");
1613 if (f) {
1614 count = fread(&val, sizeof(char), 1, f);
1615 TEST_ASSERT(count == 1, "Unable to read from param file.");
1616 fclose(f);
1617 }
1618
1619 return val == 'Y';
1620}
1621