1
2
3
4
5
6
7
8
9
10#define _GNU_SOURCE
11
12#include <inttypes.h>
13#include <stdio.h>
14#include <stdlib.h>
15#include <time.h>
16#include <poll.h>
17#include <pthread.h>
18#include <linux/userfaultfd.h>
19#include <sys/syscall.h>
20
21#include "kvm_util.h"
22#include "test_util.h"
23#include "perf_test_util.h"
24#include "guest_modes.h"
25
26#ifdef __NR_userfaultfd
27
28#ifdef PRINT_PER_PAGE_UPDATES
29#define PER_PAGE_DEBUG(...) printf(__VA_ARGS__)
30#else
31#define PER_PAGE_DEBUG(...) _no_printf(__VA_ARGS__)
32#endif
33
34#ifdef PRINT_PER_VCPU_UPDATES
35#define PER_VCPU_DEBUG(...) printf(__VA_ARGS__)
36#else
37#define PER_VCPU_DEBUG(...) _no_printf(__VA_ARGS__)
38#endif
39
40static int nr_vcpus = 1;
41static uint64_t guest_percpu_mem_size = DEFAULT_PER_VCPU_MEM_SIZE;
42static size_t demand_paging_size;
43static char *guest_data_prototype;
44
45static void *vcpu_worker(void *data)
46{
47 int ret;
48 struct perf_test_vcpu_args *vcpu_args = (struct perf_test_vcpu_args *)data;
49 int vcpu_id = vcpu_args->vcpu_id;
50 struct kvm_vm *vm = perf_test_args.vm;
51 struct kvm_run *run;
52 struct timespec start;
53 struct timespec ts_diff;
54
55 run = vcpu_state(vm, vcpu_id);
56
57 clock_gettime(CLOCK_MONOTONIC, &start);
58
59
60 ret = _vcpu_run(vm, vcpu_id);
61 TEST_ASSERT(ret == 0, "vcpu_run failed: %d\n", ret);
62 if (get_ucall(vm, vcpu_id, NULL) != UCALL_SYNC) {
63 TEST_ASSERT(false,
64 "Invalid guest sync status: exit_reason=%s\n",
65 exit_reason_str(run->exit_reason));
66 }
67
68 ts_diff = timespec_elapsed(start);
69 PER_VCPU_DEBUG("vCPU %d execution time: %ld.%.9lds\n", vcpu_id,
70 ts_diff.tv_sec, ts_diff.tv_nsec);
71
72 return NULL;
73}
74
75static int handle_uffd_page_request(int uffd_mode, int uffd, uint64_t addr)
76{
77 pid_t tid = syscall(__NR_gettid);
78 struct timespec start;
79 struct timespec ts_diff;
80 int r;
81
82 clock_gettime(CLOCK_MONOTONIC, &start);
83
84 if (uffd_mode == UFFDIO_REGISTER_MODE_MISSING) {
85 struct uffdio_copy copy;
86
87 copy.src = (uint64_t)guest_data_prototype;
88 copy.dst = addr;
89 copy.len = demand_paging_size;
90 copy.mode = 0;
91
92 r = ioctl(uffd, UFFDIO_COPY, ©);
93 if (r == -1) {
94 pr_info("Failed UFFDIO_COPY in 0x%lx from thread %d with errno: %d\n",
95 addr, tid, errno);
96 return r;
97 }
98 } else if (uffd_mode == UFFDIO_REGISTER_MODE_MINOR) {
99 struct uffdio_continue cont = {0};
100
101 cont.range.start = addr;
102 cont.range.len = demand_paging_size;
103
104 r = ioctl(uffd, UFFDIO_CONTINUE, &cont);
105 if (r == -1) {
106 pr_info("Failed UFFDIO_CONTINUE in 0x%lx from thread %d with errno: %d\n",
107 addr, tid, errno);
108 return r;
109 }
110 } else {
111 TEST_FAIL("Invalid uffd mode %d", uffd_mode);
112 }
113
114 ts_diff = timespec_elapsed(start);
115
116 PER_PAGE_DEBUG("UFFD page-in %d \t%ld ns\n", tid,
117 timespec_to_ns(ts_diff));
118 PER_PAGE_DEBUG("Paged in %ld bytes at 0x%lx from thread %d\n",
119 demand_paging_size, addr, tid);
120
121 return 0;
122}
123
124bool quit_uffd_thread;
125
126struct uffd_handler_args {
127 int uffd_mode;
128 int uffd;
129 int pipefd;
130 useconds_t delay;
131};
132
133static void *uffd_handler_thread_fn(void *arg)
134{
135 struct uffd_handler_args *uffd_args = (struct uffd_handler_args *)arg;
136 int uffd = uffd_args->uffd;
137 int pipefd = uffd_args->pipefd;
138 useconds_t delay = uffd_args->delay;
139 int64_t pages = 0;
140 struct timespec start;
141 struct timespec ts_diff;
142
143 clock_gettime(CLOCK_MONOTONIC, &start);
144 while (!quit_uffd_thread) {
145 struct uffd_msg msg;
146 struct pollfd pollfd[2];
147 char tmp_chr;
148 int r;
149 uint64_t addr;
150
151 pollfd[0].fd = uffd;
152 pollfd[0].events = POLLIN;
153 pollfd[1].fd = pipefd;
154 pollfd[1].events = POLLIN;
155
156 r = poll(pollfd, 2, -1);
157 switch (r) {
158 case -1:
159 pr_info("poll err");
160 continue;
161 case 0:
162 continue;
163 case 1:
164 break;
165 default:
166 pr_info("Polling uffd returned %d", r);
167 return NULL;
168 }
169
170 if (pollfd[0].revents & POLLERR) {
171 pr_info("uffd revents has POLLERR");
172 return NULL;
173 }
174
175 if (pollfd[1].revents & POLLIN) {
176 r = read(pollfd[1].fd, &tmp_chr, 1);
177 TEST_ASSERT(r == 1,
178 "Error reading pipefd in UFFD thread\n");
179 return NULL;
180 }
181
182 if (!(pollfd[0].revents & POLLIN))
183 continue;
184
185 r = read(uffd, &msg, sizeof(msg));
186 if (r == -1) {
187 if (errno == EAGAIN)
188 continue;
189 pr_info("Read of uffd got errno %d\n", errno);
190 return NULL;
191 }
192
193 if (r != sizeof(msg)) {
194 pr_info("Read on uffd returned unexpected size: %d bytes", r);
195 return NULL;
196 }
197
198 if (!(msg.event & UFFD_EVENT_PAGEFAULT))
199 continue;
200
201 if (delay)
202 usleep(delay);
203 addr = msg.arg.pagefault.address;
204 r = handle_uffd_page_request(uffd_args->uffd_mode, uffd, addr);
205 if (r < 0)
206 return NULL;
207 pages++;
208 }
209
210 ts_diff = timespec_elapsed(start);
211 PER_VCPU_DEBUG("userfaulted %ld pages over %ld.%.9lds. (%f/sec)\n",
212 pages, ts_diff.tv_sec, ts_diff.tv_nsec,
213 pages / ((double)ts_diff.tv_sec + (double)ts_diff.tv_nsec / 100000000.0));
214
215 return NULL;
216}
217
218static void setup_demand_paging(struct kvm_vm *vm,
219 pthread_t *uffd_handler_thread, int pipefd,
220 int uffd_mode, useconds_t uffd_delay,
221 struct uffd_handler_args *uffd_args,
222 void *hva, void *alias, uint64_t len)
223{
224 bool is_minor = (uffd_mode == UFFDIO_REGISTER_MODE_MINOR);
225 int uffd;
226 struct uffdio_api uffdio_api;
227 struct uffdio_register uffdio_register;
228 uint64_t expected_ioctls = ((uint64_t) 1) << _UFFDIO_COPY;
229
230 PER_PAGE_DEBUG("Userfaultfd %s mode, faults resolved with %s\n",
231 is_minor ? "MINOR" : "MISSING",
232 is_minor ? "UFFDIO_CONINUE" : "UFFDIO_COPY");
233
234
235 if (is_minor) {
236 size_t p;
237
238 expected_ioctls = ((uint64_t) 1) << _UFFDIO_CONTINUE;
239
240 TEST_ASSERT(alias != NULL, "Alias required for minor faults");
241 for (p = 0; p < (len / demand_paging_size); ++p) {
242 memcpy(alias + (p * demand_paging_size),
243 guest_data_prototype, demand_paging_size);
244 }
245 }
246
247 uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
248 TEST_ASSERT(uffd >= 0, "uffd creation failed, errno: %d", errno);
249
250 uffdio_api.api = UFFD_API;
251 uffdio_api.features = 0;
252 TEST_ASSERT(ioctl(uffd, UFFDIO_API, &uffdio_api) != -1,
253 "ioctl UFFDIO_API failed: %" PRIu64,
254 (uint64_t)uffdio_api.api);
255
256 uffdio_register.range.start = (uint64_t)hva;
257 uffdio_register.range.len = len;
258 uffdio_register.mode = uffd_mode;
259 TEST_ASSERT(ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) != -1,
260 "ioctl UFFDIO_REGISTER failed");
261 TEST_ASSERT((uffdio_register.ioctls & expected_ioctls) ==
262 expected_ioctls, "missing userfaultfd ioctls");
263
264 uffd_args->uffd_mode = uffd_mode;
265 uffd_args->uffd = uffd;
266 uffd_args->pipefd = pipefd;
267 uffd_args->delay = uffd_delay;
268 pthread_create(uffd_handler_thread, NULL, uffd_handler_thread_fn,
269 uffd_args);
270
271 PER_VCPU_DEBUG("Created uffd thread for HVA range [%p, %p)\n",
272 hva, hva + len);
273}
274
275struct test_params {
276 int uffd_mode;
277 useconds_t uffd_delay;
278 enum vm_mem_backing_src_type src_type;
279 bool partition_vcpu_memory_access;
280};
281
282static void run_test(enum vm_guest_mode mode, void *arg)
283{
284 struct test_params *p = arg;
285 pthread_t *vcpu_threads;
286 pthread_t *uffd_handler_threads = NULL;
287 struct uffd_handler_args *uffd_args = NULL;
288 struct timespec start;
289 struct timespec ts_diff;
290 int *pipefds = NULL;
291 struct kvm_vm *vm;
292 int vcpu_id;
293 int r;
294
295 vm = perf_test_create_vm(mode, nr_vcpus, guest_percpu_mem_size, 1,
296 p->src_type);
297
298 perf_test_args.wr_fract = 1;
299
300 demand_paging_size = get_backing_src_pagesz(p->src_type);
301
302 guest_data_prototype = malloc(demand_paging_size);
303 TEST_ASSERT(guest_data_prototype,
304 "Failed to allocate buffer for guest data pattern");
305 memset(guest_data_prototype, 0xAB, demand_paging_size);
306
307 vcpu_threads = malloc(nr_vcpus * sizeof(*vcpu_threads));
308 TEST_ASSERT(vcpu_threads, "Memory allocation failed");
309
310 perf_test_setup_vcpus(vm, nr_vcpus, guest_percpu_mem_size,
311 p->partition_vcpu_memory_access);
312
313 if (p->uffd_mode) {
314 uffd_handler_threads =
315 malloc(nr_vcpus * sizeof(*uffd_handler_threads));
316 TEST_ASSERT(uffd_handler_threads, "Memory allocation failed");
317
318 uffd_args = malloc(nr_vcpus * sizeof(*uffd_args));
319 TEST_ASSERT(uffd_args, "Memory allocation failed");
320
321 pipefds = malloc(sizeof(int) * nr_vcpus * 2);
322 TEST_ASSERT(pipefds, "Unable to allocate memory for pipefd");
323
324 for (vcpu_id = 0; vcpu_id < nr_vcpus; vcpu_id++) {
325 vm_paddr_t vcpu_gpa;
326 void *vcpu_hva;
327 void *vcpu_alias;
328 uint64_t vcpu_mem_size;
329
330
331 if (p->partition_vcpu_memory_access) {
332 vcpu_gpa = guest_test_phys_mem +
333 (vcpu_id * guest_percpu_mem_size);
334 vcpu_mem_size = guest_percpu_mem_size;
335 } else {
336 vcpu_gpa = guest_test_phys_mem;
337 vcpu_mem_size = guest_percpu_mem_size * nr_vcpus;
338 }
339 PER_VCPU_DEBUG("Added VCPU %d with test mem gpa [%lx, %lx)\n",
340 vcpu_id, vcpu_gpa, vcpu_gpa + vcpu_mem_size);
341
342
343 vcpu_hva = addr_gpa2hva(vm, vcpu_gpa);
344 vcpu_alias = addr_gpa2alias(vm, vcpu_gpa);
345
346
347
348
349
350 r = pipe2(&pipefds[vcpu_id * 2],
351 O_CLOEXEC | O_NONBLOCK);
352 TEST_ASSERT(!r, "Failed to set up pipefd");
353
354 setup_demand_paging(vm, &uffd_handler_threads[vcpu_id],
355 pipefds[vcpu_id * 2], p->uffd_mode,
356 p->uffd_delay, &uffd_args[vcpu_id],
357 vcpu_hva, vcpu_alias,
358 vcpu_mem_size);
359 }
360 }
361
362
363 sync_global_to_guest(vm, perf_test_args);
364
365 pr_info("Finished creating vCPUs and starting uffd threads\n");
366
367 clock_gettime(CLOCK_MONOTONIC, &start);
368
369 for (vcpu_id = 0; vcpu_id < nr_vcpus; vcpu_id++) {
370 pthread_create(&vcpu_threads[vcpu_id], NULL, vcpu_worker,
371 &perf_test_args.vcpu_args[vcpu_id]);
372 }
373
374 pr_info("Started all vCPUs\n");
375
376
377 for (vcpu_id = 0; vcpu_id < nr_vcpus; vcpu_id++) {
378 pthread_join(vcpu_threads[vcpu_id], NULL);
379 PER_VCPU_DEBUG("Joined thread for vCPU %d\n", vcpu_id);
380 }
381
382 ts_diff = timespec_elapsed(start);
383
384 pr_info("All vCPU threads joined\n");
385
386 if (p->uffd_mode) {
387 char c;
388
389
390 for (vcpu_id = 0; vcpu_id < nr_vcpus; vcpu_id++) {
391 r = write(pipefds[vcpu_id * 2 + 1], &c, 1);
392 TEST_ASSERT(r == 1, "Unable to write to pipefd");
393
394 pthread_join(uffd_handler_threads[vcpu_id], NULL);
395 }
396 }
397
398 pr_info("Total guest execution time: %ld.%.9lds\n",
399 ts_diff.tv_sec, ts_diff.tv_nsec);
400 pr_info("Overall demand paging rate: %f pgs/sec\n",
401 perf_test_args.vcpu_args[0].pages * nr_vcpus /
402 ((double)ts_diff.tv_sec + (double)ts_diff.tv_nsec / 100000000.0));
403
404 perf_test_destroy_vm(vm);
405
406 free(guest_data_prototype);
407 free(vcpu_threads);
408 if (p->uffd_mode) {
409 free(uffd_handler_threads);
410 free(uffd_args);
411 free(pipefds);
412 }
413}
414
415static void help(char *name)
416{
417 puts("");
418 printf("usage: %s [-h] [-m vm_mode] [-u uffd_mode] [-d uffd_delay_usec]\n"
419 " [-b memory] [-s type] [-v vcpus] [-o]\n", name);
420 guest_modes_help();
421 printf(" -u: use userfaultfd to handle vCPU page faults. Mode is a\n"
422 " UFFD registration mode: 'MISSING' or 'MINOR'.\n");
423 printf(" -d: add a delay in usec to the User Fault\n"
424 " FD handler to simulate demand paging\n"
425 " overheads. Ignored without -u.\n");
426 printf(" -b: specify the size of the memory region which should be\n"
427 " demand paged by each vCPU. e.g. 10M or 3G.\n"
428 " Default: 1G\n");
429 backing_src_help("-s");
430 printf(" -v: specify the number of vCPUs to run.\n");
431 printf(" -o: Overlap guest memory accesses instead of partitioning\n"
432 " them into a separate region of memory for each vCPU.\n");
433 puts("");
434 exit(0);
435}
436
437int main(int argc, char *argv[])
438{
439 int max_vcpus = kvm_check_cap(KVM_CAP_MAX_VCPUS);
440 struct test_params p = {
441 .src_type = DEFAULT_VM_MEM_SRC,
442 .partition_vcpu_memory_access = true,
443 };
444 int opt;
445
446 guest_modes_append_default();
447
448 while ((opt = getopt(argc, argv, "hm:u:d:b:s:v:o")) != -1) {
449 switch (opt) {
450 case 'm':
451 guest_modes_cmdline(optarg);
452 break;
453 case 'u':
454 if (!strcmp("MISSING", optarg))
455 p.uffd_mode = UFFDIO_REGISTER_MODE_MISSING;
456 else if (!strcmp("MINOR", optarg))
457 p.uffd_mode = UFFDIO_REGISTER_MODE_MINOR;
458 TEST_ASSERT(p.uffd_mode, "UFFD mode must be 'MISSING' or 'MINOR'.");
459 break;
460 case 'd':
461 p.uffd_delay = strtoul(optarg, NULL, 0);
462 TEST_ASSERT(p.uffd_delay >= 0, "A negative UFFD delay is not supported.");
463 break;
464 case 'b':
465 guest_percpu_mem_size = parse_size(optarg);
466 break;
467 case 's':
468 p.src_type = parse_backing_src_type(optarg);
469 break;
470 case 'v':
471 nr_vcpus = atoi(optarg);
472 TEST_ASSERT(nr_vcpus > 0 && nr_vcpus <= max_vcpus,
473 "Invalid number of vcpus, must be between 1 and %d", max_vcpus);
474 break;
475 case 'o':
476 p.partition_vcpu_memory_access = false;
477 break;
478 case 'h':
479 default:
480 help(argv[0]);
481 break;
482 }
483 }
484
485 if (p.uffd_mode == UFFDIO_REGISTER_MODE_MINOR &&
486 !backing_src_is_shared(p.src_type)) {
487 TEST_FAIL("userfaultfd MINOR mode requires shared memory; pick a different -s");
488 }
489
490 for_each_guest_mode(run_test, &p);
491
492 return 0;
493}
494
495#else
496
497#warning "missing __NR_userfaultfd definition"
498
499int main(void)
500{
501 print_skip("__NR_userfaultfd must be present for userfaultfd test");
502 return KSFT_SKIP;
503}
504
505#endif
506
