1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37#define _GNU_SOURCE
38#include <stdio.h>
39#include <errno.h>
40#include <unistd.h>
41#include <stdlib.h>
42#include <sys/types.h>
43#include <sys/stat.h>
44#include <fcntl.h>
45#include <time.h>
46#include <signal.h>
47#include <poll.h>
48#include <string.h>
49#include <sys/mman.h>
50#include <sys/syscall.h>
51#include <sys/ioctl.h>
52#include <sys/wait.h>
53#include <pthread.h>
54#include <linux/userfaultfd.h>
55#include <setjmp.h>
56#include <stdbool.h>
57#include <assert.h>
58
59#include "../kselftest.h"
60
61#ifdef __NR_userfaultfd
62
63static unsigned long nr_cpus, nr_pages, nr_pages_per_cpu, page_size;
64
65#define BOUNCE_RANDOM (1<<0)
66#define BOUNCE_RACINGFAULTS (1<<1)
67#define BOUNCE_VERIFY (1<<2)
68#define BOUNCE_POLL (1<<3)
69static int bounces;
70
71#define TEST_ANON 1
72#define TEST_HUGETLB 2
73#define TEST_SHMEM 3
74static int test_type;
75
76
77#define ALARM_INTERVAL_SECS 10
78static volatile bool test_uffdio_copy_eexist = true;
79static volatile bool test_uffdio_zeropage_eexist = true;
80
81static bool test_uffdio_wp = false;
82
83static bool map_shared;
84static int huge_fd;
85static char *huge_fd_off0;
86static unsigned long long *count_verify;
87static int uffd, uffd_flags, finished, *pipefd;
88static char *area_src, *area_src_alias, *area_dst, *area_dst_alias;
89static char *zeropage;
90pthread_attr_t attr;
91
92
93struct uffd_stats {
94 int cpu;
95 unsigned long missing_faults;
96 unsigned long wp_faults;
97};
98
99
100#define area_mutex(___area, ___nr) \
101 ((pthread_mutex_t *) ((___area) + (___nr)*page_size))
102
103
104
105
106#define area_count(___area, ___nr) \
107 ((volatile unsigned long long *) ((unsigned long) \
108 ((___area) + (___nr)*page_size + \
109 sizeof(pthread_mutex_t) + \
110 sizeof(unsigned long long) - 1) & \
111 ~(unsigned long)(sizeof(unsigned long long) \
112 - 1)))
113
114const char *examples =
115 "# Run anonymous memory test on 100MiB region with 99999 bounces:\n"
116 "./userfaultfd anon 100 99999\n\n"
117 "# Run share memory test on 1GiB region with 99 bounces:\n"
118 "./userfaultfd shmem 1000 99\n\n"
119 "# Run hugetlb memory test on 256MiB region with 50 bounces (using /dev/hugepages/hugefile):\n"
120 "./userfaultfd hugetlb 256 50 /dev/hugepages/hugefile\n\n"
121 "# Run the same hugetlb test but using shmem:\n"
122 "./userfaultfd hugetlb_shared 256 50 /dev/hugepages/hugefile\n\n"
123 "# 10MiB-~6GiB 999 bounces anonymous test, "
124 "continue forever unless an error triggers\n"
125 "while ./userfaultfd anon $[RANDOM % 6000 + 10] 999; do true; done\n\n";
126
127static void usage(void)
128{
129 fprintf(stderr, "\nUsage: ./userfaultfd <test type> <MiB> <bounces> "
130 "[hugetlbfs_file]\n\n");
131 fprintf(stderr, "Supported <test type>: anon, hugetlb, "
132 "hugetlb_shared, shmem\n\n");
133 fprintf(stderr, "Examples:\n\n");
134 fprintf(stderr, "%s", examples);
135 exit(1);
136}
137
138static void uffd_stats_reset(struct uffd_stats *uffd_stats,
139 unsigned long n_cpus)
140{
141 int i;
142
143 for (i = 0; i < n_cpus; i++) {
144 uffd_stats[i].cpu = i;
145 uffd_stats[i].missing_faults = 0;
146 uffd_stats[i].wp_faults = 0;
147 }
148}
149
150static void uffd_stats_report(struct uffd_stats *stats, int n_cpus)
151{
152 int i;
153 unsigned long long miss_total = 0, wp_total = 0;
154
155 for (i = 0; i < n_cpus; i++) {
156 miss_total += stats[i].missing_faults;
157 wp_total += stats[i].wp_faults;
158 }
159
160 printf("userfaults: %llu missing (", miss_total);
161 for (i = 0; i < n_cpus; i++)
162 printf("%lu+", stats[i].missing_faults);
163 printf("\b), %llu wp (", wp_total);
164 for (i = 0; i < n_cpus; i++)
165 printf("%lu+", stats[i].wp_faults);
166 printf("\b)\n");
167}
168
169static int anon_release_pages(char *rel_area)
170{
171 int ret = 0;
172
173 if (madvise(rel_area, nr_pages * page_size, MADV_DONTNEED)) {
174 perror("madvise");
175 ret = 1;
176 }
177
178 return ret;
179}
180
181static void anon_allocate_area(void **alloc_area)
182{
183 if (posix_memalign(alloc_area, page_size, nr_pages * page_size)) {
184 fprintf(stderr, "out of memory\n");
185 *alloc_area = NULL;
186 }
187}
188
189static void noop_alias_mapping(__u64 *start, size_t len, unsigned long offset)
190{
191}
192
193
194static int hugetlb_release_pages(char *rel_area)
195{
196 int ret = 0;
197
198 if (fallocate(huge_fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
199 rel_area == huge_fd_off0 ? 0 :
200 nr_pages * page_size,
201 nr_pages * page_size)) {
202 perror("fallocate");
203 ret = 1;
204 }
205
206 return ret;
207}
208
209
210static void hugetlb_allocate_area(void **alloc_area)
211{
212 void *area_alias = NULL;
213 char **alloc_area_alias;
214 *alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
215 (map_shared ? MAP_SHARED : MAP_PRIVATE) |
216 MAP_HUGETLB,
217 huge_fd, *alloc_area == area_src ? 0 :
218 nr_pages * page_size);
219 if (*alloc_area == MAP_FAILED) {
220 fprintf(stderr, "mmap of hugetlbfs file failed\n");
221 *alloc_area = NULL;
222 }
223
224 if (map_shared) {
225 area_alias = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
226 MAP_SHARED | MAP_HUGETLB,
227 huge_fd, *alloc_area == area_src ? 0 :
228 nr_pages * page_size);
229 if (area_alias == MAP_FAILED) {
230 if (munmap(*alloc_area, nr_pages * page_size) < 0)
231 perror("hugetlb munmap"), exit(1);
232 *alloc_area = NULL;
233 return;
234 }
235 }
236 if (*alloc_area == area_src) {
237 huge_fd_off0 = *alloc_area;
238 alloc_area_alias = &area_src_alias;
239 } else {
240 alloc_area_alias = &area_dst_alias;
241 }
242 if (area_alias)
243 *alloc_area_alias = area_alias;
244}
245
246static void hugetlb_alias_mapping(__u64 *start, size_t len, unsigned long offset)
247{
248 if (!map_shared)
249 return;
250
251
252
253
254
255
256 *start = (unsigned long) area_dst_alias + offset;
257}
258
259
260static int shmem_release_pages(char *rel_area)
261{
262 int ret = 0;
263
264 if (madvise(rel_area, nr_pages * page_size, MADV_REMOVE)) {
265 perror("madvise");
266 ret = 1;
267 }
268
269 return ret;
270}
271
272static void shmem_allocate_area(void **alloc_area)
273{
274 *alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
275 MAP_ANONYMOUS | MAP_SHARED, -1, 0);
276 if (*alloc_area == MAP_FAILED) {
277 fprintf(stderr, "shared memory mmap failed\n");
278 *alloc_area = NULL;
279 }
280}
281
282struct uffd_test_ops {
283 unsigned long expected_ioctls;
284 void (*allocate_area)(void **alloc_area);
285 int (*release_pages)(char *rel_area);
286 void (*alias_mapping)(__u64 *start, size_t len, unsigned long offset);
287};
288
289#define SHMEM_EXPECTED_IOCTLS ((1 << _UFFDIO_WAKE) | \
290 (1 << _UFFDIO_COPY) | \
291 (1 << _UFFDIO_ZEROPAGE))
292
293#define ANON_EXPECTED_IOCTLS ((1 << _UFFDIO_WAKE) | \
294 (1 << _UFFDIO_COPY) | \
295 (1 << _UFFDIO_ZEROPAGE) | \
296 (1 << _UFFDIO_WRITEPROTECT))
297
298static struct uffd_test_ops anon_uffd_test_ops = {
299 .expected_ioctls = ANON_EXPECTED_IOCTLS,
300 .allocate_area = anon_allocate_area,
301 .release_pages = anon_release_pages,
302 .alias_mapping = noop_alias_mapping,
303};
304
305static struct uffd_test_ops shmem_uffd_test_ops = {
306 .expected_ioctls = SHMEM_EXPECTED_IOCTLS,
307 .allocate_area = shmem_allocate_area,
308 .release_pages = shmem_release_pages,
309 .alias_mapping = noop_alias_mapping,
310};
311
312static struct uffd_test_ops hugetlb_uffd_test_ops = {
313 .expected_ioctls = UFFD_API_RANGE_IOCTLS_BASIC,
314 .allocate_area = hugetlb_allocate_area,
315 .release_pages = hugetlb_release_pages,
316 .alias_mapping = hugetlb_alias_mapping,
317};
318
319static struct uffd_test_ops *uffd_test_ops;
320
321static int my_bcmp(char *str1, char *str2, size_t n)
322{
323 unsigned long i;
324 for (i = 0; i < n; i++)
325 if (str1[i] != str2[i])
326 return 1;
327 return 0;
328}
329
330static void wp_range(int ufd, __u64 start, __u64 len, bool wp)
331{
332 struct uffdio_writeprotect prms = { 0 };
333
334
335 prms.range.start = start;
336 prms.range.len = len;
337
338 prms.mode = wp ? UFFDIO_WRITEPROTECT_MODE_WP : 0;
339
340 if (ioctl(ufd, UFFDIO_WRITEPROTECT, &prms))
341 fprintf(stderr, "clear WP failed for address 0x%Lx\n",
342 start), exit(1);
343}
344
345static void *locking_thread(void *arg)
346{
347 unsigned long cpu = (unsigned long) arg;
348 struct random_data rand;
349 unsigned long page_nr = *(&(page_nr));
350 int32_t rand_nr;
351 unsigned long long count;
352 char randstate[64];
353 unsigned int seed;
354 time_t start;
355
356 if (bounces & BOUNCE_RANDOM) {
357 seed = (unsigned int) time(NULL) - bounces;
358 if (!(bounces & BOUNCE_RACINGFAULTS))
359 seed += cpu;
360 bzero(&rand, sizeof(rand));
361 bzero(&randstate, sizeof(randstate));
362 if (initstate_r(seed, randstate, sizeof(randstate), &rand))
363 fprintf(stderr, "srandom_r error\n"), exit(1);
364 } else {
365 page_nr = -bounces;
366 if (!(bounces & BOUNCE_RACINGFAULTS))
367 page_nr += cpu * nr_pages_per_cpu;
368 }
369
370 while (!finished) {
371 if (bounces & BOUNCE_RANDOM) {
372 if (random_r(&rand, &rand_nr))
373 fprintf(stderr, "random_r 1 error\n"), exit(1);
374 page_nr = rand_nr;
375 if (sizeof(page_nr) > sizeof(rand_nr)) {
376 if (random_r(&rand, &rand_nr))
377 fprintf(stderr, "random_r 2 error\n"), exit(1);
378 page_nr |= (((unsigned long) rand_nr) << 16) <<
379 16;
380 }
381 } else
382 page_nr += 1;
383 page_nr %= nr_pages;
384
385 start = time(NULL);
386 if (bounces & BOUNCE_VERIFY) {
387 count = *area_count(area_dst, page_nr);
388 if (!count)
389 fprintf(stderr,
390 "page_nr %lu wrong count %Lu %Lu\n",
391 page_nr, count,
392 count_verify[page_nr]), exit(1);
393
394
395
396
397
398
399
400
401
402#if 1
403 if (!my_bcmp(area_dst + page_nr * page_size, zeropage,
404 page_size))
405 fprintf(stderr,
406 "my_bcmp page_nr %lu wrong count %Lu %Lu\n",
407 page_nr, count,
408 count_verify[page_nr]), exit(1);
409#else
410 unsigned long loops;
411
412 loops = 0;
413
414
415 while (!bcmp(area_dst + page_nr * page_size, zeropage,
416 page_size)) {
417 loops += 1;
418 if (loops > 10)
419 break;
420 }
421
422
423 if (loops) {
424 fprintf(stderr,
425 "page_nr %lu all zero thread %lu %p %lu\n",
426 page_nr, cpu, area_dst + page_nr * page_size,
427 loops);
428 if (loops > 10)
429 exit(1);
430 }
431#endif
432 }
433
434 pthread_mutex_lock(area_mutex(area_dst, page_nr));
435 count = *area_count(area_dst, page_nr);
436 if (count != count_verify[page_nr]) {
437 fprintf(stderr,
438 "page_nr %lu memory corruption %Lu %Lu\n",
439 page_nr, count,
440 count_verify[page_nr]), exit(1);
441 }
442 count++;
443 *area_count(area_dst, page_nr) = count_verify[page_nr] = count;
444 pthread_mutex_unlock(area_mutex(area_dst, page_nr));
445
446 if (time(NULL) - start > 1)
447 fprintf(stderr,
448 "userfault too slow %ld "
449 "possible false positive with overcommit\n",
450 time(NULL) - start);
451 }
452
453 return NULL;
454}
455
456static void retry_copy_page(int ufd, struct uffdio_copy *uffdio_copy,
457 unsigned long offset)
458{
459 uffd_test_ops->alias_mapping(&uffdio_copy->dst,
460 uffdio_copy->len,
461 offset);
462 if (ioctl(ufd, UFFDIO_COPY, uffdio_copy)) {
463
464 if (uffdio_copy->copy != -EEXIST)
465 fprintf(stderr, "UFFDIO_COPY retry error %Ld\n",
466 uffdio_copy->copy), exit(1);
467 } else {
468 fprintf(stderr, "UFFDIO_COPY retry unexpected %Ld\n",
469 uffdio_copy->copy), exit(1);
470 }
471}
472
473static int __copy_page(int ufd, unsigned long offset, bool retry)
474{
475 struct uffdio_copy uffdio_copy;
476
477 if (offset >= nr_pages * page_size)
478 fprintf(stderr, "unexpected offset %lu\n",
479 offset), exit(1);
480 uffdio_copy.dst = (unsigned long) area_dst + offset;
481 uffdio_copy.src = (unsigned long) area_src + offset;
482 uffdio_copy.len = page_size;
483 if (test_uffdio_wp)
484 uffdio_copy.mode = UFFDIO_COPY_MODE_WP;
485 else
486 uffdio_copy.mode = 0;
487 uffdio_copy.copy = 0;
488 if (ioctl(ufd, UFFDIO_COPY, &uffdio_copy)) {
489
490 if (uffdio_copy.copy != -EEXIST)
491 fprintf(stderr, "UFFDIO_COPY error %Ld\n",
492 uffdio_copy.copy), exit(1);
493 } else if (uffdio_copy.copy != page_size) {
494 fprintf(stderr, "UFFDIO_COPY unexpected copy %Ld\n",
495 uffdio_copy.copy), exit(1);
496 } else {
497 if (test_uffdio_copy_eexist && retry) {
498 test_uffdio_copy_eexist = false;
499 retry_copy_page(ufd, &uffdio_copy, offset);
500 }
501 return 1;
502 }
503 return 0;
504}
505
506static int copy_page_retry(int ufd, unsigned long offset)
507{
508 return __copy_page(ufd, offset, true);
509}
510
511static int copy_page(int ufd, unsigned long offset)
512{
513 return __copy_page(ufd, offset, false);
514}
515
516static int uffd_read_msg(int ufd, struct uffd_msg *msg)
517{
518 int ret = read(uffd, msg, sizeof(*msg));
519
520 if (ret != sizeof(*msg)) {
521 if (ret < 0) {
522 if (errno == EAGAIN)
523 return 1;
524 else
525 perror("blocking read error"), exit(1);
526 } else {
527 fprintf(stderr, "short read\n"), exit(1);
528 }
529 }
530
531 return 0;
532}
533
534static void uffd_handle_page_fault(struct uffd_msg *msg,
535 struct uffd_stats *stats)
536{
537 unsigned long offset;
538
539 if (msg->event != UFFD_EVENT_PAGEFAULT)
540 fprintf(stderr, "unexpected msg event %u\n",
541 msg->event), exit(1);
542
543 if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WP) {
544 wp_range(uffd, msg->arg.pagefault.address, page_size, false);
545 stats->wp_faults++;
546 } else {
547
548 if (bounces & BOUNCE_VERIFY &&
549 msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE)
550 fprintf(stderr, "unexpected write fault\n"), exit(1);
551
552 offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;
553 offset &= ~(page_size-1);
554
555 if (copy_page(uffd, offset))
556 stats->missing_faults++;
557 }
558}
559
560static void *uffd_poll_thread(void *arg)
561{
562 struct uffd_stats *stats = (struct uffd_stats *)arg;
563 unsigned long cpu = stats->cpu;
564 struct pollfd pollfd[2];
565 struct uffd_msg msg;
566 struct uffdio_register uffd_reg;
567 int ret;
568 char tmp_chr;
569
570 pollfd[0].fd = uffd;
571 pollfd[0].events = POLLIN;
572 pollfd[1].fd = pipefd[cpu*2];
573 pollfd[1].events = POLLIN;
574
575 for (;;) {
576 ret = poll(pollfd, 2, -1);
577 if (!ret)
578 fprintf(stderr, "poll error %d\n", ret), exit(1);
579 if (ret < 0)
580 perror("poll"), exit(1);
581 if (pollfd[1].revents & POLLIN) {
582 if (read(pollfd[1].fd, &tmp_chr, 1) != 1)
583 fprintf(stderr, "read pipefd error\n"),
584 exit(1);
585 break;
586 }
587 if (!(pollfd[0].revents & POLLIN))
588 fprintf(stderr, "pollfd[0].revents %d\n",
589 pollfd[0].revents), exit(1);
590 if (uffd_read_msg(uffd, &msg))
591 continue;
592 switch (msg.event) {
593 default:
594 fprintf(stderr, "unexpected msg event %u\n",
595 msg.event), exit(1);
596 break;
597 case UFFD_EVENT_PAGEFAULT:
598 uffd_handle_page_fault(&msg, stats);
599 break;
600 case UFFD_EVENT_FORK:
601 close(uffd);
602 uffd = msg.arg.fork.ufd;
603 pollfd[0].fd = uffd;
604 break;
605 case UFFD_EVENT_REMOVE:
606 uffd_reg.range.start = msg.arg.remove.start;
607 uffd_reg.range.len = msg.arg.remove.end -
608 msg.arg.remove.start;
609 if (ioctl(uffd, UFFDIO_UNREGISTER, &uffd_reg.range))
610 fprintf(stderr, "remove failure\n"), exit(1);
611 break;
612 case UFFD_EVENT_REMAP:
613 area_dst = (char *)(unsigned long)msg.arg.remap.to;
614 break;
615 }
616 }
617
618 return NULL;
619}
620
621pthread_mutex_t uffd_read_mutex = PTHREAD_MUTEX_INITIALIZER;
622
623static void *uffd_read_thread(void *arg)
624{
625 struct uffd_stats *stats = (struct uffd_stats *)arg;
626 struct uffd_msg msg;
627
628 pthread_mutex_unlock(&uffd_read_mutex);
629
630
631 for (;;) {
632 if (uffd_read_msg(uffd, &msg))
633 continue;
634 uffd_handle_page_fault(&msg, stats);
635 }
636
637 return NULL;
638}
639
640static void *background_thread(void *arg)
641{
642 unsigned long cpu = (unsigned long) arg;
643 unsigned long page_nr, start_nr, mid_nr, end_nr;
644
645 start_nr = cpu * nr_pages_per_cpu;
646 end_nr = (cpu+1) * nr_pages_per_cpu;
647 mid_nr = (start_nr + end_nr) / 2;
648
649
650 for (page_nr = start_nr; page_nr < mid_nr; page_nr++)
651 copy_page_retry(uffd, page_nr * page_size);
652
653
654
655
656
657
658 if (test_uffdio_wp)
659 wp_range(uffd, (unsigned long)area_dst + start_nr * page_size,
660 nr_pages_per_cpu * page_size, true);
661
662
663
664
665
666 for (page_nr = mid_nr; page_nr < end_nr; page_nr++)
667 copy_page_retry(uffd, page_nr * page_size);
668
669 return NULL;
670}
671
672static int stress(struct uffd_stats *uffd_stats)
673{
674 unsigned long cpu;
675 pthread_t locking_threads[nr_cpus];
676 pthread_t uffd_threads[nr_cpus];
677 pthread_t background_threads[nr_cpus];
678
679 finished = 0;
680 for (cpu = 0; cpu < nr_cpus; cpu++) {
681 if (pthread_create(&locking_threads[cpu], &attr,
682 locking_thread, (void *)cpu))
683 return 1;
684 if (bounces & BOUNCE_POLL) {
685 if (pthread_create(&uffd_threads[cpu], &attr,
686 uffd_poll_thread,
687 (void *)&uffd_stats[cpu]))
688 return 1;
689 } else {
690 if (pthread_create(&uffd_threads[cpu], &attr,
691 uffd_read_thread,
692 (void *)&uffd_stats[cpu]))
693 return 1;
694 pthread_mutex_lock(&uffd_read_mutex);
695 }
696 if (pthread_create(&background_threads[cpu], &attr,
697 background_thread, (void *)cpu))
698 return 1;
699 }
700 for (cpu = 0; cpu < nr_cpus; cpu++)
701 if (pthread_join(background_threads[cpu], NULL))
702 return 1;
703
704
705
706
707
708
709
710
711
712
713 if (uffd_test_ops->release_pages(area_src))
714 return 1;
715
716
717 finished = 1;
718 for (cpu = 0; cpu < nr_cpus; cpu++)
719 if (pthread_join(locking_threads[cpu], NULL))
720 return 1;
721
722 for (cpu = 0; cpu < nr_cpus; cpu++) {
723 char c;
724 if (bounces & BOUNCE_POLL) {
725 if (write(pipefd[cpu*2+1], &c, 1) != 1) {
726 fprintf(stderr, "pipefd write error\n");
727 return 1;
728 }
729 if (pthread_join(uffd_threads[cpu],
730 (void *)&uffd_stats[cpu]))
731 return 1;
732 } else {
733 if (pthread_cancel(uffd_threads[cpu]))
734 return 1;
735 if (pthread_join(uffd_threads[cpu], NULL))
736 return 1;
737 }
738 }
739
740 return 0;
741}
742
743static int userfaultfd_open(int features)
744{
745 struct uffdio_api uffdio_api;
746
747 uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
748 if (uffd < 0) {
749 fprintf(stderr,
750 "userfaultfd syscall not available in this kernel\n");
751 return 1;
752 }
753 uffd_flags = fcntl(uffd, F_GETFD, NULL);
754
755 uffdio_api.api = UFFD_API;
756 uffdio_api.features = features;
757 if (ioctl(uffd, UFFDIO_API, &uffdio_api)) {
758 fprintf(stderr, "UFFDIO_API\n");
759 return 1;
760 }
761 if (uffdio_api.api != UFFD_API) {
762 fprintf(stderr, "UFFDIO_API error %Lu\n", uffdio_api.api);
763 return 1;
764 }
765
766 return 0;
767}
768
769sigjmp_buf jbuf, *sigbuf;
770
771static void sighndl(int sig, siginfo_t *siginfo, void *ptr)
772{
773 if (sig == SIGBUS) {
774 if (sigbuf)
775 siglongjmp(*sigbuf, 1);
776 abort();
777 }
778}
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801static int faulting_process(int signal_test)
802{
803 unsigned long nr;
804 unsigned long long count;
805 unsigned long split_nr_pages;
806 unsigned long lastnr;
807 struct sigaction act;
808 unsigned long signalled = 0;
809
810 if (test_type != TEST_HUGETLB)
811 split_nr_pages = (nr_pages + 1) / 2;
812 else
813 split_nr_pages = nr_pages;
814
815 if (signal_test) {
816 sigbuf = &jbuf;
817 memset(&act, 0, sizeof(act));
818 act.sa_sigaction = sighndl;
819 act.sa_flags = SA_SIGINFO;
820 if (sigaction(SIGBUS, &act, 0)) {
821 perror("sigaction");
822 return 1;
823 }
824 lastnr = (unsigned long)-1;
825 }
826
827 for (nr = 0; nr < split_nr_pages; nr++) {
828 int steps = 1;
829 unsigned long offset = nr * page_size;
830
831 if (signal_test) {
832 if (sigsetjmp(*sigbuf, 1) != 0) {
833 if (steps == 1 && nr == lastnr) {
834 fprintf(stderr, "Signal repeated\n");
835 return 1;
836 }
837
838 lastnr = nr;
839 if (signal_test == 1) {
840 if (steps == 1) {
841
842 steps++;
843 if (copy_page(uffd, offset))
844 signalled++;
845 } else {
846
847 assert(steps == 2);
848 wp_range(uffd,
849 (__u64)area_dst +
850 offset,
851 page_size, false);
852 }
853 } else {
854 signalled++;
855 continue;
856 }
857 }
858 }
859
860 count = *area_count(area_dst, nr);
861 if (count != count_verify[nr]) {
862 fprintf(stderr,
863 "nr %lu memory corruption %Lu %Lu\n",
864 nr, count,
865 count_verify[nr]);
866 }
867
868
869
870
871 *area_count(area_dst, nr) = count;
872 }
873
874 if (signal_test)
875 return signalled != split_nr_pages;
876
877 if (test_type == TEST_HUGETLB)
878 return 0;
879
880 area_dst = mremap(area_dst, nr_pages * page_size, nr_pages * page_size,
881 MREMAP_MAYMOVE | MREMAP_FIXED, area_src);
882 if (area_dst == MAP_FAILED)
883 perror("mremap"), exit(1);
884
885 for (; nr < nr_pages; nr++) {
886 count = *area_count(area_dst, nr);
887 if (count != count_verify[nr]) {
888 fprintf(stderr,
889 "nr %lu memory corruption %Lu %Lu\n",
890 nr, count,
891 count_verify[nr]), exit(1);
892 }
893
894
895
896
897 *area_count(area_dst, nr) = count;
898 }
899
900 if (uffd_test_ops->release_pages(area_dst))
901 return 1;
902
903 for (nr = 0; nr < nr_pages; nr++) {
904 if (my_bcmp(area_dst + nr * page_size, zeropage, page_size))
905 fprintf(stderr, "nr %lu is not zero\n", nr), exit(1);
906 }
907
908 return 0;
909}
910
911static void retry_uffdio_zeropage(int ufd,
912 struct uffdio_zeropage *uffdio_zeropage,
913 unsigned long offset)
914{
915 uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start,
916 uffdio_zeropage->range.len,
917 offset);
918 if (ioctl(ufd, UFFDIO_ZEROPAGE, uffdio_zeropage)) {
919 if (uffdio_zeropage->zeropage != -EEXIST)
920 fprintf(stderr, "UFFDIO_ZEROPAGE retry error %Ld\n",
921 uffdio_zeropage->zeropage), exit(1);
922 } else {
923 fprintf(stderr, "UFFDIO_ZEROPAGE retry unexpected %Ld\n",
924 uffdio_zeropage->zeropage), exit(1);
925 }
926}
927
928static int __uffdio_zeropage(int ufd, unsigned long offset, bool retry)
929{
930 struct uffdio_zeropage uffdio_zeropage;
931 int ret;
932 unsigned long has_zeropage;
933
934 has_zeropage = uffd_test_ops->expected_ioctls & (1 << _UFFDIO_ZEROPAGE);
935
936 if (offset >= nr_pages * page_size)
937 fprintf(stderr, "unexpected offset %lu\n",
938 offset), exit(1);
939 uffdio_zeropage.range.start = (unsigned long) area_dst + offset;
940 uffdio_zeropage.range.len = page_size;
941 uffdio_zeropage.mode = 0;
942 ret = ioctl(ufd, UFFDIO_ZEROPAGE, &uffdio_zeropage);
943 if (ret) {
944
945 if (has_zeropage) {
946 if (uffdio_zeropage.zeropage == -EEXIST)
947 fprintf(stderr, "UFFDIO_ZEROPAGE -EEXIST\n"),
948 exit(1);
949 else
950 fprintf(stderr, "UFFDIO_ZEROPAGE error %Ld\n",
951 uffdio_zeropage.zeropage), exit(1);
952 } else {
953 if (uffdio_zeropage.zeropage != -EINVAL)
954 fprintf(stderr,
955 "UFFDIO_ZEROPAGE not -EINVAL %Ld\n",
956 uffdio_zeropage.zeropage), exit(1);
957 }
958 } else if (has_zeropage) {
959 if (uffdio_zeropage.zeropage != page_size) {
960 fprintf(stderr, "UFFDIO_ZEROPAGE unexpected %Ld\n",
961 uffdio_zeropage.zeropage), exit(1);
962 } else {
963 if (test_uffdio_zeropage_eexist && retry) {
964 test_uffdio_zeropage_eexist = false;
965 retry_uffdio_zeropage(ufd, &uffdio_zeropage,
966 offset);
967 }
968 return 1;
969 }
970 } else {
971 fprintf(stderr,
972 "UFFDIO_ZEROPAGE succeeded %Ld\n",
973 uffdio_zeropage.zeropage), exit(1);
974 }
975
976 return 0;
977}
978
979static int uffdio_zeropage(int ufd, unsigned long offset)
980{
981 return __uffdio_zeropage(ufd, offset, false);
982}
983
984
985static int userfaultfd_zeropage_test(void)
986{
987 struct uffdio_register uffdio_register;
988 unsigned long expected_ioctls;
989
990 printf("testing UFFDIO_ZEROPAGE: ");
991 fflush(stdout);
992
993 if (uffd_test_ops->release_pages(area_dst))
994 return 1;
995
996 if (userfaultfd_open(0) < 0)
997 return 1;
998 uffdio_register.range.start = (unsigned long) area_dst;
999 uffdio_register.range.len = nr_pages * page_size;
1000 uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
1001 if (test_uffdio_wp)
1002 uffdio_register.mode |= UFFDIO_REGISTER_MODE_WP;
1003 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
1004 fprintf(stderr, "register failure\n"), exit(1);
1005
1006 expected_ioctls = uffd_test_ops->expected_ioctls;
1007 if ((uffdio_register.ioctls & expected_ioctls) !=
1008 expected_ioctls)
1009 fprintf(stderr,
1010 "unexpected missing ioctl for anon memory\n"),
1011 exit(1);
1012
1013 if (uffdio_zeropage(uffd, 0)) {
1014 if (my_bcmp(area_dst, zeropage, page_size))
1015 fprintf(stderr, "zeropage is not zero\n"), exit(1);
1016 }
1017
1018 close(uffd);
1019 printf("done.\n");
1020 return 0;
1021}
1022
1023static int userfaultfd_events_test(void)
1024{
1025 struct uffdio_register uffdio_register;
1026 unsigned long expected_ioctls;
1027 pthread_t uffd_mon;
1028 int err, features;
1029 pid_t pid;
1030 char c;
1031 struct uffd_stats stats = { 0 };
1032
1033 printf("testing events (fork, remap, remove): ");
1034 fflush(stdout);
1035
1036 if (uffd_test_ops->release_pages(area_dst))
1037 return 1;
1038
1039 features = UFFD_FEATURE_EVENT_FORK | UFFD_FEATURE_EVENT_REMAP |
1040 UFFD_FEATURE_EVENT_REMOVE;
1041 if (userfaultfd_open(features) < 0)
1042 return 1;
1043 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
1044
1045 uffdio_register.range.start = (unsigned long) area_dst;
1046 uffdio_register.range.len = nr_pages * page_size;
1047 uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
1048 if (test_uffdio_wp)
1049 uffdio_register.mode |= UFFDIO_REGISTER_MODE_WP;
1050 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
1051 fprintf(stderr, "register failure\n"), exit(1);
1052
1053 expected_ioctls = uffd_test_ops->expected_ioctls;
1054 if ((uffdio_register.ioctls & expected_ioctls) !=
1055 expected_ioctls)
1056 fprintf(stderr,
1057 "unexpected missing ioctl for anon memory\n"),
1058 exit(1);
1059
1060 if (pthread_create(&uffd_mon, &attr, uffd_poll_thread, &stats))
1061 perror("uffd_poll_thread create"), exit(1);
1062
1063 pid = fork();
1064 if (pid < 0)
1065 perror("fork"), exit(1);
1066
1067 if (!pid)
1068 return faulting_process(0);
1069
1070 waitpid(pid, &err, 0);
1071 if (err)
1072 fprintf(stderr, "faulting process failed\n"), exit(1);
1073
1074 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
1075 perror("pipe write"), exit(1);
1076 if (pthread_join(uffd_mon, NULL))
1077 return 1;
1078
1079 close(uffd);
1080
1081 uffd_stats_report(&stats, 1);
1082
1083 return stats.missing_faults != nr_pages;
1084}
1085
1086static int userfaultfd_sig_test(void)
1087{
1088 struct uffdio_register uffdio_register;
1089 unsigned long expected_ioctls;
1090 unsigned long userfaults;
1091 pthread_t uffd_mon;
1092 int err, features;
1093 pid_t pid;
1094 char c;
1095 struct uffd_stats stats = { 0 };
1096
1097 printf("testing signal delivery: ");
1098 fflush(stdout);
1099
1100 if (uffd_test_ops->release_pages(area_dst))
1101 return 1;
1102
1103 features = UFFD_FEATURE_EVENT_FORK|UFFD_FEATURE_SIGBUS;
1104 if (userfaultfd_open(features) < 0)
1105 return 1;
1106 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
1107
1108 uffdio_register.range.start = (unsigned long) area_dst;
1109 uffdio_register.range.len = nr_pages * page_size;
1110 uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
1111 if (test_uffdio_wp)
1112 uffdio_register.mode |= UFFDIO_REGISTER_MODE_WP;
1113 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
1114 fprintf(stderr, "register failure\n"), exit(1);
1115
1116 expected_ioctls = uffd_test_ops->expected_ioctls;
1117 if ((uffdio_register.ioctls & expected_ioctls) !=
1118 expected_ioctls)
1119 fprintf(stderr,
1120 "unexpected missing ioctl for anon memory\n"),
1121 exit(1);
1122
1123 if (faulting_process(1))
1124 fprintf(stderr, "faulting process failed\n"), exit(1);
1125
1126 if (uffd_test_ops->release_pages(area_dst))
1127 return 1;
1128
1129 if (pthread_create(&uffd_mon, &attr, uffd_poll_thread, &stats))
1130 perror("uffd_poll_thread create"), exit(1);
1131
1132 pid = fork();
1133 if (pid < 0)
1134 perror("fork"), exit(1);
1135
1136 if (!pid)
1137 exit(faulting_process(2));
1138
1139 waitpid(pid, &err, 0);
1140 if (err)
1141 fprintf(stderr, "faulting process failed\n"), exit(1);
1142
1143 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
1144 perror("pipe write"), exit(1);
1145 if (pthread_join(uffd_mon, (void **)&userfaults))
1146 return 1;
1147
1148 printf("done.\n");
1149 if (userfaults)
1150 fprintf(stderr, "Signal test failed, userfaults: %ld\n",
1151 userfaults);
1152 close(uffd);
1153 return userfaults != 0;
1154}
1155
1156static int userfaultfd_stress(void)
1157{
1158 void *area;
1159 char *tmp_area;
1160 unsigned long nr;
1161 struct uffdio_register uffdio_register;
1162 unsigned long cpu;
1163 int err;
1164 struct uffd_stats uffd_stats[nr_cpus];
1165
1166 uffd_test_ops->allocate_area((void **)&area_src);
1167 if (!area_src)
1168 return 1;
1169 uffd_test_ops->allocate_area((void **)&area_dst);
1170 if (!area_dst)
1171 return 1;
1172
1173 if (userfaultfd_open(0) < 0)
1174 return 1;
1175
1176 count_verify = malloc(nr_pages * sizeof(unsigned long long));
1177 if (!count_verify) {
1178 perror("count_verify");
1179 return 1;
1180 }
1181
1182 for (nr = 0; nr < nr_pages; nr++) {
1183 *area_mutex(area_src, nr) = (pthread_mutex_t)
1184 PTHREAD_MUTEX_INITIALIZER;
1185 count_verify[nr] = *area_count(area_src, nr) = 1;
1186
1187
1188
1189
1190
1191
1192
1193 *(area_count(area_src, nr) + 1) = 1;
1194 }
1195
1196 pipefd = malloc(sizeof(int) * nr_cpus * 2);
1197 if (!pipefd) {
1198 perror("pipefd");
1199 return 1;
1200 }
1201 for (cpu = 0; cpu < nr_cpus; cpu++) {
1202 if (pipe2(&pipefd[cpu*2], O_CLOEXEC | O_NONBLOCK)) {
1203 perror("pipe");
1204 return 1;
1205 }
1206 }
1207
1208 if (posix_memalign(&area, page_size, page_size)) {
1209 fprintf(stderr, "out of memory\n");
1210 return 1;
1211 }
1212 zeropage = area;
1213 bzero(zeropage, page_size);
1214
1215 pthread_mutex_lock(&uffd_read_mutex);
1216
1217 pthread_attr_init(&attr);
1218 pthread_attr_setstacksize(&attr, 16*1024*1024);
1219
1220 err = 0;
1221 while (bounces--) {
1222 unsigned long expected_ioctls;
1223
1224 printf("bounces: %d, mode:", bounces);
1225 if (bounces & BOUNCE_RANDOM)
1226 printf(" rnd");
1227 if (bounces & BOUNCE_RACINGFAULTS)
1228 printf(" racing");
1229 if (bounces & BOUNCE_VERIFY)
1230 printf(" ver");
1231 if (bounces & BOUNCE_POLL)
1232 printf(" poll");
1233 printf(", ");
1234 fflush(stdout);
1235
1236 if (bounces & BOUNCE_POLL)
1237 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
1238 else
1239 fcntl(uffd, F_SETFL, uffd_flags & ~O_NONBLOCK);
1240
1241
1242 uffdio_register.range.start = (unsigned long) area_dst;
1243 uffdio_register.range.len = nr_pages * page_size;
1244 uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
1245 if (test_uffdio_wp)
1246 uffdio_register.mode |= UFFDIO_REGISTER_MODE_WP;
1247 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register)) {
1248 fprintf(stderr, "register failure\n");
1249 return 1;
1250 }
1251 expected_ioctls = uffd_test_ops->expected_ioctls;
1252 if ((uffdio_register.ioctls & expected_ioctls) !=
1253 expected_ioctls) {
1254 fprintf(stderr,
1255 "unexpected missing ioctl for anon memory\n");
1256 return 1;
1257 }
1258
1259 if (area_dst_alias) {
1260 uffdio_register.range.start = (unsigned long)
1261 area_dst_alias;
1262 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register)) {
1263 fprintf(stderr, "register failure alias\n");
1264 return 1;
1265 }
1266 }
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292 if (uffd_test_ops->release_pages(area_dst))
1293 return 1;
1294
1295 uffd_stats_reset(uffd_stats, nr_cpus);
1296
1297
1298 if (stress(uffd_stats))
1299 return 1;
1300
1301
1302 if (test_uffdio_wp)
1303 wp_range(uffd, (unsigned long)area_dst,
1304 nr_pages * page_size, false);
1305
1306
1307 if (ioctl(uffd, UFFDIO_UNREGISTER, &uffdio_register.range)) {
1308 fprintf(stderr, "unregister failure\n");
1309 return 1;
1310 }
1311 if (area_dst_alias) {
1312 uffdio_register.range.start = (unsigned long) area_dst;
1313 if (ioctl(uffd, UFFDIO_UNREGISTER,
1314 &uffdio_register.range)) {
1315 fprintf(stderr, "unregister failure alias\n");
1316 return 1;
1317 }
1318 }
1319
1320
1321 if (bounces & BOUNCE_VERIFY) {
1322 for (nr = 0; nr < nr_pages; nr++) {
1323 if (*area_count(area_dst, nr) != count_verify[nr]) {
1324 fprintf(stderr,
1325 "error area_count %Lu %Lu %lu\n",
1326 *area_count(area_src, nr),
1327 count_verify[nr],
1328 nr);
1329 err = 1;
1330 bounces = 0;
1331 }
1332 }
1333 }
1334
1335
1336 tmp_area = area_src;
1337 area_src = area_dst;
1338 area_dst = tmp_area;
1339
1340 tmp_area = area_src_alias;
1341 area_src_alias = area_dst_alias;
1342 area_dst_alias = tmp_area;
1343
1344 uffd_stats_report(uffd_stats, nr_cpus);
1345 }
1346
1347 if (err)
1348 return err;
1349
1350 close(uffd);
1351 return userfaultfd_zeropage_test() || userfaultfd_sig_test()
1352 || userfaultfd_events_test();
1353}
1354
1355
1356
1357
1358unsigned long default_huge_page_size(void)
1359{
1360 unsigned long hps = 0;
1361 char *line = NULL;
1362 size_t linelen = 0;
1363 FILE *f = fopen("/proc/meminfo", "r");
1364
1365 if (!f)
1366 return 0;
1367 while (getline(&line, &linelen, f) > 0) {
1368 if (sscanf(line, "Hugepagesize: %lu kB", &hps) == 1) {
1369 hps <<= 10;
1370 break;
1371 }
1372 }
1373
1374 free(line);
1375 fclose(f);
1376 return hps;
1377}
1378
1379static void set_test_type(const char *type)
1380{
1381 if (!strcmp(type, "anon")) {
1382 test_type = TEST_ANON;
1383 uffd_test_ops = &anon_uffd_test_ops;
1384
1385 test_uffdio_wp = true;
1386 } else if (!strcmp(type, "hugetlb")) {
1387 test_type = TEST_HUGETLB;
1388 uffd_test_ops = &hugetlb_uffd_test_ops;
1389 } else if (!strcmp(type, "hugetlb_shared")) {
1390 map_shared = true;
1391 test_type = TEST_HUGETLB;
1392 uffd_test_ops = &hugetlb_uffd_test_ops;
1393 } else if (!strcmp(type, "shmem")) {
1394 map_shared = true;
1395 test_type = TEST_SHMEM;
1396 uffd_test_ops = &shmem_uffd_test_ops;
1397 } else {
1398 fprintf(stderr, "Unknown test type: %s\n", type), exit(1);
1399 }
1400
1401 if (test_type == TEST_HUGETLB)
1402 page_size = default_huge_page_size();
1403 else
1404 page_size = sysconf(_SC_PAGE_SIZE);
1405
1406 if (!page_size)
1407 fprintf(stderr, "Unable to determine page size\n"),
1408 exit(2);
1409 if ((unsigned long) area_count(NULL, 0) + sizeof(unsigned long long) * 2
1410 > page_size)
1411 fprintf(stderr, "Impossible to run this test\n"), exit(2);
1412}
1413
1414static void sigalrm(int sig)
1415{
1416 if (sig != SIGALRM)
1417 abort();
1418 test_uffdio_copy_eexist = true;
1419 test_uffdio_zeropage_eexist = true;
1420 alarm(ALARM_INTERVAL_SECS);
1421}
1422
1423int main(int argc, char **argv)
1424{
1425 if (argc < 4)
1426 usage();
1427
1428 if (signal(SIGALRM, sigalrm) == SIG_ERR)
1429 fprintf(stderr, "failed to arm SIGALRM"), exit(1);
1430 alarm(ALARM_INTERVAL_SECS);
1431
1432 set_test_type(argv[1]);
1433
1434 nr_cpus = sysconf(_SC_NPROCESSORS_ONLN);
1435 nr_pages_per_cpu = atol(argv[2]) * 1024*1024 / page_size /
1436 nr_cpus;
1437 if (!nr_pages_per_cpu) {
1438 fprintf(stderr, "invalid MiB\n");
1439 usage();
1440 }
1441
1442 bounces = atoi(argv[3]);
1443 if (bounces <= 0) {
1444 fprintf(stderr, "invalid bounces\n");
1445 usage();
1446 }
1447 nr_pages = nr_pages_per_cpu * nr_cpus;
1448
1449 if (test_type == TEST_HUGETLB) {
1450 if (argc < 5)
1451 usage();
1452 huge_fd = open(argv[4], O_CREAT | O_RDWR, 0755);
1453 if (huge_fd < 0) {
1454 fprintf(stderr, "Open of %s failed", argv[3]);
1455 perror("open");
1456 exit(1);
1457 }
1458 if (ftruncate(huge_fd, 0)) {
1459 fprintf(stderr, "ftruncate %s to size 0 failed", argv[3]);
1460 perror("ftruncate");
1461 exit(1);
1462 }
1463 }
1464 printf("nr_pages: %lu, nr_pages_per_cpu: %lu\n",
1465 nr_pages, nr_pages_per_cpu);
1466 return userfaultfd_stress();
1467}
1468
1469#else
1470
1471#warning "missing __NR_userfaultfd definition"
1472
1473int main(void)
1474{
1475 printf("skip: Skipping userfaultfd test (missing __NR_userfaultfd)\n");
1476 return KSFT_SKIP;
1477}
1478
1479#endif
1480