1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Userfaultfd unit tests.
4 *
5 * Copyright (C) 2015-2023 Red Hat, Inc.
6 */
7
8 #include <asm-generic/unistd.h>
9 #include "uffd-common.h"
10
11 #include "../../../../mm/gup_test.h"
12
13 /* The unit test doesn't need a large or random size, make it 32MB for now */
14 #define UFFD_TEST_MEM_SIZE (32UL << 20)
15
16 #define MEM_ANON BIT_ULL(0)
17 #define MEM_SHMEM BIT_ULL(1)
18 #define MEM_SHMEM_PRIVATE BIT_ULL(2)
19 #define MEM_HUGETLB BIT_ULL(3)
20 #define MEM_HUGETLB_PRIVATE BIT_ULL(4)
21
22 #define MEM_ALL (MEM_ANON | MEM_SHMEM | MEM_SHMEM_PRIVATE | \
23 MEM_HUGETLB | MEM_HUGETLB_PRIVATE)
24
25 #define ALIGN_UP(x, align_to) \
26 ((__typeof__(x))((((unsigned long)(x)) + ((align_to)-1)) & ~((align_to)-1)))
27
28 struct mem_type {
29 const char *name;
30 unsigned int mem_flag;
31 uffd_test_ops_t *mem_ops;
32 bool shared;
33 };
34 typedef struct mem_type mem_type_t;
35
36 mem_type_t mem_types[] = {
37 {
38 .name = "anon",
39 .mem_flag = MEM_ANON,
40 .mem_ops = &anon_uffd_test_ops,
41 .shared = false,
42 },
43 {
44 .name = "shmem",
45 .mem_flag = MEM_SHMEM,
46 .mem_ops = &shmem_uffd_test_ops,
47 .shared = true,
48 },
49 {
50 .name = "shmem-private",
51 .mem_flag = MEM_SHMEM_PRIVATE,
52 .mem_ops = &shmem_uffd_test_ops,
53 .shared = false,
54 },
55 {
56 .name = "hugetlb",
57 .mem_flag = MEM_HUGETLB,
58 .mem_ops = &hugetlb_uffd_test_ops,
59 .shared = true,
60 },
61 {
62 .name = "hugetlb-private",
63 .mem_flag = MEM_HUGETLB_PRIVATE,
64 .mem_ops = &hugetlb_uffd_test_ops,
65 .shared = false,
66 },
67 };
68
69 /* Arguments to be passed over to each uffd unit test */
70 struct uffd_test_args {
71 mem_type_t *mem_type;
72 };
73 typedef struct uffd_test_args uffd_test_args_t;
74
75 /* Returns: UFFD_TEST_* */
76 typedef void (*uffd_test_fn)(uffd_test_args_t *);
77
78 typedef struct {
79 const char *name;
80 uffd_test_fn uffd_fn;
81 unsigned int mem_targets;
82 uint64_t uffd_feature_required;
83 uffd_test_case_ops_t *test_case_ops;
84 } uffd_test_case_t;
85
uffd_test_report(void)86 static void uffd_test_report(void)
87 {
88 printf("Userfaults unit tests: pass=%u, skip=%u, fail=%u (total=%u)\n",
89 ksft_get_pass_cnt(),
90 ksft_get_xskip_cnt(),
91 ksft_get_fail_cnt(),
92 ksft_test_num());
93 }
94
uffd_test_pass(void)95 static void uffd_test_pass(void)
96 {
97 printf("done\n");
98 ksft_inc_pass_cnt();
99 }
100
101 #define uffd_test_start(...) do { \
102 printf("Testing "); \
103 printf(__VA_ARGS__); \
104 printf("... "); \
105 fflush(stdout); \
106 } while (0)
107
108 #define uffd_test_fail(...) do { \
109 printf("failed [reason: "); \
110 printf(__VA_ARGS__); \
111 printf("]\n"); \
112 ksft_inc_fail_cnt(); \
113 } while (0)
114
uffd_test_skip(const char * message)115 static void uffd_test_skip(const char *message)
116 {
117 printf("skipped [reason: %s]\n", message);
118 ksft_inc_xskip_cnt();
119 }
120
121 /*
122 * Returns 1 if specific userfaultfd supported, 0 otherwise. Note, we'll
123 * return 1 even if some test failed as long as uffd supported, because in
124 * that case we still want to proceed with the rest uffd unit tests.
125 */
test_uffd_api(bool use_dev)126 static int test_uffd_api(bool use_dev)
127 {
128 struct uffdio_api uffdio_api;
129 int uffd;
130
131 uffd_test_start("UFFDIO_API (with %s)",
132 use_dev ? "/dev/userfaultfd" : "syscall");
133
134 if (use_dev)
135 uffd = uffd_open_dev(UFFD_FLAGS);
136 else
137 uffd = uffd_open_sys(UFFD_FLAGS);
138 if (uffd < 0) {
139 uffd_test_skip("cannot open userfaultfd handle");
140 return 0;
141 }
142
143 /* Test wrong UFFD_API */
144 uffdio_api.api = 0xab;
145 uffdio_api.features = 0;
146 if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
147 uffd_test_fail("UFFDIO_API should fail with wrong api but didn't");
148 goto out;
149 }
150
151 /* Test wrong feature bit */
152 uffdio_api.api = UFFD_API;
153 uffdio_api.features = BIT_ULL(63);
154 if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
155 uffd_test_fail("UFFDIO_API should fail with wrong feature but didn't");
156 goto out;
157 }
158
159 /* Test normal UFFDIO_API */
160 uffdio_api.api = UFFD_API;
161 uffdio_api.features = 0;
162 if (ioctl(uffd, UFFDIO_API, &uffdio_api)) {
163 uffd_test_fail("UFFDIO_API should succeed but failed");
164 goto out;
165 }
166
167 /* Test double requests of UFFDIO_API with a random feature set */
168 uffdio_api.features = BIT_ULL(0);
169 if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
170 uffd_test_fail("UFFDIO_API should reject initialized uffd");
171 goto out;
172 }
173
174 uffd_test_pass();
175 out:
176 close(uffd);
177 /* We have a valid uffd handle */
178 return 1;
179 }
180
181 /*
182 * This function initializes the global variables. TODO: remove global
183 * vars and then remove this.
184 */
185 static int
uffd_setup_environment(uffd_test_args_t * args,uffd_test_case_t * test,mem_type_t * mem_type,const char ** errmsg)186 uffd_setup_environment(uffd_test_args_t *args, uffd_test_case_t *test,
187 mem_type_t *mem_type, const char **errmsg)
188 {
189 map_shared = mem_type->shared;
190 uffd_test_ops = mem_type->mem_ops;
191 uffd_test_case_ops = test->test_case_ops;
192
193 if (mem_type->mem_flag & (MEM_HUGETLB_PRIVATE | MEM_HUGETLB))
194 page_size = default_huge_page_size();
195 else
196 page_size = psize();
197
198 nr_pages = UFFD_TEST_MEM_SIZE / page_size;
199 /* TODO: remove this global var.. it's so ugly */
200 nr_cpus = 1;
201
202 /* Initialize test arguments */
203 args->mem_type = mem_type;
204
205 return uffd_test_ctx_init(test->uffd_feature_required, errmsg);
206 }
207
uffd_feature_supported(uffd_test_case_t * test)208 static bool uffd_feature_supported(uffd_test_case_t *test)
209 {
210 uint64_t features;
211
212 if (uffd_get_features(&features))
213 return false;
214
215 return (features & test->uffd_feature_required) ==
216 test->uffd_feature_required;
217 }
218
pagemap_open(void)219 static int pagemap_open(void)
220 {
221 int fd = open("/proc/self/pagemap", O_RDONLY);
222
223 if (fd < 0)
224 err("open pagemap");
225
226 return fd;
227 }
228
229 /* This macro let __LINE__ works in err() */
230 #define pagemap_check_wp(value, wp) do { \
231 if (!!(value & PM_UFFD_WP) != wp) \
232 err("pagemap uffd-wp bit error: 0x%"PRIx64, value); \
233 } while (0)
234
235 typedef struct {
236 int parent_uffd, child_uffd;
237 } fork_event_args;
238
fork_event_consumer(void * data)239 static void *fork_event_consumer(void *data)
240 {
241 fork_event_args *args = data;
242 struct uffd_msg msg = { 0 };
243
244 /* Read until a full msg received */
245 while (uffd_read_msg(args->parent_uffd, &msg));
246
247 if (msg.event != UFFD_EVENT_FORK)
248 err("wrong message: %u\n", msg.event);
249
250 /* Just to be properly freed later */
251 args->child_uffd = msg.arg.fork.ufd;
252 return NULL;
253 }
254
255 typedef struct {
256 int gup_fd;
257 bool pinned;
258 } pin_args;
259
260 /*
261 * Returns 0 if succeed, <0 for errors. pin_pages() needs to be paired
262 * with unpin_pages(). Currently it needs to be RO longterm pin to satisfy
263 * all needs of the test cases (e.g., trigger unshare, trigger fork() early
264 * CoW, etc.).
265 */
pin_pages(pin_args * args,void * buffer,size_t size)266 static int pin_pages(pin_args *args, void *buffer, size_t size)
267 {
268 struct pin_longterm_test test = {
269 .addr = (uintptr_t)buffer,
270 .size = size,
271 /* Read-only pins */
272 .flags = 0,
273 };
274
275 if (args->pinned)
276 err("already pinned");
277
278 args->gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);
279 if (args->gup_fd < 0)
280 return -errno;
281
282 if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_START, &test)) {
283 /* Even if gup_test existed, can be an old gup_test / kernel */
284 close(args->gup_fd);
285 return -errno;
286 }
287 args->pinned = true;
288 return 0;
289 }
290
unpin_pages(pin_args * args)291 static void unpin_pages(pin_args *args)
292 {
293 if (!args->pinned)
294 err("unpin without pin first");
295 if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_STOP))
296 err("PIN_LONGTERM_TEST_STOP");
297 close(args->gup_fd);
298 args->pinned = false;
299 }
300
pagemap_test_fork(int uffd,bool with_event,bool test_pin)301 static int pagemap_test_fork(int uffd, bool with_event, bool test_pin)
302 {
303 fork_event_args args = { .parent_uffd = uffd, .child_uffd = -1 };
304 pthread_t thread;
305 pid_t child;
306 uint64_t value;
307 int fd, result;
308
309 /* Prepare a thread to resolve EVENT_FORK */
310 if (with_event) {
311 if (pthread_create(&thread, NULL, fork_event_consumer, &args))
312 err("pthread_create()");
313 }
314
315 child = fork();
316 if (!child) {
317 /* Open the pagemap fd of the child itself */
318 pin_args args = {};
319
320 fd = pagemap_open();
321
322 if (test_pin && pin_pages(&args, area_dst, page_size))
323 /*
324 * Normally when reach here we have pinned in
325 * previous tests, so shouldn't fail anymore
326 */
327 err("pin page failed in child");
328
329 value = pagemap_get_entry(fd, area_dst);
330 /*
331 * After fork(), we should handle uffd-wp bit differently:
332 *
333 * (1) when with EVENT_FORK, it should persist
334 * (2) when without EVENT_FORK, it should be dropped
335 */
336 pagemap_check_wp(value, with_event);
337 if (test_pin)
338 unpin_pages(&args);
339 /* Succeed */
340 exit(0);
341 }
342 waitpid(child, &result, 0);
343
344 if (with_event) {
345 if (pthread_join(thread, NULL))
346 err("pthread_join()");
347 if (args.child_uffd < 0)
348 err("Didn't receive child uffd");
349 close(args.child_uffd);
350 }
351
352 return result;
353 }
354
uffd_wp_unpopulated_test(uffd_test_args_t * args)355 static void uffd_wp_unpopulated_test(uffd_test_args_t *args)
356 {
357 uint64_t value;
358 int pagemap_fd;
359
360 if (uffd_register(uffd, area_dst, nr_pages * page_size,
361 false, true, false))
362 err("register failed");
363
364 pagemap_fd = pagemap_open();
365
366 /* Test applying pte marker to anon unpopulated */
367 wp_range(uffd, (uint64_t)area_dst, page_size, true);
368 value = pagemap_get_entry(pagemap_fd, area_dst);
369 pagemap_check_wp(value, true);
370
371 /* Test unprotect on anon pte marker */
372 wp_range(uffd, (uint64_t)area_dst, page_size, false);
373 value = pagemap_get_entry(pagemap_fd, area_dst);
374 pagemap_check_wp(value, false);
375
376 /* Test zap on anon marker */
377 wp_range(uffd, (uint64_t)area_dst, page_size, true);
378 if (madvise(area_dst, page_size, MADV_DONTNEED))
379 err("madvise(MADV_DONTNEED) failed");
380 value = pagemap_get_entry(pagemap_fd, area_dst);
381 pagemap_check_wp(value, false);
382
383 /* Test fault in after marker removed */
384 *area_dst = 1;
385 value = pagemap_get_entry(pagemap_fd, area_dst);
386 pagemap_check_wp(value, false);
387 /* Drop it to make pte none again */
388 if (madvise(area_dst, page_size, MADV_DONTNEED))
389 err("madvise(MADV_DONTNEED) failed");
390
391 /* Test read-zero-page upon pte marker */
392 wp_range(uffd, (uint64_t)area_dst, page_size, true);
393 *(volatile char *)area_dst;
394 /* Drop it to make pte none again */
395 if (madvise(area_dst, page_size, MADV_DONTNEED))
396 err("madvise(MADV_DONTNEED) failed");
397
398 uffd_test_pass();
399 }
400
uffd_wp_fork_test_common(uffd_test_args_t * args,bool with_event)401 static void uffd_wp_fork_test_common(uffd_test_args_t *args,
402 bool with_event)
403 {
404 int pagemap_fd;
405 uint64_t value;
406
407 if (uffd_register(uffd, area_dst, nr_pages * page_size,
408 false, true, false))
409 err("register failed");
410
411 pagemap_fd = pagemap_open();
412
413 /* Touch the page */
414 *area_dst = 1;
415 wp_range(uffd, (uint64_t)area_dst, page_size, true);
416 value = pagemap_get_entry(pagemap_fd, area_dst);
417 pagemap_check_wp(value, true);
418 if (pagemap_test_fork(uffd, with_event, false)) {
419 uffd_test_fail("Detected %s uffd-wp bit in child in present pte",
420 with_event ? "missing" : "stall");
421 goto out;
422 }
423
424 /*
425 * This is an attempt for zapping the pgtable so as to test the
426 * markers.
427 *
428 * For private mappings, PAGEOUT will only work on exclusive ptes
429 * (PM_MMAP_EXCLUSIVE) which we should satisfy.
430 *
431 * For shared, PAGEOUT may not work. Use DONTNEED instead which
432 * plays a similar role of zapping (rather than freeing the page)
433 * to expose pte markers.
434 */
435 if (args->mem_type->shared) {
436 if (madvise(area_dst, page_size, MADV_DONTNEED))
437 err("MADV_DONTNEED");
438 } else {
439 /*
440 * NOTE: ignore retval because private-hugetlb doesn't yet
441 * support swapping, so it could fail.
442 */
443 madvise(area_dst, page_size, MADV_PAGEOUT);
444 }
445
446 /* Uffd-wp should persist even swapped out */
447 value = pagemap_get_entry(pagemap_fd, area_dst);
448 pagemap_check_wp(value, true);
449 if (pagemap_test_fork(uffd, with_event, false)) {
450 uffd_test_fail("Detected %s uffd-wp bit in child in zapped pte",
451 with_event ? "missing" : "stall");
452 goto out;
453 }
454
455 /* Unprotect; this tests swap pte modifications */
456 wp_range(uffd, (uint64_t)area_dst, page_size, false);
457 value = pagemap_get_entry(pagemap_fd, area_dst);
458 pagemap_check_wp(value, false);
459
460 /* Fault in the page from disk */
461 *area_dst = 2;
462 value = pagemap_get_entry(pagemap_fd, area_dst);
463 pagemap_check_wp(value, false);
464 uffd_test_pass();
465 out:
466 if (uffd_unregister(uffd, area_dst, nr_pages * page_size))
467 err("unregister failed");
468 close(pagemap_fd);
469 }
470
uffd_wp_fork_test(uffd_test_args_t * args)471 static void uffd_wp_fork_test(uffd_test_args_t *args)
472 {
473 uffd_wp_fork_test_common(args, false);
474 }
475
uffd_wp_fork_with_event_test(uffd_test_args_t * args)476 static void uffd_wp_fork_with_event_test(uffd_test_args_t *args)
477 {
478 uffd_wp_fork_test_common(args, true);
479 }
480
uffd_wp_fork_pin_test_common(uffd_test_args_t * args,bool with_event)481 static void uffd_wp_fork_pin_test_common(uffd_test_args_t *args,
482 bool with_event)
483 {
484 int pagemap_fd;
485 pin_args pin_args = {};
486
487 if (uffd_register(uffd, area_dst, page_size, false, true, false))
488 err("register failed");
489
490 pagemap_fd = pagemap_open();
491
492 /* Touch the page */
493 *area_dst = 1;
494 wp_range(uffd, (uint64_t)area_dst, page_size, true);
495
496 /*
497 * 1. First pin, then fork(). This tests fork() special path when
498 * doing early CoW if the page is private.
499 */
500 if (pin_pages(&pin_args, area_dst, page_size)) {
501 uffd_test_skip("Possibly CONFIG_GUP_TEST missing "
502 "or unprivileged");
503 close(pagemap_fd);
504 uffd_unregister(uffd, area_dst, page_size);
505 return;
506 }
507
508 if (pagemap_test_fork(uffd, with_event, false)) {
509 uffd_test_fail("Detected %s uffd-wp bit in early CoW of fork()",
510 with_event ? "missing" : "stall");
511 unpin_pages(&pin_args);
512 goto out;
513 }
514
515 unpin_pages(&pin_args);
516
517 /*
518 * 2. First fork(), then pin (in the child, where test_pin==true).
519 * This tests COR, aka, page unsharing on private memories.
520 */
521 if (pagemap_test_fork(uffd, with_event, true)) {
522 uffd_test_fail("Detected %s uffd-wp bit when RO pin",
523 with_event ? "missing" : "stall");
524 goto out;
525 }
526 uffd_test_pass();
527 out:
528 if (uffd_unregister(uffd, area_dst, page_size))
529 err("register failed");
530 close(pagemap_fd);
531 }
532
uffd_wp_fork_pin_test(uffd_test_args_t * args)533 static void uffd_wp_fork_pin_test(uffd_test_args_t *args)
534 {
535 uffd_wp_fork_pin_test_common(args, false);
536 }
537
uffd_wp_fork_pin_with_event_test(uffd_test_args_t * args)538 static void uffd_wp_fork_pin_with_event_test(uffd_test_args_t *args)
539 {
540 uffd_wp_fork_pin_test_common(args, true);
541 }
542
check_memory_contents(char * p)543 static void check_memory_contents(char *p)
544 {
545 unsigned long i, j;
546 uint8_t expected_byte;
547
548 for (i = 0; i < nr_pages; ++i) {
549 expected_byte = ~((uint8_t)(i % ((uint8_t)-1)));
550 for (j = 0; j < page_size; j++) {
551 uint8_t v = *(uint8_t *)(p + (i * page_size) + j);
552 if (v != expected_byte)
553 err("unexpected page contents");
554 }
555 }
556 }
557
uffd_minor_test_common(bool test_collapse,bool test_wp)558 static void uffd_minor_test_common(bool test_collapse, bool test_wp)
559 {
560 unsigned long p;
561 pthread_t uffd_mon;
562 char c;
563 struct uffd_args args = { 0 };
564
565 /*
566 * NOTE: MADV_COLLAPSE is not yet compatible with WP, so testing
567 * both do not make much sense.
568 */
569 assert(!(test_collapse && test_wp));
570
571 if (uffd_register(uffd, area_dst_alias, nr_pages * page_size,
572 /* NOTE! MADV_COLLAPSE may not work with uffd-wp */
573 false, test_wp, true))
574 err("register failure");
575
576 /*
577 * After registering with UFFD, populate the non-UFFD-registered side of
578 * the shared mapping. This should *not* trigger any UFFD minor faults.
579 */
580 for (p = 0; p < nr_pages; ++p)
581 memset(area_dst + (p * page_size), p % ((uint8_t)-1),
582 page_size);
583
584 args.apply_wp = test_wp;
585 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
586 err("uffd_poll_thread create");
587
588 /*
589 * Read each of the pages back using the UFFD-registered mapping. We
590 * expect that the first time we touch a page, it will result in a minor
591 * fault. uffd_poll_thread will resolve the fault by bit-flipping the
592 * page's contents, and then issuing a CONTINUE ioctl.
593 */
594 check_memory_contents(area_dst_alias);
595
596 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
597 err("pipe write");
598 if (pthread_join(uffd_mon, NULL))
599 err("join() failed");
600
601 if (test_collapse) {
602 if (madvise(area_dst_alias, nr_pages * page_size,
603 MADV_COLLAPSE)) {
604 /* It's fine to fail for this one... */
605 uffd_test_skip("MADV_COLLAPSE failed");
606 return;
607 }
608
609 uffd_test_ops->check_pmd_mapping(area_dst,
610 nr_pages * page_size /
611 read_pmd_pagesize());
612 /*
613 * This won't cause uffd-fault - it purely just makes sure there
614 * was no corruption.
615 */
616 check_memory_contents(area_dst_alias);
617 }
618
619 if (args.missing_faults != 0 || args.minor_faults != nr_pages)
620 uffd_test_fail("stats check error");
621 else
622 uffd_test_pass();
623 }
624
uffd_minor_test(uffd_test_args_t * args)625 void uffd_minor_test(uffd_test_args_t *args)
626 {
627 uffd_minor_test_common(false, false);
628 }
629
uffd_minor_wp_test(uffd_test_args_t * args)630 void uffd_minor_wp_test(uffd_test_args_t *args)
631 {
632 uffd_minor_test_common(false, true);
633 }
634
uffd_minor_collapse_test(uffd_test_args_t * args)635 void uffd_minor_collapse_test(uffd_test_args_t *args)
636 {
637 uffd_minor_test_common(true, false);
638 }
639
640 static sigjmp_buf jbuf, *sigbuf;
641
sighndl(int sig,siginfo_t * siginfo,void * ptr)642 static void sighndl(int sig, siginfo_t *siginfo, void *ptr)
643 {
644 if (sig == SIGBUS) {
645 if (sigbuf)
646 siglongjmp(*sigbuf, 1);
647 abort();
648 }
649 }
650
651 /*
652 * For non-cooperative userfaultfd test we fork() a process that will
653 * generate pagefaults, will mremap the area monitored by the
654 * userfaultfd and at last this process will release the monitored
655 * area.
656 * For the anonymous and shared memory the area is divided into two
657 * parts, the first part is accessed before mremap, and the second
658 * part is accessed after mremap. Since hugetlbfs does not support
659 * mremap, the entire monitored area is accessed in a single pass for
660 * HUGETLB_TEST.
661 * The release of the pages currently generates event for shmem and
662 * anonymous memory (UFFD_EVENT_REMOVE), hence it is not checked
663 * for hugetlb.
664 * For signal test(UFFD_FEATURE_SIGBUS), signal_test = 1, we register
665 * monitored area, generate pagefaults and test that signal is delivered.
666 * Use UFFDIO_COPY to allocate missing page and retry. For signal_test = 2
667 * test robustness use case - we release monitored area, fork a process
668 * that will generate pagefaults and verify signal is generated.
669 * This also tests UFFD_FEATURE_EVENT_FORK event along with the signal
670 * feature. Using monitor thread, verify no userfault events are generated.
671 */
faulting_process(int signal_test,bool wp)672 static int faulting_process(int signal_test, bool wp)
673 {
674 unsigned long nr, i;
675 unsigned long long count;
676 unsigned long split_nr_pages;
677 unsigned long lastnr;
678 struct sigaction act;
679 volatile unsigned long signalled = 0;
680
681 split_nr_pages = (nr_pages + 1) / 2;
682
683 if (signal_test) {
684 sigbuf = &jbuf;
685 memset(&act, 0, sizeof(act));
686 act.sa_sigaction = sighndl;
687 act.sa_flags = SA_SIGINFO;
688 if (sigaction(SIGBUS, &act, 0))
689 err("sigaction");
690 lastnr = (unsigned long)-1;
691 }
692
693 for (nr = 0; nr < split_nr_pages; nr++) {
694 volatile int steps = 1;
695 unsigned long offset = nr * page_size;
696
697 if (signal_test) {
698 if (sigsetjmp(*sigbuf, 1) != 0) {
699 if (steps == 1 && nr == lastnr)
700 err("Signal repeated");
701
702 lastnr = nr;
703 if (signal_test == 1) {
704 if (steps == 1) {
705 /* This is a MISSING request */
706 steps++;
707 if (copy_page(uffd, offset, wp))
708 signalled++;
709 } else {
710 /* This is a WP request */
711 assert(steps == 2);
712 wp_range(uffd,
713 (__u64)area_dst +
714 offset,
715 page_size, false);
716 }
717 } else {
718 signalled++;
719 continue;
720 }
721 }
722 }
723
724 count = *area_count(area_dst, nr);
725 if (count != count_verify[nr])
726 err("nr %lu memory corruption %llu %llu\n",
727 nr, count, count_verify[nr]);
728 /*
729 * Trigger write protection if there is by writing
730 * the same value back.
731 */
732 *area_count(area_dst, nr) = count;
733 }
734
735 if (signal_test)
736 return signalled != split_nr_pages;
737
738 area_dst = mremap(area_dst, nr_pages * page_size, nr_pages * page_size,
739 MREMAP_MAYMOVE | MREMAP_FIXED, area_src);
740 if (area_dst == MAP_FAILED)
741 err("mremap");
742 /* Reset area_src since we just clobbered it */
743 area_src = NULL;
744
745 for (; nr < nr_pages; nr++) {
746 count = *area_count(area_dst, nr);
747 if (count != count_verify[nr]) {
748 err("nr %lu memory corruption %llu %llu\n",
749 nr, count, count_verify[nr]);
750 }
751 /*
752 * Trigger write protection if there is by writing
753 * the same value back.
754 */
755 *area_count(area_dst, nr) = count;
756 }
757
758 uffd_test_ops->release_pages(area_dst);
759
760 for (nr = 0; nr < nr_pages; nr++)
761 for (i = 0; i < page_size; i++)
762 if (*(area_dst + nr * page_size + i) != 0)
763 err("page %lu offset %lu is not zero", nr, i);
764
765 return 0;
766 }
767
uffd_sigbus_test_common(bool wp)768 static void uffd_sigbus_test_common(bool wp)
769 {
770 unsigned long userfaults;
771 pthread_t uffd_mon;
772 pid_t pid;
773 int err;
774 char c;
775 struct uffd_args args = { 0 };
776
777 ready_for_fork = false;
778
779 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
780
781 if (uffd_register(uffd, area_dst, nr_pages * page_size,
782 true, wp, false))
783 err("register failure");
784
785 if (faulting_process(1, wp))
786 err("faulting process failed");
787
788 uffd_test_ops->release_pages(area_dst);
789
790 args.apply_wp = wp;
791 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
792 err("uffd_poll_thread create");
793
794 while (!ready_for_fork)
795 ; /* Wait for the poll_thread to start executing before forking */
796
797 pid = fork();
798 if (pid < 0)
799 err("fork");
800
801 if (!pid)
802 exit(faulting_process(2, wp));
803
804 waitpid(pid, &err, 0);
805 if (err)
806 err("faulting process failed");
807 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
808 err("pipe write");
809 if (pthread_join(uffd_mon, (void **)&userfaults))
810 err("pthread_join()");
811
812 if (userfaults)
813 uffd_test_fail("Signal test failed, userfaults: %ld", userfaults);
814 else
815 uffd_test_pass();
816 }
817
uffd_sigbus_test(uffd_test_args_t * args)818 static void uffd_sigbus_test(uffd_test_args_t *args)
819 {
820 uffd_sigbus_test_common(false);
821 }
822
uffd_sigbus_wp_test(uffd_test_args_t * args)823 static void uffd_sigbus_wp_test(uffd_test_args_t *args)
824 {
825 uffd_sigbus_test_common(true);
826 }
827
uffd_events_test_common(bool wp)828 static void uffd_events_test_common(bool wp)
829 {
830 pthread_t uffd_mon;
831 pid_t pid;
832 int err;
833 char c;
834 struct uffd_args args = { 0 };
835
836 ready_for_fork = false;
837
838 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
839 if (uffd_register(uffd, area_dst, nr_pages * page_size,
840 true, wp, false))
841 err("register failure");
842
843 args.apply_wp = wp;
844 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
845 err("uffd_poll_thread create");
846
847 while (!ready_for_fork)
848 ; /* Wait for the poll_thread to start executing before forking */
849
850 pid = fork();
851 if (pid < 0)
852 err("fork");
853
854 if (!pid)
855 exit(faulting_process(0, wp));
856
857 waitpid(pid, &err, 0);
858 if (err)
859 err("faulting process failed");
860 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
861 err("pipe write");
862 if (pthread_join(uffd_mon, NULL))
863 err("pthread_join()");
864
865 if (args.missing_faults != nr_pages)
866 uffd_test_fail("Fault counts wrong");
867 else
868 uffd_test_pass();
869 }
870
uffd_events_test(uffd_test_args_t * args)871 static void uffd_events_test(uffd_test_args_t *args)
872 {
873 uffd_events_test_common(false);
874 }
875
uffd_events_wp_test(uffd_test_args_t * args)876 static void uffd_events_wp_test(uffd_test_args_t *args)
877 {
878 uffd_events_test_common(true);
879 }
880
retry_uffdio_zeropage(int ufd,struct uffdio_zeropage * uffdio_zeropage)881 static void retry_uffdio_zeropage(int ufd,
882 struct uffdio_zeropage *uffdio_zeropage)
883 {
884 uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start,
885 uffdio_zeropage->range.len,
886 0);
887 if (ioctl(ufd, UFFDIO_ZEROPAGE, uffdio_zeropage)) {
888 if (uffdio_zeropage->zeropage != -EEXIST)
889 err("UFFDIO_ZEROPAGE error: %"PRId64,
890 (int64_t)uffdio_zeropage->zeropage);
891 } else {
892 err("UFFDIO_ZEROPAGE error: %"PRId64,
893 (int64_t)uffdio_zeropage->zeropage);
894 }
895 }
896
do_uffdio_zeropage(int ufd,bool has_zeropage)897 static bool do_uffdio_zeropage(int ufd, bool has_zeropage)
898 {
899 struct uffdio_zeropage uffdio_zeropage = { 0 };
900 int ret;
901 __s64 res;
902
903 uffdio_zeropage.range.start = (unsigned long) area_dst;
904 uffdio_zeropage.range.len = page_size;
905 uffdio_zeropage.mode = 0;
906 ret = ioctl(ufd, UFFDIO_ZEROPAGE, &uffdio_zeropage);
907 res = uffdio_zeropage.zeropage;
908 if (ret) {
909 /* real retval in ufdio_zeropage.zeropage */
910 if (has_zeropage)
911 err("UFFDIO_ZEROPAGE error: %"PRId64, (int64_t)res);
912 else if (res != -EINVAL)
913 err("UFFDIO_ZEROPAGE not -EINVAL");
914 } else if (has_zeropage) {
915 if (res != page_size)
916 err("UFFDIO_ZEROPAGE unexpected size");
917 else
918 retry_uffdio_zeropage(ufd, &uffdio_zeropage);
919 return true;
920 } else
921 err("UFFDIO_ZEROPAGE succeeded");
922
923 return false;
924 }
925
926 /*
927 * Registers a range with MISSING mode only for zeropage test. Return true
928 * if UFFDIO_ZEROPAGE supported, false otherwise. Can't use uffd_register()
929 * because we want to detect .ioctls along the way.
930 */
931 static bool
uffd_register_detect_zeropage(int uffd,void * addr,uint64_t len)932 uffd_register_detect_zeropage(int uffd, void *addr, uint64_t len)
933 {
934 uint64_t ioctls = 0;
935
936 if (uffd_register_with_ioctls(uffd, addr, len, true,
937 false, false, &ioctls))
938 err("zeropage register fail");
939
940 return ioctls & (1 << _UFFDIO_ZEROPAGE);
941 }
942
943 /* exercise UFFDIO_ZEROPAGE */
uffd_zeropage_test(uffd_test_args_t * args)944 static void uffd_zeropage_test(uffd_test_args_t *args)
945 {
946 bool has_zeropage;
947 int i;
948
949 has_zeropage = uffd_register_detect_zeropage(uffd, area_dst, page_size);
950 if (area_dst_alias)
951 /* Ignore the retval; we already have it */
952 uffd_register_detect_zeropage(uffd, area_dst_alias, page_size);
953
954 if (do_uffdio_zeropage(uffd, has_zeropage))
955 for (i = 0; i < page_size; i++)
956 if (area_dst[i] != 0)
957 err("data non-zero at offset %d\n", i);
958
959 if (uffd_unregister(uffd, area_dst, page_size))
960 err("unregister");
961
962 if (area_dst_alias && uffd_unregister(uffd, area_dst_alias, page_size))
963 err("unregister");
964
965 uffd_test_pass();
966 }
967
uffd_register_poison(int uffd,void * addr,uint64_t len)968 static void uffd_register_poison(int uffd, void *addr, uint64_t len)
969 {
970 uint64_t ioctls = 0;
971 uint64_t expected = (1 << _UFFDIO_COPY) | (1 << _UFFDIO_POISON);
972
973 if (uffd_register_with_ioctls(uffd, addr, len, true,
974 false, false, &ioctls))
975 err("poison register fail");
976
977 if ((ioctls & expected) != expected)
978 err("registered area doesn't support COPY and POISON ioctls");
979 }
980
do_uffdio_poison(int uffd,unsigned long offset)981 static void do_uffdio_poison(int uffd, unsigned long offset)
982 {
983 struct uffdio_poison uffdio_poison = { 0 };
984 int ret;
985 __s64 res;
986
987 uffdio_poison.range.start = (unsigned long) area_dst + offset;
988 uffdio_poison.range.len = page_size;
989 uffdio_poison.mode = 0;
990 ret = ioctl(uffd, UFFDIO_POISON, &uffdio_poison);
991 res = uffdio_poison.updated;
992
993 if (ret)
994 err("UFFDIO_POISON error: %"PRId64, (int64_t)res);
995 else if (res != page_size)
996 err("UFFDIO_POISON unexpected size: %"PRId64, (int64_t)res);
997 }
998
uffd_poison_handle_fault(struct uffd_msg * msg,struct uffd_args * args)999 static void uffd_poison_handle_fault(
1000 struct uffd_msg *msg, struct uffd_args *args)
1001 {
1002 unsigned long offset;
1003
1004 if (msg->event != UFFD_EVENT_PAGEFAULT)
1005 err("unexpected msg event %u", msg->event);
1006
1007 if (msg->arg.pagefault.flags &
1008 (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR))
1009 err("unexpected fault type %llu", msg->arg.pagefault.flags);
1010
1011 offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;
1012 offset &= ~(page_size-1);
1013
1014 /* Odd pages -> copy zeroed page; even pages -> poison. */
1015 if (offset & page_size)
1016 copy_page(uffd, offset, false);
1017 else
1018 do_uffdio_poison(uffd, offset);
1019 }
1020
uffd_poison_test(uffd_test_args_t * targs)1021 static void uffd_poison_test(uffd_test_args_t *targs)
1022 {
1023 pthread_t uffd_mon;
1024 char c;
1025 struct uffd_args args = { 0 };
1026 struct sigaction act = { 0 };
1027 unsigned long nr_sigbus = 0;
1028 unsigned long nr;
1029
1030 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
1031
1032 uffd_register_poison(uffd, area_dst, nr_pages * page_size);
1033 memset(area_src, 0, nr_pages * page_size);
1034
1035 args.handle_fault = uffd_poison_handle_fault;
1036 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
1037 err("uffd_poll_thread create");
1038
1039 sigbuf = &jbuf;
1040 act.sa_sigaction = sighndl;
1041 act.sa_flags = SA_SIGINFO;
1042 if (sigaction(SIGBUS, &act, 0))
1043 err("sigaction");
1044
1045 for (nr = 0; nr < nr_pages; ++nr) {
1046 unsigned long offset = nr * page_size;
1047 const char *bytes = (const char *) area_dst + offset;
1048 const char *i;
1049
1050 if (sigsetjmp(*sigbuf, 1)) {
1051 /*
1052 * Access below triggered a SIGBUS, which was caught by
1053 * sighndl, which then jumped here. Count this SIGBUS,
1054 * and move on to next page.
1055 */
1056 ++nr_sigbus;
1057 continue;
1058 }
1059
1060 for (i = bytes; i < bytes + page_size; ++i) {
1061 if (*i)
1062 err("nonzero byte in area_dst (%p) at %p: %u",
1063 area_dst, i, *i);
1064 }
1065 }
1066
1067 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
1068 err("pipe write");
1069 if (pthread_join(uffd_mon, NULL))
1070 err("pthread_join()");
1071
1072 if (nr_sigbus != nr_pages / 2)
1073 err("expected to receive %lu SIGBUS, actually received %lu",
1074 nr_pages / 2, nr_sigbus);
1075
1076 uffd_test_pass();
1077 }
1078
1079 static void
uffd_move_handle_fault_common(struct uffd_msg * msg,struct uffd_args * args,unsigned long len)1080 uffd_move_handle_fault_common(struct uffd_msg *msg, struct uffd_args *args,
1081 unsigned long len)
1082 {
1083 unsigned long offset;
1084
1085 if (msg->event != UFFD_EVENT_PAGEFAULT)
1086 err("unexpected msg event %u", msg->event);
1087
1088 if (msg->arg.pagefault.flags &
1089 (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR | UFFD_PAGEFAULT_FLAG_WRITE))
1090 err("unexpected fault type %llu", msg->arg.pagefault.flags);
1091
1092 offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;
1093 offset &= ~(len-1);
1094
1095 if (move_page(uffd, offset, len))
1096 args->missing_faults++;
1097 }
1098
uffd_move_handle_fault(struct uffd_msg * msg,struct uffd_args * args)1099 static void uffd_move_handle_fault(struct uffd_msg *msg,
1100 struct uffd_args *args)
1101 {
1102 uffd_move_handle_fault_common(msg, args, page_size);
1103 }
1104
uffd_move_pmd_handle_fault(struct uffd_msg * msg,struct uffd_args * args)1105 static void uffd_move_pmd_handle_fault(struct uffd_msg *msg,
1106 struct uffd_args *args)
1107 {
1108 uffd_move_handle_fault_common(msg, args, read_pmd_pagesize());
1109 }
1110
1111 static void
uffd_move_test_common(uffd_test_args_t * targs,unsigned long chunk_size,void (* handle_fault)(struct uffd_msg * msg,struct uffd_args * args))1112 uffd_move_test_common(uffd_test_args_t *targs, unsigned long chunk_size,
1113 void (*handle_fault)(struct uffd_msg *msg, struct uffd_args *args))
1114 {
1115 unsigned long nr;
1116 pthread_t uffd_mon;
1117 char c;
1118 unsigned long long count;
1119 struct uffd_args args = { 0 };
1120 char *orig_area_src, *orig_area_dst;
1121 unsigned long step_size, step_count;
1122 unsigned long src_offs = 0;
1123 unsigned long dst_offs = 0;
1124
1125 /* Prevent source pages from being mapped more than once */
1126 if (madvise(area_src, nr_pages * page_size, MADV_DONTFORK))
1127 err("madvise(MADV_DONTFORK) failure");
1128
1129 if (uffd_register(uffd, area_dst, nr_pages * page_size,
1130 true, false, false))
1131 err("register failure");
1132
1133 args.handle_fault = handle_fault;
1134 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
1135 err("uffd_poll_thread create");
1136
1137 step_size = chunk_size / page_size;
1138 step_count = nr_pages / step_size;
1139
1140 if (chunk_size > page_size) {
1141 char *aligned_src = ALIGN_UP(area_src, chunk_size);
1142 char *aligned_dst = ALIGN_UP(area_dst, chunk_size);
1143
1144 if (aligned_src != area_src || aligned_dst != area_dst) {
1145 src_offs = (aligned_src - area_src) / page_size;
1146 dst_offs = (aligned_dst - area_dst) / page_size;
1147 step_count--;
1148 }
1149 orig_area_src = area_src;
1150 orig_area_dst = area_dst;
1151 area_src = aligned_src;
1152 area_dst = aligned_dst;
1153 }
1154
1155 /*
1156 * Read each of the pages back using the UFFD-registered mapping. We
1157 * expect that the first time we touch a page, it will result in a missing
1158 * fault. uffd_poll_thread will resolve the fault by moving source
1159 * page to destination.
1160 */
1161 for (nr = 0; nr < step_count * step_size; nr += step_size) {
1162 unsigned long i;
1163
1164 /* Check area_src content */
1165 for (i = 0; i < step_size; i++) {
1166 count = *area_count(area_src, nr + i);
1167 if (count != count_verify[src_offs + nr + i])
1168 err("nr %lu source memory invalid %llu %llu\n",
1169 nr + i, count, count_verify[src_offs + nr + i]);
1170 }
1171
1172 /* Faulting into area_dst should move the page or the huge page */
1173 for (i = 0; i < step_size; i++) {
1174 count = *area_count(area_dst, nr + i);
1175 if (count != count_verify[dst_offs + nr + i])
1176 err("nr %lu memory corruption %llu %llu\n",
1177 nr, count, count_verify[dst_offs + nr + i]);
1178 }
1179
1180 /* Re-check area_src content which should be empty */
1181 for (i = 0; i < step_size; i++) {
1182 count = *area_count(area_src, nr + i);
1183 if (count != 0)
1184 err("nr %lu move failed %llu %llu\n",
1185 nr, count, count_verify[src_offs + nr + i]);
1186 }
1187 }
1188 if (step_size > page_size) {
1189 area_src = orig_area_src;
1190 area_dst = orig_area_dst;
1191 }
1192
1193 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
1194 err("pipe write");
1195 if (pthread_join(uffd_mon, NULL))
1196 err("join() failed");
1197
1198 if (args.missing_faults != step_count || args.minor_faults != 0)
1199 uffd_test_fail("stats check error");
1200 else
1201 uffd_test_pass();
1202 }
1203
uffd_move_test(uffd_test_args_t * targs)1204 static void uffd_move_test(uffd_test_args_t *targs)
1205 {
1206 uffd_move_test_common(targs, page_size, uffd_move_handle_fault);
1207 }
1208
uffd_move_pmd_test(uffd_test_args_t * targs)1209 static void uffd_move_pmd_test(uffd_test_args_t *targs)
1210 {
1211 if (madvise(area_dst, nr_pages * page_size, MADV_HUGEPAGE))
1212 err("madvise(MADV_HUGEPAGE) failure");
1213 uffd_move_test_common(targs, read_pmd_pagesize(),
1214 uffd_move_pmd_handle_fault);
1215 }
1216
uffd_move_pmd_split_test(uffd_test_args_t * targs)1217 static void uffd_move_pmd_split_test(uffd_test_args_t *targs)
1218 {
1219 if (madvise(area_dst, nr_pages * page_size, MADV_NOHUGEPAGE))
1220 err("madvise(MADV_NOHUGEPAGE) failure");
1221 uffd_move_test_common(targs, read_pmd_pagesize(),
1222 uffd_move_pmd_handle_fault);
1223 }
1224
prevent_hugepages(const char ** errmsg)1225 static int prevent_hugepages(const char **errmsg)
1226 {
1227 /* This should be done before source area is populated */
1228 if (madvise(area_src, nr_pages * page_size, MADV_NOHUGEPAGE)) {
1229 /* Ignore only if CONFIG_TRANSPARENT_HUGEPAGE=n */
1230 if (errno != EINVAL) {
1231 if (errmsg)
1232 *errmsg = "madvise(MADV_NOHUGEPAGE) failed";
1233 return -errno;
1234 }
1235 }
1236 return 0;
1237 }
1238
request_hugepages(const char ** errmsg)1239 static int request_hugepages(const char **errmsg)
1240 {
1241 /* This should be done before source area is populated */
1242 if (madvise(area_src, nr_pages * page_size, MADV_HUGEPAGE)) {
1243 if (errmsg) {
1244 *errmsg = (errno == EINVAL) ?
1245 "CONFIG_TRANSPARENT_HUGEPAGE is not set" :
1246 "madvise(MADV_HUGEPAGE) failed";
1247 }
1248 return -errno;
1249 }
1250 return 0;
1251 }
1252
1253 struct uffd_test_case_ops uffd_move_test_case_ops = {
1254 .post_alloc = prevent_hugepages,
1255 };
1256
1257 struct uffd_test_case_ops uffd_move_test_pmd_case_ops = {
1258 .post_alloc = request_hugepages,
1259 };
1260
1261 /*
1262 * Test the returned uffdio_register.ioctls with different register modes.
1263 * Note that _UFFDIO_ZEROPAGE is tested separately in the zeropage test.
1264 */
1265 static void
do_register_ioctls_test(uffd_test_args_t * args,bool miss,bool wp,bool minor)1266 do_register_ioctls_test(uffd_test_args_t *args, bool miss, bool wp, bool minor)
1267 {
1268 uint64_t ioctls = 0, expected = BIT_ULL(_UFFDIO_WAKE);
1269 mem_type_t *mem_type = args->mem_type;
1270 int ret;
1271
1272 ret = uffd_register_with_ioctls(uffd, area_dst, page_size,
1273 miss, wp, minor, &ioctls);
1274
1275 /*
1276 * Handle special cases of UFFDIO_REGISTER here where it should
1277 * just fail with -EINVAL first..
1278 *
1279 * Case 1: register MINOR on anon
1280 * Case 2: register with no mode selected
1281 */
1282 if ((minor && (mem_type->mem_flag == MEM_ANON)) ||
1283 (!miss && !wp && !minor)) {
1284 if (ret != -EINVAL)
1285 err("register (miss=%d, wp=%d, minor=%d) failed "
1286 "with wrong errno=%d", miss, wp, minor, ret);
1287 return;
1288 }
1289
1290 /* UFFDIO_REGISTER should succeed, then check ioctls returned */
1291 if (miss)
1292 expected |= BIT_ULL(_UFFDIO_COPY);
1293 if (wp)
1294 expected |= BIT_ULL(_UFFDIO_WRITEPROTECT);
1295 if (minor)
1296 expected |= BIT_ULL(_UFFDIO_CONTINUE);
1297
1298 if ((ioctls & expected) != expected)
1299 err("unexpected uffdio_register.ioctls "
1300 "(miss=%d, wp=%d, minor=%d): expected=0x%"PRIx64", "
1301 "returned=0x%"PRIx64, miss, wp, minor, expected, ioctls);
1302
1303 if (uffd_unregister(uffd, area_dst, page_size))
1304 err("unregister");
1305 }
1306
uffd_register_ioctls_test(uffd_test_args_t * args)1307 static void uffd_register_ioctls_test(uffd_test_args_t *args)
1308 {
1309 int miss, wp, minor;
1310
1311 for (miss = 0; miss <= 1; miss++)
1312 for (wp = 0; wp <= 1; wp++)
1313 for (minor = 0; minor <= 1; minor++)
1314 do_register_ioctls_test(args, miss, wp, minor);
1315
1316 uffd_test_pass();
1317 }
1318
1319 uffd_test_case_t uffd_tests[] = {
1320 {
1321 /* Test returned uffdio_register.ioctls. */
1322 .name = "register-ioctls",
1323 .uffd_fn = uffd_register_ioctls_test,
1324 .mem_targets = MEM_ALL,
1325 .uffd_feature_required = UFFD_FEATURE_MISSING_HUGETLBFS |
1326 UFFD_FEATURE_MISSING_SHMEM |
1327 UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1328 UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
1329 UFFD_FEATURE_MINOR_HUGETLBFS |
1330 UFFD_FEATURE_MINOR_SHMEM,
1331 },
1332 {
1333 .name = "zeropage",
1334 .uffd_fn = uffd_zeropage_test,
1335 .mem_targets = MEM_ALL,
1336 .uffd_feature_required = 0,
1337 },
1338 {
1339 .name = "move",
1340 .uffd_fn = uffd_move_test,
1341 .mem_targets = MEM_ANON,
1342 .uffd_feature_required = UFFD_FEATURE_MOVE,
1343 .test_case_ops = &uffd_move_test_case_ops,
1344 },
1345 {
1346 .name = "move-pmd",
1347 .uffd_fn = uffd_move_pmd_test,
1348 .mem_targets = MEM_ANON,
1349 .uffd_feature_required = UFFD_FEATURE_MOVE,
1350 .test_case_ops = &uffd_move_test_pmd_case_ops,
1351 },
1352 {
1353 .name = "move-pmd-split",
1354 .uffd_fn = uffd_move_pmd_split_test,
1355 .mem_targets = MEM_ANON,
1356 .uffd_feature_required = UFFD_FEATURE_MOVE,
1357 .test_case_ops = &uffd_move_test_pmd_case_ops,
1358 },
1359 {
1360 .name = "wp-fork",
1361 .uffd_fn = uffd_wp_fork_test,
1362 .mem_targets = MEM_ALL,
1363 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1364 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
1365 },
1366 {
1367 .name = "wp-fork-with-event",
1368 .uffd_fn = uffd_wp_fork_with_event_test,
1369 .mem_targets = MEM_ALL,
1370 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1371 UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
1372 /* when set, child process should inherit uffd-wp bits */
1373 UFFD_FEATURE_EVENT_FORK,
1374 },
1375 {
1376 .name = "wp-fork-pin",
1377 .uffd_fn = uffd_wp_fork_pin_test,
1378 .mem_targets = MEM_ALL,
1379 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1380 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
1381 },
1382 {
1383 .name = "wp-fork-pin-with-event",
1384 .uffd_fn = uffd_wp_fork_pin_with_event_test,
1385 .mem_targets = MEM_ALL,
1386 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1387 UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
1388 /* when set, child process should inherit uffd-wp bits */
1389 UFFD_FEATURE_EVENT_FORK,
1390 },
1391 {
1392 .name = "wp-unpopulated",
1393 .uffd_fn = uffd_wp_unpopulated_test,
1394 .mem_targets = MEM_ANON,
1395 .uffd_feature_required =
1396 UFFD_FEATURE_PAGEFAULT_FLAG_WP | UFFD_FEATURE_WP_UNPOPULATED,
1397 },
1398 {
1399 .name = "minor",
1400 .uffd_fn = uffd_minor_test,
1401 .mem_targets = MEM_SHMEM | MEM_HUGETLB,
1402 .uffd_feature_required =
1403 UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM,
1404 },
1405 {
1406 .name = "minor-wp",
1407 .uffd_fn = uffd_minor_wp_test,
1408 .mem_targets = MEM_SHMEM | MEM_HUGETLB,
1409 .uffd_feature_required =
1410 UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM |
1411 UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1412 /*
1413 * HACK: here we leveraged WP_UNPOPULATED to detect whether
1414 * minor mode supports wr-protect. There's no feature flag
1415 * for it so this is the best we can test against.
1416 */
1417 UFFD_FEATURE_WP_UNPOPULATED,
1418 },
1419 {
1420 .name = "minor-collapse",
1421 .uffd_fn = uffd_minor_collapse_test,
1422 /* MADV_COLLAPSE only works with shmem */
1423 .mem_targets = MEM_SHMEM,
1424 /* We can't test MADV_COLLAPSE, so try our luck */
1425 .uffd_feature_required = UFFD_FEATURE_MINOR_SHMEM,
1426 },
1427 {
1428 .name = "sigbus",
1429 .uffd_fn = uffd_sigbus_test,
1430 .mem_targets = MEM_ALL,
1431 .uffd_feature_required = UFFD_FEATURE_SIGBUS |
1432 UFFD_FEATURE_EVENT_FORK,
1433 },
1434 {
1435 .name = "sigbus-wp",
1436 .uffd_fn = uffd_sigbus_wp_test,
1437 .mem_targets = MEM_ALL,
1438 .uffd_feature_required = UFFD_FEATURE_SIGBUS |
1439 UFFD_FEATURE_EVENT_FORK | UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1440 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
1441 },
1442 {
1443 .name = "events",
1444 .uffd_fn = uffd_events_test,
1445 .mem_targets = MEM_ALL,
1446 .uffd_feature_required = UFFD_FEATURE_EVENT_FORK |
1447 UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE,
1448 },
1449 {
1450 .name = "events-wp",
1451 .uffd_fn = uffd_events_wp_test,
1452 .mem_targets = MEM_ALL,
1453 .uffd_feature_required = UFFD_FEATURE_EVENT_FORK |
1454 UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE |
1455 UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1456 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
1457 },
1458 {
1459 .name = "poison",
1460 .uffd_fn = uffd_poison_test,
1461 .mem_targets = MEM_ALL,
1462 .uffd_feature_required = UFFD_FEATURE_POISON,
1463 },
1464 };
1465
usage(const char * prog)1466 static void usage(const char *prog)
1467 {
1468 printf("usage: %s [-f TESTNAME]\n", prog);
1469 puts("");
1470 puts(" -f: test name to filter (e.g., event)");
1471 puts(" -h: show the help msg");
1472 puts(" -l: list tests only");
1473 puts("");
1474 exit(KSFT_FAIL);
1475 }
1476
main(int argc,char * argv[])1477 int main(int argc, char *argv[])
1478 {
1479 int n_tests = sizeof(uffd_tests) / sizeof(uffd_test_case_t);
1480 int n_mems = sizeof(mem_types) / sizeof(mem_type_t);
1481 const char *test_filter = NULL;
1482 bool list_only = false;
1483 uffd_test_case_t *test;
1484 mem_type_t *mem_type;
1485 uffd_test_args_t args;
1486 const char *errmsg;
1487 int has_uffd, opt;
1488 int i, j;
1489
1490 while ((opt = getopt(argc, argv, "f:hl")) != -1) {
1491 switch (opt) {
1492 case 'f':
1493 test_filter = optarg;
1494 break;
1495 case 'l':
1496 list_only = true;
1497 break;
1498 case 'h':
1499 default:
1500 /* Unknown */
1501 usage(argv[0]);
1502 break;
1503 }
1504 }
1505
1506 if (!test_filter && !list_only) {
1507 has_uffd = test_uffd_api(false);
1508 has_uffd |= test_uffd_api(true);
1509
1510 if (!has_uffd) {
1511 printf("Userfaultfd not supported or unprivileged, skip all tests\n");
1512 exit(KSFT_SKIP);
1513 }
1514 }
1515
1516 for (i = 0; i < n_tests; i++) {
1517 test = &uffd_tests[i];
1518 if (test_filter && !strstr(test->name, test_filter))
1519 continue;
1520 if (list_only) {
1521 printf("%s\n", test->name);
1522 continue;
1523 }
1524 for (j = 0; j < n_mems; j++) {
1525 mem_type = &mem_types[j];
1526 if (!(test->mem_targets & mem_type->mem_flag))
1527 continue;
1528
1529 uffd_test_start("%s on %s", test->name, mem_type->name);
1530 if ((mem_type->mem_flag == MEM_HUGETLB ||
1531 mem_type->mem_flag == MEM_HUGETLB_PRIVATE) &&
1532 (default_huge_page_size() == 0)) {
1533 uffd_test_skip("huge page size is 0, feature missing?");
1534 continue;
1535 }
1536 if (!uffd_feature_supported(test)) {
1537 uffd_test_skip("feature missing");
1538 continue;
1539 }
1540 if (uffd_setup_environment(&args, test, mem_type,
1541 &errmsg)) {
1542 uffd_test_skip(errmsg);
1543 continue;
1544 }
1545 test->uffd_fn(&args);
1546 uffd_test_ctx_clear();
1547 }
1548 }
1549
1550 if (!list_only)
1551 uffd_test_report();
1552
1553 return ksft_get_fail_cnt() ? KSFT_FAIL : KSFT_PASS;
1554 }
1555
1556