1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Userfaultfd unit tests.
4 *
5 * Copyright (C) 2015-2023 Red Hat, Inc.
6 */
7
8 #include "uffd-common.h"
9
10 #include "../../../../mm/gup_test.h"
11
12 #ifdef __NR_userfaultfd
13
14 /* The unit test doesn't need a large or random size, make it 32MB for now */
15 #define UFFD_TEST_MEM_SIZE (32UL << 20)
16
17 #define MEM_ANON BIT_ULL(0)
18 #define MEM_SHMEM BIT_ULL(1)
19 #define MEM_SHMEM_PRIVATE BIT_ULL(2)
20 #define MEM_HUGETLB BIT_ULL(3)
21 #define MEM_HUGETLB_PRIVATE BIT_ULL(4)
22
23 #define MEM_ALL (MEM_ANON | MEM_SHMEM | MEM_SHMEM_PRIVATE | \
24 MEM_HUGETLB | MEM_HUGETLB_PRIVATE)
25
26 #define ALIGN_UP(x, align_to) \
27 ((__typeof__(x))((((unsigned long)(x)) + ((align_to)-1)) & ~((align_to)-1)))
28
29 struct mem_type {
30 const char *name;
31 unsigned int mem_flag;
32 uffd_test_ops_t *mem_ops;
33 bool shared;
34 };
35 typedef struct mem_type mem_type_t;
36
37 mem_type_t mem_types[] = {
38 {
39 .name = "anon",
40 .mem_flag = MEM_ANON,
41 .mem_ops = &anon_uffd_test_ops,
42 .shared = false,
43 },
44 {
45 .name = "shmem",
46 .mem_flag = MEM_SHMEM,
47 .mem_ops = &shmem_uffd_test_ops,
48 .shared = true,
49 },
50 {
51 .name = "shmem-private",
52 .mem_flag = MEM_SHMEM_PRIVATE,
53 .mem_ops = &shmem_uffd_test_ops,
54 .shared = false,
55 },
56 {
57 .name = "hugetlb",
58 .mem_flag = MEM_HUGETLB,
59 .mem_ops = &hugetlb_uffd_test_ops,
60 .shared = true,
61 },
62 {
63 .name = "hugetlb-private",
64 .mem_flag = MEM_HUGETLB_PRIVATE,
65 .mem_ops = &hugetlb_uffd_test_ops,
66 .shared = false,
67 },
68 };
69
70 /* Arguments to be passed over to each uffd unit test */
71 struct uffd_test_args {
72 mem_type_t *mem_type;
73 };
74 typedef struct uffd_test_args uffd_test_args_t;
75
76 /* Returns: UFFD_TEST_* */
77 typedef void (*uffd_test_fn)(uffd_test_args_t *);
78
79 typedef struct {
80 const char *name;
81 uffd_test_fn uffd_fn;
82 unsigned int mem_targets;
83 uint64_t uffd_feature_required;
84 uffd_test_case_ops_t *test_case_ops;
85 } uffd_test_case_t;
86
uffd_test_report(void)87 static void uffd_test_report(void)
88 {
89 printf("Userfaults unit tests: pass=%u, skip=%u, fail=%u (total=%u)\n",
90 ksft_get_pass_cnt(),
91 ksft_get_xskip_cnt(),
92 ksft_get_fail_cnt(),
93 ksft_test_num());
94 }
95
uffd_test_pass(void)96 static void uffd_test_pass(void)
97 {
98 printf("done\n");
99 ksft_inc_pass_cnt();
100 }
101
102 #define uffd_test_start(...) do { \
103 printf("Testing "); \
104 printf(__VA_ARGS__); \
105 printf("... "); \
106 fflush(stdout); \
107 } while (0)
108
109 #define uffd_test_fail(...) do { \
110 printf("failed [reason: "); \
111 printf(__VA_ARGS__); \
112 printf("]\n"); \
113 ksft_inc_fail_cnt(); \
114 } while (0)
115
uffd_test_skip(const char * message)116 static void uffd_test_skip(const char *message)
117 {
118 printf("skipped [reason: %s]\n", message);
119 ksft_inc_xskip_cnt();
120 }
121
122 /*
123 * Returns 1 if specific userfaultfd supported, 0 otherwise. Note, we'll
124 * return 1 even if some test failed as long as uffd supported, because in
125 * that case we still want to proceed with the rest uffd unit tests.
126 */
test_uffd_api(bool use_dev)127 static int test_uffd_api(bool use_dev)
128 {
129 struct uffdio_api uffdio_api;
130 int uffd;
131
132 uffd_test_start("UFFDIO_API (with %s)",
133 use_dev ? "/dev/userfaultfd" : "syscall");
134
135 if (use_dev)
136 uffd = uffd_open_dev(UFFD_FLAGS);
137 else
138 uffd = uffd_open_sys(UFFD_FLAGS);
139 if (uffd < 0) {
140 uffd_test_skip("cannot open userfaultfd handle");
141 return 0;
142 }
143
144 /* Test wrong UFFD_API */
145 uffdio_api.api = 0xab;
146 uffdio_api.features = 0;
147 if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
148 uffd_test_fail("UFFDIO_API should fail with wrong api but didn't");
149 goto out;
150 }
151
152 /* Test wrong feature bit */
153 uffdio_api.api = UFFD_API;
154 uffdio_api.features = BIT_ULL(63);
155 if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
156 uffd_test_fail("UFFDIO_API should fail with wrong feature but didn't");
157 goto out;
158 }
159
160 /* Test normal UFFDIO_API */
161 uffdio_api.api = UFFD_API;
162 uffdio_api.features = 0;
163 if (ioctl(uffd, UFFDIO_API, &uffdio_api)) {
164 uffd_test_fail("UFFDIO_API should succeed but failed");
165 goto out;
166 }
167
168 /* Test double requests of UFFDIO_API with a random feature set */
169 uffdio_api.features = BIT_ULL(0);
170 if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
171 uffd_test_fail("UFFDIO_API should reject initialized uffd");
172 goto out;
173 }
174
175 uffd_test_pass();
176 out:
177 close(uffd);
178 /* We have a valid uffd handle */
179 return 1;
180 }
181
182 /*
183 * This function initializes the global variables. TODO: remove global
184 * vars and then remove this.
185 */
186 static int
uffd_setup_environment(uffd_test_args_t * args,uffd_test_case_t * test,mem_type_t * mem_type,const char ** errmsg)187 uffd_setup_environment(uffd_test_args_t *args, uffd_test_case_t *test,
188 mem_type_t *mem_type, const char **errmsg)
189 {
190 map_shared = mem_type->shared;
191 uffd_test_ops = mem_type->mem_ops;
192 uffd_test_case_ops = test->test_case_ops;
193
194 if (mem_type->mem_flag & (MEM_HUGETLB_PRIVATE | MEM_HUGETLB))
195 page_size = default_huge_page_size();
196 else
197 page_size = psize();
198
199 nr_pages = UFFD_TEST_MEM_SIZE / page_size;
200 /* TODO: remove this global var.. it's so ugly */
201 nr_cpus = 1;
202
203 /* Initialize test arguments */
204 args->mem_type = mem_type;
205
206 return uffd_test_ctx_init(test->uffd_feature_required, errmsg);
207 }
208
uffd_feature_supported(uffd_test_case_t * test)209 static bool uffd_feature_supported(uffd_test_case_t *test)
210 {
211 uint64_t features;
212
213 if (uffd_get_features(&features))
214 return false;
215
216 return (features & test->uffd_feature_required) ==
217 test->uffd_feature_required;
218 }
219
pagemap_open(void)220 static int pagemap_open(void)
221 {
222 int fd = open("/proc/self/pagemap", O_RDONLY);
223
224 if (fd < 0)
225 err("open pagemap");
226
227 return fd;
228 }
229
230 /* This macro let __LINE__ works in err() */
231 #define pagemap_check_wp(value, wp) do { \
232 if (!!(value & PM_UFFD_WP) != wp) \
233 err("pagemap uffd-wp bit error: 0x%"PRIx64, value); \
234 } while (0)
235
236 typedef struct {
237 int parent_uffd, child_uffd;
238 } fork_event_args;
239
fork_event_consumer(void * data)240 static void *fork_event_consumer(void *data)
241 {
242 fork_event_args *args = data;
243 struct uffd_msg msg = { 0 };
244
245 ready_for_fork = true;
246
247 /* Read until a full msg received */
248 while (uffd_read_msg(args->parent_uffd, &msg));
249
250 if (msg.event != UFFD_EVENT_FORK)
251 err("wrong message: %u\n", msg.event);
252
253 /* Just to be properly freed later */
254 args->child_uffd = msg.arg.fork.ufd;
255 return NULL;
256 }
257
258 typedef struct {
259 int gup_fd;
260 bool pinned;
261 } pin_args;
262
263 /*
264 * Returns 0 if succeed, <0 for errors. pin_pages() needs to be paired
265 * with unpin_pages(). Currently it needs to be RO longterm pin to satisfy
266 * all needs of the test cases (e.g., trigger unshare, trigger fork() early
267 * CoW, etc.).
268 */
pin_pages(pin_args * args,void * buffer,size_t size)269 static int pin_pages(pin_args *args, void *buffer, size_t size)
270 {
271 struct pin_longterm_test test = {
272 .addr = (uintptr_t)buffer,
273 .size = size,
274 /* Read-only pins */
275 .flags = 0,
276 };
277
278 if (args->pinned)
279 err("already pinned");
280
281 args->gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);
282 if (args->gup_fd < 0)
283 return -errno;
284
285 if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_START, &test)) {
286 /* Even if gup_test existed, can be an old gup_test / kernel */
287 close(args->gup_fd);
288 return -errno;
289 }
290 args->pinned = true;
291 return 0;
292 }
293
unpin_pages(pin_args * args)294 static void unpin_pages(pin_args *args)
295 {
296 if (!args->pinned)
297 err("unpin without pin first");
298 if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_STOP))
299 err("PIN_LONGTERM_TEST_STOP");
300 close(args->gup_fd);
301 args->pinned = false;
302 }
303
pagemap_test_fork(int uffd,bool with_event,bool test_pin)304 static int pagemap_test_fork(int uffd, bool with_event, bool test_pin)
305 {
306 fork_event_args args = { .parent_uffd = uffd, .child_uffd = -1 };
307 pthread_t thread;
308 pid_t child;
309 uint64_t value;
310 int fd, result;
311
312 /* Prepare a thread to resolve EVENT_FORK */
313 if (with_event) {
314 ready_for_fork = false;
315 if (pthread_create(&thread, NULL, fork_event_consumer, &args))
316 err("pthread_create()");
317 while (!ready_for_fork)
318 ; /* Wait for the poll_thread to start executing before forking */
319 }
320
321 child = fork();
322 if (!child) {
323 /* Open the pagemap fd of the child itself */
324 pin_args args = {};
325
326 fd = pagemap_open();
327
328 if (test_pin && pin_pages(&args, area_dst, page_size))
329 /*
330 * Normally when reach here we have pinned in
331 * previous tests, so shouldn't fail anymore
332 */
333 err("pin page failed in child");
334
335 value = pagemap_get_entry(fd, area_dst);
336 /*
337 * After fork(), we should handle uffd-wp bit differently:
338 *
339 * (1) when with EVENT_FORK, it should persist
340 * (2) when without EVENT_FORK, it should be dropped
341 */
342 pagemap_check_wp(value, with_event);
343 if (test_pin)
344 unpin_pages(&args);
345 /* Succeed */
346 exit(0);
347 }
348 waitpid(child, &result, 0);
349
350 if (with_event) {
351 if (pthread_join(thread, NULL))
352 err("pthread_join()");
353 if (args.child_uffd < 0)
354 err("Didn't receive child uffd");
355 close(args.child_uffd);
356 }
357
358 return result;
359 }
360
uffd_wp_unpopulated_test(uffd_test_args_t * args)361 static void uffd_wp_unpopulated_test(uffd_test_args_t *args)
362 {
363 uint64_t value;
364 int pagemap_fd;
365
366 if (uffd_register(uffd, area_dst, nr_pages * page_size,
367 false, true, false))
368 err("register failed");
369
370 pagemap_fd = pagemap_open();
371
372 /* Test applying pte marker to anon unpopulated */
373 wp_range(uffd, (uint64_t)area_dst, page_size, true);
374 value = pagemap_get_entry(pagemap_fd, area_dst);
375 pagemap_check_wp(value, true);
376
377 /* Test unprotect on anon pte marker */
378 wp_range(uffd, (uint64_t)area_dst, page_size, false);
379 value = pagemap_get_entry(pagemap_fd, area_dst);
380 pagemap_check_wp(value, false);
381
382 /* Test zap on anon marker */
383 wp_range(uffd, (uint64_t)area_dst, page_size, true);
384 if (madvise(area_dst, page_size, MADV_DONTNEED))
385 err("madvise(MADV_DONTNEED) failed");
386 value = pagemap_get_entry(pagemap_fd, area_dst);
387 pagemap_check_wp(value, false);
388
389 /* Test fault in after marker removed */
390 *area_dst = 1;
391 value = pagemap_get_entry(pagemap_fd, area_dst);
392 pagemap_check_wp(value, false);
393 /* Drop it to make pte none again */
394 if (madvise(area_dst, page_size, MADV_DONTNEED))
395 err("madvise(MADV_DONTNEED) failed");
396
397 /* Test read-zero-page upon pte marker */
398 wp_range(uffd, (uint64_t)area_dst, page_size, true);
399 *(volatile char *)area_dst;
400 /* Drop it to make pte none again */
401 if (madvise(area_dst, page_size, MADV_DONTNEED))
402 err("madvise(MADV_DONTNEED) failed");
403
404 uffd_test_pass();
405 }
406
uffd_wp_fork_test_common(uffd_test_args_t * args,bool with_event)407 static void uffd_wp_fork_test_common(uffd_test_args_t *args,
408 bool with_event)
409 {
410 int pagemap_fd;
411 uint64_t value;
412
413 if (uffd_register(uffd, area_dst, nr_pages * page_size,
414 false, true, false))
415 err("register failed");
416
417 pagemap_fd = pagemap_open();
418
419 /* Touch the page */
420 *area_dst = 1;
421 wp_range(uffd, (uint64_t)area_dst, page_size, true);
422 value = pagemap_get_entry(pagemap_fd, area_dst);
423 pagemap_check_wp(value, true);
424 if (pagemap_test_fork(uffd, with_event, false)) {
425 uffd_test_fail("Detected %s uffd-wp bit in child in present pte",
426 with_event ? "missing" : "stall");
427 goto out;
428 }
429
430 /*
431 * This is an attempt for zapping the pgtable so as to test the
432 * markers.
433 *
434 * For private mappings, PAGEOUT will only work on exclusive ptes
435 * (PM_MMAP_EXCLUSIVE) which we should satisfy.
436 *
437 * For shared, PAGEOUT may not work. Use DONTNEED instead which
438 * plays a similar role of zapping (rather than freeing the page)
439 * to expose pte markers.
440 */
441 if (args->mem_type->shared) {
442 if (madvise(area_dst, page_size, MADV_DONTNEED))
443 err("MADV_DONTNEED");
444 } else {
445 /*
446 * NOTE: ignore retval because private-hugetlb doesn't yet
447 * support swapping, so it could fail.
448 */
449 madvise(area_dst, page_size, MADV_PAGEOUT);
450 }
451
452 /* Uffd-wp should persist even swapped out */
453 value = pagemap_get_entry(pagemap_fd, area_dst);
454 pagemap_check_wp(value, true);
455 if (pagemap_test_fork(uffd, with_event, false)) {
456 uffd_test_fail("Detected %s uffd-wp bit in child in zapped pte",
457 with_event ? "missing" : "stall");
458 goto out;
459 }
460
461 /* Unprotect; this tests swap pte modifications */
462 wp_range(uffd, (uint64_t)area_dst, page_size, false);
463 value = pagemap_get_entry(pagemap_fd, area_dst);
464 pagemap_check_wp(value, false);
465
466 /* Fault in the page from disk */
467 *area_dst = 2;
468 value = pagemap_get_entry(pagemap_fd, area_dst);
469 pagemap_check_wp(value, false);
470 uffd_test_pass();
471 out:
472 if (uffd_unregister(uffd, area_dst, nr_pages * page_size))
473 err("unregister failed");
474 close(pagemap_fd);
475 }
476
uffd_wp_fork_test(uffd_test_args_t * args)477 static void uffd_wp_fork_test(uffd_test_args_t *args)
478 {
479 uffd_wp_fork_test_common(args, false);
480 }
481
uffd_wp_fork_with_event_test(uffd_test_args_t * args)482 static void uffd_wp_fork_with_event_test(uffd_test_args_t *args)
483 {
484 uffd_wp_fork_test_common(args, true);
485 }
486
uffd_wp_fork_pin_test_common(uffd_test_args_t * args,bool with_event)487 static void uffd_wp_fork_pin_test_common(uffd_test_args_t *args,
488 bool with_event)
489 {
490 int pagemap_fd;
491 pin_args pin_args = {};
492
493 if (uffd_register(uffd, area_dst, page_size, false, true, false))
494 err("register failed");
495
496 pagemap_fd = pagemap_open();
497
498 /* Touch the page */
499 *area_dst = 1;
500 wp_range(uffd, (uint64_t)area_dst, page_size, true);
501
502 /*
503 * 1. First pin, then fork(). This tests fork() special path when
504 * doing early CoW if the page is private.
505 */
506 if (pin_pages(&pin_args, area_dst, page_size)) {
507 uffd_test_skip("Possibly CONFIG_GUP_TEST missing "
508 "or unprivileged");
509 close(pagemap_fd);
510 uffd_unregister(uffd, area_dst, page_size);
511 return;
512 }
513
514 if (pagemap_test_fork(uffd, with_event, false)) {
515 uffd_test_fail("Detected %s uffd-wp bit in early CoW of fork()",
516 with_event ? "missing" : "stall");
517 unpin_pages(&pin_args);
518 goto out;
519 }
520
521 unpin_pages(&pin_args);
522
523 /*
524 * 2. First fork(), then pin (in the child, where test_pin==true).
525 * This tests COR, aka, page unsharing on private memories.
526 */
527 if (pagemap_test_fork(uffd, with_event, true)) {
528 uffd_test_fail("Detected %s uffd-wp bit when RO pin",
529 with_event ? "missing" : "stall");
530 goto out;
531 }
532 uffd_test_pass();
533 out:
534 if (uffd_unregister(uffd, area_dst, page_size))
535 err("register failed");
536 close(pagemap_fd);
537 }
538
uffd_wp_fork_pin_test(uffd_test_args_t * args)539 static void uffd_wp_fork_pin_test(uffd_test_args_t *args)
540 {
541 uffd_wp_fork_pin_test_common(args, false);
542 }
543
uffd_wp_fork_pin_with_event_test(uffd_test_args_t * args)544 static void uffd_wp_fork_pin_with_event_test(uffd_test_args_t *args)
545 {
546 uffd_wp_fork_pin_test_common(args, true);
547 }
548
check_memory_contents(char * p)549 static void check_memory_contents(char *p)
550 {
551 unsigned long i, j;
552 uint8_t expected_byte;
553
554 for (i = 0; i < nr_pages; ++i) {
555 expected_byte = ~((uint8_t)(i % ((uint8_t)-1)));
556 for (j = 0; j < page_size; j++) {
557 uint8_t v = *(uint8_t *)(p + (i * page_size) + j);
558 if (v != expected_byte)
559 err("unexpected page contents");
560 }
561 }
562 }
563
uffd_minor_test_common(bool test_collapse,bool test_wp)564 static void uffd_minor_test_common(bool test_collapse, bool test_wp)
565 {
566 unsigned long p;
567 pthread_t uffd_mon;
568 char c;
569 struct uffd_args args = { 0 };
570
571 /*
572 * NOTE: MADV_COLLAPSE is not yet compatible with WP, so testing
573 * both do not make much sense.
574 */
575 assert(!(test_collapse && test_wp));
576
577 if (uffd_register(uffd, area_dst_alias, nr_pages * page_size,
578 /* NOTE! MADV_COLLAPSE may not work with uffd-wp */
579 false, test_wp, true))
580 err("register failure");
581
582 /*
583 * After registering with UFFD, populate the non-UFFD-registered side of
584 * the shared mapping. This should *not* trigger any UFFD minor faults.
585 */
586 for (p = 0; p < nr_pages; ++p)
587 memset(area_dst + (p * page_size), p % ((uint8_t)-1),
588 page_size);
589
590 args.apply_wp = test_wp;
591 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
592 err("uffd_poll_thread create");
593
594 /*
595 * Read each of the pages back using the UFFD-registered mapping. We
596 * expect that the first time we touch a page, it will result in a minor
597 * fault. uffd_poll_thread will resolve the fault by bit-flipping the
598 * page's contents, and then issuing a CONTINUE ioctl.
599 */
600 check_memory_contents(area_dst_alias);
601
602 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
603 err("pipe write");
604 if (pthread_join(uffd_mon, NULL))
605 err("join() failed");
606
607 if (test_collapse) {
608 if (madvise(area_dst_alias, nr_pages * page_size,
609 MADV_COLLAPSE)) {
610 /* It's fine to fail for this one... */
611 uffd_test_skip("MADV_COLLAPSE failed");
612 return;
613 }
614
615 uffd_test_ops->check_pmd_mapping(area_dst,
616 nr_pages * page_size /
617 read_pmd_pagesize());
618 /*
619 * This won't cause uffd-fault - it purely just makes sure there
620 * was no corruption.
621 */
622 check_memory_contents(area_dst_alias);
623 }
624
625 if (args.missing_faults != 0 || args.minor_faults != nr_pages)
626 uffd_test_fail("stats check error");
627 else
628 uffd_test_pass();
629 }
630
uffd_minor_test(uffd_test_args_t * args)631 void uffd_minor_test(uffd_test_args_t *args)
632 {
633 uffd_minor_test_common(false, false);
634 }
635
uffd_minor_wp_test(uffd_test_args_t * args)636 void uffd_minor_wp_test(uffd_test_args_t *args)
637 {
638 uffd_minor_test_common(false, true);
639 }
640
uffd_minor_collapse_test(uffd_test_args_t * args)641 void uffd_minor_collapse_test(uffd_test_args_t *args)
642 {
643 uffd_minor_test_common(true, false);
644 }
645
646 static sigjmp_buf jbuf, *sigbuf;
647
sighndl(int sig,siginfo_t * siginfo,void * ptr)648 static void sighndl(int sig, siginfo_t *siginfo, void *ptr)
649 {
650 if (sig == SIGBUS) {
651 if (sigbuf)
652 siglongjmp(*sigbuf, 1);
653 abort();
654 }
655 }
656
657 /*
658 * For non-cooperative userfaultfd test we fork() a process that will
659 * generate pagefaults, will mremap the area monitored by the
660 * userfaultfd and at last this process will release the monitored
661 * area.
662 * For the anonymous and shared memory the area is divided into two
663 * parts, the first part is accessed before mremap, and the second
664 * part is accessed after mremap. Since hugetlbfs does not support
665 * mremap, the entire monitored area is accessed in a single pass for
666 * HUGETLB_TEST.
667 * The release of the pages currently generates event for shmem and
668 * anonymous memory (UFFD_EVENT_REMOVE), hence it is not checked
669 * for hugetlb.
670 * For signal test(UFFD_FEATURE_SIGBUS), signal_test = 1, we register
671 * monitored area, generate pagefaults and test that signal is delivered.
672 * Use UFFDIO_COPY to allocate missing page and retry. For signal_test = 2
673 * test robustness use case - we release monitored area, fork a process
674 * that will generate pagefaults and verify signal is generated.
675 * This also tests UFFD_FEATURE_EVENT_FORK event along with the signal
676 * feature. Using monitor thread, verify no userfault events are generated.
677 */
faulting_process(int signal_test,bool wp)678 static int faulting_process(int signal_test, bool wp)
679 {
680 unsigned long nr, i;
681 unsigned long long count;
682 unsigned long split_nr_pages;
683 unsigned long lastnr;
684 struct sigaction act;
685 volatile unsigned long signalled = 0;
686
687 split_nr_pages = (nr_pages + 1) / 2;
688
689 if (signal_test) {
690 sigbuf = &jbuf;
691 memset(&act, 0, sizeof(act));
692 act.sa_sigaction = sighndl;
693 act.sa_flags = SA_SIGINFO;
694 if (sigaction(SIGBUS, &act, 0))
695 err("sigaction");
696 lastnr = (unsigned long)-1;
697 }
698
699 for (nr = 0; nr < split_nr_pages; nr++) {
700 volatile int steps = 1;
701 unsigned long offset = nr * page_size;
702
703 if (signal_test) {
704 if (sigsetjmp(*sigbuf, 1) != 0) {
705 if (steps == 1 && nr == lastnr)
706 err("Signal repeated");
707
708 lastnr = nr;
709 if (signal_test == 1) {
710 if (steps == 1) {
711 /* This is a MISSING request */
712 steps++;
713 if (copy_page(uffd, offset, wp))
714 signalled++;
715 } else {
716 /* This is a WP request */
717 assert(steps == 2);
718 wp_range(uffd,
719 (__u64)area_dst +
720 offset,
721 page_size, false);
722 }
723 } else {
724 signalled++;
725 continue;
726 }
727 }
728 }
729
730 count = *area_count(area_dst, nr);
731 if (count != count_verify[nr])
732 err("nr %lu memory corruption %llu %llu\n",
733 nr, count, count_verify[nr]);
734 /*
735 * Trigger write protection if there is by writing
736 * the same value back.
737 */
738 *area_count(area_dst, nr) = count;
739 }
740
741 if (signal_test)
742 return signalled != split_nr_pages;
743
744 area_dst = mremap(area_dst, nr_pages * page_size, nr_pages * page_size,
745 MREMAP_MAYMOVE | MREMAP_FIXED, area_src);
746 if (area_dst == MAP_FAILED)
747 err("mremap");
748 /* Reset area_src since we just clobbered it */
749 area_src = NULL;
750
751 for (; nr < nr_pages; nr++) {
752 count = *area_count(area_dst, nr);
753 if (count != count_verify[nr]) {
754 err("nr %lu memory corruption %llu %llu\n",
755 nr, count, count_verify[nr]);
756 }
757 /*
758 * Trigger write protection if there is by writing
759 * the same value back.
760 */
761 *area_count(area_dst, nr) = count;
762 }
763
764 uffd_test_ops->release_pages(area_dst);
765
766 for (nr = 0; nr < nr_pages; nr++)
767 for (i = 0; i < page_size; i++)
768 if (*(area_dst + nr * page_size + i) != 0)
769 err("page %lu offset %lu is not zero", nr, i);
770
771 return 0;
772 }
773
uffd_sigbus_test_common(bool wp)774 static void uffd_sigbus_test_common(bool wp)
775 {
776 unsigned long userfaults;
777 pthread_t uffd_mon;
778 pid_t pid;
779 int err;
780 char c;
781 struct uffd_args args = { 0 };
782
783 ready_for_fork = false;
784
785 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
786
787 if (uffd_register(uffd, area_dst, nr_pages * page_size,
788 true, wp, false))
789 err("register failure");
790
791 if (faulting_process(1, wp))
792 err("faulting process failed");
793
794 uffd_test_ops->release_pages(area_dst);
795
796 args.apply_wp = wp;
797 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
798 err("uffd_poll_thread create");
799
800 while (!ready_for_fork)
801 ; /* Wait for the poll_thread to start executing before forking */
802
803 pid = fork();
804 if (pid < 0)
805 err("fork");
806
807 if (!pid)
808 exit(faulting_process(2, wp));
809
810 waitpid(pid, &err, 0);
811 if (err)
812 err("faulting process failed");
813 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
814 err("pipe write");
815 if (pthread_join(uffd_mon, (void **)&userfaults))
816 err("pthread_join()");
817
818 if (userfaults)
819 uffd_test_fail("Signal test failed, userfaults: %ld", userfaults);
820 else
821 uffd_test_pass();
822 }
823
uffd_sigbus_test(uffd_test_args_t * args)824 static void uffd_sigbus_test(uffd_test_args_t *args)
825 {
826 uffd_sigbus_test_common(false);
827 }
828
uffd_sigbus_wp_test(uffd_test_args_t * args)829 static void uffd_sigbus_wp_test(uffd_test_args_t *args)
830 {
831 uffd_sigbus_test_common(true);
832 }
833
uffd_events_test_common(bool wp)834 static void uffd_events_test_common(bool wp)
835 {
836 pthread_t uffd_mon;
837 pid_t pid;
838 int err;
839 char c;
840 struct uffd_args args = { 0 };
841
842 ready_for_fork = false;
843
844 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
845 if (uffd_register(uffd, area_dst, nr_pages * page_size,
846 true, wp, false))
847 err("register failure");
848
849 args.apply_wp = wp;
850 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
851 err("uffd_poll_thread create");
852
853 while (!ready_for_fork)
854 ; /* Wait for the poll_thread to start executing before forking */
855
856 pid = fork();
857 if (pid < 0)
858 err("fork");
859
860 if (!pid)
861 exit(faulting_process(0, wp));
862
863 waitpid(pid, &err, 0);
864 if (err)
865 err("faulting process failed");
866 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
867 err("pipe write");
868 if (pthread_join(uffd_mon, NULL))
869 err("pthread_join()");
870
871 if (args.missing_faults != nr_pages)
872 uffd_test_fail("Fault counts wrong");
873 else
874 uffd_test_pass();
875 }
876
uffd_events_test(uffd_test_args_t * args)877 static void uffd_events_test(uffd_test_args_t *args)
878 {
879 uffd_events_test_common(false);
880 }
881
uffd_events_wp_test(uffd_test_args_t * args)882 static void uffd_events_wp_test(uffd_test_args_t *args)
883 {
884 uffd_events_test_common(true);
885 }
886
retry_uffdio_zeropage(int ufd,struct uffdio_zeropage * uffdio_zeropage)887 static void retry_uffdio_zeropage(int ufd,
888 struct uffdio_zeropage *uffdio_zeropage)
889 {
890 uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start,
891 uffdio_zeropage->range.len,
892 0);
893 if (ioctl(ufd, UFFDIO_ZEROPAGE, uffdio_zeropage)) {
894 if (uffdio_zeropage->zeropage != -EEXIST)
895 err("UFFDIO_ZEROPAGE error: %"PRId64,
896 (int64_t)uffdio_zeropage->zeropage);
897 } else {
898 err("UFFDIO_ZEROPAGE error: %"PRId64,
899 (int64_t)uffdio_zeropage->zeropage);
900 }
901 }
902
do_uffdio_zeropage(int ufd,bool has_zeropage)903 static bool do_uffdio_zeropage(int ufd, bool has_zeropage)
904 {
905 struct uffdio_zeropage uffdio_zeropage = { 0 };
906 int ret;
907 __s64 res;
908
909 uffdio_zeropage.range.start = (unsigned long) area_dst;
910 uffdio_zeropage.range.len = page_size;
911 uffdio_zeropage.mode = 0;
912 ret = ioctl(ufd, UFFDIO_ZEROPAGE, &uffdio_zeropage);
913 res = uffdio_zeropage.zeropage;
914 if (ret) {
915 /* real retval in ufdio_zeropage.zeropage */
916 if (has_zeropage)
917 err("UFFDIO_ZEROPAGE error: %"PRId64, (int64_t)res);
918 else if (res != -EINVAL)
919 err("UFFDIO_ZEROPAGE not -EINVAL");
920 } else if (has_zeropage) {
921 if (res != page_size)
922 err("UFFDIO_ZEROPAGE unexpected size");
923 else
924 retry_uffdio_zeropage(ufd, &uffdio_zeropage);
925 return true;
926 } else
927 err("UFFDIO_ZEROPAGE succeeded");
928
929 return false;
930 }
931
932 /*
933 * Registers a range with MISSING mode only for zeropage test. Return true
934 * if UFFDIO_ZEROPAGE supported, false otherwise. Can't use uffd_register()
935 * because we want to detect .ioctls along the way.
936 */
937 static bool
uffd_register_detect_zeropage(int uffd,void * addr,uint64_t len)938 uffd_register_detect_zeropage(int uffd, void *addr, uint64_t len)
939 {
940 uint64_t ioctls = 0;
941
942 if (uffd_register_with_ioctls(uffd, addr, len, true,
943 false, false, &ioctls))
944 err("zeropage register fail");
945
946 return ioctls & (1 << _UFFDIO_ZEROPAGE);
947 }
948
949 /* exercise UFFDIO_ZEROPAGE */
uffd_zeropage_test(uffd_test_args_t * args)950 static void uffd_zeropage_test(uffd_test_args_t *args)
951 {
952 bool has_zeropage;
953 int i;
954
955 has_zeropage = uffd_register_detect_zeropage(uffd, area_dst, page_size);
956 if (area_dst_alias)
957 /* Ignore the retval; we already have it */
958 uffd_register_detect_zeropage(uffd, area_dst_alias, page_size);
959
960 if (do_uffdio_zeropage(uffd, has_zeropage))
961 for (i = 0; i < page_size; i++)
962 if (area_dst[i] != 0)
963 err("data non-zero at offset %d\n", i);
964
965 if (uffd_unregister(uffd, area_dst, page_size))
966 err("unregister");
967
968 if (area_dst_alias && uffd_unregister(uffd, area_dst_alias, page_size))
969 err("unregister");
970
971 uffd_test_pass();
972 }
973
uffd_register_poison(int uffd,void * addr,uint64_t len)974 static void uffd_register_poison(int uffd, void *addr, uint64_t len)
975 {
976 uint64_t ioctls = 0;
977 uint64_t expected = (1 << _UFFDIO_COPY) | (1 << _UFFDIO_POISON);
978
979 if (uffd_register_with_ioctls(uffd, addr, len, true,
980 false, false, &ioctls))
981 err("poison register fail");
982
983 if ((ioctls & expected) != expected)
984 err("registered area doesn't support COPY and POISON ioctls");
985 }
986
do_uffdio_poison(int uffd,unsigned long offset)987 static void do_uffdio_poison(int uffd, unsigned long offset)
988 {
989 struct uffdio_poison uffdio_poison = { 0 };
990 int ret;
991 __s64 res;
992
993 uffdio_poison.range.start = (unsigned long) area_dst + offset;
994 uffdio_poison.range.len = page_size;
995 uffdio_poison.mode = 0;
996 ret = ioctl(uffd, UFFDIO_POISON, &uffdio_poison);
997 res = uffdio_poison.updated;
998
999 if (ret)
1000 err("UFFDIO_POISON error: %"PRId64, (int64_t)res);
1001 else if (res != page_size)
1002 err("UFFDIO_POISON unexpected size: %"PRId64, (int64_t)res);
1003 }
1004
uffd_poison_handle_fault(struct uffd_msg * msg,struct uffd_args * args)1005 static void uffd_poison_handle_fault(
1006 struct uffd_msg *msg, struct uffd_args *args)
1007 {
1008 unsigned long offset;
1009
1010 if (msg->event != UFFD_EVENT_PAGEFAULT)
1011 err("unexpected msg event %u", msg->event);
1012
1013 if (msg->arg.pagefault.flags &
1014 (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR))
1015 err("unexpected fault type %llu", msg->arg.pagefault.flags);
1016
1017 offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;
1018 offset &= ~(page_size-1);
1019
1020 /* Odd pages -> copy zeroed page; even pages -> poison. */
1021 if (offset & page_size)
1022 copy_page(uffd, offset, false);
1023 else
1024 do_uffdio_poison(uffd, offset);
1025 }
1026
uffd_poison_test(uffd_test_args_t * targs)1027 static void uffd_poison_test(uffd_test_args_t *targs)
1028 {
1029 pthread_t uffd_mon;
1030 char c;
1031 struct uffd_args args = { 0 };
1032 struct sigaction act = { 0 };
1033 unsigned long nr_sigbus = 0;
1034 unsigned long nr;
1035
1036 fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
1037
1038 uffd_register_poison(uffd, area_dst, nr_pages * page_size);
1039 memset(area_src, 0, nr_pages * page_size);
1040
1041 args.handle_fault = uffd_poison_handle_fault;
1042 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
1043 err("uffd_poll_thread create");
1044
1045 sigbuf = &jbuf;
1046 act.sa_sigaction = sighndl;
1047 act.sa_flags = SA_SIGINFO;
1048 if (sigaction(SIGBUS, &act, 0))
1049 err("sigaction");
1050
1051 for (nr = 0; nr < nr_pages; ++nr) {
1052 unsigned long offset = nr * page_size;
1053 const char *bytes = (const char *) area_dst + offset;
1054 const char *i;
1055
1056 if (sigsetjmp(*sigbuf, 1)) {
1057 /*
1058 * Access below triggered a SIGBUS, which was caught by
1059 * sighndl, which then jumped here. Count this SIGBUS,
1060 * and move on to next page.
1061 */
1062 ++nr_sigbus;
1063 continue;
1064 }
1065
1066 for (i = bytes; i < bytes + page_size; ++i) {
1067 if (*i)
1068 err("nonzero byte in area_dst (%p) at %p: %u",
1069 area_dst, i, *i);
1070 }
1071 }
1072
1073 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
1074 err("pipe write");
1075 if (pthread_join(uffd_mon, NULL))
1076 err("pthread_join()");
1077
1078 if (nr_sigbus != nr_pages / 2)
1079 err("expected to receive %lu SIGBUS, actually received %lu",
1080 nr_pages / 2, nr_sigbus);
1081
1082 uffd_test_pass();
1083 }
1084
1085 static void
uffd_move_handle_fault_common(struct uffd_msg * msg,struct uffd_args * args,unsigned long len)1086 uffd_move_handle_fault_common(struct uffd_msg *msg, struct uffd_args *args,
1087 unsigned long len)
1088 {
1089 unsigned long offset;
1090
1091 if (msg->event != UFFD_EVENT_PAGEFAULT)
1092 err("unexpected msg event %u", msg->event);
1093
1094 if (msg->arg.pagefault.flags &
1095 (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR | UFFD_PAGEFAULT_FLAG_WRITE))
1096 err("unexpected fault type %llu", msg->arg.pagefault.flags);
1097
1098 offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;
1099 offset &= ~(len-1);
1100
1101 if (move_page(uffd, offset, len))
1102 args->missing_faults++;
1103 }
1104
uffd_move_handle_fault(struct uffd_msg * msg,struct uffd_args * args)1105 static void uffd_move_handle_fault(struct uffd_msg *msg,
1106 struct uffd_args *args)
1107 {
1108 uffd_move_handle_fault_common(msg, args, page_size);
1109 }
1110
uffd_move_pmd_handle_fault(struct uffd_msg * msg,struct uffd_args * args)1111 static void uffd_move_pmd_handle_fault(struct uffd_msg *msg,
1112 struct uffd_args *args)
1113 {
1114 uffd_move_handle_fault_common(msg, args, read_pmd_pagesize());
1115 }
1116
1117 static void
uffd_move_test_common(uffd_test_args_t * targs,unsigned long chunk_size,void (* handle_fault)(struct uffd_msg * msg,struct uffd_args * args))1118 uffd_move_test_common(uffd_test_args_t *targs, unsigned long chunk_size,
1119 void (*handle_fault)(struct uffd_msg *msg, struct uffd_args *args))
1120 {
1121 unsigned long nr;
1122 pthread_t uffd_mon;
1123 char c;
1124 unsigned long long count;
1125 struct uffd_args args = { 0 };
1126 char *orig_area_src = NULL, *orig_area_dst = NULL;
1127 unsigned long step_size, step_count;
1128 unsigned long src_offs = 0;
1129 unsigned long dst_offs = 0;
1130
1131 /* Prevent source pages from being mapped more than once */
1132 if (madvise(area_src, nr_pages * page_size, MADV_DONTFORK))
1133 err("madvise(MADV_DONTFORK) failure");
1134
1135 if (uffd_register(uffd, area_dst, nr_pages * page_size,
1136 true, false, false))
1137 err("register failure");
1138
1139 args.handle_fault = handle_fault;
1140 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
1141 err("uffd_poll_thread create");
1142
1143 step_size = chunk_size / page_size;
1144 step_count = nr_pages / step_size;
1145
1146 if (chunk_size > page_size) {
1147 char *aligned_src = ALIGN_UP(area_src, chunk_size);
1148 char *aligned_dst = ALIGN_UP(area_dst, chunk_size);
1149
1150 if (aligned_src != area_src || aligned_dst != area_dst) {
1151 src_offs = (aligned_src - area_src) / page_size;
1152 dst_offs = (aligned_dst - area_dst) / page_size;
1153 step_count--;
1154 }
1155 orig_area_src = area_src;
1156 orig_area_dst = area_dst;
1157 area_src = aligned_src;
1158 area_dst = aligned_dst;
1159 }
1160
1161 /*
1162 * Read each of the pages back using the UFFD-registered mapping. We
1163 * expect that the first time we touch a page, it will result in a missing
1164 * fault. uffd_poll_thread will resolve the fault by moving source
1165 * page to destination.
1166 */
1167 for (nr = 0; nr < step_count * step_size; nr += step_size) {
1168 unsigned long i;
1169
1170 /* Check area_src content */
1171 for (i = 0; i < step_size; i++) {
1172 count = *area_count(area_src, nr + i);
1173 if (count != count_verify[src_offs + nr + i])
1174 err("nr %lu source memory invalid %llu %llu\n",
1175 nr + i, count, count_verify[src_offs + nr + i]);
1176 }
1177
1178 /* Faulting into area_dst should move the page or the huge page */
1179 for (i = 0; i < step_size; i++) {
1180 count = *area_count(area_dst, nr + i);
1181 if (count != count_verify[dst_offs + nr + i])
1182 err("nr %lu memory corruption %llu %llu\n",
1183 nr, count, count_verify[dst_offs + nr + i]);
1184 }
1185
1186 /* Re-check area_src content which should be empty */
1187 for (i = 0; i < step_size; i++) {
1188 count = *area_count(area_src, nr + i);
1189 if (count != 0)
1190 err("nr %lu move failed %llu %llu\n",
1191 nr, count, count_verify[src_offs + nr + i]);
1192 }
1193 }
1194 if (chunk_size > page_size) {
1195 area_src = orig_area_src;
1196 area_dst = orig_area_dst;
1197 }
1198
1199 if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
1200 err("pipe write");
1201 if (pthread_join(uffd_mon, NULL))
1202 err("join() failed");
1203
1204 if (args.missing_faults != step_count || args.minor_faults != 0)
1205 uffd_test_fail("stats check error");
1206 else
1207 uffd_test_pass();
1208 }
1209
uffd_move_test(uffd_test_args_t * targs)1210 static void uffd_move_test(uffd_test_args_t *targs)
1211 {
1212 uffd_move_test_common(targs, page_size, uffd_move_handle_fault);
1213 }
1214
uffd_move_pmd_test(uffd_test_args_t * targs)1215 static void uffd_move_pmd_test(uffd_test_args_t *targs)
1216 {
1217 if (madvise(area_dst, nr_pages * page_size, MADV_HUGEPAGE))
1218 err("madvise(MADV_HUGEPAGE) failure");
1219 uffd_move_test_common(targs, read_pmd_pagesize(),
1220 uffd_move_pmd_handle_fault);
1221 }
1222
uffd_move_pmd_split_test(uffd_test_args_t * targs)1223 static void uffd_move_pmd_split_test(uffd_test_args_t *targs)
1224 {
1225 if (madvise(area_dst, nr_pages * page_size, MADV_NOHUGEPAGE))
1226 err("madvise(MADV_NOHUGEPAGE) failure");
1227 uffd_move_test_common(targs, read_pmd_pagesize(),
1228 uffd_move_pmd_handle_fault);
1229 }
1230
prevent_hugepages(const char ** errmsg)1231 static int prevent_hugepages(const char **errmsg)
1232 {
1233 /* This should be done before source area is populated */
1234 if (madvise(area_src, nr_pages * page_size, MADV_NOHUGEPAGE)) {
1235 /* Ignore only if CONFIG_TRANSPARENT_HUGEPAGE=n */
1236 if (errno != EINVAL) {
1237 if (errmsg)
1238 *errmsg = "madvise(MADV_NOHUGEPAGE) failed";
1239 return -errno;
1240 }
1241 }
1242 return 0;
1243 }
1244
request_hugepages(const char ** errmsg)1245 static int request_hugepages(const char **errmsg)
1246 {
1247 /* This should be done before source area is populated */
1248 if (madvise(area_src, nr_pages * page_size, MADV_HUGEPAGE)) {
1249 if (errmsg) {
1250 *errmsg = (errno == EINVAL) ?
1251 "CONFIG_TRANSPARENT_HUGEPAGE is not set" :
1252 "madvise(MADV_HUGEPAGE) failed";
1253 }
1254 return -errno;
1255 }
1256 return 0;
1257 }
1258
1259 struct uffd_test_case_ops uffd_move_test_case_ops = {
1260 .post_alloc = prevent_hugepages,
1261 };
1262
1263 struct uffd_test_case_ops uffd_move_test_pmd_case_ops = {
1264 .post_alloc = request_hugepages,
1265 };
1266
1267 /*
1268 * Test the returned uffdio_register.ioctls with different register modes.
1269 * Note that _UFFDIO_ZEROPAGE is tested separately in the zeropage test.
1270 */
1271 static void
do_register_ioctls_test(uffd_test_args_t * args,bool miss,bool wp,bool minor)1272 do_register_ioctls_test(uffd_test_args_t *args, bool miss, bool wp, bool minor)
1273 {
1274 uint64_t ioctls = 0, expected = BIT_ULL(_UFFDIO_WAKE);
1275 mem_type_t *mem_type = args->mem_type;
1276 int ret;
1277
1278 ret = uffd_register_with_ioctls(uffd, area_dst, page_size,
1279 miss, wp, minor, &ioctls);
1280
1281 /*
1282 * Handle special cases of UFFDIO_REGISTER here where it should
1283 * just fail with -EINVAL first..
1284 *
1285 * Case 1: register MINOR on anon
1286 * Case 2: register with no mode selected
1287 */
1288 if ((minor && (mem_type->mem_flag == MEM_ANON)) ||
1289 (!miss && !wp && !minor)) {
1290 if (ret != -EINVAL)
1291 err("register (miss=%d, wp=%d, minor=%d) failed "
1292 "with wrong errno=%d", miss, wp, minor, ret);
1293 return;
1294 }
1295
1296 /* UFFDIO_REGISTER should succeed, then check ioctls returned */
1297 if (miss)
1298 expected |= BIT_ULL(_UFFDIO_COPY);
1299 if (wp)
1300 expected |= BIT_ULL(_UFFDIO_WRITEPROTECT);
1301 if (minor)
1302 expected |= BIT_ULL(_UFFDIO_CONTINUE);
1303
1304 if ((ioctls & expected) != expected)
1305 err("unexpected uffdio_register.ioctls "
1306 "(miss=%d, wp=%d, minor=%d): expected=0x%"PRIx64", "
1307 "returned=0x%"PRIx64, miss, wp, minor, expected, ioctls);
1308
1309 if (uffd_unregister(uffd, area_dst, page_size))
1310 err("unregister");
1311 }
1312
uffd_register_ioctls_test(uffd_test_args_t * args)1313 static void uffd_register_ioctls_test(uffd_test_args_t *args)
1314 {
1315 int miss, wp, minor;
1316
1317 for (miss = 0; miss <= 1; miss++)
1318 for (wp = 0; wp <= 1; wp++)
1319 for (minor = 0; minor <= 1; minor++)
1320 do_register_ioctls_test(args, miss, wp, minor);
1321
1322 uffd_test_pass();
1323 }
1324
1325 uffd_test_case_t uffd_tests[] = {
1326 {
1327 /* Test returned uffdio_register.ioctls. */
1328 .name = "register-ioctls",
1329 .uffd_fn = uffd_register_ioctls_test,
1330 .mem_targets = MEM_ALL,
1331 .uffd_feature_required = UFFD_FEATURE_MISSING_HUGETLBFS |
1332 UFFD_FEATURE_MISSING_SHMEM |
1333 UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1334 UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
1335 UFFD_FEATURE_MINOR_HUGETLBFS |
1336 UFFD_FEATURE_MINOR_SHMEM,
1337 },
1338 {
1339 .name = "zeropage",
1340 .uffd_fn = uffd_zeropage_test,
1341 .mem_targets = MEM_ALL,
1342 .uffd_feature_required = 0,
1343 },
1344 {
1345 .name = "move",
1346 .uffd_fn = uffd_move_test,
1347 .mem_targets = MEM_ANON,
1348 .uffd_feature_required = UFFD_FEATURE_MOVE,
1349 .test_case_ops = &uffd_move_test_case_ops,
1350 },
1351 {
1352 .name = "move-pmd",
1353 .uffd_fn = uffd_move_pmd_test,
1354 .mem_targets = MEM_ANON,
1355 .uffd_feature_required = UFFD_FEATURE_MOVE,
1356 .test_case_ops = &uffd_move_test_pmd_case_ops,
1357 },
1358 {
1359 .name = "move-pmd-split",
1360 .uffd_fn = uffd_move_pmd_split_test,
1361 .mem_targets = MEM_ANON,
1362 .uffd_feature_required = UFFD_FEATURE_MOVE,
1363 .test_case_ops = &uffd_move_test_pmd_case_ops,
1364 },
1365 {
1366 .name = "wp-fork",
1367 .uffd_fn = uffd_wp_fork_test,
1368 .mem_targets = MEM_ALL,
1369 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1370 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
1371 },
1372 {
1373 .name = "wp-fork-with-event",
1374 .uffd_fn = uffd_wp_fork_with_event_test,
1375 .mem_targets = MEM_ALL,
1376 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1377 UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
1378 /* when set, child process should inherit uffd-wp bits */
1379 UFFD_FEATURE_EVENT_FORK,
1380 },
1381 {
1382 .name = "wp-fork-pin",
1383 .uffd_fn = uffd_wp_fork_pin_test,
1384 .mem_targets = MEM_ALL,
1385 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1386 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
1387 },
1388 {
1389 .name = "wp-fork-pin-with-event",
1390 .uffd_fn = uffd_wp_fork_pin_with_event_test,
1391 .mem_targets = MEM_ALL,
1392 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1393 UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
1394 /* when set, child process should inherit uffd-wp bits */
1395 UFFD_FEATURE_EVENT_FORK,
1396 },
1397 {
1398 .name = "wp-unpopulated",
1399 .uffd_fn = uffd_wp_unpopulated_test,
1400 .mem_targets = MEM_ANON,
1401 .uffd_feature_required =
1402 UFFD_FEATURE_PAGEFAULT_FLAG_WP | UFFD_FEATURE_WP_UNPOPULATED,
1403 },
1404 {
1405 .name = "minor",
1406 .uffd_fn = uffd_minor_test,
1407 .mem_targets = MEM_SHMEM | MEM_HUGETLB,
1408 .uffd_feature_required =
1409 UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM,
1410 },
1411 {
1412 .name = "minor-wp",
1413 .uffd_fn = uffd_minor_wp_test,
1414 .mem_targets = MEM_SHMEM | MEM_HUGETLB,
1415 .uffd_feature_required =
1416 UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM |
1417 UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1418 /*
1419 * HACK: here we leveraged WP_UNPOPULATED to detect whether
1420 * minor mode supports wr-protect. There's no feature flag
1421 * for it so this is the best we can test against.
1422 */
1423 UFFD_FEATURE_WP_UNPOPULATED,
1424 },
1425 {
1426 .name = "minor-collapse",
1427 .uffd_fn = uffd_minor_collapse_test,
1428 /* MADV_COLLAPSE only works with shmem */
1429 .mem_targets = MEM_SHMEM,
1430 /* We can't test MADV_COLLAPSE, so try our luck */
1431 .uffd_feature_required = UFFD_FEATURE_MINOR_SHMEM,
1432 },
1433 {
1434 .name = "sigbus",
1435 .uffd_fn = uffd_sigbus_test,
1436 .mem_targets = MEM_ALL,
1437 .uffd_feature_required = UFFD_FEATURE_SIGBUS |
1438 UFFD_FEATURE_EVENT_FORK,
1439 },
1440 {
1441 .name = "sigbus-wp",
1442 .uffd_fn = uffd_sigbus_wp_test,
1443 .mem_targets = MEM_ALL,
1444 .uffd_feature_required = UFFD_FEATURE_SIGBUS |
1445 UFFD_FEATURE_EVENT_FORK | UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1446 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
1447 },
1448 {
1449 .name = "events",
1450 .uffd_fn = uffd_events_test,
1451 .mem_targets = MEM_ALL,
1452 .uffd_feature_required = UFFD_FEATURE_EVENT_FORK |
1453 UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE,
1454 },
1455 {
1456 .name = "events-wp",
1457 .uffd_fn = uffd_events_wp_test,
1458 .mem_targets = MEM_ALL,
1459 .uffd_feature_required = UFFD_FEATURE_EVENT_FORK |
1460 UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE |
1461 UFFD_FEATURE_PAGEFAULT_FLAG_WP |
1462 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
1463 },
1464 {
1465 .name = "poison",
1466 .uffd_fn = uffd_poison_test,
1467 .mem_targets = MEM_ALL,
1468 .uffd_feature_required = UFFD_FEATURE_POISON,
1469 },
1470 };
1471
usage(const char * prog)1472 static void usage(const char *prog)
1473 {
1474 printf("usage: %s [-f TESTNAME]\n", prog);
1475 puts("");
1476 puts(" -f: test name to filter (e.g., event)");
1477 puts(" -h: show the help msg");
1478 puts(" -l: list tests only");
1479 puts("");
1480 exit(KSFT_FAIL);
1481 }
1482
main(int argc,char * argv[])1483 int main(int argc, char *argv[])
1484 {
1485 int n_tests = sizeof(uffd_tests) / sizeof(uffd_test_case_t);
1486 int n_mems = sizeof(mem_types) / sizeof(mem_type_t);
1487 const char *test_filter = NULL;
1488 bool list_only = false;
1489 uffd_test_case_t *test;
1490 mem_type_t *mem_type;
1491 uffd_test_args_t args;
1492 const char *errmsg;
1493 int has_uffd, opt;
1494 int i, j;
1495
1496 while ((opt = getopt(argc, argv, "f:hl")) != -1) {
1497 switch (opt) {
1498 case 'f':
1499 test_filter = optarg;
1500 break;
1501 case 'l':
1502 list_only = true;
1503 break;
1504 case 'h':
1505 default:
1506 /* Unknown */
1507 usage(argv[0]);
1508 break;
1509 }
1510 }
1511
1512 if (!test_filter && !list_only) {
1513 has_uffd = test_uffd_api(false);
1514 has_uffd |= test_uffd_api(true);
1515
1516 if (!has_uffd) {
1517 printf("Userfaultfd not supported or unprivileged, skip all tests\n");
1518 exit(KSFT_SKIP);
1519 }
1520 }
1521
1522 for (i = 0; i < n_tests; i++) {
1523 test = &uffd_tests[i];
1524 if (test_filter && !strstr(test->name, test_filter))
1525 continue;
1526 if (list_only) {
1527 printf("%s\n", test->name);
1528 continue;
1529 }
1530 for (j = 0; j < n_mems; j++) {
1531 mem_type = &mem_types[j];
1532 if (!(test->mem_targets & mem_type->mem_flag))
1533 continue;
1534
1535 uffd_test_start("%s on %s", test->name, mem_type->name);
1536 if ((mem_type->mem_flag == MEM_HUGETLB ||
1537 mem_type->mem_flag == MEM_HUGETLB_PRIVATE) &&
1538 (default_huge_page_size() == 0)) {
1539 uffd_test_skip("huge page size is 0, feature missing?");
1540 continue;
1541 }
1542 if (!uffd_feature_supported(test)) {
1543 uffd_test_skip("feature missing");
1544 continue;
1545 }
1546 if (uffd_setup_environment(&args, test, mem_type,
1547 &errmsg)) {
1548 uffd_test_skip(errmsg);
1549 continue;
1550 }
1551 test->uffd_fn(&args);
1552 uffd_test_ctx_clear();
1553 }
1554 }
1555
1556 if (!list_only)
1557 uffd_test_report();
1558
1559 return ksft_get_fail_cnt() ? KSFT_FAIL : KSFT_PASS;
1560 }
1561
1562 #else /* __NR_userfaultfd */
1563
1564 #warning "missing __NR_userfaultfd definition"
1565
main(void)1566 int main(void)
1567 {
1568 printf("Skipping %s (missing __NR_userfaultfd)\n", __file__);
1569 return KSFT_SKIP;
1570 }
1571
1572 #endif /* __NR_userfaultfd */
1573