1 // SPDX-License-Identifier: GPL-2.0-only
5  *  Copyright (C) 2015-2023  Red Hat, Inc.
8 #include "uffd-common.h"
27 	((__typeof__(x))((((unsigned long)(x)) + ((align_to)-1)) & ~((align_to)-1)))
51 		.name = "shmem-private",
63 		.name = "hugetlb-private",
190 	map_shared = mem_type->shared;  in uffd_setup_environment()
191 	uffd_test_ops = mem_type->mem_ops;  in uffd_setup_environment()
192 	uffd_test_case_ops = test->test_case_ops;  in uffd_setup_environment()
194 	if (mem_type->mem_flag & (MEM_HUGETLB_PRIVATE | MEM_HUGETLB))  in uffd_setup_environment()
204 	args->mem_type = mem_type;  in uffd_setup_environment()
206 	return uffd_test_ctx_init(test->uffd_feature_required, errmsg);  in uffd_setup_environment()
216 	return (features & test->uffd_feature_required) ==  in uffd_feature_supported()
217 	    test->uffd_feature_required;  in uffd_feature_supported()
231 #define  pagemap_check_wp(value, wp) do {				\  argument
232 		if (!!(value & PM_UFFD_WP) != wp)			\
233 			err("pagemap uffd-wp bit error: 0x%"PRIx64, value); \
248 	while (uffd_read_msg(args->parent_uffd, &msg));  in fork_event_consumer()
254 	args->child_uffd = msg.arg.fork.ufd;  in fork_event_consumer()
274 		/* Read-only pins */  in pin_pages()
278 	if (args->pinned)  in pin_pages()
281 	args->gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);  in pin_pages()
282 	if (args->gup_fd < 0)  in pin_pages()
283 		return -errno;  in pin_pages()
285 	if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_START, &test)) {  in pin_pages()
287 		close(args->gup_fd);  in pin_pages()
288 		return -errno;  in pin_pages()
290 	args->pinned = true;  in pin_pages()
296 	if (!args->pinned)  in unpin_pages()
298 	if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_STOP))  in unpin_pages()
300 	close(args->gup_fd);  in unpin_pages()
301 	args->pinned = false;  in unpin_pages()
306 	fork_event_args args = { .parent_uffd = uffd, .child_uffd = -1 };  in pagemap_test_fork()
337 		 * After fork(), we should handle uffd-wp bit differently:  in pagemap_test_fork()
397 	/* Test read-zero-page upon pte marker */  in uffd_wp_unpopulated_test()
425 		uffd_test_fail("Detected %s uffd-wp bit in child in present pte",  in uffd_wp_fork_test_common()
441 	if (args->mem_type->shared) {  in uffd_wp_fork_test_common()
446 		 * NOTE: ignore retval because private-hugetlb doesn't yet  in uffd_wp_fork_test_common()
452 	/* Uffd-wp should persist even swapped out */  in uffd_wp_fork_test_common()
456 		uffd_test_fail("Detected %s uffd-wp bit in child in zapped pte",  in uffd_wp_fork_test_common()
515 		uffd_test_fail("Detected %s uffd-wp bit in early CoW of fork()",  in uffd_wp_fork_pin_test_common()
528 		uffd_test_fail("Detected %s uffd-wp bit when RO pin",  in uffd_wp_fork_pin_test_common()
555 		expected_byte = ~((uint8_t)(i % ((uint8_t)-1)));  in check_memory_contents()
572 	 * NOTE: MADV_COLLAPSE is not yet compatible with WP, so testing  in uffd_minor_test_common()
578 			  /* NOTE! MADV_COLLAPSE may not work with uffd-wp */  in uffd_minor_test_common()
583 	 * After registering with UFFD, populate the non-UFFD-registered side of  in uffd_minor_test_common()
587 		memset(area_dst + (p * page_size), p % ((uint8_t)-1),  in uffd_minor_test_common()
595 	 * Read each of the pages back using the UFFD-registered mapping. We  in uffd_minor_test_common()
597 	 * fault. uffd_poll_thread will resolve the fault by bit-flipping the  in uffd_minor_test_common()
615 		uffd_test_ops->check_pmd_mapping(area_dst,  in uffd_minor_test_common()
619 		 * This won't cause uffd-fault - it purely just makes sure there  in uffd_minor_test_common()
658  * For non-cooperative userfaultfd test we fork() a process that will
673  * test robustness use case - we release monitored area, fork a process
678 static int faulting_process(int signal_test, bool wp)  in faulting_process()  argument
696 		lastnr = (unsigned long)-1;  in faulting_process()
713 						if (copy_page(uffd, offset, wp))  in faulting_process()
716 						/* This is a WP request */  in faulting_process()
764 	uffd_test_ops->release_pages(area_dst);  in faulting_process()
774 static void uffd_sigbus_test_common(bool wp)  in uffd_sigbus_test_common()  argument
788 			  true, wp, false))  in uffd_sigbus_test_common()
791 	if (faulting_process(1, wp))  in uffd_sigbus_test_common()
794 	uffd_test_ops->release_pages(area_dst);  in uffd_sigbus_test_common()
796 	args.apply_wp = wp;  in uffd_sigbus_test_common()
808 		exit(faulting_process(2, wp));  in uffd_sigbus_test_common()
834 static void uffd_events_test_common(bool wp)  in uffd_events_test_common()  argument
846 			  true, wp, false))  in uffd_events_test_common()
849 	args.apply_wp = wp;  in uffd_events_test_common()
861 		exit(faulting_process(0, wp));  in uffd_events_test_common()
890 	uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start,  in retry_uffdio_zeropage()
891 				     uffdio_zeropage->range.len,  in retry_uffdio_zeropage()
894 		if (uffdio_zeropage->zeropage != -EEXIST)  in retry_uffdio_zeropage()
896 			    (int64_t)uffdio_zeropage->zeropage);  in retry_uffdio_zeropage()
899 		    (int64_t)uffdio_zeropage->zeropage);  in retry_uffdio_zeropage()
918 		else if (res != -EINVAL)  in do_uffdio_zeropage()
919 			err("UFFDIO_ZEROPAGE not -EINVAL");  in do_uffdio_zeropage()
963 				err("data non-zero at offset %d\n", i);  in uffd_zeropage_test()
1010 	if (msg->event != UFFD_EVENT_PAGEFAULT)  in uffd_poison_handle_fault()
1011 		err("unexpected msg event %u", msg->event);  in uffd_poison_handle_fault()
1013 	if (msg->arg.pagefault.flags &  in uffd_poison_handle_fault()
1015 		err("unexpected fault type %llu", msg->arg.pagefault.flags);  in uffd_poison_handle_fault()
1017 	offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;  in uffd_poison_handle_fault()
1018 	offset &= ~(page_size-1);  in uffd_poison_handle_fault()
1020 	/* Odd pages -> copy zeroed page; even pages -> poison. */  in uffd_poison_handle_fault()
1091 	if (msg->event != UFFD_EVENT_PAGEFAULT)  in uffd_move_handle_fault_common()
1092 		err("unexpected msg event %u", msg->event);  in uffd_move_handle_fault_common()
1094 	if (msg->arg.pagefault.flags &  in uffd_move_handle_fault_common()
1096 		err("unexpected fault type %llu", msg->arg.pagefault.flags);  in uffd_move_handle_fault_common()
1098 	offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;  in uffd_move_handle_fault_common()
1099 	offset &= ~(len-1);  in uffd_move_handle_fault_common()
1102 		args->missing_faults++;  in uffd_move_handle_fault_common()
1151 			src_offs = (aligned_src - area_src) / page_size;  in uffd_move_test_common()
1152 			dst_offs = (aligned_dst - area_dst) / page_size;  in uffd_move_test_common()
1153 			step_count--;  in uffd_move_test_common()
1162 	 * Read each of the pages back using the UFFD-registered mapping. We  in uffd_move_test_common()
1170 		/* Check area_src content */  in uffd_move_test_common()
1186 		/* Re-check area_src content which should be empty */  in uffd_move_test_common()
1239 			return -errno;  in prevent_hugepages()
1254 		return -errno;  in request_hugepages()
1272 do_register_ioctls_test(uffd_test_args_t *args, bool miss, bool wp, bool minor)  in do_register_ioctls_test()  argument
1275 	mem_type_t *mem_type = args->mem_type;  in do_register_ioctls_test()
1279 					miss, wp, minor, &ioctls);  in do_register_ioctls_test()
1283 	 * just fail with -EINVAL first..  in do_register_ioctls_test()
1288 	if ((minor && (mem_type->mem_flag == MEM_ANON)) ||  in do_register_ioctls_test()
1289 	    (!miss && !wp && !minor)) {  in do_register_ioctls_test()
1290 		if (ret != -EINVAL)  in do_register_ioctls_test()
1291 			err("register (miss=%d, wp=%d, minor=%d) failed "  in do_register_ioctls_test()
1292 			    "with wrong errno=%d", miss, wp, minor, ret);  in do_register_ioctls_test()
1299 	if (wp)  in do_register_ioctls_test()
1306 		    "(miss=%d, wp=%d, minor=%d): expected=0x%"PRIx64", "  in do_register_ioctls_test()
1307 		    "returned=0x%"PRIx64, miss, wp, minor, expected, ioctls);  in do_register_ioctls_test()
1315 	int miss, wp, minor;  in uffd_register_ioctls_test()  local
1318 		for (wp = 0; wp <= 1; wp++)  in uffd_register_ioctls_test()
1320 				do_register_ioctls_test(args, miss, wp, minor);  in uffd_register_ioctls_test()
1328 		.name = "register-ioctls",
1352 		.name = "move-pmd",
1359 		.name = "move-pmd-split",
1366 		.name = "wp-fork",
1373 		.name = "wp-fork-with-event",
1378 		/* when set, child process should inherit uffd-wp bits */
1382 		.name = "wp-fork-pin",
1389 		.name = "wp-fork-pin-with-event",
1394 		/* when set, child process should inherit uffd-wp bits */
1398 		.name = "wp-unpopulated",
1412 		.name = "minor-wp",
1420 		 * minor mode supports wr-protect.  There's no feature flag
1426 		.name = "minor-collapse",
1441 		.name = "sigbus-wp",
1456 		.name = "events-wp",
1474 	printf("usage: %s [-f TESTNAME]\n", prog);  in usage()
1476 	puts(" -f: test name to filter (e.g., event)");  in usage()
1477 	puts(" -h: show the help msg");  in usage()
1478 	puts(" -l: list tests only");  in usage()
1496 	while ((opt = getopt(argc, argv, "f:hl")) != -1) {  in main()
1524 		if (test_filter && !strstr(test->name, test_filter))  in main()
1527 			printf("%s\n", test->name);  in main()
1532 			if (!(test->mem_targets & mem_type->mem_flag))  in main()
1535 			uffd_test_start("%s on %s", test->name, mem_type->name);  in main()
1536 			if ((mem_type->mem_flag == MEM_HUGETLB ||  in main()
1537 			    mem_type->mem_flag == MEM_HUGETLB_PRIVATE) &&  in main()
1551 			test->uffd_fn(&args);  in main()