1 // SPDX-License-Identifier: GPL-2.0-only
7  * This test allocates two virtual areas and bounces the physical
11  * There are three threads running per CPU:
13  * 1) one per-CPU thread takes a per-page pthread_mutex in a random
15  *    area_src), and increments a per-page counter in the same page,
18  * 2) another per-CPU thread handles the userfaults generated by
22  * 3) one last per-CPU thread transfers the memory in the background
24  *    2). Each cpu thread takes cares of transferring a portion of the
32  * per-CPU threads 1 by triggering userfaults inside
37 #include "uffd-common.h"
47 /* defined globally for this particular test as the sigalrm handler
63 	"# Run anonymous memory test on 100MiB region with 99999 bounces:\n"
64 	"./uffd-stress anon 100 99999\n\n"
65 	"# Run share memory test on 1GiB region with 99 bounces:\n"
66 	"./uffd-stress shmem 1000 99\n\n"
67 	"# Run hugetlb memory test on 256MiB region with 50 bounces:\n"
68 	"./uffd-stress hugetlb 256 50\n\n"
69 	"# Run the same hugetlb test but using private file:\n"
70 	"./uffd-stress hugetlb-private 256 50\n\n"
71 	"# 10MiB-~6GiB 999 bounces anonymous test, "
73 	"while ./uffd-stress anon $[RANDOM % 6000 + 10] 999; do true; done\n\n";
77 	fprintf(stderr, "\nUsage: ./uffd-stress <test type> <MiB> <bounces>\n\n");  in usage()
78 	fprintf(stderr, "Supported <test type>: anon, hugetlb, "  in usage()
79 		"hugetlb-private, shmem, shmem-private\n\n");  in usage()
91 		args[i].cpu = i;  in uffd_stats_reset()
92 		args[i].apply_wp = gopts->test_uffdio_wp;  in uffd_stats_reset()
103 	uffd_global_test_opts_t *gopts = args->gopts;  in locking_thread()
104 	unsigned long cpu = (unsigned long) args->cpu;  in locking_thread()  local
109 		page_nr = -bounces;  in locking_thread()
111 			page_nr += cpu * gopts->nr_pages_per_cpu;  in locking_thread()
114 	while (!gopts->finished) {  in locking_thread()
120 		page_nr %= gopts->nr_pages;  in locking_thread()
121 		pthread_mutex_lock(area_mutex(gopts->area_dst, page_nr, gopts));  in locking_thread()
122 		count = *area_count(gopts->area_dst, page_nr, gopts);  in locking_thread()
123 		if (count != gopts->count_verify[page_nr])  in locking_thread()
125 			    page_nr, count, gopts->count_verify[page_nr]);  in locking_thread()
127 		*area_count(gopts->area_dst, page_nr, gopts) = gopts->count_verify[page_nr] = count;  in locking_thread()
128 		pthread_mutex_unlock(area_mutex(gopts->area_dst, page_nr, gopts));  in locking_thread()
136 	return __copy_page(gopts, offset, true, gopts->test_uffdio_wp);  in copy_page_retry()
144 	uffd_global_test_opts_t *gopts = args->gopts;  in uffd_read_thread()
162 	uffd_global_test_opts_t *gopts = args->gopts;  in background_thread()
163 	unsigned long cpu = (unsigned long) args->cpu;  in background_thread()  local
166 	start_nr = cpu * gopts->nr_pages_per_cpu;  in background_thread()
167 	end_nr = (cpu+1) * gopts->nr_pages_per_cpu;  in background_thread()
172 		copy_page_retry(gopts, page_nr * gopts->page_size);  in background_thread()
175 	 * If we need to test uffd-wp, set it up now.  Then we'll have  in background_thread()
177 	 * can be write-protected for testing  in background_thread()
179 	if (gopts->test_uffdio_wp)  in background_thread()
180 		wp_range(gopts->uffd, (unsigned long)gopts->area_dst + start_nr * gopts->page_size,  in background_thread()
181 			gopts->nr_pages_per_cpu * gopts->page_size, true);  in background_thread()
188 		copy_page_retry(gopts, page_nr * gopts->page_size);  in background_thread()
195 	unsigned long cpu;  in stress()  local
196 	uffd_global_test_opts_t *gopts = args->gopts;  in stress()
197 	pthread_t locking_threads[gopts->nr_parallel];  in stress()
198 	pthread_t uffd_threads[gopts->nr_parallel];  in stress()
199 	pthread_t background_threads[gopts->nr_parallel];  in stress()
201 	gopts->finished = 0;  in stress()
202 	for (cpu = 0; cpu < gopts->nr_parallel; cpu++) {  in stress()
203 		if (pthread_create(&locking_threads[cpu], &attr,  in stress()
204 				   locking_thread, (void *)&args[cpu]))  in stress()
207 			if (pthread_create(&uffd_threads[cpu],  in stress()
210 					   (void *) &args[cpu]))  in stress()
213 			if (pthread_create(&uffd_threads[cpu], &attr,  in stress()
215 					   (void *)&args[cpu]))  in stress()
219 		if (pthread_create(&background_threads[cpu], &attr,  in stress()
220 				   background_thread, (void *)&args[cpu]))  in stress()
223 	for (cpu = 0; cpu < gopts->nr_parallel; cpu++)  in stress()
224 		if (pthread_join(background_threads[cpu], NULL))  in stress()
232 	 * area_src (but they're guaranteed to get -EEXIST from  in stress()
236 	uffd_test_ops->release_pages(gopts, gopts->area_src);  in stress()
238 	gopts->finished = 1;  in stress()
239 	for (cpu = 0; cpu < gopts->nr_parallel; cpu++)  in stress()
240 		if (pthread_join(locking_threads[cpu], NULL))  in stress()
243 	for (cpu = 0; cpu < gopts->nr_parallel; cpu++) {  in stress()
246 			if (write(gopts->pipefd[cpu*2+1], &c, 1) != 1)  in stress()
248 			if (pthread_join(uffd_threads[cpu],  in stress()
249 					 (void *)&args[cpu]))  in stress()
252 			if (pthread_cancel(uffd_threads[cpu]))  in stress()
254 			if (pthread_join(uffd_threads[cpu], NULL))  in stress()
266 	struct uffd_args args[gopts->nr_parallel];  in userfaultfd_stress()
267 	uint64_t mem_size = gopts->nr_pages * gopts->page_size;  in userfaultfd_stress()
270 	memset(args, 0, sizeof(struct uffd_args) * gopts->nr_parallel);  in userfaultfd_stress()
272 	if (features & UFFD_FEATURE_WP_UNPOPULATED && gopts->test_type == TEST_ANON)  in userfaultfd_stress()
278 	if (posix_memalign(&area, gopts->page_size, gopts->page_size))  in userfaultfd_stress()
281 	bzero(zeropage, gopts->page_size);  in userfaultfd_stress()
288 	while (bounces--) {  in userfaultfd_stress()
304 			fcntl(gopts->uffd, F_SETFL, gopts->uffd_flags | O_NONBLOCK);  in userfaultfd_stress()
306 			fcntl(gopts->uffd, F_SETFL, gopts->uffd_flags & ~O_NONBLOCK);  in userfaultfd_stress()
309 		if (uffd_register(gopts->uffd, gopts->area_dst, mem_size,  in userfaultfd_stress()
310 				  true, gopts->test_uffdio_wp, false))  in userfaultfd_stress()
313 		if (gopts->area_dst_alias) {  in userfaultfd_stress()
314 			if (uffd_register(gopts->uffd, gopts->area_dst_alias, mem_size,  in userfaultfd_stress()
315 					  true, gopts->test_uffdio_wp, false))  in userfaultfd_stress()
327 		 * return -EEXIST). The problem comes at the next  in userfaultfd_stress()
332 		 * area_src would lead to -EEXIST failure during the  in userfaultfd_stress()
343 		uffd_test_ops->release_pages(gopts, gopts->area_dst);  in userfaultfd_stress()
345 		uffd_stats_reset(gopts, args, gopts->nr_parallel);  in userfaultfd_stress()
354 		if (gopts->test_uffdio_wp)  in userfaultfd_stress()
355 			wp_range(gopts->uffd, (unsigned long)gopts->area_dst,  in userfaultfd_stress()
356 				 gopts->nr_pages * gopts->page_size, false);  in userfaultfd_stress()
359 		if (uffd_unregister(gopts->uffd, gopts->area_dst, mem_size))  in userfaultfd_stress()
361 		if (gopts->area_dst_alias) {  in userfaultfd_stress()
362 			if (uffd_unregister(gopts->uffd, gopts->area_dst_alias, mem_size))  in userfaultfd_stress()
368 			for (nr = 0; nr < gopts->nr_pages; nr++)  in userfaultfd_stress()
369 				if (*area_count(gopts->area_dst, nr, gopts) !=  in userfaultfd_stress()
370 						gopts->count_verify[nr])  in userfaultfd_stress()
372 					    *area_count(gopts->area_src, nr, gopts),  in userfaultfd_stress()
373 					    gopts->count_verify[nr], nr);  in userfaultfd_stress()
376 		swap(gopts->area_src, gopts->area_dst);  in userfaultfd_stress()
378 		swap(gopts->area_src_alias, gopts->area_dst_alias);  in userfaultfd_stress()
380 		uffd_stats_report(args, gopts->nr_parallel);  in userfaultfd_stress()
390 		gopts->test_type = TEST_ANON;  in set_test_type()
393 		gopts->test_type = TEST_HUGETLB;  in set_test_type()
395 		gopts->map_shared = true;  in set_test_type()
396 	} else if (!strcmp(type, "hugetlb-private")) {  in set_test_type()
397 		gopts->test_type = TEST_HUGETLB;  in set_test_type()
400 		gopts->map_shared = true;  in set_test_type()
401 		gopts->test_type = TEST_SHMEM;  in set_test_type()
403 	} else if (!strcmp(type, "shmem-private")) {  in set_test_type()
404 		gopts->test_type = TEST_SHMEM;  in set_test_type()
413 	if (!gopts->test_type)  in parse_test_type_arg()
414 		err("failed to parse test type argument: '%s'", raw_type);  in parse_test_type_arg()
416 	if (gopts->test_type == TEST_HUGETLB)  in parse_test_type_arg()
417 		gopts->page_size = default_huge_page_size();  in parse_test_type_arg()
419 		gopts->page_size = sysconf(_SC_PAGE_SIZE);  in parse_test_type_arg()
421 	if (!gopts->page_size)  in parse_test_type_arg()
424 	    > gopts->page_size)  in parse_test_type_arg()
425 		err("Impossible to run this test");  in parse_test_type_arg()
428 	 * Whether we can test certain features depends not just on test type,  in parse_test_type_arg()
436 	gopts->test_uffdio_wp = gopts->test_uffdio_wp &&  in parse_test_type_arg()
439 	if (gopts->test_type != TEST_ANON && !(features & UFFD_FEATURE_WP_HUGETLBFS_SHMEM))  in parse_test_type_arg()
440 		gopts->test_uffdio_wp = false;  in parse_test_type_arg()
442 	close(gopts->uffd);  in parse_test_type_arg()
443 	gopts->uffd = -1;  in parse_test_type_arg()
450 	gopts->test_uffdio_copy_eexist = true;  in sigalrm()
476 		gopts->nr_parallel = 32;  in main()
478 		gopts->nr_parallel = nr_cpus;  in main()
483 	 * Ensure nr_parallel - 1 hugepages on top of that to account  in main()
486 	if (gopts->test_type == TEST_HUGETLB &&  in main()
487 	   get_free_hugepages() < 2 * (bytes / gopts->page_size) + gopts->nr_parallel - 1) {  in main()
492 	gopts->nr_pages_per_cpu = bytes / gopts->page_size / gopts->nr_parallel;  in main()
493 	if (!gopts->nr_pages_per_cpu) {  in main()
494 		_err("pages_per_cpu = 0, cannot test (%lu / %lu / %lu)",  in main()
495 			bytes, gopts->page_size, gopts->nr_parallel);  in main()
504 	gopts->nr_pages = gopts->nr_pages_per_cpu * gopts->nr_parallel;  in main()
507 	       gopts->nr_pages, gopts->nr_pages_per_cpu);  in main()
517 	printf("skip: Skipping userfaultfd test (missing __NR_userfaultfd)\n");  in main()